From arigo at codespeak.net Sun Nov 1 08:55:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 08:55:50 +0100 (CET) Subject: [pypy-svn] r68882 - pypy/branch/logging2/pypy/jit/metainterp Message-ID: <20091101075550.405E1168079@codespeak.net> Author: arigo Date: Sun Nov 1 08:55:49 2009 New Revision: 68882 Modified: pypy/branch/logging2/pypy/jit/metainterp/compile.py pypy/branch/logging2/pypy/jit/metainterp/optimize.py pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py pypy/branch/logging2/pypy/jit/metainterp/warmstate.py Log: Add debug_start/debug_stop events. This duplicates the job done by jitprof, which should eventually be removed, but it's a bit of a mess right now. Modified: pypy/branch/logging2/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/compile.py Sun Nov 1 08:55:49 2009 @@ -2,6 +2,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import debug_start, debug_stop from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -92,11 +93,16 @@ globaldata.loopnumbering += 1 metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd, loop) loop.check_consistency() - metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, loop.token) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, + loop.token) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): @@ -109,12 +115,16 @@ def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): n = faildescr.get_index() metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd) TreeLoop.check_consistency_of(inputargs, operations) pass - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if not we_are_translated(): metainterp_sd.stats.compiled() Modified: pypy/branch/logging2/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/optimize.py Sun Nov 1 08:55:49 2009 @@ -1,3 +1,5 @@ +from pypy.rlib.debug import debug_start, debug_stop + # ____________________________________________________________ from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder @@ -5,6 +7,13 @@ from pypy.jit.metainterp.specnode import equals_specnodes def optimize_loop(metainterp_sd, old_loop_tokens, loop): + debug_start("jit-optimize") + try: + return _optimize_loop(metainterp_sd, old_loop_tokens, loop) + finally: + debug_stop("jit-optimize") + +def _optimize_loop(metainterp_sd, old_loop_tokens, loop): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) finder = PerfectSpecializationFinder(cpu) @@ -21,6 +30,13 @@ from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 def optimize_bridge(metainterp_sd, old_loop_tokens, bridge): + debug_start("jit-optimize") + try: + return _optimize_bridge(metainterp_sd, old_loop_tokens, bridge) + finally: + debug_stop("jit-optimize") + +def _optimize_bridge(metainterp_sd, old_loop_tokens, bridge): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(bridge.inputargs, bridge.operations) finder = BridgeSpecializationFinder(cpu) Modified: pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/pyjitpl.py Sun Nov 1 08:55:49 2009 @@ -1018,9 +1018,9 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: + debug_print(self.jit_starting_line) self._setup_class_sizes() self.cpu.setup_once() - debug_print(self.jit_starting_line) if not self.profiler.initialized: self.profiler.start() self.profiler.initialized = True @@ -1365,8 +1365,8 @@ raise def compile_and_run_once(self, *args): - self.staticdata._setup_once() debug_start('jit-tracing') + self.staticdata._setup_once() self.create_empty_history() try: return self._compile_and_run_once(*args) Modified: pypy/branch/logging2/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/warmstate.py Sun Nov 1 08:55:49 2009 @@ -10,6 +10,7 @@ from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell +from pypy.rlib.debug import debug_start, debug_stop from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -229,7 +230,9 @@ # ---------- execute assembler ---------- while True: # until interrupted by an exception metainterp_sd.profiler.start_running() + debug_start("jit-running") fail_index = metainterp_sd.cpu.execute_token(loop_token) + debug_stop("jit-running") metainterp_sd.profiler.end_running() fail_descr = globaldata.get_fail_descr_from_number(fail_index) loop_token = fail_descr.handle_fail(metainterp_sd) From arigo at codespeak.net Sun Nov 1 10:46:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 10:46:45 +0100 (CET) Subject: [pypy-svn] r68883 - pypy/branch/logging2/pypy/translator/c/gcc Message-ID: <20091101094645.5308E168077@codespeak.net> Author: arigo Date: Sun Nov 1 10:46:44 2009 New Revision: 68883 Modified: pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py Log: Add an assert. Modified: pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py Sun Nov 1 10:46:44 2009 @@ -1297,6 +1297,7 @@ f = open(fn, 'r') firstline = f.readline() f.seek(0) + assert firstline, "file %r is empty!" % (fn,) if firstline.startswith('seen_main = '): tracker.reload_raw_table(f) f.close() From arigo at codespeak.net Sun Nov 1 10:50:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 10:50:01 +0100 (CET) Subject: [pypy-svn] r68884 - pypy/branch/logging2/pypy/translator/c/gcc Message-ID: <20091101095001.2C63B168077@codespeak.net> Author: arigo Date: Sun Nov 1 10:50:00 2009 New Revision: 68884 Modified: pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py Log: Add an instruction. Modified: pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/logging2/pypy/translator/c/gcc/trackgcroot.py Sun Nov 1 10:50:00 2009 @@ -588,7 +588,7 @@ # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', - 'bswap', 'bt', + 'bswap', 'bt', 'rdtsc', # zero-extending moves should not produce GC pointers 'movz', ]) From arigo at codespeak.net Sun Nov 1 11:15:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 11:15:20 +0100 (CET) Subject: [pypy-svn] r68885 - in pypy/branch/logging2/pypy/tool: . test Message-ID: <20091101101520.90DF8168077@codespeak.net> Author: arigo Date: Sun Nov 1 11:15:20 2009 New Revision: 68885 Added: pypy/branch/logging2/pypy/tool/test/test_logparser.py (contents, props changed) Modified: pypy/branch/logging2/pypy/tool/logparser.py Log: Incorporate the JIT in the numbers, and display summary histograms. Modified: pypy/branch/logging2/pypy/tool/logparser.py ============================================================================== --- pypy/branch/logging2/pypy/tool/logparser.py (original) +++ pypy/branch/logging2/pypy/tool/logparser.py Sun Nov 1 11:15:20 2009 @@ -33,14 +33,65 @@ def getsubcategories(log): return [entry for entry in log if entry[0] != 'debug_print'] +def gettimebounds(log): + # returns (mintime, maxtime) + maincats = getsubcategories(log) + return (maincats[0][1], maincats[-1][2]) + +def gettotaltimes(log): + # returns a dict {'label' or None: totaltime} + def rectime(category1, timestart1, timestop1, subcats): + substartstop = [] + for entry in getsubcategories(subcats): + rectime(*entry) + substartstop.append(entry[1:3]) # (start, stop) + # compute the total time for category1 as the part of the + # interval [timestart1, timestop1] that is not covered by + # any interval from one of the subcats. + mytime = 0 + substartstop.sort() + for substart, substop in substartstop: + if substart >= timestop1: + break + if substart > timestart1: + mytime += substart - timestart1 + if timestart1 < substop: + timestart1 = substop + if timestart1 < timestop1: + mytime += timestop1 - timestart1 + # + try: + result[category1] += mytime + except KeyError: + result[category1] = mytime + # + result = {} + timestart0, timestop0 = gettimebounds(log) + rectime(None, timestart0, timestop0, log) + return result + # ____________________________________________________________ COLORS = { + None: (248, 248, 248), '': (160, 160, 160), - 'gc-': (192, 0, 64), + 'gc-': (224, 0, 0), 'gc-minor': (192, 0, 16), 'gc-collect': (255, 0, 0), + 'jit-': (0, 224, 0), + 'jit-running': (192, 255, 160), + 'jit-tracing': (0, 255, 0), + 'jit-optimize': (160, 255, 0), + 'jit-backend': (0, 255, 144), + 'jit-blackhole': (0, 160, 0), + } +SUMMARY = { + None: 'normal execution', + '': 'other', + 'gc-': 'gc', + 'jit-': 'jit', + 'jit-running': 'jit-running', } def getcolor(category): @@ -73,14 +124,25 @@ _cache[text] = sx, sy, texthoriz, textvert return _cache[text] +def bevelrect(draw, (x1, y1, x2, y2), color): + if x2 <= x1: + x2 = x1 + 1 # minimal width + elif x2 >= x1 + 4: + draw.line((x1, y1+1, x1, y2-1), fill=getlightercolor(color)) + x1 += 1 + x2 -= 1 + draw.line((x2, y1+1, x2, y2-1), fill=getdarkercolor(color)) + draw.line((x1, y1, x2-1, y1), fill=getlightercolor(color)) + y1 += 1 + y2 -= 1 + draw.line((x1, y2, x2-1, y2), fill=getdarkercolor(color)) + draw.rectangle((x1, y1, x2-1, y2-1), fill=color) + +# ---------- -def get_timeline_image(log, width=900, height=150): +def get_timeline_image(log, width, height): from PIL import Image, ImageDraw - width = int(width) - height = int(height) - maincats = getsubcategories(log) - timestart0 = maincats[0][1] - timestop0 = maincats[-1][2] + timestart0, timestop0 = gettimebounds(log) assert timestop0 > timestart0 timefactor = float(width) / (timestop0 - timestart0) # @@ -91,21 +153,13 @@ x2 = int((timestop1 - timestart0) * timefactor) y1 = (height - subheight) / 2 y2 = y1 + subheight + y1 = int(y1) + y2 = int(y2) color = getcolor(category1) if firstx1 is None: firstx1 = x1 - if x2 <= x1: - x2 = x1 + 1 # minimal width - elif x2 >= x1 + 4: - draw.line([x1, y1+1, x1, y2-1], fill=getlightercolor(color)) - x1 += 1 - x2 -= 1 - draw.line([x2, y1+1, x2, y2-1], fill=getdarkercolor(color)) - draw.line([x1, y1, x2-1, y1], fill=getlightercolor(color)) - y1 += 1 - y2 -= 1 - draw.line([x1, y2, x2-1, y2], fill=getdarkercolor(color)) - draw.rectangle([x1, y1, x2-1, y2-1], fill=color) + bevelrect(draw, (x1, y1, x2, y2), color) + subcats = getsubcategories(subcats) if subcats: x2 = recdraw(subcats, subheight * 0.94) - 1 sx, sy, texthoriz, textvert = getlabel(category1) @@ -115,13 +169,131 @@ image.paste(textvert, (x1+1, y1+5), textvert) return firstx1 # - image = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + image = Image.new("RGBA", (width, height), (255, 255, 255, 0)) draw = ImageDraw.Draw(image) - recdraw(maincats, height) + recdraw(getsubcategories(log), height) return image -def draw_timeline_image(log, output=None, **kwds): - image = get_timeline_image(log, **kwds) +# ---------- + +def render_histogram(times, time0, labels, width, barheight): + # Render a histogram showing horizontal time bars are given by the + # 'times' dictionary. Each entry has the label specified by 'labels', + # or by default the key used in 'times'. + from PIL import Image, ImageDraw + times = [(time, key) for (key, time) in times.items()] + times.sort() + times.reverse() + images = [] + for time, key in times: + fraction = float(time) / time0 + if fraction < 0.01: + break + color = getcolor(key) + image = Image.new("RGBA", (width, barheight), (255, 255, 255, 0)) + draw = ImageDraw.Draw(image) + x2 = int(fraction * width) + bevelrect(draw, (0, 0, x2, barheight), color) + # draw the labels "x%" and "key" + percent = "%.1f%%" % (100.0 * fraction,) + s1x, s1y, textpercent, vtextpercent = getlabel(percent) + s2x, _, textlabel, _ = getlabel(labels.get(key, key)) + t1x = 5 + if t1x + s1x >= x2 - 3: + if t1x + s1y < x2 - 3: + textpercent = vtextpercent + s1x = s1y + else: + t1x = x2 + 6 + t2x = t1x + s1x + 12 + if t2x + s2x >= x2 - 3: + t2x = max(t2x, x2 + 8) + image.paste(textpercent, (t1x, 5), textpercent) + image.paste(textlabel, (t2x, 5), textlabel) + images.append(image) + return combine(images, spacing=0, border=1, horizontal=False) + +def get_timesummary_single_image(totaltimes, totaltime0, componentdict, + width, barheight): + # Compress the totaltimes dict so that its only entries left are + # from componentdict. We do that by taking the times assigned to + # subkeys in totaltimes and adding them to the superkeys specified + # in componentdict. + totaltimes = totaltimes.copy() + for key, value in totaltimes.items(): + if key in componentdict: + continue + del totaltimes[key] + if key is not None: + while key not in componentdict: + key = key[:-1] + try: + totaltimes[key] += value + except KeyError: + totaltimes[key] = value + return render_histogram(totaltimes, totaltime0, componentdict, + width, barheight) + +def get_timesummary_image(log, summarywidth, summarybarheight): + timestart0, timestop0 = gettimebounds(log) + totaltime0 = timestop0 - timestart0 + totaltimes = gettotaltimes(log) + spacing = 50 + width = (summarywidth - spacing) // 2 + img1 = get_timesummary_single_image(totaltimes, totaltime0, SUMMARY, + width, summarybarheight) + if None in totaltimes: + del totaltimes[None] + img2 = render_histogram(totaltimes, totaltime0, {}, + width, summarybarheight) + return combine([img1, img2], spacing=spacing, horizontal=True) + +# ---------- + +def combine(imagelist, spacing=50, border=0, horizontal=False): + if len(imagelist) <= 1 and not border: + return imagelist[0] + from PIL import Image, ImageDraw + wlist = [image.size[0] for image in imagelist] + hlist = [image.size[1] for image in imagelist] + if horizontal: + w = sum(wlist) + spacing*(len(imagelist)-1) + h = max(hlist) + else: + w = max(wlist) + h = sum(hlist) + spacing*(len(imagelist)-1) + w += 2*border + h += 2*border + bigimage = Image.new("RGBA", (w, h), (255, 255, 255, 0)) + if border: + draw = ImageDraw.Draw(bigimage) + draw.rectangle((0, 0, w-1, border-1), fill=(0, 0, 0)) + draw.rectangle((0, h-border, w-1, h-1), fill=(0, 0, 0)) + draw.rectangle((0, 0, border-1, h-1), fill=(0, 0, 0)) + draw.rectangle((w-1, 0, w-border, h-1), fill=(0, 0, 0)) + x = border + y = border + for image in imagelist: + bigimage.paste(image, (x, y)) + if horizontal: + x += image.size[0] + spacing + else: + y += image.size[1] + spacing + return bigimage + +def draw_timeline_image(log, output=None, mainwidth=3000, mainheight=150, + summarywidth=850, summarybarheight=40): + mainwidth = int(mainwidth) + mainheight = int(mainheight) + summarywidth = int(summarywidth) + summarybarheight = int(summarybarheight) + images = [] + if mainwidth > 0 and mainheight > 0: + images.append(get_timeline_image(log, mainwidth, mainheight)) + if summarywidth > 0 and summarybarheight > 0: + images.append(get_timesummary_image(log, summarywidth, + summarybarheight)) + image = combine(images, horizontal=False) if output is None: image.save(sys.stdout, format='png') else: @@ -131,7 +303,10 @@ ACTIONS = { - 'draw-time': (draw_timeline_image, ['width=', 'height=', 'output=']), + 'draw-time': (draw_timeline_image, ['output=', + 'mainwidth=', 'mainheight=', + 'summarywidth=', 'summarybarheight=', + ]), } if __name__ == '__main__': Added: pypy/branch/logging2/pypy/tool/test/test_logparser.py ============================================================================== --- (empty file) +++ pypy/branch/logging2/pypy/tool/test/test_logparser.py Sun Nov 1 11:15:20 2009 @@ -0,0 +1,13 @@ +from pypy.tool.logparser import * + +def test_gettotaltimes(): + result = gettotaltimes([ + ('foo', 2, 17, [ + ('bar', 4, 5, []), + ('bar', 7, 9, []), + ]), + ('bar', 20, 30, []), + ]) + assert result == {None: 3, # the hole between 17 and 20 + 'foo': 15 - 1 - 2, + 'bar': 1 + 2 + 10} From arigo at codespeak.net Sun Nov 1 14:41:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 14:41:12 +0100 (CET) Subject: [pypy-svn] r68887 - in pypy/branch/logging2/pypy/jit/metainterp: . test Message-ID: <20091101134112.24ED9168020@codespeak.net> Author: arigo Date: Sun Nov 1 14:41:11 2009 New Revision: 68887 Modified: pypy/branch/logging2/pypy/jit/metainterp/jitprof.py pypy/branch/logging2/pypy/jit/metainterp/test/test_jitprof.py Log: Disable timing the "Running asm" and "Blackhole" parts. Thanks mwhudson for leading me to notice that it actually makes a serious slowdown, particularly on virtual machines, to call time.time() extremely often. This is checked in in the logger2 branch because the same info can still be obtained via logging if needed. Modified: pypy/branch/logging2/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/jitprof.py Sun Nov 1 14:41:11 2009 @@ -91,7 +91,7 @@ def start(self): self.starttime = self.timer() self.t1 = self.starttime - self.times = [0, 0, 0, 0] + self.times = [0, 0] self.counters = [0] * ncounters self.calls = [[0, 0], [0, 0], [0, 0]] self.current = [] @@ -130,11 +130,11 @@ def start_backend(self): self._start(BACKEND) def end_backend(self): self._end (BACKEND) - def start_running(self): self._start(RUNNING) - def end_running(self): self._end (RUNNING) + def start_running(self): self.count(RUNNING) + def end_running(self): pass - def start_blackhole(self): self._start(BLACKHOLE) - def end_blackhole(self): self._end (BLACKHOLE) + def start_blackhole(self): self.count(BLACKHOLE) + def end_blackhole(self): pass def count(self, kind, inc=1): self.counters[kind] += inc @@ -153,8 +153,8 @@ calls = self.calls self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) - self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) - self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) + self._print_intline("Running asm", cnt[RUNNING]) + self._print_intline("Blackhole", cnt[BLACKHOLE]) line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) os.write(2, line) self._print_intline("ops", cnt[OPS]) Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_jitprof.py Sun Nov 1 14:41:11 2009 @@ -6,9 +6,11 @@ from pypy.jit.metainterp.jitprof import * class FakeProfiler(Profiler): - def __init__(self): + def start(self): self.counter = 123456 + Profiler.start(self) self.events = [] + self.times = [0, 0, 0, 0] def timer(self): self.counter += 1 @@ -22,6 +24,12 @@ Profiler._end(self, event) self.events.append(~event) + def start_running(self): self._start(RUNNING) + def end_running(self): self._end(RUNNING) + + def start_blackhole(self): self._start(BLACKHOLE) + def end_blackhole(self): self._end(BLACKHOLE) + class ProfilerMixin(LLJitMixin): def meta_interp(self, *args, **kwds): kwds = kwds.copy() From arigo at codespeak.net Sun Nov 1 15:09:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 15:09:16 +0100 (CET) Subject: [pypy-svn] r68888 - in pypy/branch/logging2/pypy/translator: c/src platform Message-ID: <20091101140916.0454F168078@codespeak.net> Author: arigo Date: Sun Nov 1 15:09:16 2009 New Revision: 68888 Modified: pypy/branch/logging2/pypy/translator/c/src/asm_gcc_x86.h pypy/branch/logging2/pypy/translator/c/src/debug.h pypy/branch/logging2/pypy/translator/platform/linux.py Log: Port READ_TIMESTAMP to other platforms. Modified: pypy/branch/logging2/pypy/translator/c/src/asm_gcc_x86.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/asm_gcc_x86.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/asm_gcc_x86.h Sun Nov 1 15:09:16 2009 @@ -50,6 +50,10 @@ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ +/* Pentium only! */ +#define READ_TIMESTAMP(val) \ + asm volatile("rdtsc" : "=A" (val)) + /* prototypes */ Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Sun Nov 1 15:09:16 2009 @@ -99,9 +99,24 @@ } -/* XXXXXXXXXX x86 Pentium only! */ -#define READ_TIMESTAMP(val) \ - __asm__ __volatile__("rdtsc" : "=A" (val)) +#ifndef READ_TIMESTAMP +/* asm_xxx.h may contain a specific implementation of READ_TIMESTAMP. + * This is the default generic timestamp implementation. + */ +# ifdef WIN32 +# define READ_TIMESTAMP(val) QueryPerformanceCounter(&(val)) +# else +# include +# define READ_TIMESTAMP(val) (val) = pypy_read_timestamp() + + static long long pypy_read_timestamp(void) + { + struct timespec tspec; + clock_gettime(CLOCK_MONOTONIC, &tspec); + return ((long long)tspec.tv_sec) * 1000000000LL + tspec.tv_nsec; + } +# endif +#endif static bool_t startswith(const char *str, const char *substr) Modified: pypy/branch/logging2/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/logging2/pypy/translator/platform/linux.py (original) +++ pypy/branch/logging2/pypy/translator/platform/linux.py Sun Nov 1 15:09:16 2009 @@ -8,7 +8,7 @@ class Linux(BasePosix): name = "linux" - link_flags = ['-pthread'] + link_flags = ['-pthread', '-lrt'] cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] standalone_only = [] shared_only = [] From arigo at codespeak.net Sun Nov 1 15:35:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 15:35:03 +0100 (CET) Subject: [pypy-svn] r68889 - pypy/branch/logging2/pypy/jit/metainterp/test Message-ID: <20091101143503.B9710168075@codespeak.net> Author: arigo Date: Sun Nov 1 15:35:03 2009 New Revision: 68889 Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_loop.py Log: Remove this test which no longer makes sense. Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_loop.py Sun Nov 1 15:35:03 2009 @@ -594,25 +594,6 @@ res = self.meta_interp(f, [200]) - def test_dump_storage(self): - import os - from pypy.tool.udir import udir - logfile = udir.join('resume.log') - os.environ['PYPYJITRESUMELOG'] = str(logfile) - try: - jitdriver = JitDriver(greens = [], reds = ['i', 'n']) - - def f(n): - i = 0 - while i < n: - jitdriver.can_enter_jit(n=n, i=i) - jitdriver.jit_merge_point(n=n, i=i) - i += 1 - return i - res = self.meta_interp(f, [10]) - data = logfile.read() # assert did not crash - finally: - del os.environ['PYPYJITRESUMELOG'] class TestOOtype(LoopTest, OOJitMixin): pass From arigo at codespeak.net Sun Nov 1 15:35:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 15:35:09 +0100 (CET) Subject: [pypy-svn] r68890 - pypy/branch/logging2/pypy/jit/metainterp/test Message-ID: <20091101143509.06115168079@codespeak.net> Author: arigo Date: Sun Nov 1 15:35:09 2009 New Revision: 68890 Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_logger.py Log: Fix tests. Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_logger.py Sun Nov 1 15:35:09 2009 @@ -1,4 +1,5 @@ - +import sys +from pypy.rlib import debug from pypy.jit.metainterp.test.oparser import pure_parse from pypy.jit.metainterp import logger from pypy.jit.metainterp.typesystem import llhelper @@ -9,12 +10,20 @@ class Descr(AbstractDescr): pass +def capturing(func, *args, **kwds): + log_stream = StringIO() + debug._stderr = log_stream + try: + func(*args, **kwds) + finally: + debug._stderr = sys.stderr + return log_stream.getvalue() + class Logger(logger.Logger): def log_loop(self, loop, namespace={}): - self.log_stream = StringIO() self.namespace = namespace - logger.Logger.log_loop(self, loop.inputargs, loop.operations) - return self.log_stream.getvalue() + return capturing(logger.Logger.log_loop, self, + loop.inputargs, loop.operations) def repr_of_descr(self, descr): for k, v in self.namespace.items(): @@ -116,16 +125,12 @@ def test_intro_loop(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_loop([], [], 1, "foo") - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# Loop1 (foo), 0 ops" + output = capturing(bare_logger.log_loop, [], [], 1, "foo") + assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_bridge([], [], 3) - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# bridge out of Guard3, 0 ops" + output = capturing(bare_logger.log_bridge, [], [], 3) + assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) From arigo at codespeak.net Sun Nov 1 16:18:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 16:18:59 +0100 (CET) Subject: [pypy-svn] r68891 - pypy/branch/logging2/pypy/jit/metainterp/test Message-ID: <20091101151859.79959168076@codespeak.net> Author: arigo Date: Sun Nov 1 16:18:58 2009 New Revision: 68891 Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_virtual.py pypy/branch/logging2/pypy/jit/metainterp/test/test_warmspot.py Log: Fix a test. Skip two tests because they are testing details that changed about a feature that is soon obsolete anyway. Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_virtual.py Sun Nov 1 16:18:58 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.lltypesystem import lltype, rclass @@ -123,7 +124,8 @@ def test_two_loops_with_escaping_virtual(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) def externfn(node): - llop.debug_print(lltype.Void, node) + llop.debug_print(lltype.Void, compute_unique_id(node), + node.value, node.extra) return node.value * 2 def f(n): node = self._new() Modified: pypy/branch/logging2/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/logging2/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/logging2/pypy/jit/metainterp/test/test_warmspot.py Sun Nov 1 16:18:58 2009 @@ -131,6 +131,7 @@ assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge def test_static_debug_level(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -177,6 +178,7 @@ assert not "Running asm" in err def test_set_param_debug(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler From arigo at codespeak.net Sun Nov 1 16:38:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 16:38:22 +0100 (CET) Subject: [pypy-svn] r68892 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091101153822.CC51A168078@codespeak.net> Author: arigo Date: Sun Nov 1 16:38:22 2009 New Revision: 68892 Modified: pypy/trunk/pypy/jit/metainterp/test/test_vlist.py Log: Remove the py.test.xfail and replace it with a skip. It took me a while to figure out that the test is not actually supposed to completely fail, but just the check_loops() at the end. The construction works fine, it's just that it needs a residual call so far. Modified: pypy/trunk/pypy/jit/metainterp/test/test_vlist.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_vlist.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_vlist.py Sun Nov 1 16:38:22 2009 @@ -97,9 +97,9 @@ assert res == f(10) self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0) - @py.test.mark.xfail def test_vlist_alloc_and_set(self): - # this fails, because [non-null] * n is not supported yet + # the check_loops fails, because [non-null] * n is not supported yet + # (it is implemented as a residual call) jitdriver = JitDriver(greens = [], reds = ['n']) def f(n): l = [1] * 20 @@ -116,6 +116,7 @@ res = self.meta_interp(f, [10], listops=True) assert res == f(10) + py.test.skip("'[non-null] * n' gives a residual call so far") self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0) def test_append_pop(self): From arigo at codespeak.net Sun Nov 1 17:12:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 17:12:32 +0100 (CET) Subject: [pypy-svn] r68893 - in pypy/branch/logging2/pypy: jit/backend/test rlib translator/cli Message-ID: <20091101161232.ACFB1168075@codespeak.net> Author: arigo Date: Sun Nov 1 17:12:32 2009 New Revision: 68893 Modified: pypy/branch/logging2/pypy/jit/backend/test/support.py pypy/branch/logging2/pypy/rlib/debug.py pypy/branch/logging2/pypy/translator/cli/metavm.py pypy/branch/logging2/pypy/translator/cli/opcodes.py Log: Tweaks to appear to make the CLI JIT tests pass. Modified: pypy/branch/logging2/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/logging2/pypy/jit/backend/test/support.py (original) +++ pypy/branch/logging2/pypy/jit/backend/test/support.py Sun Nov 1 17:12:32 2009 @@ -1,6 +1,6 @@ import py import sys -from pypy.jit.metainterp.history import log +from pypy.rlib.debug import debug_print from pypy.translator.translator import TranslationContext class BaseCompiledMixin(object): @@ -119,10 +119,10 @@ cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() exe_name = cbuilder.compile() - log('---------- Test starting ----------') + debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) res = int(stdout) - log('---------- Test done (%d) ----------' % (res,)) + debug_print('---------- Test done (%d) ----------' % (res,)) return res class CliCompiledMixin(BaseCompiledMixin): Modified: pypy/branch/logging2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/logging2/pypy/rlib/debug.py (original) +++ pypy/branch/logging2/pypy/rlib/debug.py Sun Nov 1 17:12:32 2009 @@ -45,11 +45,13 @@ _log = None # patched from tests to be an object of class DebugLog # or compatible +_stderr = sys.stderr # alternatively, this is patched from tests + # (redirects debug_print(), but not debug_start/stop) def debug_print(*args): for arg in args: - print >> sys.stderr, arg, - print >> sys.stderr + print >> _stderr, arg, + print >> _stderr if _log is not None: _log.debug_print(*args) Modified: pypy/branch/logging2/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/logging2/pypy/translator/cli/metavm.py (original) +++ pypy/branch/logging2/pypy/translator/cli/metavm.py Sun Nov 1 17:12:32 2009 @@ -270,6 +270,10 @@ generator.ilasm.call('void [pypylib]pypy.runtime.Utils::debug_print(%s)' % signature) +class _HaveDebugPrints(MicroInstruction): + def render(self, generator, op): + generator.ilasm.load_const(ootype.Bool, True) + OOTYPE_TO_MNEMONIC = { ootype.Bool: 'i1', @@ -306,3 +310,4 @@ SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() DebugPrint = _DebugPrint() +HaveDebugPrints = _HaveDebugPrints() Modified: pypy/branch/logging2/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/logging2/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/logging2/pypy/translator/cli/opcodes.py Sun Nov 1 17:12:32 2009 @@ -1,7 +1,8 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, DebugPrint + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, \ + DebugPrint, HaveDebugPrints from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -77,6 +78,9 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'debug_print': [DebugPrint], + 'debug_start': Ignore, + 'debug_stop': Ignore, + 'have_debug_prints': [HaveDebugPrints], 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, From arigo at codespeak.net Sun Nov 1 17:38:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 17:38:51 +0100 (CET) Subject: [pypy-svn] r68894 - pypy/branch/logging2/pypy/translator/c/src Message-ID: <20091101163851.E0DAD168075@codespeak.net> Author: arigo Date: Sun Nov 1 17:38:50 2009 New Revision: 68894 Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h Log: Using the CLOCK_THREAD_CPUTIME_ID appears better, at least from the Linux doc; and it is also much faster on my laptop. Note that this code is not used by default on x86/GCC, where we just use asm("rdtsc"). Modified: pypy/branch/logging2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/logging2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/logging2/pypy/translator/c/src/debug.h Sun Nov 1 17:38:50 2009 @@ -112,7 +112,7 @@ static long long pypy_read_timestamp(void) { struct timespec tspec; - clock_gettime(CLOCK_MONOTONIC, &tspec); + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &tspec); return ((long long)tspec.tv_sec) * 1000000000LL + tspec.tv_nsec; } # endif From arigo at codespeak.net Sun Nov 1 17:56:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 17:56:12 +0100 (CET) Subject: [pypy-svn] r68895 - pypy/branch/logging2/pypy/doc/config Message-ID: <20091101165612.D1B51168075@codespeak.net> Author: arigo Date: Sun Nov 1 17:56:12 2009 New Revision: 68895 Added: pypy/branch/logging2/pypy/doc/config/translation.gcremovetypeptr.txt - copied unchanged from r68885, pypy/branch/logging2/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/branch/logging2/pypy/doc/config/translation.log.txt (contents, props changed) Removed: pypy/branch/logging2/pypy/doc/config/translation.gcconfig.debugprint.txt pypy/branch/logging2/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/branch/logging2/pypy/doc/config/translation.gcconfig.txt Log: Update the docs about the config options. Added: pypy/branch/logging2/pypy/doc/config/translation.log.txt ============================================================================== --- (empty file) +++ pypy/branch/logging2/pypy/doc/config/translation.log.txt Sun Nov 1 17:56:12 2009 @@ -0,0 +1,5 @@ +Include debug prints in the translation. + +These must be enabled by setting the PYPYLOG environment variable. +The exact set of features supported by PYPYLOG is described in +pypy/translation/c/src/debug.h. From arigo at codespeak.net Sun Nov 1 17:59:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 17:59:06 +0100 (CET) Subject: [pypy-svn] r68896 - pypy/branch/logging2/pypy/doc/config Message-ID: <20091101165906.4AB45168075@codespeak.net> Author: arigo Date: Sun Nov 1 17:59:05 2009 New Revision: 68896 Modified: pypy/branch/logging2/pypy/doc/config/ (props changed) Log: svn:ignore the style.css file too. From arigo at codespeak.net Sun Nov 1 18:10:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 18:10:42 +0100 (CET) Subject: [pypy-svn] r68897 - pypy/branch/logging2/pypy/tool Message-ID: <20091101171042.84E98168016@codespeak.net> Author: arigo Date: Sun Nov 1 18:10:41 2009 New Revision: 68897 Modified: pypy/branch/logging2/pypy/tool/logparser.py Log: During parsing, check that the numbers are always increasing. Modified: pypy/branch/logging2/pypy/tool/logparser.py ============================================================================== --- pypy/branch/logging2/pypy/tool/logparser.py (original) +++ pypy/branch/logging2/pypy/tool/logparser.py Sun Nov 1 18:10:41 2009 @@ -14,19 +14,28 @@ def parse_log_file(filename): r_start = re.compile(r"\[([0-9a-f]+)\] \{([\w-]+)$") r_stop = re.compile(r"\[([0-9a-f]+)\] ([\w-]+)\}$") + lasttime = 0 log = DebugLog() f = open(filename, 'r') for line in f: line = line.rstrip() match = r_start.match(line) if match: - log.debug_start(match.group(2), time=int(match.group(1), 16)) - continue - match = r_stop.match(line) - if match: - log.debug_stop(match.group(2), time=int(match.group(1), 16)) - continue - log.debug_print(line) + record = log.debug_start + else: + match = r_stop.match(line) + if match: + record = log.debug_stop + else: + log.debug_print(line) + continue + time = int(int(match.group(1), 16)) + if time < lasttime: + raise Exception("The time decreases! The log file may have been" + " produced on a multi-CPU machine and the process" + " moved between CPUs.") + lasttime = time + record(match.group(2), time=int(match.group(1), 16)) f.close() return log From arigo at codespeak.net Sun Nov 1 18:17:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 18:17:55 +0100 (CET) Subject: [pypy-svn] r68898 - in pypy/branch/logging2/pypy/jit/tool: . test Message-ID: <20091101171755.D06AC168076@codespeak.net> Author: arigo Date: Sun Nov 1 18:17:55 2009 New Revision: 68898 Modified: pypy/branch/logging2/pypy/jit/tool/jitoutput.py pypy/branch/logging2/pypy/jit/tool/test/test_jitoutput.py Log: Fix this too. Modified: pypy/branch/logging2/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/branch/logging2/pypy/jit/tool/jitoutput.py (original) +++ pypy/branch/logging2/pypy/jit/tool/jitoutput.py Sun Nov 1 18:17:55 2009 @@ -10,9 +10,9 @@ REGEXES = [ (('tracing_no', 'tracing_time'), '^Tracing:\s+([\d.]+)\s+([\d.]+)$'), (('backend_no', 'backend_time'), '^Backend:\s+([\d.]+)\s+([\d.]+)$'), - (('asm_no', 'asm_time'), '^Running asm:\s+([\d.]+)\s+([\d.]+)$'), - (('blackhole_no', 'blackhole_time'), - '^Blackhole:\s+([\d.]+)\s+([\d.]+)$'), + (('asm_no',), '^Running asm:\s+([\d.]+)$'), + (('blackhole_no',), + '^Blackhole:\s+([\d.]+)$'), (None, '^TOTAL.*$'), (('ops.total',), '^ops:\s+(\d+)$'), (('ops.calls',), '^\s+calls:\s+(\d+)$'), Modified: pypy/branch/logging2/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/logging2/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/logging2/pypy/jit/tool/test/test_jitoutput.py Sun Nov 1 18:17:55 2009 @@ -51,8 +51,8 @@ DATA = '''Tracing: 1 0.006992 Backend: 1 0.000525 -Running asm: 1 0.016957 -Blackhole: 1 0.000233 +Running asm: 1 +Blackhole: 1 TOTAL: 0.025532 ops: 2 calls: 1 @@ -75,9 +75,7 @@ assert info.tracing_no == 1 assert info.tracing_time == 0.006992 assert info.asm_no == 1 - assert info.asm_time == 0.016957 assert info.blackhole_no == 1 - assert info.blackhole_time == 0.000233 assert info.backend_no == 1 assert info.backend_time == 0.000525 assert info.ops.total == 2 From arigo at codespeak.net Sun Nov 1 19:08:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:08:09 +0100 (CET) Subject: [pypy-svn] r68899 - pypy/branch/logging2/pypy/rpython/lltypesystem/test Message-ID: <20091101180809.8D03D168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:08:08 2009 New Revision: 68899 Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/test/test_lloperation.py Log: Fix test. Modified: pypy/branch/logging2/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/branch/logging2/pypy/rpython/lltypesystem/test/test_lloperation.py Sun Nov 1 19:08:08 2009 @@ -127,7 +127,7 @@ def test_llinterp_complete(): for opname, llop in LL_OPERATIONS.items(): - if llop.canfold: + if llop.canrun: continue if opname.startswith('gc_x_'): continue # ignore experimental stuff From arigo at codespeak.net Sun Nov 1 19:41:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:41:20 +0100 (CET) Subject: [pypy-svn] r68900 - pypy/branch/logging2/pypy/rpython/memory/gc Message-ID: <20091101184120.83D44168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:41:20 2009 New Revision: 68900 Modified: pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py Log: Disable reading the time via time.time() from the GC. It crashes translator/sandbox/test/test_pypy_interact.py. For now the PYPYLOGs have all the timing information that we need. Modified: pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/logging2/pypy/rpython/memory/gc/semispace.py Sun Nov 1 19:41:20 2009 @@ -12,7 +12,7 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase -import sys, os, time +import sys, os first_gcflag = 1 << 16 GCFLAG_FORWARDED = first_gcflag @@ -55,14 +55,14 @@ MovingGCBase.__init__(self, config, chunk_size) def setup(self): - self.total_collection_time = 0.0 + #self.total_collection_time = 0.0 self.total_collection_count = 0 self.space_size = self.param_space_size self.max_space_size = self.param_max_space_size self.red_zone = 0 - self.program_start_time = time.time() + #self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -219,7 +219,7 @@ start_usage = self.free - self.tospace debug_print("| used before collection: ", start_usage, "bytes") - start_time = time.time() + #start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -250,11 +250,11 @@ self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if have_debug_prints(): - end_time = time.time() - elapsed_time = end_time - start_time - self.total_collection_time += elapsed_time + #end_time = time.time() + #elapsed_time = end_time - start_time + #self.total_collection_time += elapsed_time self.total_collection_count += 1 - total_program_time = end_time - self.program_start_time + #total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace debug_print("| used after collection: ", end_usage, "bytes") @@ -264,16 +264,16 @@ self.space_size, "bytes") debug_print("| fraction of semispace now used: ", end_usage * 100.0 / self.space_size, "%") - ct = self.total_collection_time + #ct = self.total_collection_time cc = self.total_collection_count debug_print("| number of semispace_collects: ", cc) - debug_print("| i.e.: ", - cc / total_program_time, "per second") - debug_print("| total time in semispace_collect: ", - ct, "seconds") - debug_print("| i.e.: ", - ct * 100.0 / total_program_time, "%") + #debug_print("| i.e.: ", + # cc / total_program_time, "per second") + #debug_print("| total time in semispace_collect: ", + # ct, "seconds") + #debug_print("| i.e.: ", + # ct * 100.0 / total_program_time, "%") debug_print("`----------------------------------------------") debug_stop("gc-collect") From arigo at codespeak.net Sun Nov 1 19:43:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:43:58 +0100 (CET) Subject: [pypy-svn] r68901 - pypy/branch/logging2/pypy/translator/c/test Message-ID: <20091101184358.E748D168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:43:58 2009 New Revision: 68901 Modified: pypy/branch/logging2/pypy/translator/c/test/test_newgc.py Log: Fix test. Modified: pypy/branch/logging2/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/logging2/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/logging2/pypy/translator/c/test/test_newgc.py Sun Nov 1 19:43:58 2009 @@ -39,8 +39,7 @@ t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, - removetypeptr=cls.removetypeptr, - debugprint=True) + gcremovetypeptr=cls.removetypeptr) t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() From arigo at codespeak.net Sun Nov 1 19:57:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:57:22 +0100 (CET) Subject: [pypy-svn] r68902 - in pypy/trunk/pypy: config doc/config jit/backend/test jit/backend/x86/test jit/metainterp jit/metainterp/test jit/tool jit/tool/test rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test tool tool/test translator/c translator/c/gcc translator/c/src translator/c/test translator/cli translator/platform Message-ID: <20091101185722.ABC28168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:57:20 2009 New Revision: 68902 Added: pypy/trunk/pypy/doc/config/translation.gcremovetypeptr.txt - copied unchanged from r68901, pypy/branch/logging2/pypy/doc/config/translation.gcremovetypeptr.txt pypy/trunk/pypy/doc/config/translation.log.txt - copied unchanged from r68901, pypy/branch/logging2/pypy/doc/config/translation.log.txt pypy/trunk/pypy/tool/logparser.py - copied unchanged from r68901, pypy/branch/logging2/pypy/tool/logparser.py pypy/trunk/pypy/tool/test/test_logparser.py - copied unchanged from r68901, pypy/branch/logging2/pypy/tool/test/test_logparser.py pypy/trunk/pypy/translator/c/src/debug.h - copied unchanged from r68901, pypy/branch/logging2/pypy/translator/c/src/debug.h Removed: pypy/trunk/pypy/doc/config/translation.gcconfig.debugprint.txt pypy/trunk/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/trunk/pypy/doc/config/translation.gcconfig.txt Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/doc/config/ (props changed) pypy/trunk/pypy/jit/backend/test/support.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/jitprof.py pypy/trunk/pypy/jit/metainterp/logger.py pypy/trunk/pypy/jit/metainterp/optimize.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/policy.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py pypy/trunk/pypy/jit/metainterp/test/test_logger.py pypy/trunk/pypy/jit/metainterp/test/test_loop.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/jit/metainterp/test/test_virtual.py pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py pypy/trunk/pypy/jit/metainterp/warmstate.py pypy/trunk/pypy/jit/tool/jitoutput.py pypy/trunk/pypy/jit/tool/test/test_jitoutput.py pypy/trunk/pypy/rlib/debug.py pypy/trunk/pypy/rlib/test/test_debug.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py pypy/trunk/pypy/rpython/memory/gc/generation.py pypy/trunk/pypy/rpython/memory/gc/hybrid.py pypy/trunk/pypy/rpython/memory/gc/marksweep.py pypy/trunk/pypy/rpython/memory/gc/semispace.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py pypy/trunk/pypy/translator/c/funcgen.py pypy/trunk/pypy/translator/c/gc.py pypy/trunk/pypy/translator/c/gcc/trackgcroot.py pypy/trunk/pypy/translator/c/genc.py pypy/trunk/pypy/translator/c/src/asm_gcc_x86.h pypy/trunk/pypy/translator/c/src/g_include.h pypy/trunk/pypy/translator/c/test/test_newgc.py pypy/trunk/pypy/translator/c/test/test_standalone.py pypy/trunk/pypy/translator/cli/metavm.py pypy/trunk/pypy/translator/cli/opcodes.py pypy/trunk/pypy/translator/platform/linux.py Log: Merge the 'logging2' branch. Add a general way to record logging events to pypy.rlib.debug. In addition to debug_print() there is now debug_start() and debug_stop() to mark the start/stop of a category. svn merge -r68854:68901 svn+ssh://codespeak.net/svn/pypy/branch/logging2 Use it in the GC (replaces the gcconfig.debugprint config option) and in the JIT (similar to jitprof.py, which is not dead yet but obsoleting). To enable logging run the compiled program with 'PYPYLOG=:file'. To enable lightweight logging only, suitable for profiling, run it with 'PYPYLOG=file'. See translator/c/src/debug.h for more options. To turn the profiling logs into nice graphics, see pypy/tool/logparser.py. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Sun Nov 1 19:57:20 2009 @@ -40,6 +40,9 @@ }, cmdline="-b --backend"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", + default=True, cmdline="--log"), + # gc ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", @@ -66,12 +69,8 @@ "ref": [("translation.gcrootfinder", "n/a")], "none": [("translation.gcrootfinder", "n/a")], }), - OptionDescription("gcconfig", "Configure garbage collectors", [ - BoolOption("debugprint", "Turn on debug printing for the GC", - default=False), - BoolOption("removetypeptr", "Remove the typeptr from every object", - default=False, cmdline="--gcremovetypeptr"), - ]), + BoolOption("gcremovetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", ["n/a", "shadowstack", "asmgcc"], @@ -97,7 +96,7 @@ BoolOption("jit", "generate a JIT", default=False, requires=[("translation.thread", False), - ("translation.gcconfig.removetypeptr", False)], + ("translation.gcremovetypeptr", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), @@ -359,7 +358,7 @@ elif word == 'jit': config.translation.suggest(jit=True) elif word == 'removetypeptr': - config.translation.gcconfig.suggest(removetypeptr=True) + config.translation.suggest(gcremovetypeptr=True) else: raise ValueError(word) Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Sun Nov 1 19:57:20 2009 @@ -1,6 +1,6 @@ import py import sys -from pypy.jit.metainterp.history import log +from pypy.rlib.debug import debug_print from pypy.translator.translator import TranslationContext class BaseCompiledMixin(object): @@ -119,10 +119,10 @@ cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() exe_name = cbuilder.compile() - log('---------- Test starting ----------') + debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) res = int(stdout) - log('---------- Test done (%d) ----------' % (res,)) + debug_print('---------- Test done (%d) ----------' % (res,)) return res class CliCompiledMixin(BaseCompiledMixin): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Sun Nov 1 19:57:20 2009 @@ -15,6 +15,7 @@ from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.x86.regalloc import X86StackManager +from pypy.tool.udir import udir stack_pos = X86StackManager.stack_pos @@ -77,7 +78,6 @@ # t = TranslationContext() t.config.translation.gc = gc - t.config.translation.gcconfig.debugprint = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) @@ -92,7 +92,8 @@ def run(cbuilder, args=''): # - data = cbuilder.cmdexec(args) + pypylog = udir.join('test_zrpy_gc.log') + data = cbuilder.cmdexec(args, env={'PYPYLOG': str(pypylog)}) return data.strip() def compile_and_run(f, gc, **kwds): @@ -164,7 +165,9 @@ cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) def run(self, name, n=2000): - res = self.cbuilder.cmdexec("%s %d" %(name, n)) + pypylog = udir.join('TestCompileHybrid.log') + res = self.cbuilder.cmdexec("%s %d" %(name, n), + env={'PYPYLOG': str(pypylog)}) assert int(res) == 20 def run_orig(self, name, n, x): Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Sun Nov 1 19:57:20 2009 @@ -2,10 +2,11 @@ from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import debug_start, debug_stop from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.history import TreeLoop, log, Box, History, LoopToken +from pypy.jit.metainterp.history import TreeLoop, Box, History, LoopToken from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt from pypy.jit.metainterp.history import BoxPtr, BoxObj, BoxFloat, Const from pypy.jit.metainterp import history @@ -92,11 +93,16 @@ globaldata.loopnumbering += 1 metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd, loop) loop.check_consistency() - metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, loop.token) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, + loop.token) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): @@ -109,12 +115,16 @@ def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): n = faildescr.get_index() metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd) TreeLoop.check_consistency_of(inputargs, operations) pass - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if not we_are_translated(): metainterp_sd.stats.compiled() Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Sun Nov 1 19:57:20 2009 @@ -3,18 +3,13 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic -from pypy.rlib.objectmodel import compute_hash +from pypy.rlib.objectmodel import compute_hash, compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.tool.uid import uid from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('compiler') -py.log.setconsumer('compiler', ansi_log) - # ____________________________________________________________ INT = 'i' @@ -69,14 +64,9 @@ except AttributeError: return box.value -class ReprRPython: - def __init__(self): - self.seen = {} - def repr_rpython(self, box, typechars): - n = self.seen.setdefault(box, len(self.seen)) - return '%s/%s%d' % (box._get_hash_(), typechars, n) - -repr_rpython = ReprRPython().repr_rpython +def repr_rpython(box, typechars): + return '%s/%s%d' % (box._get_hash_(), typechars, + compute_unique_id(box)) class AbstractValue(object): Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Sun Nov 1 19:57:20 2009 @@ -91,7 +91,7 @@ def start(self): self.starttime = self.timer() self.t1 = self.starttime - self.times = [0, 0, 0, 0] + self.times = [0, 0] self.counters = [0] * ncounters self.calls = [[0, 0], [0, 0], [0, 0]] self.current = [] @@ -130,11 +130,11 @@ def start_backend(self): self._start(BACKEND) def end_backend(self): self._end (BACKEND) - def start_running(self): self._start(RUNNING) - def end_running(self): self._end (RUNNING) + def start_running(self): self.count(RUNNING) + def end_running(self): pass - def start_blackhole(self): self._start(BLACKHOLE) - def end_blackhole(self): self._end (BLACKHOLE) + def start_blackhole(self): self.count(BLACKHOLE) + def end_blackhole(self): pass def count(self, kind, inc=1): self.counters[kind] += inc @@ -153,8 +153,8 @@ calls = self.calls self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) - self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) - self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) + self._print_intline("Running asm", cnt[RUNNING]) + self._print_intline("Blackhole", cnt[BLACKHOLE]) line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) os.write(2, line) self._print_intline("ops", cnt[OPS]) Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Sun Nov 1 19:57:20 2009 @@ -1,48 +1,43 @@ import os -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ BoxInt, ConstAddr, ConstFloat, BoxFloat, AbstractFailDescr -from pypy.rlib.streamio import open_file_as_stream class Logger(object): def __init__(self, ts, guard_number=False): - self.log_stream = None self.ts = ts self.guard_number=guard_number - def create_log(self, extension='.ops'): - if self.log_stream is not None: - return self.log_stream - s = os.environ.get('PYPYJITLOG') - if not s: - return None - s += extension - try: - self.log_stream = open_file_as_stream(s, 'w') - except OSError: - os.write(2, "could not create log file\n") - return None - return self.log_stream - def log_loop(self, inputargs, operations, number=0, type=None): - if self.log_stream is None: + if not have_debug_prints(): return - if type is not None: - self.log_stream.write("# Loop%d (%s), %d ops\n" % (number, - type, - len(operations))) - self._log_operations(inputargs, operations, {}) + if type is None: + debug_start("jit-log-noopt-loop") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-loop") + else: + debug_start("jit-log-opt-loop") + debug_print("# Loop", number, ":", type, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-loop") def log_bridge(self, inputargs, operations, number=-1): - if self.log_stream is None: + if not have_debug_prints(): return - if number != -1: - self.log_stream.write("# bridge out of Guard%d, %d ops\n" % (number, - len(operations))) - self._log_operations(inputargs, operations, {}) - + if number == -1: + debug_start("jit-log-noopt-bridge") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-bridge") + else: + debug_start("jit-log-opt-bridge") + debug_print("# bridge out of Guard", number, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-bridge") def repr_of_descr(self, descr): return descr.repr_of_descr() @@ -70,15 +65,18 @@ else: return '?' - def _log_operations(self, inputargs, operations, memo): + def _log_operations(self, inputargs, operations): + if not have_debug_prints(): + return + memo = {} if inputargs is not None: args = ", ".join([self.repr_of_arg(memo, arg) for arg in inputargs]) - self.log_stream.write('[' + args + ']\n') + debug_print('[' + args + ']') for i in range(len(operations)): op = operations[i] if op.opnum == rop.DEBUG_MERGE_POINT: loc = op.args[0]._get_str() - self.log_stream.write("debug_merge_point('%s')\n" % (loc,)) + debug_print("debug_merge_point('%s')" % (loc,)) continue args = ", ".join([self.repr_of_arg(memo, arg) for arg in op.args]) if op.result is not None: @@ -99,6 +97,5 @@ for arg in op.fail_args]) + ']' else: fail_args = '' - self.log_stream.write(res + op.getopname() + - '(' + args + ')' + fail_args + '\n') - self.log_stream.flush() + debug_print(res + op.getopname() + + '(' + args + ')' + fail_args) Modified: pypy/trunk/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimize.py Sun Nov 1 19:57:20 2009 @@ -1,3 +1,5 @@ +from pypy.rlib.debug import debug_start, debug_stop + # ____________________________________________________________ from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder @@ -5,6 +7,13 @@ from pypy.jit.metainterp.specnode import equals_specnodes def optimize_loop(metainterp_sd, old_loop_tokens, loop): + debug_start("jit-optimize") + try: + return _optimize_loop(metainterp_sd, old_loop_tokens, loop) + finally: + debug_stop("jit-optimize") + +def _optimize_loop(metainterp_sd, old_loop_tokens, loop): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) finder = PerfectSpecializationFinder(cpu) @@ -21,6 +30,13 @@ from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 def optimize_bridge(metainterp_sd, old_loop_tokens, bridge): + debug_start("jit-optimize") + try: + return _optimize_bridge(metainterp_sd, old_loop_tokens, bridge) + finally: + debug_stop("jit-optimize") + +def _optimize_bridge(metainterp_sd, old_loop_tokens, bridge): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(bridge.inputargs, bridge.operations) finder = BridgeSpecializationFinder(cpu) Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sun Nov 1 19:57:20 2009 @@ -518,8 +518,7 @@ def store_final_boxes_in_guard(self, op): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo, - self.metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp Modified: pypy/trunk/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/policy.py (original) +++ pypy/trunk/pypy/jit/metainterp/policy.py Sun Nov 1 19:57:20 2009 @@ -77,8 +77,9 @@ getkind(v.concretetype, supports_floats) getkind(op.result.concretetype, supports_floats) except NotImplementedError, e: - history.log.WARNING('%s, ignoring graph' % (e,)) - history.log.WARNING(' %s' % (graph,)) + from pypy.jit.metainterp.codewriter import log + log.WARNING('%s, ignoring graph' % (e,)) + log.WARNING(' %s' % (graph,)) return True return False Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sun Nov 1 19:57:20 2009 @@ -3,7 +3,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box @@ -875,12 +875,10 @@ self.pc = pc self.exception_target = exception_target self.env = env - if self.metainterp.staticdata.state.debug_level >= DEBUG_DETAILED: - values = ' '.join([box.repr_rpython() for box in self.env]) - log = self.metainterp.staticdata.log - log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, - self.pc, values, - self.exception_target)) + ## values = ' '.join([box.repr_rpython() for box in self.env]) + ## log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, + ## self.pc, values, + ## self.exception_target)) def run_one_step(self): # Execute the frame forward. This method contains a loop that leaves @@ -1020,16 +1018,13 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: + debug_print(self.jit_starting_line) self._setup_class_sizes() self.cpu.setup_once() - self.log(self.jit_starting_line) if not self.profiler.initialized: self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.logger_noopt.create_log('.noopt') - self.logger_ops.create_log('.ops') - self.globaldata.storedebug = os.environ.get('PYPYJITRESUMELOG') def _setup_class_sizes(self): class_sizes = {} @@ -1081,12 +1076,8 @@ # ---------------- logging ------------------------ - def log(self, msg, event_kind='info'): - if self.state.debug_level > DEBUG_PROFILE: - if not we_are_translated(): - getattr(history.log, event_kind)(msg) - else: - debug_print(msg) + def log(self, msg): + debug_print(msg) # ____________________________________________________________ @@ -1139,12 +1130,6 @@ def is_blackholing(self): return self.history is None - def blackholing_text(self): - if self.history is None: - return " (BlackHole)" - else: - return "" - def newframe(self, jitcode): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -1336,9 +1321,11 @@ op.name = self.framestack[-1].jitcode.name def switch_to_blackhole(self): + debug_print('~~~ ABORTING TRACING') + debug_stop('jit-tracing') + debug_start('jit-blackhole') self.history = None # start blackholing self.staticdata.stats.aborted() - self.staticdata.log('~~~ ABORTING TRACING', event_kind='event') self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() @@ -1353,8 +1340,6 @@ # Execute the frames forward until we raise a DoneWithThisFrame, # a ContinueRunningNormally, or a GenerateMergePoint exception. self.staticdata.stats.entered() - self.staticdata.log('~~~ ENTER' + self.blackholing_text(), - event_kind='event') try: while True: self.framestack[-1].run_one_step() @@ -1366,8 +1351,6 @@ self.staticdata.profiler.end_blackhole() else: self.staticdata.profiler.end_tracing() - self.staticdata.log('~~~ LEAVE' + self.blackholing_text(), - event_kind='event') def interpret(self): if we_are_translated(): @@ -1378,11 +1361,22 @@ except: import sys if sys.exc_info()[0] is not None: - history.log.info(sys.exc_info()[0].__name__) + codewriter.log.info(sys.exc_info()[0].__name__) raise def compile_and_run_once(self, *args): - self.staticdata.log('Switching from interpreter to compiler') + debug_start('jit-tracing') + self.staticdata._setup_once() + self.create_empty_history() + try: + return self._compile_and_run_once(*args) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _compile_and_run_once(self, *args): original_boxes = self.initialize_state_from_start(*args) self.current_merge_points = [(original_boxes, 0)] num_green_args = self.staticdata.num_green_args @@ -1398,9 +1392,24 @@ return self.designate_target_loop(gmp) def handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase assert isinstance(key, compile.ResumeGuardDescr) - resumedescr = self.initialize_state_from_guard_failure(key) + warmrunnerstate = self.staticdata.state + must_compile = warmrunnerstate.must_compile_from_failure(key) + if must_compile: + debug_start('jit-tracing') + else: + debug_start('jit-blackhole') + self.initialize_state_from_guard_failure(key, must_compile) + try: + return self._handle_guard_failure(key) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _handle_guard_failure(self, key): + from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1418,7 +1427,7 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(resumedescr) + warmrunnerstate.reset_counter_from_failure(key) raise def forget_consts(self, boxes, startindex=0): @@ -1580,9 +1589,7 @@ def initialize_state_from_start(self, *args): self.in_recursion = -1 # always one portal around - self.staticdata._setup_once() self.staticdata.profiler.start_tracing() - self.create_empty_history() num_green_args = self.staticdata.num_green_args original_boxes = [] self._initialize_from_start(original_boxes, num_green_args, *args) @@ -1594,12 +1601,11 @@ self.initialize_virtualizable(original_boxes) return original_boxes - def initialize_state_from_guard_failure(self, resumedescr): + def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around inputargs = self.load_values_from_failure(resumedescr) warmrunnerstate = self.staticdata.state - must_compile = warmrunnerstate.must_compile_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) self.history.inputargs = inputargs @@ -1608,7 +1614,6 @@ self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true self.rebuild_state_after_failure(resumedescr, inputargs) - return resumedescr def load_values_from_failure(self, resumedescr): cpu = self.cpu Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Sun Nov 1 19:57:20 2009 @@ -4,6 +4,8 @@ from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, @@ -177,10 +179,9 @@ class ResumeDataVirtualAdder(object): - def __init__(self, storage, memo, debug_storage=None): + def __init__(self, storage, memo): self.storage = storage self.memo = memo - self.debug_storage = debug_storage #self.virtuals = [] #self.vfieldboxes = [] @@ -256,8 +257,8 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if self.debug_storage: - dump_storage(self.debug_storage, storage, liveboxes) + if have_debug_prints(): + dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -307,6 +308,13 @@ self.fielddescrs[i], box, fieldbox) + def debug_prints(self): + assert len(self.fielddescrs) == len(self.fieldnums) + for i in range(len(self.fielddescrs)): + debug_print("\t\t", + str(self.fielddescrs[i]), + str(untag(self.fieldnums[i]))) + class VirtualInfo(AbstractVirtualStructInfo): def __init__(self, known_class, fielddescrs): AbstractVirtualStructInfo.__init__(self, fielddescrs) @@ -316,11 +324,9 @@ return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, None, self.known_class) - def repr_rpython(self): - return 'VirtualInfo("%s", %s, %s)' % ( - self.known_class, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvirtualinfo", self.known_class.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VStructInfo(AbstractVirtualStructInfo): def __init__(self, typedescr, fielddescrs): @@ -330,11 +336,9 @@ def allocate(self, metainterp): return metainterp.execute_and_record(rop.NEW, self.typedescr) - def repr_rpython(self): - return 'VStructInfo("%s", %s, %s)' % ( - self.typedescr, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvstructinfo", self.typedescr.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): @@ -354,10 +358,10 @@ self.arraydescr, box, ConstInt(i), itembox) - def repr_rpython(self): - return 'VArrayInfo("%s", %s)' % ( - self.arraydescr, - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvarrayinfo", self.arraydescr) + for i in self.fieldnums: + debug_print("\t\t", str(untag(i))) def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): @@ -423,38 +427,31 @@ # ____________________________________________________________ -def dump_storage(logname, storage, liveboxes): +def dump_storage(storage, liveboxes): "For profiling only." - import os - from pypy.rlib import objectmodel - assert logname is not None # annotator hack - fd = os.open(logname, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) - os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) + from pypy.rlib.objectmodel import compute_unique_id + debug_start("jit-resume") + debug_print('Log storage', compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list - while True: - os.write(fd, '\t("%s", %d, %d) at %xd,\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target, - objectmodel.compute_unique_id(frameinfo))) + while frameinfo is not None: + try: + jitcodename = frameinfo.jitcode.name + except AttributeError: + jitcodename = str(compute_unique_id(frameinfo.jitcode)) + debug_print('\tjitcode/pc', jitcodename, + frameinfo.pc, frameinfo.exception_target, + 'at', compute_unique_id(frameinfo)) frameinfo = frameinfo.prev - if frameinfo is None: - break - os.write(fd, '\t],\n\t[\n') numb = storage.rd_numb - while True: - os.write(fd, '\t\t%s at %xd,\n' % ([untag(i) for i in numb.nums], - objectmodel.compute_unique_id(numb))) + while numb is not None: + debug_print('\tnumb', str([untag(i) for i in numb.nums]), + 'at', compute_unique_id(numb)) numb = numb.prev - if numb is None: - break - os.write(fd, '\t], [\n') for const in storage.rd_consts: - os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - os.write(fd, '\t"%s",\n' % (box.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - os.write(fd, '\t%s,\n' % (virtual.repr_rpython(),)) - os.write(fd, '\t])\n') - os.close(fd) + virtual.debug_prints() + debug_stop("jit-resume") Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Sun Nov 1 19:57:20 2009 @@ -22,8 +22,7 @@ if op.is_guard(): descr = op.descr assert isinstance(descr, compile.ResumeGuardDescr) - modifier = resume.ResumeDataVirtualAdder(descr, memo, - metainterp_sd.globaldata.storedebug) + modifier = resume.ResumeDataVirtualAdder(descr, memo) newboxes = modifier.finish(EMPTY_VALUES) descr.store_final_boxes(op, newboxes) newoperations.append(op) Modified: pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py Sun Nov 1 19:57:20 2009 @@ -6,9 +6,11 @@ from pypy.jit.metainterp.jitprof import * class FakeProfiler(Profiler): - def __init__(self): + def start(self): self.counter = 123456 + Profiler.start(self) self.events = [] + self.times = [0, 0, 0, 0] def timer(self): self.counter += 1 @@ -22,6 +24,12 @@ Profiler._end(self, event) self.events.append(~event) + def start_running(self): self._start(RUNNING) + def end_running(self): self._end(RUNNING) + + def start_blackhole(self): self._start(BLACKHOLE) + def end_blackhole(self): self._end(BLACKHOLE) + class ProfilerMixin(LLJitMixin): def meta_interp(self, *args, **kwds): kwds = kwds.copy() Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Sun Nov 1 19:57:20 2009 @@ -1,4 +1,5 @@ - +import sys +from pypy.rlib import debug from pypy.jit.metainterp.test.oparser import pure_parse from pypy.jit.metainterp import logger from pypy.jit.metainterp.typesystem import llhelper @@ -9,12 +10,20 @@ class Descr(AbstractDescr): pass +def capturing(func, *args, **kwds): + log_stream = StringIO() + debug._stderr = log_stream + try: + func(*args, **kwds) + finally: + debug._stderr = sys.stderr + return log_stream.getvalue() + class Logger(logger.Logger): def log_loop(self, loop, namespace={}): - self.log_stream = StringIO() self.namespace = namespace - logger.Logger.log_loop(self, loop.inputargs, loop.operations) - return self.log_stream.getvalue() + return capturing(logger.Logger.log_loop, self, + loop.inputargs, loop.operations) def repr_of_descr(self, descr): for k, v in self.namespace.items(): @@ -116,16 +125,12 @@ def test_intro_loop(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_loop([], [], 1, "foo") - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# Loop1 (foo), 0 ops" + output = capturing(bare_logger.log_loop, [], [], 1, "foo") + assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_bridge([], [], 3) - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# bridge out of Guard3, 0 ops" + output = capturing(bare_logger.log_bridge, [], [], 3) + assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) Modified: pypy/trunk/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_loop.py Sun Nov 1 19:57:20 2009 @@ -594,25 +594,6 @@ res = self.meta_interp(f, [200]) - def test_dump_storage(self): - import os - from pypy.tool.udir import udir - logfile = udir.join('resume.log') - os.environ['PYPYJITRESUMELOG'] = str(logfile) - try: - jitdriver = JitDriver(greens = [], reds = ['i', 'n']) - - def f(n): - i = 0 - while i < n: - jitdriver.can_enter_jit(n=n, i=i) - jitdriver.jit_merge_point(n=n, i=i) - i += 1 - return i - res = self.meta_interp(f, [10]) - data = logfile.read() # assert did not crash - finally: - del os.environ['PYPYJITRESUMELOG'] class TestOOtype(LoopTest, OOJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Sun Nov 1 19:57:20 2009 @@ -8,7 +8,9 @@ from pypy.jit.metainterp import executor class Storage: - pass + rd_frame_info_list = None + rd_numb = None + rd_consts = [] def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -591,6 +593,7 @@ snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) snapshot = Snapshot(snapshot, [b1, b2, b3]) storage.rd_snapshot = snapshot + storage.rd_frame_info_list = None return storage def test_virtual_adder_int_constants(): @@ -761,6 +764,7 @@ [b4s, c1s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -807,6 +811,7 @@ [c1s, b4s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None b4t = BoxPtr() Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtual.py Sun Nov 1 19:57:20 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.lltypesystem import lltype, rclass @@ -123,7 +124,8 @@ def test_two_loops_with_escaping_virtual(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) def externfn(node): - llop.debug_print(lltype.Void, node) + llop.debug_print(lltype.Void, compute_unique_id(node), + node.value, node.extra) return node.value * 2 def f(n): node = self._new() Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmspot.py Sun Nov 1 19:57:20 2009 @@ -131,6 +131,7 @@ assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge def test_static_debug_level(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -177,6 +178,7 @@ assert not "Running asm" in err def test_set_param_debug(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Sun Nov 1 19:57:20 2009 @@ -10,6 +10,7 @@ from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell +from pypy.rlib.debug import debug_start, debug_stop from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -229,7 +230,9 @@ # ---------- execute assembler ---------- while True: # until interrupted by an exception metainterp_sd.profiler.start_running() + debug_start("jit-running") fail_index = metainterp_sd.cpu.execute_token(loop_token) + debug_stop("jit-running") metainterp_sd.profiler.end_running() fail_descr = globaldata.get_fail_descr_from_number(fail_index) loop_token = fail_descr.handle_fail(metainterp_sd) Modified: pypy/trunk/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/trunk/pypy/jit/tool/jitoutput.py (original) +++ pypy/trunk/pypy/jit/tool/jitoutput.py Sun Nov 1 19:57:20 2009 @@ -10,9 +10,9 @@ REGEXES = [ (('tracing_no', 'tracing_time'), '^Tracing:\s+([\d.]+)\s+([\d.]+)$'), (('backend_no', 'backend_time'), '^Backend:\s+([\d.]+)\s+([\d.]+)$'), - (('asm_no', 'asm_time'), '^Running asm:\s+([\d.]+)\s+([\d.]+)$'), - (('blackhole_no', 'blackhole_time'), - '^Blackhole:\s+([\d.]+)\s+([\d.]+)$'), + (('asm_no',), '^Running asm:\s+([\d.]+)$'), + (('blackhole_no',), + '^Blackhole:\s+([\d.]+)$'), (None, '^TOTAL.*$'), (('ops.total',), '^ops:\s+(\d+)$'), (('ops.calls',), '^\s+calls:\s+(\d+)$'), Modified: pypy/trunk/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/trunk/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/trunk/pypy/jit/tool/test/test_jitoutput.py Sun Nov 1 19:57:20 2009 @@ -51,8 +51,8 @@ DATA = '''Tracing: 1 0.006992 Backend: 1 0.000525 -Running asm: 1 0.016957 -Blackhole: 1 0.000233 +Running asm: 1 +Blackhole: 1 TOTAL: 0.025532 ops: 2 calls: 1 @@ -75,9 +75,7 @@ assert info.tracing_no == 1 assert info.tracing_time == 0.006992 assert info.asm_no == 1 - assert info.asm_time == 0.016957 assert info.blackhole_no == 1 - assert info.blackhole_time == 0.000233 assert info.backend_no == 1 assert info.backend_time == 0.000525 assert info.ops.total == 2 Modified: pypy/trunk/pypy/rlib/debug.py ============================================================================== --- pypy/trunk/pypy/rlib/debug.py (original) +++ pypy/trunk/pypy/rlib/debug.py Sun Nov 1 19:57:20 2009 @@ -1,4 +1,4 @@ - +import sys, time from pypy.rpython.extregistry import ExtRegistryEntry def ll_assert(x, msg): @@ -20,11 +20,40 @@ hop.genop('debug_assert', vlist) +class DebugLog(list): + def debug_print(self, *args): + self.append(('debug_print',) + args) + def debug_start(self, category, time=None): + self.append(('debug_start', category, time)) + def debug_stop(self, category, time=None): + for i in xrange(len(self)-1, -1, -1): + if self[i][0] == 'debug_start': + assert self[i][1] == category, ( + "nesting error: starts with %r but stops with %r" % + (self[i][1], category)) + starttime = self[i][2] + if starttime is not None or time is not None: + self[i:] = [(category, starttime, time, self[i+1:])] + else: + self[i:] = [(category, self[i+1:])] + return + assert False, ("nesting error: no start corresponding to stop %r" % + (category,)) + def __repr__(self): + import pprint + return pprint.pformat(list(self)) + +_log = None # patched from tests to be an object of class DebugLog + # or compatible +_stderr = sys.stderr # alternatively, this is patched from tests + # (redirects debug_print(), but not debug_start/stop) + def debug_print(*args): - import sys for arg in args: - print >> sys.stderr, arg, - print >> sys.stderr + print >> _stderr, arg, + print >> _stderr + if _log is not None: + _log.debug_print(*args) class Entry(ExtRegistryEntry): _about_ = debug_print @@ -35,7 +64,72 @@ def specialize_call(self, hop): vlist = hop.inputargs(*hop.args_r) hop.exception_cannot_occur() - hop.genop('debug_print', vlist) + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop('debug_print', vlist) + + +if sys.stderr.isatty(): + _start_colors_1 = "\033[1m\033[31m" + _start_colors_2 = "\033[31m" + _stop_colors = "\033[0m" +else: + _start_colors_1 = "" + _start_colors_2 = "" + _stop_colors = "" + +def debug_start(category): + print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + category, _stop_colors) + if _log is not None: + _log.debug_start(category) + +def debug_stop(category): + print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + category, _stop_colors) + if _log is not None: + _log.debug_stop(category) + +class Entry(ExtRegistryEntry): + _about_ = debug_start, debug_stop + + def compute_result_annotation(self, s_category): + return None + + def specialize_call(self, hop): + fn = self.instance + string_repr = hop.rtyper.type_system.rstr.string_repr + vlist = hop.inputargs(string_repr) + hop.exception_cannot_occur() + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop(fn.__name__, vlist) + + +def have_debug_prints(): + # returns True if the next calls to debug_print show up, + # and False if they would not have any effect. + return True + +class Entry(ExtRegistryEntry): + _about_ = have_debug_prints + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + t = self.bookkeeper.annotator.translator + if t.config.translation.log: + return annmodel.s_Bool + else: + return self.bookkeeper.immutablevalue(False) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + t = hop.rtyper.annotator.translator + hop.exception_cannot_occur() + if t.config.translation.log: + return hop.genop('have_debug_prints', [], resulttype=lltype.Bool) + else: + return hop.inputconst(lltype.Bool, False) def llinterpcall(RESTYPE, pythonfunction, *args): Modified: pypy/trunk/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_debug.py (original) +++ pypy/trunk/pypy/rlib/test/test_debug.py Sun Nov 1 19:57:20 2009 @@ -1,6 +1,9 @@ import py from pypy.rlib.debug import check_annotation, make_sure_not_resized +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints +from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -37,3 +40,46 @@ py.test.raises(TooLateForChange, interpret, f, [], list_comprehension_operations=True) + + +class DebugTests: + + def test_debug_print_start_stop(self): + def f(x): + debug_start("mycat") + debug_print("foo", 2, "bar", x) + debug_stop("mycat") + return have_debug_prints() + + try: + debug._log = dlog = debug.DebugLog() + res = f(3) + assert res == True + finally: + debug._log = None + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] + + try: + debug._log = dlog = debug.DebugLog() + res = self.interpret(f, [3]) + assert res == True + finally: + debug._log = None + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] + + +class TestLLType(DebugTests): + def interpret(self, f, args): + return interpret(f, args, type_system='lltype') + +class TestOOType(DebugTests): + def interpret(self, f, args): + return interpret(f, args, type_system='ootype') Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Sun Nov 1 19:57:20 2009 @@ -522,20 +522,6 @@ from pypy.translator.tool.lltracker import track track(*ll_objects) - def op_debug_print(self, *ll_args): - from pypy.rpython.lltypesystem.rstr import STR - line = [] - for arg in ll_args: - T = lltype.typeOf(arg) - if T == lltype.Ptr(STR): - arg = ''.join(arg.chars) - line.append(str(arg)) - line = ' '.join(line) - print line - tracer = self.llinterpreter.tracer - if tracer: - tracer.dump('\n[debug] %s\n' % (line,)) - def op_debug_pdb(self, *ll_args): if self.llinterpreter.tracer: self.llinterpreter.tracer.flush() Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Sun Nov 1 19:57:20 2009 @@ -523,6 +523,9 @@ # __________ debugging __________ 'debug_view': LLOp(), 'debug_print': LLOp(canrun=True), + 'debug_start': LLOp(canrun=True), + 'debug_stop': LLOp(canrun=True), + 'have_debug_prints': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Sun Nov 1 19:57:20 2009 @@ -3,6 +3,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import opimpls +from pypy.rlib import debug # ____________________________________________________________ # Implementation of the 'canfold' operations @@ -411,10 +412,26 @@ raise TypeError("cannot fold getfield on mutable array") return p[index] +def _normalize(x): + if not isinstance(x, str): + TYPE = lltype.typeOf(x) + if (isinstance(TYPE, lltype.Ptr) and TYPE.TO._name == 'rpy_string' + or getattr(TYPE, '_name', '') == 'String'): # ootype + from pypy.rpython.annlowlevel import hlstr + return hlstr(x) + return x + def op_debug_print(*args): - for arg in args: - print arg, - print + debug.debug_print(*map(_normalize, args)) + +def op_debug_start(category): + debug.debug_start(_normalize(category)) + +def op_debug_stop(category): + debug.debug_stop(_normalize(category)) + +def op_have_debug_prints(): + return debug.have_debug_prints() def op_gc_stack_bottom(): pass # marker for trackgcroot.py Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_lloperation.py Sun Nov 1 19:57:20 2009 @@ -127,7 +127,7 @@ def test_llinterp_complete(): for opname, llop in LL_OPERATIONS.items(): - if llop.canfold: + if llop.canrun: continue if opname.startswith('gc_x_'): continue # ignore experimental stuff Modified: pypy/trunk/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/generation.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/generation.py Sun Nov 1 19:57:20 2009 @@ -7,6 +7,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living @@ -85,8 +86,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size( - self.config.gcconfig.debugprint) + newsize = estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -102,6 +102,7 @@ self.nursery_free = NULL def set_nursery_size(self, newsize): + debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: @@ -116,13 +117,12 @@ while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) - llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", - self.largest_young_fixedsize) - llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", - self.largest_young_var_basesize) - llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) + debug_print("nursery_size =", newsize) + debug_print("largest_young_fixedsize =", + self.largest_young_fixedsize) + debug_print("largest_young_var_basesize =", + self.largest_young_var_basesize) + debug_print("nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) @@ -131,6 +131,7 @@ # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() + debug_stop("gc-set-nursery-size") @staticmethod def get_young_fixedsize(nursery_size): @@ -249,11 +250,7 @@ self.weakrefs_grow_older() self.ids_grow_older() self.reset_nursery() - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) - if self.config.gcconfig.debugprint and not size_changing: - llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size) def make_a_copy(self, obj, objsize): tid = self.header(obj).tid @@ -330,10 +327,9 @@ ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "--- minor collect ---") - llop.debug_print(lltype.Void, "nursery:", - self.nursery, "to", self.nursery_top) + debug_start("gc-minor") + debug_print("--- minor collect ---") + debug_print("nursery:", self.nursery, "to", self.nursery_top) # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() @@ -347,10 +343,9 @@ self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "survived (fraction of the size):", - float(scan - beginning) / self.nursery_size) + debug_print("survived (fraction of the size):", + float(scan - beginning) / self.nursery_size) + debug_stop("gc-minor") #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either @@ -376,8 +371,7 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) + debug_print("collect_oldrefs_to_nursery", count) def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -588,19 +582,18 @@ pass return -1 -def best_nursery_size_for_L2cache(L2cache, debugprint=False): - if debugprint: - llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) +def best_nursery_size_for_L2cache(L2cache): # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2 if sys.platform == 'linux2': - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = sys.maxint try: fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) @@ -648,10 +641,13 @@ if number < L2cache: L2cache = number + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") + if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1 @@ -676,10 +672,11 @@ rffi.INT, sandboxsafe=True) - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = 0 l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') try: @@ -703,14 +700,16 @@ lltype.free(len_p, flavor='raw') finally: lltype.free(l2cache_p, flavor='raw') + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1 else: - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): return -1 # XXX implement me for other platforms Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Sun Nov 1 19:57:20 2009 @@ -8,7 +8,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import rffi @@ -117,8 +118,7 @@ def setup(self): self.large_objects_collect_trigger = self.param_space_size - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 @@ -271,12 +271,12 @@ def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "allocated", - self._initial_trigger - - self.large_objects_collect_trigger, - "bytes, triggering full collection") + debug_start("gc-rawsize-collect") + debug_print("allocated", (self._initial_trigger - + self.large_objects_collect_trigger), + "bytes, triggering full collection") self.semispace_collect() + debug_stop("gc-rawsize-collect") def malloc_varsize_marknsweep(self, totalsize, resizable=False): # In order to free the large objects from time to time, we @@ -341,9 +341,8 @@ None) ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at start") - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count = 0 - self._nonmoving_copy_size = 0 + self._nonmoving_copy_count = 0 + self._nonmoving_copy_size = 0 def _set_gcflag_unvisited(self, obj, ignored): ll_assert(not (self.header(obj).tid & GCFLAG_UNVISITED), @@ -419,9 +418,8 @@ newaddr = self.allocate_external_object(totalsize_incl_hash) if not newaddr: return llmemory.NULL # can't raise MemoryError during a collect() - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count += 1 - self._nonmoving_copy_size += raw_malloc_usage(totalsize) + self._nonmoving_copy_count += 1 + self._nonmoving_copy_size += raw_malloc_usage(totalsize) llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) # check if we need to write a hash value at the end of the new obj @@ -464,11 +462,9 @@ def finished_full_collect(self): ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at end") - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hybrid] made nonmoving: ", - self._nonmoving_copy_size, "bytes in", - self._nonmoving_copy_count, "objs") + debug_print("| [hybrid] made nonmoving: ", + self._nonmoving_copy_size, "bytes in", + self._nonmoving_copy_count, "objs") # sweep the nonmarked rawmalloced objects if self.is_collecting_gen3(): self.sweep_rawmalloced_objects(generation=3) @@ -479,8 +475,7 @@ self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation @@ -513,17 +508,18 @@ surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 + debug = have_debug_prints() while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: - if self.config.gcconfig.debugprint: + if debug: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: - if self.config.gcconfig.debugprint: + if debug: alive_count+=1 alive_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) if generation == 3: @@ -554,17 +550,14 @@ self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving now alive: ", - alive_size, "bytes in", - alive_count, "objs") - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving freed: ", - dead_size, "bytes in", - dead_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving now alive: ", + alive_size, "bytes in", + alive_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving freed: ", + dead_size, "bytes in", + dead_count, "objs") def id(self, ptr): obj = llmemory.cast_ptr_to_adr(ptr) Modified: pypy/trunk/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/marksweep.py Sun Nov 1 19:57:20 2009 @@ -8,6 +8,7 @@ from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.gc.base import GCBase @@ -242,8 +243,7 @@ # call __del__, move the object to the list of object-without-del import time from pypy.rpython.lltypesystem.lloperation import llop - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, 'collecting...') + debug_start("gc-collect") start_time = time.time() self.collect_in_progress = True size_gc_header = self.gcheaderbuilder.size_gc_header @@ -406,31 +406,22 @@ 256 * 1024 * 1024) self.total_collection_time += collect_time self.prev_collect_end_time = end_time - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) + debug_print(" malloced since previous collection:", + old_malloced, "bytes") + debug_print(" heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + debug_print(" freed: ", + freed_size, "bytes") + debug_print(" new heap usage: ", + curr_heap_size, "bytes") + debug_print(" total time spent collecting: ", + self.total_collection_time, "seconds") + debug_print(" collecting time: ", + collect_time) + debug_print(" computing time: ", + collect_time) + debug_print(" new threshold: ", + self.bytes_malloced_threshold) ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size @@ -463,8 +454,9 @@ #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) finalizer(obj) if not self.collect_in_progress: # another collection was caused? - llop.debug_print(lltype.Void, "outer collect interrupted " - "by recursive collect") + debug_print("outer collect interrupted " + "by recursive collect") + debug_stop("gc-collect") return if not last: if self.malloced_objects_with_finalizer == next: @@ -480,6 +472,7 @@ last.next = lltype.nullptr(self.HDR) hdr = next self.collect_in_progress = False + debug_stop("gc-collect") def _mark_root(self, root): # 'root' is the address of the GCPTR gcobjectaddr = root.address[0] Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Sun Nov 1 19:57:20 2009 @@ -6,12 +6,13 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rlib.objectmodel import free_non_gc_object -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase -import sys, os, time +import sys, os first_gcflag = 1 << 16 GCFLAG_FORWARDED = first_gcflag @@ -54,15 +55,14 @@ MovingGCBase.__init__(self, config, chunk_size) def setup(self): - self.total_collection_time = 0.0 + #self.total_collection_time = 0.0 self.total_collection_count = 0 self.space_size = self.param_space_size self.max_space_size = self.param_max_space_size self.red_zone = 0 - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + #self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -74,7 +74,7 @@ self.objects_with_weakrefs = self.AddressStack() def _teardown(self): - llop.debug_print(lltype.Void, "Teardown") + debug_print("Teardown") llarena.arena_free(self.fromspace) llarena.arena_free(self.tospace) @@ -213,18 +213,13 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") - start_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used before collection: ", - start_usage, "bytes") - start_time = time.time() - else: - start_time = 0 # Help the flow space - start_usage = 0 # Help the flow space + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection ------------------") + start_usage = self.free - self.tospace + debug_print("| used before collection: ", + start_usage, "bytes") + #start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -254,41 +249,33 @@ self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) - if self.config.gcconfig.debugprint: - end_time = time.time() - elapsed_time = end_time - start_time - self.total_collection_time += elapsed_time + if have_debug_prints(): + #end_time = time.time() + #elapsed_time = end_time - start_time + #self.total_collection_time += elapsed_time self.total_collection_count += 1 - total_program_time = end_time - self.program_start_time + #total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used after collection: ", - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| freed: ", - start_usage - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| size of each semispace: ", - self.space_size, "bytes") - llop.debug_print(lltype.Void, - "| fraction of semispace now used: ", - end_usage * 100.0 / self.space_size, "%") - ct = self.total_collection_time + debug_print("| used after collection: ", + end_usage, "bytes") + debug_print("| freed: ", + start_usage - end_usage, "bytes") + debug_print("| size of each semispace: ", + self.space_size, "bytes") + debug_print("| fraction of semispace now used: ", + end_usage * 100.0 / self.space_size, "%") + #ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of semispace_collects: ", - cc) - llop.debug_print(lltype.Void, - "| i.e.: ", - cc / total_program_time, "per second") - llop.debug_print(lltype.Void, - "| total time in semispace_collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| i.e.: ", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + debug_print("| number of semispace_collects: ", + cc) + #debug_print("| i.e.: ", + # cc / total_program_time, "per second") + #debug_print("| total time in semispace_collect: ", + # ct, "seconds") + #debug_print("| i.e.: ", + # ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def starting_full_collect(self): pass # hook for the HybridGC Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Sun Nov 1 19:57:20 2009 @@ -130,7 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcconfig.removetypeptr: + if translator.config.translation.gcremovetypeptr: lltype2vtable = translator.rtyper.lltype2vtable else: lltype2vtable = {} @@ -888,7 +888,7 @@ def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_getfield_typeptr(hop) else: GCTransformer.gct_getfield(self, hop) @@ -896,7 +896,7 @@ def gct_setfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_setfield_typeptr(hop) else: GCTransformer.gct_setfield(self, hop) @@ -995,7 +995,7 @@ def __init__(self, config): from pypy.rpython.memory.gc.base import choose_gc_from_config try: - assert not config.translation.gcconfig.removetypeptr + assert not config.translation.gcremovetypeptr except AttributeError: # for some tests pass GCClass, _ = choose_gc_from_config(config) Modified: pypy/trunk/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/gctypelayout.py Sun Nov 1 19:57:20 2009 @@ -219,8 +219,7 @@ type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id # store the vtable of the type (if any) immediately thereafter - # (note that if gcconfig.removetypeptr is False, lltype2vtable - # is empty) + # (note that if gcremovetypeptr is False, lltype2vtable is empty) vtable = self.lltype2vtable.get(TYPE, None) if vtable is not None: # check that if we have a vtable, we are not varsize Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Sun Nov 1 19:57:20 2009 @@ -23,7 +23,7 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname - t.config.translation.gcconfig.removetypeptr = True + t.config.translation.gcremovetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) Modified: pypy/trunk/pypy/translator/c/funcgen.py ============================================================================== --- pypy/trunk/pypy/translator/c/funcgen.py (original) +++ pypy/trunk/pypy/translator/c/funcgen.py Sun Nov 1 19:57:20 2009 @@ -714,6 +714,7 @@ from pypy.rpython.lltypesystem.rstr import STR format = [] argv = [] + free_line = "" for arg in op.args: T = arg.concretetype if T == Ptr(STR): @@ -722,6 +723,7 @@ else: format.append('%s') argv.append('RPyString_AsCharP(%s)' % self.expr(arg)) + free_line = "RPyString_FreeCache();" continue elif T == Signed: format.append('%d') @@ -741,9 +743,22 @@ else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) - return "fprintf(stderr, %s%s); RPyString_FreeCache();" % ( - c_string_constant(' '.join(format) + '\n\000'), - ''.join([', ' + s for s in argv])) + argv.insert(0, c_string_constant(' '.join(format) + '\n')) + return ( + "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" + % (', '.join(argv), free_line)) + + def OP_DEBUG_START(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_START(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) + + def OP_DEBUG_STOP(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_STOP(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/trunk/pypy/translator/c/gc.py ============================================================================== --- pypy/trunk/pypy/translator/c/gc.py (original) +++ pypy/trunk/pypy/translator/c/gc.py Sun Nov 1 19:57:20 2009 @@ -352,7 +352,7 @@ def need_no_typeptr(self): config = self.db.translator.config - return config.translation.gcconfig.removetypeptr + return config.translation.gcremovetypeptr def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Sun Nov 1 19:57:20 2009 @@ -588,7 +588,7 @@ # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', - 'bswap', 'bt', + 'bswap', 'bt', 'rdtsc', # zero-extending moves should not produce GC pointers 'movz', ]) @@ -1297,6 +1297,7 @@ f = open(fn, 'r') firstline = f.readline() f.seek(0) + assert firstline, "file %r is empty!" % (fn,) if firstline.startswith('seen_main = '): tracker.reload_raw_table(f) f.close() Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Sun Nov 1 19:57:20 2009 @@ -430,12 +430,14 @@ bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) - def cmdexec(self, args='', env=None): + def cmdexec(self, args='', env=None, err=False): assert self._compiled res = self.translator.platform.execute(self.executable_name, args, env=env) if res.returncode != 0: raise Exception("Returned %d" % (res.returncode,)) + if err: + return res.out, res.err return res.out def compile(self): Modified: pypy/trunk/pypy/translator/c/src/asm_gcc_x86.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/asm_gcc_x86.h (original) +++ pypy/trunk/pypy/translator/c/src/asm_gcc_x86.h Sun Nov 1 19:57:20 2009 @@ -50,6 +50,10 @@ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ +/* Pentium only! */ +#define READ_TIMESTAMP(val) \ + asm volatile("rdtsc" : "=A" (val)) + /* prototypes */ Modified: pypy/trunk/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/g_include.h (original) +++ pypy/trunk/pypy/translator/c/src/g_include.h Sun Nov 1 19:57:20 2009 @@ -51,6 +51,7 @@ /*** modules ***/ #ifdef HAVE_RTYPER /* only if we have an RTyper */ # include "src/rtyper.h" +# include "src/debug.h" #ifndef AVR # include "src/ll_os.h" # include "src/ll_strtod.h" Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Sun Nov 1 19:57:20 2009 @@ -39,8 +39,7 @@ t = Translation(main, standalone=True, gc=cls.gcpolicy, policy=annpolicy.StrictAnnotatorPolicy(), taggedpointers=cls.taggedpointers, - removetypeptr=cls.removetypeptr, - debugprint=True) + gcremovetypeptr=cls.removetypeptr) t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_standalone.py (original) +++ pypy/trunk/pypy/translator/c/test/test_standalone.py Sun Nov 1 19:57:20 2009 @@ -2,7 +2,8 @@ import sys, os, re from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.debug import ll_assert, debug_print +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo @@ -11,9 +12,22 @@ from pypy.tool.autopath import pypydir -class TestStandalone(object): +class StandaloneTests(object): config = None - + + def compile(self, entry_point): + t = TranslationContext(self.config) + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source() + cbuilder.compile() + return t, cbuilder + + +class TestStandalone(StandaloneTests): + def test_hello_world(self): def entry_point(argv): os.write(1, "hello world\n") @@ -23,13 +37,7 @@ os.write(1, " '" + str(s) + "'\n") return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''') @@ -43,13 +51,7 @@ print [len(s) for s in argv] return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello simpler world\n''' '''argument count: 2\n''' @@ -130,13 +132,7 @@ print m, x return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert map(float, data.split()) == [0.0, 0.0] @@ -173,13 +169,8 @@ os.setpgrp() return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) + cbuilder.cmdexec("") def test_profopt_mac_osx_bug(self): @@ -223,12 +214,7 @@ print "BAD POS" os.close(fd) return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.strip() == "OK" @@ -270,6 +256,128 @@ assert " ll_strtod.h" in makefile assert " ll_strtod.o" in makefile + def test_debug_print_start_stop(self): + def entry_point(argv): + x = "got:" + debug_start ("mycat") + if have_debug_prints(): x += "b" + debug_print ("foo", 2, "bar", 3) + debug_start ("cat2") + if have_debug_prints(): x += "c" + debug_print ("baz") + debug_stop ("cat2") + if have_debug_prints(): x += "d" + debug_print ("bok") + debug_stop ("mycat") + if have_debug_prints(): x += "a" + debug_print("toplevel") + os.write(1, x + '.\n') + return 0 + t, cbuilder = self.compile(entry_point) + # check with PYPYLOG undefined + out, err = cbuilder.cmdexec("", err=True, env={}) + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err + # check with PYPYLOG defined to an empty string (same as undefined) + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err + # check with PYPYLOG=:- (means print to stderr) + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ':-'}) + assert out.strip() == 'got:bcda.' + assert 'toplevel' in err + assert '{mycat' in err + assert 'mycat}' in err + assert 'foo 2 bar 3' in err + assert '{cat2' in err + assert 'cat2}' in err + assert 'baz' in err + assert 'bok' in err + # check with PYPYLOG=:somefilename + path = udir.join('test_debug_xxx.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:bcda.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data + assert '{cat2' in data + assert 'cat2}' in data + assert 'baz' in data + assert 'bok' in data + # check with PYPYLOG=somefilename + path = udir.join('test_debug_xxx_prof.log') + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': str(path)}) + assert out.strip() == 'got:a.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' not in data + assert '{cat2' in data + assert 'cat2}' in data + assert 'baz' not in data + assert 'bok' not in data + # check with PYPYLOG=myc:somefilename (includes mycat but not cat2) + path = udir.join('test_debug_xxx_myc.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'myc:%s' % path}) + assert out.strip() == 'got:bda.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data + assert 'cat2' not in data + assert 'baz' not in data + assert 'bok' in data + # check with PYPYLOG=cat:somefilename (includes cat2 but not mycat) + path = udir.join('test_debug_xxx_cat.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'cat:%s' % path}) + assert out.strip() == 'got:a.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in path.read() + assert 'mycat' not in path.read() + assert 'foo 2 bar 3' not in path.read() + assert 'cat2' not in data # because it is nested + assert 'baz' not in data + assert 'bok' not in data + # + # finally, check compiling with logging disabled + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.log = False + self.config = config + t, cbuilder = self.compile(entry_point) + path = udir.join('test_debug_does_not_show_up.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:.' + assert not err + assert path.check(file=0) + + class TestMaemo(TestStandalone): def setup_class(cls): from pypy.translator.platform.maemo import check_scratchbox Modified: pypy/trunk/pypy/translator/cli/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/cli/metavm.py (original) +++ pypy/trunk/pypy/translator/cli/metavm.py Sun Nov 1 19:57:20 2009 @@ -270,6 +270,10 @@ generator.ilasm.call('void [pypylib]pypy.runtime.Utils::debug_print(%s)' % signature) +class _HaveDebugPrints(MicroInstruction): + def render(self, generator, op): + generator.ilasm.load_const(ootype.Bool, True) + OOTYPE_TO_MNEMONIC = { ootype.Bool: 'i1', @@ -306,3 +310,4 @@ SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() DebugPrint = _DebugPrint() +HaveDebugPrints = _HaveDebugPrints() Modified: pypy/trunk/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/trunk/pypy/translator/cli/opcodes.py (original) +++ pypy/trunk/pypy/translator/cli/opcodes.py Sun Nov 1 19:57:20 2009 @@ -1,7 +1,8 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, DebugPrint + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, \ + DebugPrint, HaveDebugPrints from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -77,6 +78,9 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'debug_print': [DebugPrint], + 'debug_start': Ignore, + 'debug_stop': Ignore, + 'have_debug_prints': [HaveDebugPrints], 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Sun Nov 1 19:57:20 2009 @@ -8,7 +8,7 @@ class Linux(BasePosix): name = "linux" - link_flags = ['-pthread'] + link_flags = ['-pthread', '-lrt'] cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] standalone_only = [] shared_only = [] From arigo at codespeak.net Sun Nov 1 19:57:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:57:42 +0100 (CET) Subject: [pypy-svn] r68903 - pypy/branch/logging2 Message-ID: <20091101185742.52469168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:57:41 2009 New Revision: 68903 Removed: pypy/branch/logging2/ Log: Remove merged branch. From arigo at codespeak.net Sun Nov 1 19:57:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 1 Nov 2009 19:57:54 +0100 (CET) Subject: [pypy-svn] r68904 - pypy/branch/logging Message-ID: <20091101185754.7B6B2168076@codespeak.net> Author: arigo Date: Sun Nov 1 19:57:53 2009 New Revision: 68904 Removed: pypy/branch/logging/ Log: Remove the other branch too. From afa at codespeak.net Sun Nov 1 23:13:27 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 1 Nov 2009 23:13:27 +0100 (CET) Subject: [pypy-svn] r68905 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091101221327.C09D6168078@codespeak.net> Author: afa Date: Sun Nov 1 23:13:24 2009 New Revision: 68905 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: The code generated by the msvc compiler is inconsistent after a never-returning function: sometimes esp is adjusted, sometimes not. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Sun Nov 1 23:13:24 2009 @@ -635,7 +635,7 @@ else: target = match.group(1) if target in FUNCTIONS_NOT_RETURNING: - return InsnStop() + return [InsnStop(), InsnCannotFollowEsp()] if sys.platform == 'win32' and target == '__alloca': # in functions with large stack requirements, windows # needs a call to _alloca(), to turn reserved pages From cfbolz at codespeak.net Mon Nov 2 12:04:36 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 2 Nov 2009 12:04:36 +0100 (CET) Subject: [pypy-svn] r68908 - pypy/extradoc/planning Message-ID: <20091102110436.14189168079@codespeak.net> Author: cfbolz Date: Mon Nov 2 12:04:34 2009 New Revision: 68908 Modified: pypy/extradoc/planning/jit.txt Log: these things are don Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 2 12:04:34 2009 @@ -3,13 +3,9 @@ - sort out a benchmark infrastructure. graphs! -- nightly run of checks on output of pypy-c-jit: we need tests for - pypy-jit behaviour that explicitely check whether the loops make - sense - - lose less information across residual calls -- improve on predictability: delegate the caching, don't trace into signals ... but produce just a conditional call +- improve on predictability: don't trace into signals ... but produce just a conditional call - we need to be able to not inline when the traces are too long, this would need being able to start tracing from function starts, the latter From cfbolz at codespeak.net Mon Nov 2 12:04:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 2 Nov 2009 12:04:54 +0100 (CET) Subject: [pypy-svn] r68909 - pypy/trunk/pypy/translator/backendopt Message-ID: <20091102110454.9EBA016807B@codespeak.net> Author: cfbolz Date: Mon Nov 2 12:04:53 2009 New Revision: 68909 Modified: pypy/trunk/pypy/translator/backendopt/inline.py Log: use a set instead of a dict (mostly just because) Modified: pypy/trunk/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/inline.py (original) +++ pypy/trunk/pypy/translator/backendopt/inline.py Mon Nov 2 12:04:53 2009 @@ -631,7 +631,7 @@ count), True def inlinable_static_callers(graphs): - ok_to_call = dict.fromkeys(graphs) + ok_to_call = set(graphs) result = [] for parentgraph in graphs: for block in parentgraph.iterblocks(): From cfbolz at codespeak.net Mon Nov 2 14:21:56 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 2 Nov 2009 14:21:56 +0100 (CET) Subject: [pypy-svn] r68910 - in pypy/trunk/pypy: interpreter interpreter/test module/pypyjit/test objspace objspace/std Message-ID: <20091102132156.0532E168079@codespeak.net> Author: cfbolz Date: Mon Nov 2 14:21:55 2009 New Revision: 68910 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/test/test_function.py pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py pypy/trunk/pypy/objspace/descroperation.py pypy/trunk/pypy/objspace/std/callmethod.py Log: Prevent the changing of the func_code attribute of builtin functions. This makes some shortcuts less beautiful, but produces one guard and one getfield less a bit everywhere. Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Mon Nov 2 14:21:55 2009 @@ -16,12 +16,20 @@ funccallunrolling = unrolling_iterable(range(4)) + at jit.purefunction +def _get_immutable_code(func): + assert not func.can_change_code + return func.code + class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, and an arbitrary 'closure' passed to the code object.""" - def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): + can_change_code = True + + def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, + forcename=None): self.space = space self.name = forcename or code.co_name self.w_doc = None # lazily read from code.getdocstring() @@ -48,7 +56,12 @@ return self.getcode().funcrun_obj(self, w_obj, args) def getcode(self): - return jit.hint(self.code, promote=True) + if jit.we_are_jitted(): + if not self.can_change_code: + self = jit.hint(self, promote=True) + return _get_immutable_code(self) + return jit.hint(self.code, promote=True) + return self.code def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway @@ -368,6 +381,9 @@ def fset_func_code(space, self, w_code): from pypy.interpreter.pycode import PyCode + if not self.can_change_code: + raise OperationError(space.w_TypeError, + space.wrap("Cannot change code attribute of builtin functions")) code = space.interp_w(Code, w_code) closure_len = 0 if self.closure: @@ -568,7 +584,11 @@ "'%s' object is not callable" % typename)) return space.wrap(ClassMethod(w_function)) +class FunctionWithFixedCode(Function): + can_change_code = False + class BuiltinFunction(Function): + can_change_code = False def __init__(self, func): assert isinstance(func, Function) Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Mon Nov 2 14:21:55 2009 @@ -16,6 +16,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import eval from pypy.interpreter.function import Function, Method, ClassMethod +from pypy.interpreter.function import FunctionWithFixedCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, Signature @@ -788,7 +789,7 @@ space = cache.space defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code - fn = Function(space, code, None, defs, forcename = gateway.name) + fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) if not space.config.translating: # for tests and py.py fn._freeze_() if gateway.as_classmethod: Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Mon Nov 2 14:21:55 2009 @@ -56,7 +56,14 @@ #print "loaded", w_value # obscure func = space.interpclass_w(w_value) - if type(func) is Function: + # the idea of the following code is that all functions that are + # directly in a mixed-module are "builtin", e.g. they get a + # special type without a __get__ + # note that this is not just all functions that contain a + # builtin code object, as e.g. methods of builtin types have to + # be normal Functions to get the correct binding behaviour + if (isinstance(func, Function) and + type(func) is not BuiltinFunction): try: bltin = func._builtinversion_ except AttributeError: Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Mon Nov 2 14:21:55 2009 @@ -84,7 +84,12 @@ return f() # a closure raises(ValueError, "f.func_code = h.func_code") - + def test_write_code_builtin_forbidden(self): + def f(*args): + return 42 + raises(TypeError, "dir.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") + class AppTestFunction: def test_simple_call(self): Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 2 14:21:55 2009 @@ -252,8 +252,7 @@ assert len(callisinstance.get_opnames("guard")) <= 2 bytecode, = self.get_by_bytecode("STORE_ATTR") - # XXX where does that come from? - assert bytecode.get_opnames() == ["getfield_gc", "guard_value"] + assert bytecode.get_opnames() == [] def test_mixed_type_loop(self): self.run_source(''' @@ -272,7 +271,28 @@ bytecode, = self.get_by_bytecode("BINARY_ADD") assert not bytecode.get_opnames("call") assert not bytecode.get_opnames("new") - assert len(bytecode.get_opnames("guard")) <= 3 + assert len(bytecode.get_opnames("guard")) <= 2 + + def test_call_builtin_function(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 2 + l = [] + while i < n: + i += 1 + l.append(i) + return i, len(l) + ''', + ([20], (20, 18)), + ([31], (31, 29))) + + bytecode, = self.get_by_bytecode("CALL_METHOD") + assert len(bytecode.get_opnames("new_with_vtable")) == 1 # the forcing of the int + assert len(bytecode.get_opnames("call")) == 1 # the call to append + assert len(bytecode.get_opnames("guard")) == 1 # guard_no_exception after the call + class AppTestJIT(PyPyCJITTests): def setup_class(cls): @@ -288,7 +308,7 @@ class TestJIT(PyPyCJITTests): def setup_class(cls): if option.pypy_c is None: - py.test.skip("pass --pypy-c!") + py.test.skip("pass --pypy!") cls.tmpdir = udir.join('pypy-jit') cls.tmpdir.ensure(dir=1) cls.counter = 0 Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Mon Nov 2 14:21:55 2009 @@ -1,7 +1,7 @@ import operator from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace -from pypy.interpreter.function import Function, Method +from pypy.interpreter.function import Function, Method, FunctionWithFixedCode from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import default_identity_hash from pypy.tool.sourcetools import compile2, func_with_new_name @@ -81,7 +81,7 @@ def get_and_call_args(space, w_descr, w_obj, args): descr = space.interpclass_w(w_descr) # a special case for performance and to avoid infinite recursion - if type(descr) is Function: + if isinstance(descr, Function): return descr.call_obj_args(w_obj, args) else: w_impl = space.get(w_descr, w_obj) @@ -89,8 +89,15 @@ def get_and_call_function(space, w_descr, w_obj, *args_w): descr = space.interpclass_w(w_descr) + typ = type(descr) # a special case for performance and to avoid infinite recursion - if type(descr) is Function: + if typ is Function or typ is FunctionWithFixedCode: + # isinstance(typ, Function) would not be correct here: + # for a BuiltinFunction we must not use that shortcut, because a + # builtin function binds differently than a normal function + # see test_builtin_as_special_method_is_not_bound + # in interpreter/test/test_function.py + # the fastcall paths are purely for performance, but the resulting # increase of speed is huge return descr.funccall(w_obj, *args_w) Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Mon Nov 2 14:21:55 2009 @@ -55,14 +55,16 @@ # this handles directly the common case # module.function(args..) w_value = w_obj.getdictvalue(space, w_name) - elif type(w_descr) is function.Function: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) - if w_value is None: - # fast method path: a function object in the class, - # nothing in the instance - f.pushvalue(w_descr) - f.pushvalue(w_obj) - return + else: + typ = type(w_descr) + if typ is function.Function or typ is function.FunctionWithFixedCode: + w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + if w_value is None: + # fast method path: a function object in the class, + # nothing in the instance + f.pushvalue(w_descr) + f.pushvalue(w_obj) + return if w_value is None: w_value = space.getattr(w_obj, w_name) f.pushvalue(w_value) @@ -89,7 +91,8 @@ w_getattribute = space.lookup(w_obj, '__getattribute__') if w_getattribute is object_getattribute(space): w_descr = space.lookup(w_obj, methname) - if type(w_descr) is function.Function: + typ = type(w_descr) + if typ is function.Function or typ is function.FunctionWithFixedCode: w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) if w_value is None: # fast method path: a function object in the class, From arigo at codespeak.net Mon Nov 2 14:36:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 14:36:19 +0100 (CET) Subject: [pypy-svn] r68911 - pypy/trunk/pypy/translator/c/src Message-ID: <20091102133619.A48AA16807B@codespeak.net> Author: arigo Date: Mon Nov 2 14:36:18 2009 New Revision: 68911 Modified: pypy/trunk/pypy/translator/c/src/debug.h Log: Remove includes that are no longer needed (and that go in the way of Windows compilation). Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Mon Nov 2 14:36:18 2009 @@ -43,9 +43,7 @@ /* implementations */ #ifndef PYPY_NOT_MAIN_FILE -#include #include -#include int pypy_ignoring_nested_prints = 0; FILE *pypy_debug_file = NULL; From cfbolz at codespeak.net Mon Nov 2 15:28:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 2 Nov 2009 15:28:39 +0100 (CET) Subject: [pypy-svn] r68912 - pypy/branch/shrink-multidict/pypy/objspace/std Message-ID: <20091102142839.4475916807A@codespeak.net> Author: cfbolz Date: Mon Nov 2 15:28:38 2009 New Revision: 68912 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Log: running the tests is generally a good idea Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Mon Nov 2 15:28:38 2009 @@ -98,7 +98,7 @@ self.impl_setitem_str(attr, w_value) return True w_dict = self.getdict() - w_dict.setitem(w_attr, w_value) + w_dict.setitem_str(attr, w_value) return True def deldictvalue(self, space, w_attr): From arigo at codespeak.net Mon Nov 2 16:09:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 16:09:45 +0100 (CET) Subject: [pypy-svn] r68914 - in pypy/trunk/pypy/tool: . test Message-ID: <20091102150945.320CD168079@codespeak.net> Author: arigo Date: Mon Nov 2 16:09:44 2009 New Revision: 68914 Modified: pypy/trunk/pypy/tool/logparser.py pypy/trunk/pypy/tool/test/test_logparser.py Log: Add a function with a test. Add another test for logparser. Modified: pypy/trunk/pypy/tool/logparser.py ============================================================================== --- pypy/trunk/pypy/tool/logparser.py (original) +++ pypy/trunk/pypy/tool/logparser.py Mon Nov 2 16:09:44 2009 @@ -39,6 +39,20 @@ f.close() return log +def extract_category(log, catprefix='', toplevel=False): + got = [] + resulttext = [] + for entry in log: + if entry[0] == 'debug_print': + resulttext.append(entry[1]) + else: + got.extend(extract_category( + entry[3], catprefix, toplevel=entry[0].startswith(catprefix))) + if toplevel: + resulttext.append('') + got.insert(0, '\n'.join(resulttext)) + return got + def getsubcategories(log): return [entry for entry in log if entry[0] != 'debug_print'] Modified: pypy/trunk/pypy/tool/test/test_logparser.py ============================================================================== --- pypy/trunk/pypy/tool/test/test_logparser.py (original) +++ pypy/trunk/pypy/tool/test/test_logparser.py Mon Nov 2 16:09:44 2009 @@ -1,5 +1,54 @@ +from pypy.tool.udir import udir from pypy.tool.logparser import * + +globalpath = udir.join('test_logparser.log') +globalpath.write("""\ +test1 +[12a0] {foo +test2a +test2b +[12b0] {bar +test3 +[12e0] bar} +test4 +[12e5] {bar +test5a +test5b +[12e6] bar} +test6 +[12f0] foo} +test7 +""") + + +def test_parse_log_file(): + log = parse_log_file(str(globalpath)) + assert log == [ + ('debug_print', 'test1'), + ('foo', 0x12a0, 0x12f0, [ + ('debug_print', 'test2a'), + ('debug_print', 'test2b'), + ('bar', 0x12b0, 0x12e0, [ + ('debug_print', 'test3')]), + ('debug_print', 'test4'), + ('bar', 0x12e5, 0x12e6, [ + ('debug_print', 'test5a'), + ('debug_print', 'test5b')]), + ('debug_print', 'test6')]), + ('debug_print', 'test7')] + +def test_extract_category(): + log = parse_log_file(str(globalpath)) + catbar = list(extract_category(log, 'bar')) + assert catbar == ["test3\n", "test5a\ntest5b\n"] + assert catbar == list(extract_category(log, 'ba')) + catfoo = list(extract_category(log, 'foo')) + assert catfoo == ["test2a\ntest2b\ntest4\ntest6\n"] + assert catfoo == list(extract_category(log, 'f')) + catall = list(extract_category(log, '')) + assert catall == catfoo + catbar + def test_gettotaltimes(): result = gettotaltimes([ ('foo', 2, 17, [ From arigo at codespeak.net Mon Nov 2 16:14:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 16:14:14 +0100 (CET) Subject: [pypy-svn] r68915 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091102151414.AFD3D1680A3@codespeak.net> Author: arigo Date: Mon Nov 2 16:14:14 2009 New Revision: 68915 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix test_pypy_c.py. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 2 16:14:14 2009 @@ -55,22 +55,21 @@ # we don't have os.popen() yet on pypy-c... if sys.platform.startswith('win'): py.test.skip("XXX this is not Windows-friendly") - child_stdin, child_stdout = os.popen2('PYPYJITLOG="%s" "%s" "%s"' % ( + child_stdin, child_stdout = os.popen2('PYPYLOG=":%s" "%s" "%s"' % ( logfilepath, self.pypy_c, filepath)) child_stdin.close() result = child_stdout.read() child_stdout.close() assert result assert result.splitlines()[-1].strip() == 'OK :-)' - assert logfilepath.check() - opslogfile = logfilepath.new(ext='.log.ops') - self.parse_loops(opslogfile) + self.parse_loops(logfilepath) def parse_loops(self, opslogfile): - from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops + from pypy.jit.metainterp.test.oparser import parse + from pypy.tool import logparser assert opslogfile.check() - logs = opslogfile.read() - parts = split_logs_into_loops(logs) + log = logparser.parse_log_file(str(opslogfile)) + parts = logparser.extract_category(log, 'jit-log-opt-') # skip entry bridges, they can contain random things self.loops = [parse(part, no_namespace=True) for part in parts if "entry bridge" not in part] From arigo at codespeak.net Mon Nov 2 16:21:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 16:21:32 +0100 (CET) Subject: [pypy-svn] r68916 - in pypy/trunk/pypy/jit: backend tool Message-ID: <20091102152132.52DAF168079@codespeak.net> Author: arigo Date: Mon Nov 2 16:21:31 2009 New Revision: 68916 Added: pypy/trunk/pypy/jit/tool/autopath.py - copied unchanged from r68910, pypy/trunk/pypy/jit/backend/autopath.py pypy/trunk/pypy/jit/tool/loopviewer.py - copied, changed from r68910, pypy/trunk/pypy/jit/backend/loopviewer.py pypy/trunk/pypy/jit/tool/showstats.py - copied, changed from r68910, pypy/trunk/pypy/jit/backend/showstats.py Removed: pypy/trunk/pypy/jit/backend/loopviewer.py pypy/trunk/pypy/jit/backend/showstats.py Log: Move these tools to jit/tool/. Fix them. Copied: pypy/trunk/pypy/jit/tool/loopviewer.py (from r68910, pypy/trunk/pypy/jit/backend/loopviewer.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/loopviewer.py (original) +++ pypy/trunk/pypy/jit/tool/loopviewer.py Mon Nov 2 16:21:31 2009 @@ -5,13 +5,14 @@ import autopath import py import sys -from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops +from pypy.tool import logparser +from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.history import ConstInt from pypy.rpython.lltypesystem import llmemory, lltype def main(loopnum, loopfile): - data = py.path.local(loopfile).read() - loops = split_logs_into_loops(data) + log = logparser.parse_log_file(loopfile) + loops = logparser.extract_category(log, "jit-log-opt-") inp = loops[loopnum] loop = parse(inp, no_namespace=True) loop.show() Copied: pypy/trunk/pypy/jit/tool/showstats.py (from r68910, pypy/trunk/pypy/jit/backend/showstats.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/showstats.py (original) +++ pypy/trunk/pypy/jit/tool/showstats.py Mon Nov 2 16:21:31 2009 @@ -1,12 +1,14 @@ #!/usr/bin/env python import autopath import sys, py -from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops +from pypy.tool import logparser +from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import lltype, llmemory def main(argv): - parts = split_logs_into_loops(py.path.local(argv[0]).read()) + log = logparser.parse_log_file(argv[0]) + parts = logparser.extract_category(log, "jit-log-opt-") for oplist in parts: loop = parse(oplist, no_namespace=True) num_ops = 0 From arigo at codespeak.net Mon Nov 2 16:23:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 16:23:56 +0100 (CET) Subject: [pypy-svn] r68917 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091102152356.1CB5C168007@codespeak.net> Author: arigo Date: Mon Nov 2 16:23:55 2009 New Revision: 68917 Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Log: Remove this function, no longer needed (use pypy.tool.logparser.extract_category). Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Mon Nov 2 16:23:55 2009 @@ -314,23 +314,6 @@ return parse(*args, **kwds) -def split_logs_into_loops(text): - lines = text.splitlines() - parts = [] - last_with_hash = False - lines_of_part = [] - for i, line in enumerate(lines): - if (line.startswith("[") and last_with_hash - and len(lines_of_part) > 1): - parts.append("\n".join(lines_of_part[:-1])) - lines_of_part = [lines_of_part[-1], line] - else: - lines_of_part.append(line) - last_with_hash = line.startswith("#") - parts.append("\n".join(lines_of_part)) - return parts - - def _box_counter_more_than(s): if s.isdigit(): Box._counter = max(Box._counter, int(s)+1) Modified: pypy/trunk/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_oparser.py Mon Nov 2 16:23:55 2009 @@ -1,7 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.metainterp.test.oparser import parse, split_logs_into_loops +from pypy.jit.metainterp.test.oparser import parse from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import AbstractDescr, BoxInt, LoopToken,\ BoxFloat @@ -161,55 +161,7 @@ loop = parse(x) # assert did not explode -examplelog = '''\ -# Loop0 (loop), 12 ops -[i0, i1] -debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstClass(cls2), i0, descr=) -guard_no_exception(, descr=) [i0, i1, i3] -i5 = int_add(i1, 2) -i7 = call(ConstClass(cls6), i0, descr=) -p9 = guard_exception(4, descr=) [i5, i0, i7] -i11 = int_sub(i5, 1) -i12 = int_sub(i0, 1) -i14 = int_gt(i12, 3) -guard_true(i14, descr=) [i11, i12] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i12, i11, descr=) -# Loop1 (entry bridge), 12 ops -[i0, i1] -debug_merge_point('(no jitdriver.get_printable_location!)') -i3 = call(ConstClass(cls2), i0, descr=) -p5 = guard_exception(4, descr=) [i0, i1, i3] -i7 = int_add(i1, 1) -i9 = call(ConstClass(cls8), i0, descr=) -p11 = guard_exception(4, descr=) [i7, i0, i9] -i12 = int_sub(i7, 1) -i13 = int_sub(i0, 1) -i15 = int_gt(i13, 3) -guard_true(i15, descr=) [i12, i13] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i13, i12, descr=) -# bridge out of Guard5, 10 ops -[i0, i1, i2] -p4 = guard_exception(4, descr=) [i0, i1, i2] -i6 = int_add(i1, 1) -i8 = call(ConstClass(cls7), i0, descr=) -p10 = guard_exception(4, descr=) [i6, i0, i8] -i11 = int_sub(i6, 1) -i12 = int_sub(i0, 1) -i14 = int_gt(i12, 3) -guard_true(i14, descr=) [i11, i12] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i12, i11, descr=) -# bridge out of Guard9, 6 ops -[i0, i1, i2] -i4 = int_add(i0, 2) -i6 = int_sub(i1, 1) -i8 = int_gt(i6, 3) -guard_true(i8, descr=) [i4, i6] -debug_merge_point('(no jitdriver.get_printable_location!)') -jump(i6, i4, descr=) +example_loop_log = '''\ # bridge out of Guard12, 6 ops [i0, i1, i2] i4 = int_add(i0, 2) @@ -220,18 +172,5 @@ jump(i6, i4, descr=) ''' -def test_split_logs_into_loops(): - parts = split_logs_into_loops(examplelog) - assert len(parts) == 5 - assert "\n".join(parts) == examplelog.strip() - for part, typ in zip(parts, - ["Loop0", "Loop1", - "bridge out of Guard5", - "bridge out of Guard9", - "bridge out of Guard12"]): - assert part.startswith("# %s" % typ) - def test_parse_no_namespace(): - parts = split_logs_into_loops(examplelog) - for part in parts: - loop = parse(part, no_namespace=True) + loop = parse(example_loop_log, no_namespace=True) From antocuni at codespeak.net Mon Nov 2 16:46:32 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 2 Nov 2009 16:46:32 +0100 (CET) Subject: [pypy-svn] r68918 - pypy/trunk/pypy/jit/backend/cli/test Message-ID: <20091102154632.AA182168077@codespeak.net> Author: antocuni Date: Mon Nov 2 16:46:31 2009 New Revision: 68918 Modified: pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py Log: this is no longer needed, as this test does not xfail() anymore Modified: pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py Mon Nov 2 16:46:31 2009 @@ -7,6 +7,4 @@ # for the individual tests see # ====> ../../../metainterp/test/test_vlist.py - # disable the xfail() - def test_vlist_alloc_and_set(self): - test_vlist.TestOOtype.test_vlist_alloc_and_set(self) + pass From arigo at codespeak.net Mon Nov 2 16:56:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 2 Nov 2009 16:56:38 +0100 (CET) Subject: [pypy-svn] r68919 - pypy/trunk/pypy/translator/c Message-ID: <20091102155638.1ED02168077@codespeak.net> Author: arigo Date: Mon Nov 2 16:56:37 2009 New Revision: 68919 Modified: pypy/trunk/pypy/translator/c/genc.py Log: Don't print a meaningless line "struct None;" to structdef.h. Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Mon Nov 2 16:56:37 2009 @@ -703,7 +703,7 @@ if hasattr(node, 'forward_decl'): if node.forward_decl: print >> f, node.forward_decl - else: + elif node.name is not None: print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: From cfbolz at codespeak.net Mon Nov 2 18:33:08 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 2 Nov 2009 18:33:08 +0100 (CET) Subject: [pypy-svn] r68923 - in pypy/branch/shrink-multidict/pypy/objspace/std: . test Message-ID: <20091102173308.E47A7168010@codespeak.net> Author: cfbolz Date: Mon Nov 2 18:33:08 2009 New Revision: 68923 Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Log: A bug found by test_descr.py: setting the dict of an object with a devolved dict runs into an assertion. Modified: pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py Mon Nov 2 18:33:08 2009 @@ -124,9 +124,12 @@ cell.invalidate() for k in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(k) + self._clear_fields() + return self + + def _clear_fields(self): self.content = None self.unshadowed_builtins = None - return self class ModuleDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): Modified: pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py Mon Nov 2 18:33:08 2009 @@ -370,9 +370,12 @@ r_dict_content = self.initialize_as_rdict() for k, w_v in self.content.items(): r_dict_content[self.space.wrap(k)] = w_v - self.content = None + self._clear_fields() return self + def _clear_fields(self): + self.content = None + class StrIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) Modified: pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py Mon Nov 2 18:33:08 2009 @@ -61,6 +61,8 @@ "init_dictattributes") make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, "make_rdict") + clear_fields = func_with_new_name(dictimplclass._clear_fields.im_func, + "clear_fields") class InlineDictMixin(object): @@ -116,7 +118,7 @@ return True def setdict(self, space, w_dict): - make_rdict(self) # invalidate attributes on self + self._clear_fields() # invalidate attributes on self self.w__dict__ = check_new_dictionary(space, w_dict) def _as_rdict(self): @@ -125,6 +127,8 @@ def initialize_as_rdict(self): return self.getdict().initialize_as_rdict() + + _clear_fields = clear_fields for methname, _ in implementation_methods: implname = "impl_" + methname Modified: pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py Mon Nov 2 18:33:08 2009 @@ -163,10 +163,12 @@ r_dict_content = self.initialize_as_rdict() for k, i in self.structure.keys.items(): r_dict_content[self.space.wrap(k)] = self.entries[i] - self.structure = None - self.entries = None + self._clear_fields() return self + def _clear_fields(self): + self.structure = None + self.entries = None class SharedIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): Modified: pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py Mon Nov 2 18:33:08 2009 @@ -40,7 +40,7 @@ def test_getdict(self): obj = self.make_obj() w_dict = obj.getdict() - assert obj.getdict() or w_dict # always get the same dict + assert obj.getdict() is w_dict # always get the same dict assert obj.w__dict__ is w_dict assert w_dict.getitem("hello") == 1 @@ -50,6 +50,22 @@ assert obj.getdictvalue(self.fakespace, "hello") == 4 assert obj.getdictvalue(self.fakespace, "world") == 5 + def test_setdict(self): + obj1 = self.make_obj() + w_dict1 = obj1.getdict() + obj2 = self.make_obj() + w_dict2 = obj2.getdict() + w_dict2.setitem(4, 1) # devolve dict + w_dict2.setitem(5, 2) + obj2.setdict(self.space, w_dict1) + assert obj2.getdictvalue(self.fakespace, "hello") == 1 + assert obj2.getdictvalue(self.fakespace, "world") == 2 + obj1.setdictvalue(self.fakespace, "hello", 4) + obj1.setdictvalue(self.fakespace, "world", 5) + assert obj2.getdictvalue(self.fakespace, "hello") == 4 + assert obj2.getdictvalue(self.fakespace, "world") == 5 + + def test_dict_devolves_via_dict(self): obj = self.make_obj() w_dict = obj.getdict() From arigo at codespeak.net Tue Nov 3 09:30:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 09:30:32 +0100 (CET) Subject: [pypy-svn] r68924 - pypy/trunk/pypy/doc/config Message-ID: <20091103083032.11E73168012@codespeak.net> Author: arigo Date: Tue Nov 3 09:30:31 2009 New Revision: 68924 Modified: pypy/trunk/pypy/doc/config/opt.txt Log: Briefly mention "--opt=jit" here. Modified: pypy/trunk/pypy/doc/config/opt.txt ============================================================================== --- pypy/trunk/pypy/doc/config/opt.txt (original) +++ pypy/trunk/pypy/doc/config/opt.txt Tue Nov 3 09:30:31 2009 @@ -18,6 +18,7 @@ `--opt=mem` minimize the run-time RAM consumption (in-progress) `--opt=2` all optimizations on; good run-time performance `--opt=3` same as `--opt=2`; remove asserts; gcc profiling `(**)`_ + `--opt=jit` includes the JIT and tweak other optimizations for it ============= ======================================================== .. _`(*)`: From cfbolz at codespeak.net Tue Nov 3 10:18:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 10:18:39 +0100 (CET) Subject: [pypy-svn] r68925 - pypy/trunk/pypy/interpreter/astcompiler Message-ID: <20091103091839.D5515168015@codespeak.net> Author: cfbolz Date: Tue Nov 3 10:18:38 2009 New Revision: 68925 Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py Log: typo in docstring Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/assemble.py Tue Nov 3 10:18:38 2009 @@ -363,7 +363,7 @@ def assemble(self): """Build a PyCode object.""" - # Unless it's interactive, every code object must in an a return. + # Unless it's interactive, every code object must end in a return. if not self.current_block.have_return: self.use_next_block() if self.add_none_to_final_return: From cfbolz at codespeak.net Tue Nov 3 10:24:28 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 10:24:28 +0100 (CET) Subject: [pypy-svn] r68926 - in pypy/trunk/pypy: interpreter interpreter/astcompiler interpreter/test module/pypyjit module/pypyjit/test objspace/std Message-ID: <20091103092428.60D3C16801D@codespeak.net> Author: cfbolz Date: Tue Nov 3 10:24:27 2009 New Revision: 68926 Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/interpreter/test/test_code.py pypy/trunk/pypy/module/pypyjit/interp_jit.py pypy/trunk/pypy/module/pypyjit/test/test_can_inline.py pypy/trunk/pypy/objspace/std/celldict.py pypy/trunk/pypy/objspace/std/marshal_impl.py Log: Add two new flags to code objects: CO_CONTAINSLOOP, CO_CONTAINSGLOBALS. CO_CONTAINSLOOP is used by the JIT to figure out which functions to inline without having to go over the bytecode every time again. CO_CONTAINSGLOBALS is used by the celldict implementation to check whether the function contains a LOAD_GLOBAL or LOAD_NAME. If neither is there, executing the function can proceed without finding the globals cache. Modified: pypy/trunk/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/consts.py Tue Nov 3 10:24:27 2009 @@ -9,6 +9,8 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 +CO_CONTAINSLOOP = 0x0080 +CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Tue Nov 3 10:24:27 2009 @@ -11,23 +11,20 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.baseobjspace import ObjSpace, W_Root +from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, + CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, + CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash +from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT # helper def unpack_str_tuple(space,w_str_tuple): return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] -# code object contants, for co_flags below -CO_OPTIMIZED = 0x0001 -CO_NEWLOCALS = 0x0002 -CO_VARARGS = 0x0004 -CO_VARKEYWORDS = 0x0008 -CO_NESTED = 0x0010 -CO_GENERATOR = 0x0020 # Magic numbers for the bytecode version in code objects. # See comments in pypy/module/__builtin__/importing. @@ -87,6 +84,7 @@ self._initialize() def _initialize(self): + self._init_flags() # Precompute what arguments need to be copied into cellvars self._args_as_cellvars = [] @@ -123,6 +121,24 @@ from pypy.objspace.std.celldict import init_code init_code(self) + def _init_flags(self): + co_code = self.co_code + next_instr = 0 + while next_instr < len(co_code): + opcode = ord(co_code[next_instr]) + next_instr += 1 + if opcode >= HAVE_ARGUMENT: + next_instr += 2 + while opcode == opcodedesc.EXTENDED_ARG.index: + opcode = ord(co_code[next_instr]) + next_instr += 3 + if opcode == opcodedesc.JUMP_ABSOLUTE.index: + self.co_flags |= CO_CONTAINSLOOP + elif opcode == opcodedesc.LOAD_GLOBAL.index: + self.co_flags |= CO_CONTAINSGLOBALS + elif opcode == opcodedesc.LOAD_NAME.index: + self.co_flags |= CO_CONTAINSGLOBALS + co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace def signature(self): @@ -163,18 +179,6 @@ list(code.co_cellvars), hidden_applevel, cpython_magic) - def _code_new_w(space, argcount, nlocals, stacksize, flags, - code, consts, names, varnames, filename, - name, firstlineno, lnotab, freevars, cellvars, - hidden_applevel=False): - """Initialize a new code objects from parameters given by - the pypy compiler""" - return PyCode(space, argcount, nlocals, stacksize, flags, code, - consts[:], names, varnames, filename, name, firstlineno, - lnotab, freevars, cellvars, hidden_applevel) - - _code_new_w = staticmethod(_code_new_w) - def _compute_flatcall(self): # Speed hack! Modified: pypy/trunk/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_code.py (original) +++ pypy/trunk/pypy/interpreter/test/test_code.py Tue Nov 3 10:24:27 2009 @@ -188,3 +188,31 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 + # check for CO_CONTAINSLOOP + assert not f.func_code.co_flags & 0x0080 + # check for CO_CONTAINSGLOBALS + assert not f.func_code.co_flags & 0x0800 + + + exec """if 1: + def f(): + return [l for l in range(100)] + def g(): + return [l for l in [1, 2, 3, 4]] +""" + + # check for CO_CONTAINSLOOP + assert f.func_code.co_flags & 0x0080 + assert g.func_code.co_flags & 0x0080 + # check for CO_CONTAINSGLOBALS + assert f.func_code.co_flags & 0x0800 + assert not g.func_code.co_flags & 0x0800 + + exec """if 1: + b = 2 + def f(x): + exec "a = 1"; + return a + b + x +""" + # check for CO_CONTAINSGLOBALS + assert f.func_code.co_flags & 0x0800 Modified: pypy/trunk/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/trunk/pypy/module/pypyjit/interp_jit.py Tue Nov 3 10:24:27 2009 @@ -11,12 +11,11 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, Arguments from pypy.interpreter.eval import Frame -from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS +from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.function import Function from pypy.interpreter.pyopcode import ExitFrame from pypy.rpython.annlowlevel import cast_base_ptr_to_instance -from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT from opcode import opmap from pypy.rlib.objectmodel import we_are_translated @@ -28,19 +27,7 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] def can_inline(next_instr, bytecode): - co_code = bytecode.co_code - next_instr = 0 - while next_instr < len(co_code): - opcode = ord(co_code[next_instr]) - next_instr += 1 - if opcode >= HAVE_ARGUMENT: - next_instr += 2 - while opcode == opcodedesc.EXTENDED_ARG.index: - opcode = ord(co_code[next_instr]) - next_instr += 3 - if opcode == JUMP_ABSOLUTE: - return False - return True + return not bool(bytecode.co_flags & CO_CONTAINSLOOP) def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names Modified: pypy/trunk/pypy/module/pypyjit/test/test_can_inline.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_can_inline.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_can_inline.py Tue Nov 3 10:24:27 2009 @@ -1,14 +1,28 @@ - +from pypy.interpreter import pycode from pypy.module.pypyjit.interp_jit import can_inline +class FakeSpace(object): + class config: + class objspace: + class std: + withcelldict = True + wrap = new_interned_str = unwrap = lambda _, x: x + def fromcache(self, X): + return X(self) + + def test_one(): + space = FakeSpace() def f(): pass - assert can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert can_inline(0, code) def f(): while i < 0: pass - assert not can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert not can_inline(0, code) def f(a, b): return a + b - assert can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert can_inline(0, code) Modified: pypy/trunk/pypy/objspace/std/celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/celldict.py (original) +++ pypy/trunk/pypy/objspace/std/celldict.py Tue Nov 3 10:24:27 2009 @@ -1,3 +1,4 @@ +from pypy.interpreter.pycode import CO_CONTAINSGLOBALS from pypy.objspace.std.dictmultiobject import DictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash @@ -201,14 +202,19 @@ getcache_slow._dont_inline_ = True def init_code(code): - code.globalcacheholder = GlobalCacheHolder(code.space) + if code.co_flags & CO_CONTAINSGLOBALS: + code.globalcacheholder = GlobalCacheHolder(code.space) + else: + code.globalcacheholder = None def get_global_cache(space, code, w_globals): from pypy.interpreter.pycode import PyCode assert isinstance(code, PyCode) holder = code.globalcacheholder - return holder.getcache(space, code, w_globals) + if holder is not None: + return holder.getcache(space, code, w_globals) + return None def getimplementation(w_dict): if type(w_dict) is W_DictMultiObject: Modified: pypy/trunk/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/trunk/pypy/objspace/std/marshal_impl.py (original) +++ pypy/trunk/pypy/objspace/std/marshal_impl.py Tue Nov 3 10:24:27 2009 @@ -442,9 +442,9 @@ name = unmarshal_str(u) firstlineno = u.get_int() lnotab = unmarshal_str(u) - code = PyCode._code_new_w(space, argcount, nlocals, stacksize, flags, - code, consts_w, names, varnames, filename, - name, firstlineno, lnotab, freevars, cellvars) + code = PyCode(space, argcount, nlocals, stacksize, flags, + code, consts_w[:], names, varnames, filename, + name, firstlineno, lnotab, freevars, cellvars) return space.wrap(code) register(TYPE_CODE, unmarshal_pycode) From cfbolz at codespeak.net Tue Nov 3 10:28:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 10:28:06 +0100 (CET) Subject: [pypy-svn] r68927 - pypy/extradoc/planning Message-ID: <20091103092806.6F019168013@codespeak.net> Author: cfbolz Date: Tue Nov 3 10:28:06 2009 New Revision: 68927 Modified: pypy/extradoc/planning/jit.txt Log: those two are done now Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 3 10:28:06 2009 @@ -23,10 +23,6 @@ - goal: on average <=5 guards per original bytecode -- don't look for the global-lookup-cache when a function contains no - LOAD_GLOBAL -- it's a bit silly that it's possible to change the code objects of builtin - functions - raising an exception tends to escape frames, due to the traceback capturing - prevent jitting really general */** calls - sharing dict needs to cope with dels From arigo at codespeak.net Tue Nov 3 11:41:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 11:41:38 +0100 (CET) Subject: [pypy-svn] r68928 - pypy/branch/jit-removetypeptr Message-ID: <20091103104138.67840168016@codespeak.net> Author: arigo Date: Tue Nov 3 11:41:36 2009 New Revision: 68928 Added: pypy/branch/jit-removetypeptr/ - copied from r68927, pypy/trunk/ Log: A branch to implement gcremovetypeptr with the jit. From arigo at codespeak.net Tue Nov 3 11:42:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 11:42:37 +0100 (CET) Subject: [pypy-svn] r68929 - in pypy/branch/jit-removetypeptr/pypy: config jit/backend/llsupport jit/backend/llsupport/test jit/backend/x86 jit/backend/x86/test rpython rpython/lltypesystem rpython/memory rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test translator/c translator/c/test Message-ID: <20091103104237.56F1E168016@codespeak.net> Author: arigo Date: Tue Nov 3 11:42:36 2009 New Revision: 68929 Modified: pypy/branch/jit-removetypeptr/pypy/config/translationoption.py pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/gc.py pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/assembler.py pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/ri386setup.py pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/jit-removetypeptr/pypy/rpython/llinterp.py pypy/branch/jit-removetypeptr/pypy/rpython/lltypesystem/lloperation.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/framework.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/transform.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctypelayout.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_gctypelayout.py pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/jit-removetypeptr/pypy/translator/c/database.py pypy/branch/jit-removetypeptr/pypy/translator/c/node.py pypy/branch/jit-removetypeptr/pypy/translator/c/test/test_newgc.py Log: Implement jit support for gcremovetypeptr. Modified: pypy/branch/jit-removetypeptr/pypy/config/translationoption.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/config/translationoption.py (original) +++ pypy/branch/jit-removetypeptr/pypy/config/translationoption.py Tue Nov 3 11:42:36 2009 @@ -95,8 +95,7 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False), - ("translation.gcremovetypeptr", False)], + requires=[("translation.thread", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/gc.py Tue Nov 3 11:42:36 2009 @@ -325,8 +325,7 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.layoutbuilder = framework.JITTransformerLayoutBuilder( - gcdescr.config) + self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/llmodel.py Tue Nov 3 11:42:36 2009 @@ -28,8 +28,11 @@ else: translator = None self.gc_ll_descr = get_ll_description(gcdescr, translator) - self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, - 'typeptr', + if translator and translator.config.translation.gcremovetypeptr: + self.vtable_offset = None + else: + self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, + 'typeptr', translate_support_code) self._setup_prebuilt_error('ovf', OverflowError) self._setup_prebuilt_error('zer', ZeroDivisionError) @@ -424,8 +427,9 @@ classint = classbox.getint() descrsize = self.class_sizes[classint] res = self.gc_ll_descr.gc_malloc(descrsize) - as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) - as_array[self.vtable_offset/WORD] = classint + if self.vtable_offset is not None: + as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + as_array[self.vtable_offset/WORD] = classint return BoxPtr(res) def do_new_array(self, countbox, arraydescr): Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/llsupport/test/test_gc.py Tue Nov 3 11:42:36 2009 @@ -147,19 +147,20 @@ class TestFramework: def setup_method(self, meth): - class FakeTranslator: - pass - class config: + class config_: class translation: gc = 'hybrid' gcrootfinder = 'asmgcc' gctransformer = 'framework' + gcremovetypeptr = False + class FakeTranslator: + config = config_ class FakeCPU: def cast_adr_to_int(self, adr): ptr = llmemory.cast_adr_to_ptr(adr, gc_ll_descr.WB_FUNCPTR) assert ptr._obj._callable == llop1._write_barrier_failing_case return 42 - gcdescr = get_description(config) + gcdescr = get_description(config_) translator = FakeTranslator() llop1 = FakeLLOp() gc_ll_descr = GcLLDescr_framework(gcdescr, FakeTranslator(), llop1) Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/assembler.py Tue Nov 3 11:42:36 2009 @@ -518,7 +518,8 @@ self.set_vtable(eax, loc_vtable) def set_vtable(self, loc, loc_vtable): - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + if self.cpu.vtable_offset is not None: + self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -713,7 +714,24 @@ def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset - self.mc.CMP(mem(locs[0], offset), locs[1]) + if offset is not None: + self.mc.CMP(mem(locs[0], offset), locs[1]) + else: + # XXX hard-coded assumption: to go from an object to its class + # we use the following algorithm: + # - read the typeid from mem(locs[0]), i.e. at offset 0 + # - keep the lower 16 bits read there + # - multiply by 4 and use it as an offset in type_info_group. + loc = locs[1] + assert isinstance(loc, IMM32) + classptr = loc.value + # here, we have to go back from 'classptr' to the value expected + # from reading the 16 bits in the object header + type_info_group = llop.gc_get_type_info_group(llmemory.Address) + type_info_group = rffi.cast(lltype.Signed, type_info_group) + expected_typeid = (classptr - type_info_group) >> 2 + self.mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + # return self.implement_guard(addr, self.mc.JNE) def _no_const_locs(self, args): Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/ri386setup.py Tue Nov 3 11:42:36 2009 @@ -310,6 +310,11 @@ CMP = Instruction() CMP.common_modes(7) +# special mode for comparing a 16-bit operand with an immediate +CMP16 = Instruction() +CMP16.mode2(MODRM, IMM32, ['\x66', '\x81', orbyte(7<<3), modrm(1), + immediate(2,'h')]) + NOP = Instruction() NOP.mode0(['\x90']) Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Tue Nov 3 11:42:36 2009 @@ -218,6 +218,8 @@ return [] # MOV [constant-address], accum if instrname == "MOV16": return [] # skipped + if instrname == "CMP16": + return [] # skipped if instrname == "LEA": if (args[1][1].__class__ != i386.MODRM or args[1][1].is_register()): Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Nov 3 11:42:36 2009 @@ -78,6 +78,7 @@ # t = TranslationContext() t.config.translation.gc = gc + t.config.translation.gcremovetypeptr = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) Modified: pypy/branch/jit-removetypeptr/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/llinterp.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/llinterp.py Tue Nov 3 11:42:36 2009 @@ -890,6 +890,9 @@ def op_gc_stack_bottom(self): pass # marker for trackgcroot.py + def op_gc_get_type_info_group(self): + raise NotImplementedError("gc_get_type_info_group") + def op_do_malloc_fixedsize_clear(self): raise NotImplementedError("do_malloc_fixedsize_clear") Modified: pypy/branch/jit-removetypeptr/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/lltypesystem/lloperation.py Tue Nov 3 11:42:36 2009 @@ -432,6 +432,7 @@ 'do_malloc_fixedsize_clear': LLOp(canunwindgc=True), 'do_malloc_varsize_clear': LLOp(canunwindgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), + 'gc_get_type_info_group': LLOp(sideeffects=False), # __________ GC operations __________ Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/gc/test/test_direct.py Tue Nov 3 11:42:36 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder(self.GCClass, {}) + self.layoutbuilder = TypeLayoutBuilder(self.GCClass) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/framework.py Tue Nov 3 11:42:36 2009 @@ -130,12 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcremovetypeptr: - lltype2vtable = translator.rtyper.lltype2vtable - else: - lltype2vtable = {} - self.layoutbuilder = TransformerLayoutBuilder(GCClass, - lltype2vtable) + self.layoutbuilder = TransformerLayoutBuilder(translator, GCClass) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id @@ -508,11 +503,16 @@ self.write_typeid_list() return newgcdependencies - def get_final_dependencies(self): - # returns an iterator enumerating the type_info_group's members, - # to make sure that they are all followed (only a part of them - # might have been followed by a previous enum_dependencies()). - return iter(self.layoutbuilder.type_info_group.members) + def get_finish_tables(self): + # We must first make sure that the type_info_group's members + # are all followed. Do it repeatedly while new members show up. + # Once it is really done, do finish_tables(). + seen = 0 + while seen < len(self.layoutbuilder.type_info_group.members): + curtotal = len(self.layoutbuilder.type_info_group.members) + yield self.layoutbuilder.type_info_group.members[seen:curtotal] + seen = curtotal + yield self.finish_tables() def write_typeid_list(self): """write out the list of type ids together with some info""" @@ -821,6 +821,9 @@ if hasattr(self.root_walker, 'thread_die_ptr'): hop.genop("direct_call", [self.root_walker.thread_die_ptr]) + def gct_gc_get_type_info_group(self, hop): + return hop.cast_result(self.c_type_info_group) + def gct_malloc_nonmovable_varsize(self, hop): TYPE = hop.spaceop.result.concretetype if self.gcdata.gc.can_malloc_nonmovable(): @@ -964,6 +967,16 @@ class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): + def __init__(self, translator, GCClass=None): + if GCClass is None: + from pypy.rpython.memory.gc.base import choose_gc_from_config + GCClass, _ = choose_gc_from_config(translator.config) + if translator.config.translation.gcremovetypeptr: + lltype2vtable = translator.rtyper.lltype2vtable + else: + lltype2vtable = None + super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable) + def has_finalizer(self, TYPE): rtti = get_rtti(TYPE) return rtti is not None and hasattr(rtti._obj, 'destructor_funcptr') @@ -990,18 +1003,6 @@ return fptr -class JITTransformerLayoutBuilder(TransformerLayoutBuilder): - # for the JIT: currently does not support removetypeptr - def __init__(self, config): - from pypy.rpython.memory.gc.base import choose_gc_from_config - try: - assert not config.translation.gcremovetypeptr - except AttributeError: # for some tests - pass - GCClass, _ = choose_gc_from_config(config) - TransformerLayoutBuilder.__init__(self, GCClass, {}) - - def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: previous_steps = [] Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctransform/transform.py Tue Nov 3 11:42:36 2009 @@ -312,12 +312,12 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies - def get_final_dependencies(self): - pass - def finish_tables(self): pass + def get_finish_tables(self): + return self.finish_tables + def finish(self, backendopt=True): self.finish_helpers(backendopt=backendopt) self.finish_tables() Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/gctypelayout.py Tue Nov 3 11:42:36 2009 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup +from pypy.rpython.lltypesystem import rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert @@ -169,7 +170,7 @@ size_of_fixed_type_info = llmemory.sizeof(GCData.TYPE_INFO) - def __init__(self, GCClass, lltype2vtable): + def __init__(self, GCClass, lltype2vtable=None): self.GCClass = GCClass self.lltype2vtable = lltype2vtable self.make_type_info_group() @@ -218,16 +219,28 @@ # store it type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id - # store the vtable of the type (if any) immediately thereafter - # (note that if gcremovetypeptr is False, lltype2vtable is empty) - vtable = self.lltype2vtable.get(TYPE, None) - if vtable is not None: - # check that if we have a vtable, we are not varsize - assert lltype.typeOf(fullinfo) == GCData.TYPE_INFO_PTR - vtable = lltype.normalizeptr(vtable) - self.type_info_group.add_member(vtable) + self.add_vtable_after_typeinfo(TYPE) return type_id + def add_vtable_after_typeinfo(self, TYPE): + # if gcremovetypeptr is False, then lltype2vtable is None and it + # means that we don't have to store the vtables in type_info_group. + if self.lltype2vtable is None: + return + # does the type have a vtable? + vtable = self.lltype2vtable.get(TYPE, None) + if vtable is not None: + # yes. check that in this case, we are not varsize + assert not TYPE._is_varsize() + vtable = lltype.normalizeptr(vtable) + self.type_info_group.add_member(vtable) + else: + # no vtable from lltype2vtable -- double-check to be sure + # that it's not a subclass of OBJECT. + while isinstance(TYPE, lltype.GcStruct): + assert TYPE is not rclass.OBJECT + _, TYPE = TYPE._first_struct() + def get_info(self, type_id): return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group._as_ptr(), Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_gctypelayout.py Tue Nov 3 11:42:36 2009 @@ -37,7 +37,7 @@ for T, c in [(GC_S, 0), (GC_S2, 2), (GC_A, 0), (GC_A2, 0), (GC_S3, 2)]: assert len(offsets_to_gc_pointers(T)) == c -def test_layout_builder(lltype2vtable={}): +def test_layout_builder(lltype2vtable=None): # XXX a very minimal test layoutbuilder = TypeLayoutBuilder(FakeGC, lltype2vtable) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: @@ -67,7 +67,7 @@ layoutbuilder.size_of_fixed_type_info) def test_constfold(): - layoutbuilder = TypeLayoutBuilder(FakeGC, {}) + layoutbuilder = TypeLayoutBuilder(FakeGC) tid1 = layoutbuilder.get_type_id(GC_A) tid2 = layoutbuilder.get_type_id(GC_S3) class MockGC: Modified: pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/rpython/memory/test/test_transformed_gc.py Tue Nov 3 11:42:36 2009 @@ -709,9 +709,7 @@ if jit2gc: return jit2gc['layoutbuilder'] GCClass = cls.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) + layoutbuilder = framework.TransformerLayoutBuilder(translator, GCClass) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/database.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/database.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/database.py Tue Nov 3 11:42:36 2009 @@ -288,10 +288,8 @@ finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) if self.gctransformer: - finish_callbacks.append(('GC transformer: tracking vtables', - self.gctransformer.get_final_dependencies)) finish_callbacks.append(('GC transformer: finished tables', - self.gctransformer.finish_tables)) + self.gctransformer.get_finish_tables())) def add_dependencies(newdependencies): for value in newdependencies: @@ -336,8 +334,18 @@ if finish_callbacks: logmsg, finish = finish_callbacks.pop(0) - newdependencies = finish() - log.database(logmsg) + if not hasattr(finish, 'next'): + newdependencies = finish() + else: + # if 'finish' is a generator, consume the next element + # and put the generator again in the queue + try: + newdependencies = finish.next() + finish_callbacks.insert(0, (None, finish)) + except StopIteration: + newdependencies = None + if logmsg: + log.database(logmsg) if newdependencies: add_dependencies(newdependencies) continue # progress - follow all dependencies again Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/node.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/node.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/node.py Tue Nov 3 11:42:36 2009 @@ -960,8 +960,8 @@ def enum_dependencies(self): # note: for the group used by the GC, it can grow during this phase, - # which means that we might not return all members yet. This is - # fixed by finish_tables() in rpython/memory/gctransform/framework.py + # which means that we might not return all members yet. This is fixed + # by get_final_dependencies() in rpython.memory.gctransform.framework for member in self.obj.members: yield member._as_ptr() Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/test/test_newgc.py Tue Nov 3 11:42:36 2009 @@ -949,6 +949,19 @@ class TestHybridGCRemoveTypePtr(TestHybridGC): removetypeptr = True + def definestr_str_instance(cls): + class AbcDef: + pass + def fn(_): + a = AbcDef() + return str(a) + return fn + + def test_str_instance(self): + res = self.run('str_instance') + assert res.startswith('<') and res.endswith('>') + assert 'AbcDef' in res + class TestMarkCompactGC(TestSemiSpaceGC): gcpolicy = "markcompact" From cfbolz at codespeak.net Tue Nov 3 11:42:44 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 11:42:44 +0100 (CET) Subject: [pypy-svn] r68930 - in pypy/trunk/pypy: config doc/config interpreter interpreter/test jit/tl module/__builtin__ module/__builtin__/test module/operator module/sys module/sys/test objspace objspace/std objspace/std/test Message-ID: <20091103104244.399BC168020@codespeak.net> Author: cfbolz Date: Tue Nov 3 11:42:42 2009 New Revision: 68930 Added: pypy/trunk/pypy/doc/config/objspace.std.withinlineddict.txt (contents, props changed) pypy/trunk/pypy/objspace/std/inlinedict.py (contents, props changed) pypy/trunk/pypy/objspace/std/test/test_inlinedict.py (contents, props changed) Removed: pypy/trunk/pypy/doc/config/objspace.std.withbucketdict.txt pypy/trunk/pypy/objspace/std/dictbucket.py pypy/trunk/pypy/objspace/std/test/test_dictbucket.py Modified: pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/main.py pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/interpreter/test/test_function.py pypy/trunk/pypy/interpreter/test/test_typedef.py pypy/trunk/pypy/interpreter/typedef.py pypy/trunk/pypy/jit/tl/pypyjit.py pypy/trunk/pypy/module/__builtin__/interp_classobj.py pypy/trunk/pypy/module/__builtin__/test/test_classobj.py pypy/trunk/pypy/module/operator/__init__.py pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/module/sys/test/test_sysmodule.py pypy/trunk/pypy/objspace/descroperation.py pypy/trunk/pypy/objspace/std/callmethod.py pypy/trunk/pypy/objspace/std/celldict.py pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/dicttype.py pypy/trunk/pypy/objspace/std/marshal_impl.py pypy/trunk/pypy/objspace/std/model.py pypy/trunk/pypy/objspace/std/objectobject.py pypy/trunk/pypy/objspace/std/objecttype.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/proxyobject.py pypy/trunk/pypy/objspace/std/sharingdict.py pypy/trunk/pypy/objspace/std/test/test_celldict.py pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py pypy/trunk/pypy/objspace/std/test/test_obj.py pypy/trunk/pypy/objspace/std/test/test_shadowtracking.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/objspace/std/typetype.py pypy/trunk/pypy/objspace/taint.py Log: Merge the shrink-multidict branch: ------------------------------------------------------------------------ r68599 | cfbolz | 2009-10-17 18:49:36 +0200 (Sat, 17 Oct 2009) | 2 lines Changed paths: A /pypy/branch/shrink-multidict (from /pypy/trunk:68598) resurrect the branch to try again ------------------------------------------------------------------------ r68749 | cfbolz | 2009-10-26 14:46:32 +0100 (Mon, 26 Oct 2009) | 5 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py M /pypy/branch/shrink-multidict/pypy/interpreter/typedef.py M /pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py M /pypy/branch/shrink-multidict/pypy/module/__builtin__/test/test_classobj.py M /pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py D /pypy/branch/shrink-multidict/pypy/objspace/std/dictbucket.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dicttype.py M /pypy/branch/shrink-multidict/pypy/objspace/std/marshal_impl.py M /pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py M /pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_celldict.py D /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictbucket.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py M /pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py Refactor the multidict to use one level of indirection less. Instead of having an indirection, all W_DictMultiObjects have an attribute r_dict_content that is usually None. When the dict devolves, it is set to an r_dict and used instead of the normal attributes. ------------------------------------------------------------------------ r68751 | cfbolz | 2009-10-26 15:28:38 +0100 (Mon, 26 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/objectobject.py don't use * import ------------------------------------------------------------------------ r68752 | cfbolz | 2009-10-26 15:34:37 +0100 (Mon, 26 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_obj.py uh? ------------------------------------------------------------------------ r68753 | cfbolz | 2009-10-26 15:34:58 +0100 (Mon, 26 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/objecttype.py remove old commented out code ------------------------------------------------------------------------ r68754 | cfbolz | 2009-10-26 15:43:38 +0100 (Mon, 26 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py A number of fixes. ------------------------------------------------------------------------ r68759 | cfbolz | 2009-10-26 16:54:49 +0100 (Mon, 26 Oct 2009) | 12 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/config/pypyoption.py M /pypy/branch/shrink-multidict/pypy/interpreter/test/test_typedef.py M /pypy/branch/shrink-multidict/pypy/interpreter/typedef.py A /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/model.py M /pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py M /pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py A /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_shadowtracking.py Make it possible for objects to not have an applevel dict at all: - the W_UserDictObjectObject gets the attributes of a StrDictImplementation inlined. As long as nobody accesses the applevel .__dict__ attribute, only those are used - if the __dict__ is requested, we create a dict that forwards all requests to the instance - if the __dict__ devolves (e.g. a non-string-key is added), the content is stored into the r_dict_content of the forwarding dict instance. In this case the information on the object is invalid. ------------------------------------------------------------------------ r68763 | cfbolz | 2009-10-26 18:42:39 +0100 (Mon, 26 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/interpreter/function.py M /pypy/branch/shrink-multidict/pypy/interpreter/test/test_function.py the repr of bound methods was wrong (and quite confusing) ------------------------------------------------------------------------ r68776 | cfbolz | 2009-10-27 10:01:24 +0100 (Tue, 27 Oct 2009) | 4 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py M /pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py M /pypy/branch/shrink-multidict/pypy/interpreter/typedef.py M /pypy/branch/shrink-multidict/pypy/objspace/descroperation.py M /pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py M /pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py M /pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py A new shortcut in the objspace: finditem_str that takes an unwrapped string as the item to find. Please don't ask me how I managed to triplicate the content of inlinedict.py and test_inlinedict.py ------------------------------------------------------------------------ r68789 | cfbolz | 2009-10-27 14:16:35 +0100 (Tue, 27 Oct 2009) | 2 lines Changed paths: D /pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withbucketdict.txt this option is gone ------------------------------------------------------------------------ r68792 | cfbolz | 2009-10-27 14:27:30 +0100 (Tue, 27 Oct 2009) | 2 lines Changed paths: A /pypy/branch/shrink-multidict/pypy/doc/config/objspace.std.withinlineddict.txt document the new option ------------------------------------------------------------------------ r68819 | cfbolz | 2009-10-28 15:02:40 +0100 (Wed, 28 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/module/sys/__init__.py M /pypy/branch/shrink-multidict/pypy/module/sys/test/test_sysmodule.py bah, there were no tests for sys.exc_value, .exc_type and .exc_traceback ------------------------------------------------------------------------ r68820 | cfbolz | 2009-10-28 15:26:31 +0100 (Wed, 28 Oct 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py M /pypy/branch/shrink-multidict/pypy/interpreter/main.py M /pypy/branch/shrink-multidict/pypy/interpreter/mixedmodule.py M /pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py M /pypy/branch/shrink-multidict/pypy/module/operator/__init__.py M /pypy/branch/shrink-multidict/pypy/module/sys/__init__.py M /pypy/branch/shrink-multidict/pypy/objspace/descroperation.py M /pypy/branch/shrink-multidict/pypy/objspace/std/callmethod.py M /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py M /pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/typeobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/typetype.py M /pypy/branch/shrink-multidict/pypy/objspace/taint.py make getdictvalue take an unwrapped string and kill getdictvalue_w ------------------------------------------------------------------------ r68821 | cfbolz | 2009-10-28 16:21:35 +0100 (Wed, 28 Oct 2009) | 3 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/interpreter/baseobjspace.py M /pypy/branch/shrink-multidict/pypy/interpreter/pyopcode.py M /pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit.py M /pypy/branch/shrink-multidict/pypy/jit/tl/pypyjit_demo.py M /pypy/branch/shrink-multidict/pypy/module/__builtin__/interp_classobj.py M /pypy/branch/shrink-multidict/pypy/objspace/descroperation.py M /pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/objspace.py M /pypy/branch/shrink-multidict/pypy/objspace/std/proxyobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/taint.py M /pypy/branch/shrink-multidict/pypy/translator/goal/gcbench.py Change setdictvalue to also take a string. Now it's asymmetric to deldictvalue, I might not care though. ------------------------------------------------------------------------ r68912 | cfbolz | 2009-11-02 15:28:38 +0100 (Mon, 02 Nov 2009) | 2 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py running the tests is generally a good idea ------------------------------------------------------------------------ r68923 | cfbolz | 2009-11-02 18:33:08 +0100 (Mon, 02 Nov 2009) | 3 lines Changed paths: M /pypy/branch/shrink-multidict/pypy/objspace/std/celldict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/dictmultiobject.py M /pypy/branch/shrink-multidict/pypy/objspace/std/inlinedict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/sharingdict.py M /pypy/branch/shrink-multidict/pypy/objspace/std/test/test_inlinedict.py A bug found by test_descr.py: setting the dict of an object with a devolved dict runs into an assertion. ------------------------------------------------------------------------ Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Tue Nov 3 11:42:42 2009 @@ -236,10 +236,10 @@ "about dictionaries", default=False), - BoolOption("withbucketdict", - "use dictionaries with chained hash tables " - "(default is open addressing)", - default=False), + BoolOption("withinlineddict", + "make instances more compact by revoming a level of indirection", + default=False, + requires=[("objspace.std.withshadowtracking", False)]), BoolOption("withrangelist", "enable special range list implementation that does not " @@ -362,6 +362,7 @@ if type_system != 'ootype': config.objspace.std.suggest(withsharingdict=True) config.objspace.std.suggest(withcelldict=True) + config.objspace.std.suggest(withinlineddict=True) def enable_allworkingmodules(config): Added: pypy/trunk/pypy/doc/config/objspace.std.withinlineddict.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/objspace.std.withinlineddict.txt Tue Nov 3 11:42:42 2009 @@ -0,0 +1,2 @@ +Make instances smaller by creating the __dict__ only when somebody actually +accesses it. Also makes attribute accesses a tiny bit faster. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Tue Nov 3 11:42:42 2009 @@ -20,26 +20,24 @@ in a 'normal' object space like StdObjSpace.""" __slots__ = () _settled_ = True + user_overridden_class = False def getdict(self): return None - def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) - - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): w_dict = self.getdict() if w_dict is not None: - return space.finditem(w_dict, w_attr) + return space.finditem_str(w_dict, attr) return None - def getdictvalue_attr_is_in_class(self, space, w_attr): - return self.getdictvalue(space, w_attr) + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue(space, attr) - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): w_dict = self.getdict() if w_dict is not None: - space.set_str_keyed_item(w_dict, w_attr, w_value, shadows_type) + space.set_str_keyed_item(w_dict, attr, w_value, shadows_type) return True return False @@ -281,7 +279,7 @@ self.timer.stop("startup " + modname) def finish(self): - w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') + w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module @@ -550,12 +548,6 @@ # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. - #def is_(self, w_x, w_y): -- not really useful. Must be subclassed - # "'x is y'." - # w_id_x = self.id(w_x) - # w_id_y = self.id(w_y) - # return self.eq(w_id_x, w_id_y) - def not_(self, w_obj): return self.wrap(not self.is_true(w_obj)) @@ -571,8 +563,11 @@ """shortcut for space.int_w(space.hash(w_obj))""" return self.int_w(self.hash(w_obj)) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): - return self.setitem(w_obj, w_key, w_value) + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + return self.setitem(w_obj, self.wrap(key), w_value) + + def finditem_str(self, w_obj, key): + return self.finditem(w_obj, self.wrap(key)) def finditem(self, w_obj, w_key): try: @@ -777,7 +772,7 @@ w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) for w_supertype in self.unpackiterable(w_mro): - w_value = w_supertype.getdictvalue_w(self, name) + w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value return None Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Tue Nov 3 11:42:42 2009 @@ -494,9 +494,8 @@ return space.wrap(s) else: objrepr = space.str_w(space.repr(self.w_instance)) - info = 'bound method %s.%s of %s' % (typename, name, objrepr) - # info = "method %s of %s object" % (name, typename) - return self.w_instance.getrepr(self.space, info) + s = '' % (typename, name, objrepr) + return space.wrap(s) def descr_method_getattribute(self, w_attr): space = self.space Modified: pypy/trunk/pypy/interpreter/main.py ============================================================================== --- pypy/trunk/pypy/interpreter/main.py (original) +++ pypy/trunk/pypy/interpreter/main.py Tue Nov 3 11:42:42 2009 @@ -149,11 +149,11 @@ w_traceback) # call sys.excepthook if present - w_hook = space.sys.getdictvalue_w(space, 'excepthook') + w_hook = space.sys.getdictvalue(space, 'excepthook') if w_hook is not None: # hack: skip it if it wasn't modified by the user, # to do instead the faster verbose/nonverbose thing below - w_original = space.sys.getdictvalue_w(space, '__excepthook__') + w_original = space.sys.getdictvalue(space, '__excepthook__') if w_original is None or not space.is_w(w_hook, w_original): space.call_function(w_hook, w_type, w_value, w_traceback) return False # done Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Tue Nov 3 11:42:42 2009 @@ -32,7 +32,7 @@ def get(self, name): space = self.space - w_value = self.getdictvalue_w(space, name) + w_value = self.getdictvalue(space, name) if w_value is None: raise OperationError(space.w_AttributeError, space.wrap(name)) return w_value @@ -41,40 +41,41 @@ w_builtin = self.get(name) return self.space.call_function(w_builtin, *args_w) - def getdictvalue(self, space, w_name): - w_value = space.finditem(self.w_dict, w_name) + def getdictvalue(self, space, name): + w_value = space.finditem_str(self.w_dict, name) if self.lazy and w_value is None: - name = space.str_w(w_name) - w_name = space.new_interned_w_str(w_name) - try: - loader = self.loaders[name] - except KeyError: - return None - else: - #print "trying to load", name - w_value = loader(space) - #print "loaded", w_value - # obscure - func = space.interpclass_w(w_value) - # the idea of the following code is that all functions that are - # directly in a mixed-module are "builtin", e.g. they get a - # special type without a __get__ - # note that this is not just all functions that contain a - # builtin code object, as e.g. methods of builtin types have to - # be normal Functions to get the correct binding behaviour - if (isinstance(func, Function) and - type(func) is not BuiltinFunction): - try: - bltin = func._builtinversion_ - except AttributeError: - bltin = BuiltinFunction(func) - bltin.w_module = self.w_name - func._builtinversion_ = bltin - bltin.name = name - w_value = space.wrap(bltin) - space.setitem(self.w_dict, w_name, w_value) + return self._load_lazily(space, name) return w_value + def _load_lazily(self, space, name): + w_name = space.new_interned_str(name) + try: + loader = self.loaders[name] + except KeyError: + return None + else: + w_value = loader(space) + func = space.interpclass_w(w_value) + # the idea of the following code is that all functions that are + # directly in a mixed-module are "builtin", e.g. they get a + # special type without a __get__ + # note that this is not just all functions that contain a + # builtin code object, as e.g. methods of builtin types have to + # be normal Functions to get the correct binding behaviour + if (isinstance(func, Function) and + type(func) is not BuiltinFunction): + try: + bltin = func._builtinversion_ + except AttributeError: + bltin = BuiltinFunction(func) + bltin.w_module = self.w_name + func._builtinversion_ = bltin + bltin.name = name + w_value = space.wrap(bltin) + space.setitem(self.w_dict, w_name, w_value) + return w_value + + def getdict(self): if self.lazy: space = self.space Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Tue Nov 3 11:42:42 2009 @@ -619,9 +619,9 @@ f.pushvalue(w_newclass) def STORE_NAME(f, varindex, *ignored): - w_varname = f.getname_w(varindex) + varname = f.getname_u(varindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_locals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_locals, varname, w_newvalue) def DELETE_NAME(f, varindex, *ignored): w_varname = f.getname_w(varindex) @@ -656,9 +656,9 @@ f.space.delattr(w_obj, w_attributename) def STORE_GLOBAL(f, nameindex, *ignored): - w_varname = f.getname_w(nameindex) + varname = f.getname_u(nameindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_globals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_globals, varname, w_newvalue) def DELETE_GLOBAL(f, nameindex, *ignored): w_varname = f.getname_w(nameindex) @@ -673,25 +673,24 @@ return f.LOAD_GLOBAL(nameindex) # fall-back - def _load_global(f, w_varname): - w_value = f.space.finditem(f.w_globals, w_varname) + def _load_global(f, varname): + w_value = f.space.finditem_str(f.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins - w_value = f.get_builtin().getdictvalue(f.space, w_varname) + w_value = f.get_builtin().getdictvalue(f.space, varname) if w_value is None: - f._load_global_failed(w_varname) + f._load_global_failed(varname) return w_value _load_global._always_inline_ = True - def _load_global_failed(f, w_varname): - varname = f.space.str_w(w_varname) + def _load_global_failed(f, varname): message = "global name '%s' is not defined" % varname raise OperationError(f.space.w_NameError, f.space.wrap(message)) _load_global_failed._dont_inline_ = True def LOAD_GLOBAL(f, nameindex, *ignored): - f.pushvalue(f._load_global(f.getname_w(nameindex))) + f.pushvalue(f._load_global(f.getname_u(nameindex))) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(f, varindex, *ignored): @@ -782,7 +781,7 @@ else: w_flag = None - w_import = f.get_builtin().getdictvalue_w(f.space, '__import__') + w_import = f.get_builtin().getdictvalue(f.space, '__import__') if w_import is None: raise OperationError(space.w_ImportError, space.wrap("__import__ not found")) @@ -989,8 +988,8 @@ def CALL_LIKELY_BUILTIN(f, oparg, *ignored): # overridden by faster version in the standard object space. from pypy.module.__builtin__ import OPTIMIZED_BUILTINS - w_varname = f.space.wrap(OPTIMIZED_BUILTINS[oparg >> 8]) - w_function = f._load_global(w_varname) + varname = OPTIMIZED_BUILTINS[oparg >> 8] + w_function = f._load_global(varname) nargs = oparg&0xFF try: w_result = f.space.call_valuestack(w_function, nargs, f) Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Tue Nov 3 11:42:42 2009 @@ -382,11 +382,13 @@ pass assert repr(A.f) == "" assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") def test_method_call(self): Modified: pypy/trunk/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_typedef.py (original) +++ pypy/trunk/pypy/interpreter/test/test_typedef.py Tue Nov 3 11:42:42 2009 @@ -125,7 +125,7 @@ checks[2], checks[3])) subclasses = {} for key, subcls in typedef._subclass_cache.items(): - cls = key[0] + cls = key[1] subclasses.setdefault(cls, {}) subclasses[cls][subcls] = True for cls, set in subclasses.items(): Modified: pypy/trunk/pypy/interpreter/typedef.py ============================================================================== --- pypy/trunk/pypy/interpreter/typedef.py (original) +++ pypy/trunk/pypy/interpreter/typedef.py Tue Nov 3 11:42:42 2009 @@ -97,23 +97,24 @@ # / \ # 5 6 -def get_unique_interplevel_subclass(cls, hasdict, wants_slots, needsdel=False, - weakrefable=False): +def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots, + needsdel=False, weakrefable=False): "NOT_RPYTHON: initialization-time only" if hasattr(cls, '__del__') and getattr(cls, "handle_del_manually", False): needsdel = False assert cls.typedef.acceptable_as_base_class - key = cls, hasdict, wants_slots, needsdel, weakrefable + key = config, cls, hasdict, wants_slots, needsdel, weakrefable try: return _subclass_cache[key] except KeyError: - subcls = _getusercls(cls, hasdict, wants_slots, needsdel, weakrefable) + subcls = _getusercls(config, cls, hasdict, wants_slots, needsdel, + weakrefable) _subclass_cache[key] = subcls return subcls get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _subclass_cache = {} -def enum_interplevel_subclasses(cls): +def enum_interplevel_subclasses(config, cls): """Return a list of all the extra interp-level subclasses of 'cls' that can be built by get_unique_interplevel_subclass().""" result = [] @@ -121,14 +122,13 @@ for flag2 in (False, True): for flag3 in (False, True): for flag4 in (False, True): - result.append(get_unique_interplevel_subclass(cls, flag1, - flag2, flag3, - flag4)) + result.append(get_unique_interplevel_subclass( + config, cls, flag1, flag2, flag3, flag4)) result = dict.fromkeys(result) assert len(result) <= 6 return result.keys() -def _getusercls(cls, wants_dict, wants_slots, wants_del, weakrefable): +def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable): typedef = cls.typedef if wants_dict and typedef.hasdict: wants_dict = False @@ -136,47 +136,47 @@ if wants_del: if wants_dict: # case 5. Parent class is 3. - parentcls = get_unique_interplevel_subclass(cls, True, True, + parentcls = get_unique_interplevel_subclass(config, cls, True, True, False, True) else: # case 6. Parent class is 4. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, True) - return _usersubclswithfeature(parentcls, "del") + return _usersubclswithfeature(config, parentcls, "del") elif wants_dict: if wants_slots: # case 3. Parent class is 1. - parentcls = get_unique_interplevel_subclass(cls, True, False, + parentcls = get_unique_interplevel_subclass(config, cls, True, False, False, True) - return _usersubclswithfeature(parentcls, "slots") + return _usersubclswithfeature(config, parentcls, "slots") else: # case 1 (we need to add weakrefable unless it's already in 'cls') if not typedef.weakrefable: - return _usersubclswithfeature(cls, "user", "dict", "weakref") + return _usersubclswithfeature(config, cls, "user", "dict", "weakref") else: - return _usersubclswithfeature(cls, "user", "dict") + return _usersubclswithfeature(config, cls, "user", "dict") else: if weakrefable and not typedef.weakrefable: # case 4. Parent class is 2. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, False) - return _usersubclswithfeature(parentcls, "weakref") + return _usersubclswithfeature(config, parentcls, "weakref") else: # case 2 (if the base is already weakrefable, case 2 == case 4) - return _usersubclswithfeature(cls, "user", "slots") + return _usersubclswithfeature(config, cls, "user", "slots") -def _usersubclswithfeature(parentcls, *features): - key = parentcls, features +def _usersubclswithfeature(config, parentcls, *features): + key = config, parentcls, features try: return _usersubclswithfeature_cache[key] except KeyError: - subcls = _builduserclswithfeature(parentcls, *features) + subcls = _builduserclswithfeature(config, parentcls, *features) _usersubclswithfeature_cache[key] = subcls return subcls _usersubclswithfeature_cache = {} _allusersubcls_cache = {} -def _builduserclswithfeature(supercls, *features): +def _builduserclswithfeature(config, supercls, *features): "NOT_RPYTHON: initialization-time only" name = supercls.__name__ name += ''.join([name.capitalize() for name in features]) @@ -190,6 +190,8 @@ if "user" in features: # generic feature needed by all subcls class Proto(object): + user_overridden_class = True + def getclass(self, space): return hint(self.w__class__, promote=True) @@ -242,7 +244,16 @@ return self.slots_w[index] add(Proto) - if "dict" in features: + wantdict = "dict" in features + if wantdict and config.objspace.std.withinlineddict: + from pypy.objspace.std.objectobject import W_ObjectObject + from pypy.objspace.std.inlinedict import make_mixin + if supercls is W_ObjectObject: + Mixin = make_mixin(config) + add(Mixin) + wantdict = False + + if wantdict: class Proto(object): def getdict(self): return self.w__dict__ @@ -253,32 +264,22 @@ def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space, - sharing=True) - elif space.config.objspace.std.withshadowtracking: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space) - self.w__dict__.implementation = \ - dictmultiobject.ShadowDetectingDictImplementation( - space, w_subtype) - else: - self.w__dict__ = space.newdict() + self.w__dict__ = space.newdict( + instance=True, classofinstance=w_subtype) self.user_setup_slots(w_subtype.nslots) def setclass(self, space, w_subtype): # only used by descr_set___class__ self.w__class__ = w_subtype if space.config.objspace.std.withshadowtracking: - self.w__dict__.implementation.set_shadows_anything() + self.w__dict__.set_shadows_anything() - def getdictvalue_attr_is_in_class(self, space, w_name): + def getdictvalue_attr_is_in_class(self, space, name): w_dict = self.w__dict__ if space.config.objspace.std.withshadowtracking: - if not w_dict.implementation.shadows_anything(): + if not w_dict.shadows_anything(): return None - return space.finditem(w_dict, w_name) + return space.finditem_str(w_dict, name) add(Proto) Modified: pypy/trunk/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/trunk/pypy/jit/tl/pypyjit.py (original) +++ pypy/trunk/pypy/jit/tl/pypyjit.py Tue Nov 3 11:42:42 2009 @@ -41,6 +41,7 @@ config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') +config.objspace.std.withinlineddict = True if BACKEND == 'c': config.objspace.std.multimethods = 'mrd' Modified: pypy/trunk/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/trunk/pypy/module/__builtin__/interp_classobj.py Tue Nov 3 11:42:42 2009 @@ -304,12 +304,7 @@ class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - w_dict = dictmultiobject.W_DictMultiObject(space, - sharing=True) - else: - w_dict = space.newdict() + w_dict = space.newdict(instance=True) assert isinstance(w_class, W_ClassObject) self.w_class = w_class self.w_dict = w_dict @@ -388,7 +383,7 @@ if w_meth is not None: space.call_function(w_meth, w_name, w_value) else: - self.setdictvalue(space, w_name, w_value) + self.setdictvalue(space, name, w_value) def descr_delattr(self, space, w_name): name = unwrap_attr(space, w_name) Modified: pypy/trunk/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_classobj.py Tue Nov 3 11:42:42 2009 @@ -772,8 +772,7 @@ def is_sharing(space, w_inst): from pypy.objspace.std.sharingdict import SharedDictImplementation, W_DictMultiObject w_d = w_inst.getdict() - return space.wrap(isinstance(w_d, W_DictMultiObject) and - isinstance(w_d.implementation, SharedDictImplementation)) + return space.wrap(isinstance(w_d, SharedDictImplementation)) cls.w_is_sharing = cls.space.wrap(gateway.interp2app(is_sharing)) Modified: pypy/trunk/pypy/module/operator/__init__.py ============================================================================== --- pypy/trunk/pypy/module/operator/__init__.py (original) +++ pypy/trunk/pypy/module/operator/__init__.py Tue Nov 3 11:42:42 2009 @@ -9,7 +9,7 @@ def __init__(self, space, w_name): def create_lambda(name, alsoname): - return lambda space : self.getdictvalue(space, space.wrap(alsoname)) + return lambda space : self.getdictvalue(space, alsoname) MixedModule.__init__(self, space, w_name) for name, alsoname in self.mapping.iteritems(): Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Tue Nov 3 11:42:42 2009 @@ -100,12 +100,11 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): """ specialize access to dynamic exc_* attributes. """ - value = MixedModule.getdictvalue(self, space, w_attr) + value = MixedModule.getdictvalue(self, space, attr) if value is not None: return value - attr = space.str_w(w_attr) if attr == 'exc_type': operror = space.getexecutioncontext().sys_exc_info() if operror is None: Modified: pypy/trunk/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/trunk/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/trunk/pypy/module/sys/test/test_sysmodule.py Tue Nov 3 11:42:42 2009 @@ -65,6 +65,26 @@ assert exc_val2 ==e2 assert tb2.tb_lineno - tb.tb_lineno == 5 + def test_dynamic_attributes(self): + try: + raise Exception + except Exception,e: + import sys + exc_type = sys.exc_type + exc_val = sys.exc_value + tb = sys.exc_traceback + try: + raise Exception # 7 lines below the previous one + except Exception,e2: + exc_type2 = sys.exc_type + exc_val2 = sys.exc_value + tb2 = sys.exc_traceback + assert exc_type ==Exception + assert exc_val ==e + assert exc_type2 ==Exception + assert exc_val2 ==e2 + assert tb2.tb_lineno - tb.tb_lineno == 7 + def test_exc_info_normalization(self): import sys try: Modified: pypy/trunk/pypy/objspace/descroperation.py ============================================================================== --- pypy/trunk/pypy/objspace/descroperation.py (original) +++ pypy/trunk/pypy/objspace/descroperation.py Tue Nov 3 11:42:42 2009 @@ -36,9 +36,9 @@ if w_descr is not None: if space.is_data_descr(w_descr): return space.get(w_descr, w_obj) - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) else: - w_value = w_obj.getdictvalue(space, w_name) + w_value = w_obj.getdictvalue(space, name) if w_value is not None: return w_value if w_descr is not None: @@ -54,7 +54,7 @@ space.set(w_descr, w_obj, w_value) return shadows_type = True - if w_obj.setdictvalue(space, w_name, w_value, shadows_type): + if w_obj.setdictvalue(space, name, w_value, shadows_type): return raiseattrerror(space, w_obj, name, w_descr) Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Tue Nov 3 11:42:42 2009 @@ -54,11 +54,11 @@ if w_descr is None: # this handles directly the common case # module.function(args..) - w_value = w_obj.getdictvalue(space, w_name) + w_value = w_obj.getdictvalue(space, name) else: typ = type(w_descr) if typ is function.Function or typ is function.FunctionWithFixedCode: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) if w_value is None: # fast method path: a function object in the class, # nothing in the instance @@ -87,16 +87,16 @@ """An optimized version of space.call_method() based on the same principle as above. """ - w_name = space.wrap(methname) w_getattribute = space.lookup(w_obj, '__getattribute__') if w_getattribute is object_getattribute(space): w_descr = space.lookup(w_obj, methname) typ = type(w_descr) if typ is function.Function or typ is function.FunctionWithFixedCode: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, methname) if w_value is None: # fast method path: a function object in the class, # nothing in the instance return space.call_function(w_descr, w_obj, *arg_w) + w_name = space.wrap(methname) w_meth = space.getattr(w_obj, w_name) return space.call_function(w_meth, *arg_w) Modified: pypy/trunk/pypy/objspace/std/celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/celldict.py (original) +++ pypy/trunk/pypy/objspace/std/celldict.py Tue Nov 3 11:42:42 2009 @@ -1,5 +1,4 @@ from pypy.interpreter.pycode import CO_CONTAINSGLOBALS -from pypy.objspace.std.dictmultiobject import DictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib import jit @@ -16,7 +15,7 @@ def __repr__(self): return "" % (self.w_value, ) -class ModuleDictImplementation(DictImplementation): +class ModuleDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} @@ -45,24 +44,21 @@ w_value = cell.invalidate() cell = impl.content[name] = ModuleCell(w_value) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - name = self.space.str_w(w_key) + def impl_setitem_str(self, name, w_value, shadows_type=True): self.getcell(name).w_value = w_value if name in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(name) del self.unshadowed_builtins[name] - return self - - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -72,84 +68,71 @@ raise KeyError cell.invalidate() del self.content[key] - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - res = self.getcell(space.str_w(w_lookup), False) - if res is None: - return None - return res.w_value + return self.impl_getitem_str(space.str_w(w_lookup)) + elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) - - def iteritems(self): - return ModuleDictItemIteratorImplementation(self.space, self) + return self._as_rdict().getitem(w_lookup) - def iterkeys(self): - return ModuleDictKeyIteratorImplementation(self.space, self) + def impl_getitem_str(self, lookup): + res = self.getcell(lookup, False) + if res is None: + return None + return res.w_value - def itervalues(self): - return ModuleDictValueIteratorImplementation(self.space, self) + def impl_iter(self): + return ModuleDictIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return [cell.w_value for cell in self.content.itervalues()] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), cell.w_value]) for (key, cell) in self.content.iteritems()] - def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + def impl_clear(self): + # inefficient, but who cares for k, cell in self.content.iteritems(): - newimpl.setitem(self.space.wrap(k), cell.w_value) cell.invalidate() for k in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(k) - return newimpl - -# grrrrr. just a copy-paste from StrKeyIteratorImplementation in dictmultiobject -class ModuleDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() + self.content.clear() + self.unshadowed_builtins.clear() - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None -class ModuleDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + for k, cell in self.content.iteritems(): + r_dict_content[self.space.wrap(k)] = cell.w_value + cell.invalidate() + for k in self.unshadowed_builtins: + self.invalidate_unshadowed_builtin(k) + self._clear_fields() + return self - def next_entry(self): - # note that this 'for' loop only runs once, at most - for cell in self.iterator: - return cell.w_value - else: - return None + def _clear_fields(self): + self.content = None + self.unshadowed_builtins = None -class ModuleDictItemIteratorImplementation(IteratorImplementation): +class ModuleDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() @@ -157,14 +140,9 @@ def next_entry(self): # note that this 'for' loop only runs once, at most for key, cell in self.iterator: - return self.space.newtuple([self.space.wrap(key), cell.w_value]) + return (self.space.wrap(key), cell.w_value) else: - return None - - - - - + return None, None class State(object): @@ -217,8 +195,8 @@ return None def getimplementation(w_dict): - if type(w_dict) is W_DictMultiObject: - return w_dict.implementation + if type(w_dict) is ModuleDictImplementation and w_dict.r_dict_content is None: + return w_dict else: return None @@ -251,5 +229,5 @@ if cell is not None: f.cache_for_globals[nameindex] = cell return cell.w_value - return f._load_global(f.getname_w(nameindex)) + return f._load_global(f.getname_u(nameindex)) load_global_fill_cache._dont_inline_ = True Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Tue Nov 3 11:42:42 2009 @@ -22,93 +22,246 @@ space.is_w(w_lookup_type, space.w_float) ) +class W_DictMultiObject(W_Object): + from pypy.objspace.std.dicttype import dict_typedef as typedef -# DictImplementation lattice + r_dict_content = None -# a dictionary starts with an EmptyDictImplementation, and moves down -# in this list: -# -# EmptyDictImplementation -# / \ -# | | -# StrDictImplementation | -# \ / -# RDictImplementation -# -# (in addition, any dictionary can go back to EmptyDictImplementation) + @staticmethod + def allocate_and_init_instance(space, w_type=None, module=False, + instance=False, classofinstance=None, + from_strdict_shared=None): + if from_strdict_shared is not None: + assert w_type is None + assert not module and not instance and classofinstance is None + w_self = StrDictImplementation(space) + w_self.content = from_strdict_shared + return w_self + if space.config.objspace.std.withcelldict and module: + from pypy.objspace.std.celldict import ModuleDictImplementation + assert w_type is None + return ModuleDictImplementation(space) + elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: + assert w_type is None + return WaryDictImplementation(space) + elif space.config.objspace.std.withdictmeasurement: + assert w_type is None + return MeasuringDictImplementation(space) + elif space.config.objspace.std.withsharingdict and instance: + from pypy.objspace.std.sharingdict import SharedDictImplementation + assert w_type is None + return SharedDictImplementation(space) + elif (space.config.objspace.std.withshadowtracking and instance and + classofinstance is not None): + assert w_type is None + return ShadowDetectingDictImplementation(space, classofinstance) + elif instance: + assert w_type is None + return StrDictImplementation(space) + else: + if w_type is None: + w_type = space.w_dict + w_self = space.allocate_instance(W_DictMultiObject, w_type) + W_DictMultiObject.__init__(w_self, space) + w_self.initialize_as_rdict() + return w_self -class DictImplementation(object): - - def get(self, w_lookup): + def __init__(self, space): + self.space = space + + def initialize_as_rdict(self): + assert self.r_dict_content is None + self.r_dict_content = r_dict(self.space.eq_w, self.space.hash_w) + return self.r_dict_content + + + def initialize_content(w_self, list_pairs_w): + for w_k, w_v in list_pairs_w: + w_self.setitem(w_k, w_v) + + def __repr__(w_self): + """ representation for debugging purposes """ + return "%s()" % (w_self.__class__.__name__, ) + + def unwrap(w_dict, space): + result = {} + items = w_dict.items() + for w_pair in items: + key, val = space.unwrap(w_pair) + result[key] = val + return result + + def missing_method(w_dict, space, w_key): + if not space.is_w(space.type(w_dict), space.w_dict): + w_missing = space.lookup(w_dict, "__missing__") + if w_missing is None: + return None + return space.call_function(w_missing, w_dict, w_key) + else: + return None + + def set_str_keyed_item(w_dict, key, w_value, shadows_type=True): + w_dict.setitem_str(key, w_value, shadows_type) + + # _________________________________________________________________ + # implementation methods + def impl_getitem(self, w_key): #return w_value or None raise NotImplementedError("abstract base class") - def setitem_str(self, w_key, w_value, shadows_type=True): - #return implementation + def impl_getitem_str(self, w_key): + #return w_value or None raise NotImplementedError("abstract base class") - def setitem(self, w_key, w_value): - #return implementation + def impl_setitem_str(self, key, w_value, shadows_type=True): raise NotImplementedError("abstract base class") - def delitem(self, w_key): - #return implementation - raise NotImplementedError("abstract base class") - - def length(self): + def impl_setitem(self, w_key, w_value): raise NotImplementedError("abstract base class") - def iteritems(self): + def impl_delitem(self, w_key): raise NotImplementedError("abstract base class") - def iterkeys(self): + + def impl_length(self): raise NotImplementedError("abstract base class") - def itervalues(self): + + def impl_iter(self): raise NotImplementedError("abstract base class") + def impl_clear(self): + raise NotImplementedError("abstract base class") - def keys(self): - iterator = self.iterkeys() + def impl_keys(self): + iterator = self.impl_iter() result = [] while 1: - w_key = iterator.next() + w_key, w_value = iterator.next() if w_key is not None: result.append(w_key) else: return result - def values(self): - iterator = self.itervalues() + def impl_values(self): + iterator = self.impl_iter() result = [] while 1: - w_value = iterator.next() + w_key, w_value = iterator.next() if w_value is not None: result.append(w_value) else: return result - def items(self): - iterator = self.iteritems() + def impl_items(self): + iterator = self.impl_iter() result = [] while 1: - w_item = iterator.next() - if w_item is not None: - result.append(w_item) + w_key, w_value = iterator.next() + if w_key is not None: + result.append(self.space.newtuple([w_key, w_value])) else: return result -# the following method only makes sense when the option to use the -# CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen -# by the annotator - def get_builtin_indexed(self, i): - w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.get(w_key) + # the following method only makes sense when the option to use the + # CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen + # by the annotator + def impl_get_builtin_indexed(self, i): + key = OPTIMIZED_BUILTINS[i] + return self.impl_getitem_str(key) + + # this method will only be seen whan a certain config option is used + def impl_shadows_anything(self): + return True + + def impl_set_shadows_anything(self): + pass + + # _________________________________________________________________ + # fallback implementation methods + + def impl_fallback_setitem(self, w_key, w_value): + self.r_dict_content[w_key] = w_value + + def impl_fallback_setitem_str(self, key, w_value, shadows_type=True): + return self.impl_fallback_setitem(self.space.wrap(key), w_value) -# this method will only be seen whan a certain config option is used - def shadows_anything(self): + def impl_fallback_delitem(self, w_key): + del self.r_dict_content[w_key] + + def impl_fallback_length(self): + return len(self.r_dict_content) + + def impl_fallback_getitem(self, w_key): + return self.r_dict_content.get(w_key, None) + + def impl_fallback_getitem_str(self, key): + return self.r_dict_content.get(self.space.wrap(key), None) + + def impl_fallback_iter(self): + return RDictIteratorImplementation(self.space, self) + + def impl_fallback_keys(self): + return self.r_dict_content.keys() + def impl_fallback_values(self): + return self.r_dict_content.values() + def impl_fallback_items(self): + return [self.space.newtuple([w_key, w_val]) + for w_key, w_val in self.r_dict_content.iteritems()] + + def impl_fallback_clear(self): + self.r_dict_content.clear() + + def impl_fallback_get_builtin_indexed(self, i): + key = OPTIMIZED_BUILTINS[i] + return self.impl_fallback_getitem_str(key) + + def impl_fallback_shadows_anything(self): return True - def set_shadows_anything(self): + def impl_fallback_set_shadows_anything(self): pass +implementation_methods = [ + ("getitem", 1), + ("getitem_str", 1), + ("length", 0), + ("setitem_str", 3), + ("setitem", 2), + ("delitem", 1), + ("iter", 0), + ("items", 0), + ("values", 0), + ("keys", 0), + ("clear", 0), + ("get_builtin_indexed", 1), + ("shadows_anything", 0), + ("set_shadows_anything", 0), +] + + +def _make_method(name, implname, fallback, numargs): + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def %s(self, %s): + if self.r_dict_content is not None: + return self.%s(%s) + return self.%s(%s)""" % (name, args, fallback, args, implname, args) + d = {} + exec py.code.Source(code).compile() in d + implementation_method = d[name] + implementation_method.func_defaults = getattr(W_DictMultiObject, implname).func_defaults + return implementation_method + +def _install_methods(): + for name, numargs in implementation_methods: + implname = "impl_" + name + fallbackname = "impl_fallback_" + name + func = _make_method(name, implname, fallbackname, numargs) + setattr(W_DictMultiObject, name, func) +_install_methods() + +registerimplementation(W_DictMultiObject) + +# DictImplementation lattice +# XXX fix me + # Iterator Implementation base classes class IteratorImplementation(object): @@ -120,19 +273,19 @@ def next(self): if self.dictimplementation is None: - return None + return None, None if self.len != self.dictimplementation.length(): self.len = -1 # Make this error state sticky raise OperationError(self.space.w_RuntimeError, self.space.wrap("dictionary changed size during iteration")) # look for the next entry - w_result = self.next_entry() - if w_result is not None: + if self.pos < self.len: + result = self.next_entry() self.pos += 1 - return w_result + return result # no more entries self.dictimplementation = None - return None + return None, None def next_entry(self): """ Purely abstract method @@ -148,170 +301,92 @@ # concrete subclasses of the above -class EmptyDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - - def get(self, w_lookup): - space = self.space - if not _is_str(space, w_lookup) and not _is_sane_hash(space, - space.type(w_lookup)): - # give hash a chance to raise an exception - space.hash(w_lookup) - return None - - def setitem(self, w_key, w_value): - space = self.space - if _is_str(space, w_key): - return StrDictImplementation(space).setitem_str(w_key, w_value) - else: - return space.DefaultDictImpl(space).setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - return StrDictImplementation(self.space).setitem_str(w_key, w_value) - #return SmallStrDictImplementation(self.space, w_key, w_value) - - def delitem(self, w_key): - space = self.space - if not _is_str(space, w_key) and not _is_sane_hash(space, - space.type(w_key)): - # count hash - space.hash(w_key) - raise KeyError - - def length(self): - return 0 - - def iteritems(self): - return EmptyIteratorImplementation(self.space, self) - def iterkeys(self): - return EmptyIteratorImplementation(self.space, self) - def itervalues(self): - return EmptyIteratorImplementation(self.space, self) - - def keys(self): - return [] - def values(self): - return [] - def items(self): - return [] - -class EmptyIteratorImplementation(IteratorImplementation): - def next_entry(self): - return None - - -class StrDictImplementation(DictImplementation): +class StrDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - self.content[self.space.str_w(w_key)] = w_value - return self + def impl_setitem_str(self, key, w_value, shadows_type=True): + self.content[key] = w_value - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): del self.content[space.str_w(w_key)] - if self.content: - return self - else: - return space.emptydictimpl + return elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem_str(self, key): + return self.content.get(key, None) + + def impl_getitem(self, w_key): space = self.space # -- This is called extremely often. Hack for performance -- - if type(w_lookup) is space.StringObjectCls: - return self.content.get(w_lookup.unwrap(space), None) + if type(w_key) is space.StringObjectCls: + return self.impl_getitem_str(w_key.unwrap(space)) # -- End of performance hack -- - w_lookup_type = space.type(w_lookup) + w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_str): - return self.content.get(space.str_w(w_lookup), None) + return self.impl_getitem_str(space.str_w(w_key)) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_key) - def iteritems(self): - return StrItemIteratorImplementation(self.space, self) + def impl_iter(self): + return StrIteratorImplementation(self.space, self) - def iterkeys(self): - return StrKeyIteratorImplementation(self.space, self) - - def itervalues(self): - return StrValueIteratorImplementation(self.space, self) - - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return self.content.values() - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), w_value]) for (key, w_value) in self.content.iteritems()] + def impl_clear(self): + self.content.clear() + def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + r_dict_content = self.initialize_as_rdict() for k, w_v in self.content.items(): - newimpl.setitem(self.space.wrap(k), w_v) - return newimpl - -# the following are very close copies of the base classes above - -class StrKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None - -class StrValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() + r_dict_content[self.space.wrap(k)] = w_v + self._clear_fields() + return self - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None + def _clear_fields(self): + self.content = None -class StrItemIteratorImplementation(IteratorImplementation): +class StrIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for key, w_value in self.iterator: - return self.space.newtuple([self.space.wrap(key), w_value]) + for str, w_value in self.iterator: + return self.space.wrap(str), w_value else: - return None + return None, None class ShadowDetectingDictImplementation(StrDictImplementation): @@ -324,29 +399,29 @@ else: self._shadows_anything = False - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): if shadows_type: self._shadows_anything = True - return StrDictImplementation.setitem_str( - self, w_key, w_value, shadows_type) + StrDictImplementation.impl_setitem_str( + self, key, w_value, shadows_type) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): if not self._shadows_anything: w_obj = self.w_type.lookup(space.str_w(w_key)) if w_obj is not None: self._shadows_anything = True - return StrDictImplementation.setitem_str( - self, w_key, w_value, False) + StrDictImplementation.impl_setitem_str( + self, self.space.str_w(w_key), w_value, False) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def shadows_anything(self): + def impl_shadows_anything(self): return (self._shadows_anything or self.w_type.version_tag is not self.original_version_tag) - def set_shadows_anything(self): + def impl_set_shadows_anything(self): self._shadows_anything = True class WaryDictImplementation(StrDictImplementation): @@ -354,15 +429,13 @@ StrDictImplementation.__init__(self, space) self.shadowed = [None] * len(BUILTIN_TO_INDEX) - def setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = w_value self.content[key] = w_value - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -371,96 +444,30 @@ i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = None - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def get_builtin_indexed(self, i): + def impl_get_builtin_indexed(self, i): return self.shadowed[i] -class RDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - self.content = r_dict(space.eq_w, space.hash_w) - - def __repr__(self): - return "%s<%s>" % (self.__class__.__name__, self.content) - - def setitem(self, w_key, w_value): - self.content[w_key] = w_value - return self - - def setitem_str(self, w_key, w_value, shadows_type=True): - return self.setitem(w_key, w_value) - - def delitem(self, w_key): - del self.content[w_key] - if self.content: - return self - else: - return self.space.emptydictimpl - - def length(self): - return len(self.content) - def get(self, w_lookup): - return self.content.get(w_lookup, None) - - def iteritems(self): - return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): - return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): - return RDictValueIteratorImplementation(self.space, self) - - def keys(self): - return self.content.keys() - def values(self): - return self.content.values() - def items(self): - return [self.space.newtuple([w_key, w_val]) - for w_key, w_val in self.content.iteritems()] - - -class RDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_key in self.iterator: - return w_key - else: - return None - -class RDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None - -class RDictItemIteratorImplementation(IteratorImplementation): +class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iteritems() + self.iterator = dictimplementation.r_dict_content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for w_key, w_value in self.iterator: - return self.space.newtuple([w_key, w_value]) + for item in self.iterator: + return item else: - return None + return None, None +# XXX fix this thing import time, py class DictInfo(object): @@ -518,7 +525,7 @@ def __del__(self): self.info.lifetime = time.time() - self.info.createtime -class MeasuringDictImplementation(DictImplementation): +class MeasuringDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = r_dict(space.eq_w, space.hash_w) @@ -544,7 +551,7 @@ else: self.info.misses += 1 - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): if not self.info.seen_non_string_in_write and not self._is_str(w_key): self.info.seen_non_string_in_write = True self.info.size_on_non_string_seen_in_write = len(self.content) @@ -552,11 +559,10 @@ self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) - return self - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): self.info.setitem_strs += 1 - return self.setitem(w_key, w_value) - def delitem(self, w_key): + self.impl_setitem(self.space.wrap(key), w_value) + def impl_delitem(self, w_key): if not self.info.seen_non_string_in_write \ and not self.info.seen_non_string_in_read_first \ and not self._is_str(w_key): @@ -565,38 +571,39 @@ self.info.delitems += 1 self.info.writes += 1 del self.content[w_key] - return self - def length(self): + def impl_length(self): self.info.lengths += 1 return len(self.content) - def get(self, w_lookup): + def impl_getitem_str(self, key): + return self.impl_getitem(self.space.wrap(key)) + def impl_getitem(self, w_key): self.info.gets += 1 - self._read(w_lookup) - return self.content.get(w_lookup, None) + self._read(w_key) + return self.content.get(w_key, None) - def iteritems(self): + def impl_iteritems(self): self.info.iteritems += 1 self.info.iterations += 1 return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): + def impl_iterkeys(self): self.info.iterkeys += 1 self.info.iterations += 1 return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): + def impl_itervalues(self): self.info.itervalues += 1 self.info.iterations += 1 return RDictValueIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): self.info.keys += 1 self.info.listings += 1 return self.content.keys() - def values(self): + def impl_values(self): self.info.values += 1 self.info.listings += 1 return self.content.values() - def items(self): + def impl_items(self): self.info.items += 1 self.info.listings += 1 return [self.space.newtuple([w_key, w_val]) @@ -630,71 +637,6 @@ os.close(fd) os.write(2, "Reporting done.\n") -class W_DictMultiObject(W_Object): - from pypy.objspace.std.dicttype import dict_typedef as typedef - - def __init__(w_self, space, module=False, sharing=False): - if space.config.objspace.std.withcelldict and module: - from pypy.objspace.std.celldict import ModuleDictImplementation - w_self.implementation = ModuleDictImplementation(space) - elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: - w_self.implementation = WaryDictImplementation(space) - elif space.config.objspace.std.withdictmeasurement: - w_self.implementation = MeasuringDictImplementation(space) - elif space.config.objspace.std.withsharingdict and sharing: - from pypy.objspace.std.sharingdict import SharedDictImplementation - w_self.implementation = SharedDictImplementation(space) - else: - w_self.implementation = space.emptydictimpl - w_self.space = space - - def initialize_content(w_self, list_pairs_w): - impl = w_self.implementation - for w_k, w_v in list_pairs_w: - impl = impl.setitem(w_k, w_v) - w_self.implementation = impl - - def initialize_from_strdict_shared(w_self, strdict): - impl = StrDictImplementation(w_self.space) - impl.content = strdict - w_self.implementation = impl - - def __repr__(w_self): - """ representation for debugging purposes """ - return "%s(%s)" % (w_self.__class__.__name__, w_self.implementation) - - def unwrap(w_dict, space): - result = {} - items = w_dict.implementation.items() - for w_pair in items: - key, val = space.unwrap(w_pair) - result[key] = val - return result - - def missing_method(w_dict, space, w_key): - if not space.is_w(space.type(w_dict), space.w_dict): - w_missing = space.lookup(w_dict, "__missing__") - if w_missing is None: - return None - return space.call_function(w_missing, w_dict, w_key) - else: - return None - - def len(w_self): - return w_self.implementation.length() - - def get(w_dict, w_key, w_default): - w_value = w_dict.implementation.get(w_key) - if w_value is not None: - return w_value - else: - return w_default - - def set_str_keyed_item(w_dict, w_key, w_value, shadows_type=True): - w_dict.implementation = w_dict.implementation.setitem_str( - w_key, w_value, shadows_type) - -registerimplementation(W_DictMultiObject) init_signature = Signature(['seq_or_map'], None, 'kwargs') @@ -715,7 +657,7 @@ raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) w_k, w_v = pair - w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) + w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): from pypy.objspace.std.dicttype import update1 @@ -724,71 +666,67 @@ from pypy.objspace.std.dicttype import update1 update1(space, w_dict, w_kwds) -def getitem__DictMulti_ANY(space, w_dict, w_lookup): - w_value = w_dict.implementation.get(w_lookup) +def getitem__DictMulti_ANY(space, w_dict, w_key): + w_value = w_dict.getitem(w_key) if w_value is not None: return w_value - w_missing_item = w_dict.missing_method(space, w_lookup) + w_missing_item = w_dict.missing_method(space, w_key) if w_missing_item is not None: return w_missing_item - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def setitem__DictMulti_ANY_ANY(space, w_dict, w_newkey, w_newvalue): - w_dict.implementation = w_dict.implementation.setitem(w_newkey, w_newvalue) + w_dict.setitem(w_newkey, w_newvalue) -def delitem__DictMulti_ANY(space, w_dict, w_lookup): +def delitem__DictMulti_ANY(space, w_dict, w_key): try: - w_dict.implementation = w_dict.implementation.delitem(w_lookup) + w_dict.delitem(w_key) except KeyError: - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def len__DictMulti(space, w_dict): - return space.wrap(w_dict.implementation.length()) + return space.wrap(w_dict.length()) -def contains__DictMulti_ANY(space, w_dict, w_lookup): - return space.newbool(w_dict.implementation.get(w_lookup) is not None) +def contains__DictMulti_ANY(space, w_dict, w_key): + return space.newbool(w_dict.getitem(w_key) is not None) dict_has_key__DictMulti_ANY = contains__DictMulti_ANY def iter__DictMulti(space, w_dict): - return W_DictMultiIterObject(space, w_dict.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_dict.iter(), KEYSITER) def eq__DictMulti_DictMulti(space, w_left, w_right): if space.is_w(w_left, w_right): return space.w_True - if w_left.implementation.length() != w_right.implementation.length(): + if w_left.length() != w_right.length(): return space.w_False - iteratorimplementation = w_left.implementation.iteritems() + iteratorimplementation = w_left.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) - w_rightval = w_right.implementation.get(w_key) + w_rightval = w_right.getitem(w_key) if w_rightval is None: return space.w_False if not space.eq_w(w_val, w_rightval): return space.w_False return space.w_True -def characterize(space, aimpl, bimpl): +def characterize(space, w_a, w_b): """ (similar to CPython) returns the smallest key in acontent for which b's value is different or absent and this value """ w_smallest_diff_a_key = None w_its_value = None - iteratorimplementation = aimpl.iteritems() + iteratorimplementation = w_a.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) if w_smallest_diff_a_key is None or space.is_true(space.lt(w_key, w_smallest_diff_a_key)): - w_bvalue = bimpl.get(w_key) + w_bvalue = w_b.getitem(w_key) if w_bvalue is None: w_its_value = w_val w_smallest_diff_a_key = w_key @@ -800,18 +738,16 @@ def lt__DictMulti_DictMulti(space, w_left, w_right): # Different sizes, no problem - leftimpl = w_left.implementation - rightimpl = w_right.implementation - if leftimpl.length() < rightimpl.length(): + if w_left.length() < w_right.length(): return space.w_True - if leftimpl.length() > rightimpl.length(): + if w_left.length() > w_right.length(): return space.w_False # Same size - w_leftdiff, w_leftval = characterize(space, leftimpl, rightimpl) + w_leftdiff, w_leftval = characterize(space, w_left, w_right) if w_leftdiff is None: return space.w_False - w_rightdiff, w_rightval = characterize(space, rightimpl, leftimpl) + w_rightdiff, w_rightval = characterize(space, w_right, w_left) if w_rightdiff is None: # w_leftdiff is not None, w_rightdiff is None return space.w_True @@ -824,47 +760,51 @@ def dict_copy__DictMulti(space, w_self): from pypy.objspace.std.dicttype import update1 - w_new = W_DictMultiObject(space) + w_new = W_DictMultiObject.allocate_and_init_instance(space) update1(space, w_new, w_self) return w_new def dict_items__DictMulti(space, w_self): - return space.newlist(w_self.implementation.items()) + return space.newlist(w_self.items()) def dict_keys__DictMulti(space, w_self): - return space.newlist(w_self.implementation.keys()) + return space.newlist(w_self.keys()) def dict_values__DictMulti(space, w_self): - return space.newlist(w_self.implementation.values()) + return space.newlist(w_self.values()) def dict_iteritems__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iteritems()) + return W_DictMultiIterObject(space, w_self.iter(), ITEMSITER) def dict_iterkeys__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_self.iter(), KEYSITER) def dict_itervalues__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.itervalues()) + return W_DictMultiIterObject(space, w_self.iter(), VALUESITER) def dict_clear__DictMulti(space, w_self): - w_self.implementation = space.emptydictimpl + w_self.clear() -def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): - return w_dict.get(w_lookup, w_default) +def dict_get__DictMulti_ANY_ANY(space, w_dict, w_key, w_default): + w_value = w_dict.getitem(w_key) + if w_value is not None: + return w_value + else: + return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): defaults = space.unpackiterable(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) - w_item = w_dict.implementation.get(w_key) + w_item = w_dict.getitem(w_key) if w_item is None: if len_defaults > 0: return defaults[0] else: space.raise_key_error(w_key) else: - w_dict.implementation.delitem(w_key) + w_dict.delitem(w_key) return w_item app = gateway.applevel(''' @@ -895,7 +835,7 @@ dictrepr = app.interphook("dictrepr") def repr__DictMulti(space, w_dict): - if w_dict.implementation.length() == 0: + if w_dict.length() == 0: return space.wrap('{}') ec = space.getexecutioncontext() w_currently_in_repr = ec._py_repr @@ -908,12 +848,17 @@ # Iteration +KEYSITER = 0 +ITEMSITER = 1 +VALUESITER = 2 + class W_DictMultiIterObject(W_Object): from pypy.objspace.std.dicttype import dictiter_typedef as typedef - def __init__(w_self, space, iteratorimplementation): + def __init__(w_self, space, iteratorimplementation, itertype): w_self.space = space w_self.iteratorimplementation = iteratorimplementation + w_self.itertype = itertype registerimplementation(W_DictMultiIterObject) @@ -922,16 +867,19 @@ def next__DictMultiIterObject(space, w_dictiter): iteratorimplementation = w_dictiter.iteratorimplementation - w_result = iteratorimplementation.next() - if w_result is not None: - return w_result + w_key, w_value = iteratorimplementation.next() + if w_key is not None: + itertype = w_dictiter.itertype + if itertype == KEYSITER: + return w_key + elif itertype == VALUESITER: + return w_value + elif itertype == ITEMSITER: + return space.newtuple([w_key, w_value]) + else: + assert 0, "should be unreachable" raise OperationError(space.w_StopIteration, space.w_None) -# XXX __length_hint__() -##def len__DictMultiIterObject(space, w_dictiter): -## iteratorimplementation = w_dictiter.iteratorimplementation -## return space.wrap(iteratorimplementation.length()) - # ____________________________________________________________ from pypy.objspace.std import dicttype Modified: pypy/trunk/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dicttype.py (original) +++ pypy/trunk/pypy/objspace/std/dicttype.py Tue Nov 3 11:42:42 2009 @@ -141,8 +141,8 @@ # ____________________________________________________________ def descr__new__(space, w_dicttype, __args__): - w_obj = space.allocate_instance(space.DictObjectCls, w_dicttype) - space.DictObjectCls.__init__(w_obj, space) + from pypy.objspace.std.dictmultiobject import W_DictMultiObject + w_obj = W_DictMultiObject.allocate_and_init_instance(space, w_dicttype) return w_obj # ____________________________________________________________ Added: pypy/trunk/pypy/objspace/std/inlinedict.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/objspace/std/inlinedict.py Tue Nov 3 11:42:42 2009 @@ -0,0 +1,139 @@ +import py +from pypy.interpreter.typedef import check_new_dictionary +from pypy.objspace.std.dictmultiobject import W_DictMultiObject +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.dictmultiobject import IteratorImplementation +from pypy.objspace.std.dictmultiobject import implementation_methods +from pypy.tool.sourcetools import func_with_new_name + +def make_mixin(config): + if config.objspace.std.withsharingdict: + from pypy.objspace.std.sharingdict import SharedDictImplementation + return make_inlinedict_mixin(SharedDictImplementation, "structure") + else: + return make_inlinedict_mixin(StrDictImplementation, "content") + +def make_indirection_method(methname, numargs): + # *args don't work, the call normalization gets confused + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def f(self, %s): + return self.w_obj.%s(%s) +""" % (args, methname, args) + d = {} + exec py.code.Source(code).compile() in d + func = d["f"] + func.func_name = methname + "_indirect" + func.func_defaults = getattr(W_DictMultiObject, methname).func_defaults + return func + +def make_inlinedict_mixin(dictimplclass, attrname): + assert dictimplclass.__base__ is W_DictMultiObject + class IndirectionIterImplementation(IteratorImplementation): + def __init__(self, space, dictimpl, itemlist): + IteratorImplementation.__init__(self, space, dictimpl) + self.itemlist = itemlist + + def next_entry(self): + return self.itemlist[self.pos] + + class IndirectionDictImplementation(W_DictMultiObject): + def __init__(self, space, w_obj): + self.space = space + self.w_obj = w_obj + + def impl_iter(self): + # XXX sucky + items = [] + for w_item in self.impl_items(): + w_key, w_value = self.space.viewiterable(w_item) + items.append((w_key, w_value)) + return IndirectionIterImplementation(self.space, self, items) + + IndirectionDictImplementation.__name__ = "IndirectionDictImplementation" + dictimplclass.__name__ + + for methname, numargs in implementation_methods: + implname = "impl_" + methname + if implname != "impl_iter": + setattr(IndirectionDictImplementation, implname, + make_indirection_method(implname, numargs)) + + init_dictattributes = func_with_new_name(dictimplclass.__init__.im_func, + "init_dictattributes") + make_rdict = func_with_new_name(dictimplclass._as_rdict.im_func, + "make_rdict") + clear_fields = func_with_new_name(dictimplclass._clear_fields.im_func, + "clear_fields") + + class InlineDictMixin(object): + + def user_setup(self, space, w_subtype): + self.space = space + self.w__class__ = w_subtype + self.w__dict__ = None + init_dictattributes(self, space) + assert getattr(self, attrname) is not None + self.user_setup_slots(w_subtype.nslots) + + def getdict(self): + w__dict__ = self.w__dict__ + if w__dict__ is None: + w__dict__ = IndirectionDictImplementation(self.space, self) + self.w__dict__ = w__dict__ + assert isinstance(w__dict__, W_DictMultiObject) + return w__dict__ + + def _inlined_dict_valid(self): + return getattr(self, attrname) is not None + + def getdictvalue(self, space, attr): + if self._inlined_dict_valid(): + return self.impl_getitem_str(attr) + w_dict = self.getdict() + return w_dict.getitem_str(attr) + + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue(space, attr) + + def setdictvalue(self, space, attr, w_value, shadows_type=True): + if self._inlined_dict_valid(): + # XXX don't ignore shadows_type + self.impl_setitem_str(attr, w_value) + return True + w_dict = self.getdict() + w_dict.setitem_str(attr, w_value) + return True + + def deldictvalue(self, space, w_attr): + if self._inlined_dict_valid(): + try: + self.impl_delitem(w_attr) + except KeyError: + return False + return True + w_dict = self.getdict() + try: + w_dict.delitem(w_attr) + except KeyError: + return False + return True + + def setdict(self, space, w_dict): + self._clear_fields() # invalidate attributes on self + self.w__dict__ = check_new_dictionary(space, w_dict) + + def _as_rdict(self): + make_rdict(self) + return self.getdict() + + def initialize_as_rdict(self): + return self.getdict().initialize_as_rdict() + + _clear_fields = clear_fields + + for methname, _ in implementation_methods: + implname = "impl_" + methname + meth = func_with_new_name(getattr(dictimplclass, implname).im_func, + implname) + if not hasattr(InlineDictMixin, implname): + setattr(InlineDictMixin, implname, meth) + return InlineDictMixin Modified: pypy/trunk/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/trunk/pypy/objspace/std/marshal_impl.py (original) +++ pypy/trunk/pypy/objspace/std/marshal_impl.py Tue Nov 3 11:42:42 2009 @@ -355,7 +355,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) - for w_tuple in w_dict.implementation.items(): + for w_tuple in w_dict.items(): w_key, w_value = space.viewiterable(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) Modified: pypy/trunk/pypy/objspace/std/model.py ============================================================================== --- pypy/trunk/pypy/objspace/std/model.py (original) +++ pypy/trunk/pypy/objspace/std/model.py Tue Nov 3 11:42:42 2009 @@ -26,6 +26,7 @@ def __init__(self, config): """NOT_RPYTHON: inititialization only""" + self.config = config # All the Python types that we want to provide in this StdObjSpace class result: from pypy.objspace.std.objecttype import object_typedef @@ -250,7 +251,7 @@ for cls in self.typeorder: if (hasattr(cls, 'typedef') and cls.typedef is not None and cls.typedef.acceptable_as_base_class): - subclslist = enum_interplevel_subclasses(cls) + subclslist = enum_interplevel_subclasses(self.config, cls) for subcls in subclslist: if cls in subcls.__bases__: # only direct subclasses # for user subclasses we only accept "generic" Modified: pypy/trunk/pypy/objspace/std/objectobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objectobject.py (original) +++ pypy/trunk/pypy/objspace/std/objectobject.py Tue Nov 3 11:42:42 2009 @@ -1,4 +1,4 @@ -from pypy.objspace.std.objspace import * +from pypy.objspace.std.objspace import W_Object, register_all class W_ObjectObject(W_Object): Modified: pypy/trunk/pypy/objspace/std/objecttype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objecttype.py (original) +++ pypy/trunk/pypy/objspace/std/objecttype.py Tue Nov 3 11:42:42 2009 @@ -63,7 +63,6 @@ space.wrap("default __new__ takes " "no parameters")) w_obj = space.allocate_instance(W_ObjectObject, w_type) - #W_ObjectObject.__init__(w_obj) return w_obj def descr__init__(space, w_obj, __args__): Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Tue Nov 3 11:42:42 2009 @@ -140,13 +140,13 @@ w_globals = f.w_globals num = oparg >> 8 assert isinstance(w_globals, W_DictMultiObject) - w_value = w_globals.implementation.get_builtin_indexed(num) + w_value = w_globals.get_builtin_indexed(num) if w_value is None: builtins = f.get_builtin() assert isinstance(builtins, Module) w_builtin_dict = builtins.w_dict assert isinstance(w_builtin_dict, W_DictMultiObject) - w_value = w_builtin_dict.implementation.get_builtin_indexed(num) + w_value = w_builtin_dict.get_builtin_indexed(num) ## if w_value is not None: ## print "CALL_LIKELY_BUILTIN fast" if w_value is None: @@ -240,16 +240,9 @@ self.FrameClass = StdObjSpaceFrame - # XXX store the dict class on the space to access it in various places + # store the dict class on the space to access it in various places from pypy.objspace.std import dictmultiobject self.DictObjectCls = dictmultiobject.W_DictMultiObject - self.emptydictimpl = dictmultiobject.EmptyDictImplementation(self) - if self.config.objspace.std.withbucketdict: - from pypy.objspace.std import dictbucket - self.DefaultDictImpl = dictbucket.BucketDictImplementation - else: - self.DefaultDictImpl = dictmultiobject.RDictImplementation - assert self.DictObjectCls in self.model.typeorder from pypy.objspace.std import tupleobject self.TupleObjectCls = tupleobject.W_TupleObject @@ -577,11 +570,13 @@ from pypy.objspace.std.listobject import W_ListObject return W_ListObject(list_w) - def newdict(self, module=False): + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): from pypy.objspace.std.dictmultiobject import W_DictMultiObject - if module: - return W_DictMultiObject(self, module=True) - return W_DictMultiObject(self) + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) @@ -615,12 +610,14 @@ user-defined type, without actually __init__ializing the instance.""" w_type = self.gettypeobject(cls.typedef) if self.is_w(w_type, w_subtype): - instance = instantiate(cls) + instance = instantiate(cls) elif cls.typedef.acceptable_as_base_class: # the purpose of the above check is to avoid the code below # to be annotated at all for 'cls' if it is not necessary w_subtype = w_type.check_user_subclass(w_subtype) - subcls = get_unique_interplevel_subclass(cls, w_subtype.hasdict, w_subtype.nslots != 0, w_subtype.needsdel, w_subtype.weakrefable) + subcls = get_unique_interplevel_subclass( + self.config, cls, w_subtype.hasdict, w_subtype.nslots != 0, + w_subtype.needsdel, w_subtype.weakrefable) instance = instantiate(subcls) instance.user_setup(self, w_subtype) else: @@ -672,7 +669,6 @@ return self.int_w(l_w[0]), self.int_w(l_w[1]), self.int_w(l_w[2]) def is_(self, w_one, w_two): - # XXX a bit of hacking to gain more speed if w_one is w_two: return self.w_True return self.w_False @@ -712,7 +708,7 @@ e = None if w_descr is not None: if not self.is_data_descr(w_descr): - w_value = w_obj.getdictvalue_attr_is_in_class(self, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(self, name) if w_value is not None: return w_value try: @@ -721,7 +717,7 @@ if not e.match(self, self.w_AttributeError): raise else: - w_value = w_obj.getdictvalue(self, w_name) + w_value = w_obj.getdictvalue(self, name) if w_value is not None: return w_value @@ -733,18 +729,27 @@ else: raiseattrerror(self, w_obj, name) + def finditem_str(self, w_obj, key): + # performance shortcut to avoid creating the OperationError(KeyError) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem_str(key) + return ObjSpace.finditem_str(self, w_obj, key) + def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: - return w_obj.get(w_key, None) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: - w_obj.set_str_keyed_item(w_key, w_value, shadows_type) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + w_obj.set_str_keyed_item(key, w_value, shadows_type) else: - self.setitem(w_obj, w_key, w_value) + self.setitem(w_obj, self.wrap(key), w_value) def getindex_w(self, w_obj, w_exception, objdescr=None): # Performance shortcut for the common case of w_obj being an int. Modified: pypy/trunk/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/proxyobject.py (original) +++ pypy/trunk/pypy/objspace/std/proxyobject.py Tue Nov 3 11:42:42 2009 @@ -34,19 +34,19 @@ raise OperationError(space.w_TypeError, space.wrap("You cannot override __class__ for transparent proxies")) - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): try: return space.call_function(self.w_controller, space.wrap('__getattribute__'), - w_attr) + space.wrap(attr)) except OperationError, e: if not e.match(space, space.w_AttributeError): raise return None - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): try: space.call_function(self.w_controller, space.wrap('__setattr__'), - w_attr, w_value) + space.wrap(attr), w_value) return True except OperationError, e: if not e.match(space, space.w_AttributeError): @@ -64,19 +64,12 @@ return False def getdict(self): - return self.getdictvalue(self.space, self.space.wrap('__dict__')) + return self.getdictvalue(self.space, '__dict__') def setdict(self, space, w_dict): - if not self.setdictvalue(space, space.wrap('__dict__'), w_dict): + if not self.setdictvalue(space, '__dict__', w_dict): baseobjspace.W_Root.setdict(self, space, w_dict) -## def __getattr__(self, attr): -## # NOT_RPYTHON -## try: -## return self.getdictvalue(self.space, self.space.wrap(attr)) -## except OperationError, e: -## raise AttributeError(attr) - W_Transparent.__name__ = name return W_Transparent Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Tue Nov 3 11:42:42 2009 @@ -1,4 +1,3 @@ -from pypy.objspace.std.dictmultiobject import DictImplementation, StrDictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib.jit import purefunction, hint, we_are_jitted, unroll_safe @@ -75,41 +74,42 @@ self.emptylist = [] -class SharedDictImplementation(DictImplementation): +class SharedDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.structure = space.fromcache(State).empty_structure self.entries = space.fromcache(State).emptylist - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - lookup = space.str_w(w_lookup) - i = self.structure.lookup_position(lookup) - if i == -1: - return None - return self.entries[i] + return self.impl_getitem_str(space.str_w(w_lookup)) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_lookup) - def setitem(self, w_key, w_value): + def impl_getitem_str(self, lookup): + i = self.structure.lookup_position(lookup) + if i == -1: + return None + return self.entries[i] + + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) @unroll_safe - def setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = self.structure.lookup_position(key) if i != -1: self.entries[i] = w_value - return self + return new_structure = self.structure.get_next_structure(key) if new_structure.length > len(self.entries): new_entries = [None] * new_structure.size_estimate() @@ -120,9 +120,8 @@ self.entries[new_structure.length - 1] = w_value assert self.structure.length + 1 == new_structure.length self.structure = new_structure - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -131,64 +130,47 @@ key == self.structure.last_key): self.entries[self.structure.length - 1] = None self.structure = self.structure.back_struct - return self - return self._as_rdict().delitem(w_key) + return + self._as_rdict().delitem(w_key) elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return self.structure.length - def iteritems(self): - return SharedItemIteratorImplementation(self.space, self) - - def iterkeys(self): - return SharedKeyIteratorImplementation(self.space, self) + def impl_iter(self): + return SharedIteratorImplementation(self.space, self) - def itervalues(self): - return SharedValueIteratorImplementation(self.space, self) - - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] + for (key, item) in self.structure.keys.iteritems()] - def values(self): + def impl_values(self): return self.entries[:self.structure.length] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), self.entries[item]]) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] - - def _as_rdict(self, as_strdict=False): - if as_strdict: - newimpl = StrDictImplementation(self.space) - else: - newimpl = self.space.DefaultDictImpl(self.space) + for (key, item) in self.structure.keys.iteritems()] + def impl_clear(self): + space = self.space + self.structure = space.fromcache(State).empty_structure + self.entries = space.fromcache(State).emptylist + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() for k, i in self.structure.keys.items(): - if i >= 0: - newimpl.setitem_str(self.space.wrap(k), self.entries[i]) - return newimpl + r_dict_content[self.space.wrap(k)] = self.entries[i] + self._clear_fields() + return self + def _clear_fields(self): + self.structure = None + self.entries = None -class SharedValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.values = dictimplementation.entries - - def next(self): - if self.pos < self.len: - return self.values[self.pos] - else: - self.values = None - return None - -class SharedItemIteratorImplementation(IteratorImplementation): +class SharedIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.structure.keys.iteritems() @@ -198,19 +180,6 @@ assert isinstance(implementation, SharedDictImplementation) for key, index in self.iterator: w_value = implementation.entries[index] - return self.space.newtuple([self.space.wrap(key), w_value]) - else: - return None - -class SharedKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.structure.keys.iteritems() - - def next_entry(self): - implementation = self.dictimplementation - assert isinstance(implementation, SharedDictImplementation) - for key, index in self.iterator: - return self.space.wrap(key) + return self.space.wrap(key), w_value else: - return None + return None, None Modified: pypy/trunk/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_celldict.py Tue Nov 3 11:42:42 2009 @@ -26,7 +26,7 @@ def rescue_builtins(space): w_dict = space.builtin.getdict() content = {} - for key, cell in w_dict.implementation.content.iteritems(): + for key, cell in w_dict.content.iteritems(): newcell = ModuleCell() newcell.w_value = cell.w_value content[key] = newcell @@ -35,9 +35,9 @@ cls.w_rescue_builtins = cls.space.wrap(rescue_builtins) def restore_builtins(space): w_dict = space.builtin.getdict() - if not isinstance(w_dict.implementation, ModuleDictImplementation): - w_dict.implementation = ModuleDictImplementation(space) - w_dict.implementation.content = stored_builtins.pop() + assert isinstance(w_dict, ModuleDictImplementation) + w_dict.content = stored_builtins.pop() + w_dict.fallback = None restore_builtins = gateway.interp2app(restore_builtins) cls.w_restore_builtins = cls.space.wrap(restore_builtins) Modified: pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py Tue Nov 3 11:42:42 2009 @@ -1,10 +1,10 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ - EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ - MeasuringDictImplementation + StrDictImplementation from pypy.objspace.std.celldict import ModuleDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation from pypy.conftest import gettestobjspace @@ -15,13 +15,13 @@ def test_empty(self): space = self.space - d = self.space.DictObjectCls(space) + d = self.space.newdict() assert not self.space.is_true(d) def test_nonempty(self): space = self.space wNone = space.w_None - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content([(wNone, wNone)]) assert space.is_true(d) i = space.getitem(d, wNone) @@ -32,7 +32,7 @@ space = self.space wk1 = space.wrap('key') wone = space.wrap(1) - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content([(space.wrap('zero'),space.wrap(0))]) space.setitem(d,wk1,wone) wback = space.getitem(d,wk1) @@ -41,7 +41,7 @@ def test_delitem(self): space = self.space wk1 = space.wrap('key') - d = self.space.DictObjectCls(space) + d = self.space.newdict() d.initialize_content( [(space.wrap('zero'),space.wrap(0)), (space.wrap('one'),space.wrap(1)), (space.wrap('two'),space.wrap(2))]) @@ -52,7 +52,7 @@ space.getitem,d,space.wrap('one')) def test_wrap_dict(self): - assert isinstance(self.space.wrap({}), self.space.DictObjectCls) + assert isinstance(self.space.wrap({}), W_DictMultiObject) def test_dict_compare(self): @@ -134,8 +134,7 @@ space = self.space w = space.wrap d = {"a": w(1), "b": w(2)} - w_d = space.DictObjectCls(space) - w_d.initialize_from_strdict_shared(d) + w_d = space.newdict(from_strdict_shared=d) assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) @@ -143,8 +142,7 @@ space = self.space w = space.wrap d = {"a": w(1), "b": w(2)} - w_d = space.DictObjectCls(space) - w_d.initialize_from_strdict_shared(d) + w_d = space.newdict(from_strdict_shared=d) assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) d["c"] = w(41) @@ -552,7 +550,11 @@ raises(KeyError, "d['def']") -class C: pass + +class FakeString(str): + def unwrap(self, space): + self.unwrapped = True + return str(self) # the minimal 'space' needed to use a W_DictMultiObject class FakeSpace: @@ -587,9 +589,25 @@ def newtuple(self, l): return tuple(l) + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) + + def allocate_instance(self, cls, type): + return object.__new__(cls) + + def fromcache(self, cls): + return cls(self) + w_StopIteration = StopIteration w_None = None - StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem + StringObjectCls = FakeString + w_dict = None + iter = iter + viewiterable = list class Config: @@ -599,6 +617,7 @@ withsharingdict = False withsmalldicts = False withcelldict = False + withshadowtracking = False class opcodes: CALL_LIKELY_BUILTIN = False @@ -608,143 +627,147 @@ class TestDictImplementation: def setup_method(self,method): self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DefaultDictImpl = RDictImplementation def test_stressdict(self): from random import randint - d = self.space.DictObjectCls(self.space) + d = self.space.newdict() N = 10000 pydict = {} for i in range(N): x = randint(-N, N) setitem__DictMulti_ANY_ANY(self.space, d, x, i) pydict[x] = i - for x in pydict: - assert pydict[x] == getitem__DictMulti_ANY(self.space, d, x) + for key, value in pydict.iteritems(): + assert value == getitem__DictMulti_ANY(self.space, d, key) -class TestRDictImplementation: - ImplementionClass = RDictImplementation - DevolvedClass = RDictImplementation - EmptyClass = EmptyDictImplementation - DefaultDictImpl = RDictImplementation +class BaseTestRDictImplementation: def setup_method(self,method): - self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DefaultDictImpl = self.DefaultDictImpl - self.string = self.space.wrap("fish") - self.string2 = self.space.wrap("fish2") + self.fakespace = FakeSpace() + self.string = self.fakespace.wrap("fish") + self.string2 = self.fakespace.wrap("fish2") self.impl = self.get_impl() def get_impl(self): - "Needs to be empty, or one entry with key self.string" - return self.ImplementionClass(self.space) + return self.ImplementionClass(self.fakespace) + + def fill_impl(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + + def check_not_devolved(self): + assert self.impl.r_dict_content is None def test_setitem(self): - assert self.impl.setitem(self.string, 1000) is self.impl + self.impl.setitem(self.string, 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 + self.check_not_devolved() def test_setitem_str(self): - assert self.impl.setitem_str(self.space.str_w(self.string), 1000) is self.impl + self.impl.setitem_str(self.fakespace.str_w(self.string), 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 + self.check_not_devolved() def test_delitem(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() assert self.impl.length() == 2 - newimpl = self.impl.delitem(self.string) + self.impl.delitem(self.string2) assert self.impl.length() == 1 - assert newimpl is self.impl - newimpl = self.impl.delitem(self.string2) + self.impl.delitem(self.string) assert self.impl.length() == 0 - assert isinstance(newimpl, self.EmptyClass) + self.check_not_devolved() def test_keys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() keys = self.impl.keys() keys.sort() assert keys == [self.string, self.string2] + self.check_not_devolved() def test_values(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() values = self.impl.values() values.sort() assert values == [1000, 2000] + self.check_not_devolved() def test_items(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() items = self.impl.items() items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() - def test_iterkeys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iterkeys() - keys = [] - while 1: - key = iteratorimplementation.next() - if key is None: - break - keys.append(key) - keys.sort() - assert keys == [self.string, self.string2] - - def test_itervalues(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.itervalues() - values = [] - while 1: - value = iteratorimplementation.next() - if value is None: - break - values.append(value) - values.sort() - assert values == [1000, 2000] - - def test_iteritems(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iteritems() + def test_iter(self): + self.fill_impl() + iteratorimplementation = self.impl.iter() items = [] while 1: item = iteratorimplementation.next() - if item is None: + if item == (None, None): break items.append(item) items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() def test_devolve(self): impl = self.impl for x in xrange(100): - impl = impl.setitem(self.space.str_w(str(x)), x) - impl = impl.setitem(x, x) - assert isinstance(impl, self.DevolvedClass) + impl.setitem(self.fakespace.str_w(str(x)), x) + impl.setitem(x, x) + assert impl.r_dict_content is not None -class TestStrDictImplementation(TestRDictImplementation): +class TestStrDictImplementation(BaseTestRDictImplementation): ImplementionClass = StrDictImplementation -class TestMeasuringDictImplementation(TestRDictImplementation): - ImplementionClass = MeasuringDictImplementation - DevolvedClass = MeasuringDictImplementation - EmptyClass = MeasuringDictImplementation + def test_str_shortcut(self): + self.fill_impl() + s = FakeString(self.string) + assert self.impl.getitem(s) == 1000 + assert s.unwrapped + +## class TestMeasuringDictImplementation(BaseTestRDictImplementation): +## ImplementionClass = MeasuringDictImplementation +## DevolvedClass = MeasuringDictImplementation + +class TestModuleDictImplementation(BaseTestRDictImplementation): + ImplementionClass = ModuleDictImplementation -class TestModuleDictImplementation(TestRDictImplementation): +class TestModuleDictImplementationWithBuiltinNames(BaseTestRDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation -class TestModuleDictImplementationWithBuiltinNames(TestRDictImplementation): + string = "int" + string2 = "isinstance" + +class TestSharedDictImplementation(BaseTestRDictImplementation): + ImplementionClass = SharedDictImplementation + + +class BaseTestDevolvedDictImplementation(BaseTestRDictImplementation): + def fill_impl(self): + BaseTestRDictImplementation.fill_impl(self) + self.impl._as_rdict() + + def check_not_devolved(self): + pass + +class TestDevolvedStrDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = StrDictImplementation + +class TestDevolvedModuleDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = ModuleDictImplementation + +class TestDevolvedModuleDictImplementationWithBuiltinNames(BaseTestDevolvedDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation string = "int" string2 = "isinstance" +class TestDevolvedSharedDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = SharedDictImplementation + Added: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Tue Nov 3 11:42:42 2009 @@ -0,0 +1,127 @@ +from pypy.conftest import gettestobjspace +from pypy.objspace.std.inlinedict import make_inlinedict_mixin +from pypy.objspace.std.dictmultiobject import StrDictImplementation +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace +from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation + +class FakeSubtype: + nslots = 0 + +class TestMixin(object): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + fakespace = FakeSpace() + + def make_obj(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + obj.setdictvalue(self.fakespace, "hello", 1) + obj.setdictvalue(self.fakespace, "world", 2) + assert obj._inlined_dict_valid() + assert obj.w__dict__ is None + return obj + + def test_setgetdel_dictvalue(self): + obj = self.make_obj() + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, "bla") is None + assert not obj.deldictvalue(self.fakespace, "bla") + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + + + def test_getdict(self): + obj = self.make_obj() + w_dict = obj.getdict() + assert obj.getdict() is w_dict # always get the same dict + assert obj.w__dict__ is w_dict + + assert w_dict.getitem("hello") == 1 + assert w_dict.getitem("world") == 2 + w_dict.setitem("hello", 4) + w_dict.setitem("world", 5) + assert obj.getdictvalue(self.fakespace, "hello") == 4 + assert obj.getdictvalue(self.fakespace, "world") == 5 + + def test_setdict(self): + obj1 = self.make_obj() + w_dict1 = obj1.getdict() + obj2 = self.make_obj() + w_dict2 = obj2.getdict() + w_dict2.setitem(4, 1) # devolve dict + w_dict2.setitem(5, 2) + obj2.setdict(self.space, w_dict1) + assert obj2.getdictvalue(self.fakespace, "hello") == 1 + assert obj2.getdictvalue(self.fakespace, "world") == 2 + obj1.setdictvalue(self.fakespace, "hello", 4) + obj1.setdictvalue(self.fakespace, "world", 5) + assert obj2.getdictvalue(self.fakespace, "hello") == 4 + assert obj2.getdictvalue(self.fakespace, "world") == 5 + + + def test_dict_devolves_via_dict(self): + obj = self.make_obj() + w_dict = obj.getdict() + w_dict.setitem(4, 1) + w_dict.setitem(5, 2) + assert dict(w_dict.r_dict_content) == {4: 1, 5: 2, "hello": 1, "world": 2} + assert obj.getdictvalue(self.fakespace, "hello") == 1 + assert obj.getdictvalue(self.fakespace, "world") == 2 + assert obj.getdictvalue(self.fakespace, 4) == 1 + assert obj.getdictvalue(self.fakespace, 5) == 2 + obj.deldictvalue(self.fakespace, "world") + assert obj.getdictvalue(self.fakespace, "world") is None + obj.deldictvalue(self.fakespace, "hello") + assert obj.getdictvalue(self.fakespace, "hello") is None + +class TestMixinShared(TestMixin): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + +class TestIndirectDict(BaseTestRDictImplementation): + Mixin = make_inlinedict_mixin(StrDictImplementation, "content") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + def get_impl(self): + obj = self.FakeObject() + obj.user_setup(self.fakespace, FakeSubtype) + return obj.getdict() + + +class TestIndirectDictShared(TestIndirectDict): + Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") + class FakeObject(Mixin): + def user_setup_slots(self, nslots): + pass + + + + +class TestInlineDict(object): + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withinlineddict": True}) + + def test_simple(self): + w_a = self.space.appexec([], """(): + class A(object): + pass + a = A() + a.x = 12 + a.y = 13 + return a + """) + assert w_a.w__dict__ is None + assert self.space.int_w(w_a.content['x']) == 12 + assert self.space.int_w(w_a.content['y']) == 13 + Modified: pypy/trunk/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_obj.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_obj.py Tue Nov 3 11:42:42 2009 @@ -1,4 +1,3 @@ -# -*- coding: iso-8859-1 -*- from pypy.conftest import option class AppTestObject: Modified: pypy/trunk/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_shadowtracking.py Tue Nov 3 11:42:42 2009 @@ -13,15 +13,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.g = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.f = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_shadowing_via__dict__(self): space = self.space @@ -32,15 +32,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["g"] = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["f"] = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__dict__(self): space = self.space @@ -51,11 +51,11 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__ = {} """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__class__(self): space = self.space @@ -66,14 +66,14 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): class B(object): def g(self): return 42 a.__class__ = B """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing_the_type(self): space = self.space @@ -84,13 +84,13 @@ a.x = 72 return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() w_x = space.appexec([w_inst], """(a): a.__class__.x = 42 return a.x """) assert space.unwrap(w_x) == 72 - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() class AppTestShadowTracking(object): def setup_class(cls): Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Tue Nov 3 11:42:42 2009 @@ -131,10 +131,7 @@ def compute_default_mro(w_self): return compute_C3_mro(w_self.space, w_self) - def getdictvalue(w_self, space, w_attr): - return w_self.getdictvalue_w(space, space.str_w(w_attr)) - - def getdictvalue_w(w_self, space, attr): + def getdictvalue(w_self, space, attr): w_value = w_self.dict_w.get(attr, None) if w_self.lazyloaders and w_value is None: if attr in w_self.lazyloaders: @@ -172,7 +169,7 @@ if w_class is w_starttype: look = True elif look: - w_value = w_class.getdictvalue_w(space, name) + w_value = w_class.getdictvalue(space, name) if w_value is not None: return w_value return None @@ -182,7 +179,7 @@ def _lookup(w_self, key): space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_value return None @@ -193,7 +190,7 @@ # attribute was found space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_class, w_value return None, None @@ -273,7 +270,7 @@ "NOT_RPYTHON. Forces the lazy attributes to be computed." if 'lazyloaders' in w_self.__dict__: for attr in w_self.lazyloaders.keys(): - w_self.getdictvalue_w(w_self.space, attr) + w_self.getdictvalue(w_self.space, attr) del w_self.lazyloaders return False @@ -281,8 +278,7 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - newdic = space.DictObjectCls(space) - newdic.initialize_from_strdict_shared(w_self.dict_w) + newdic = space.newdict(from_strdict_shared=w_self.dict_w) return W_DictProxyObject(newdic) def unwrap(w_self, space): Modified: pypy/trunk/pypy/objspace/std/typetype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typetype.py (original) +++ pypy/trunk/pypy/objspace/std/typetype.py Tue Nov 3 11:42:42 2009 @@ -174,7 +174,7 @@ return space.wrap("""type(object) -> the object's type type(name, bases, dict) -> a new type""") w_type = _check(space, w_type) - w_result = w_type.getdictvalue_w(space, '__doc__') + w_result = w_type.getdictvalue(space, '__doc__') if w_result is None: return space.w_None else: Modified: pypy/trunk/pypy/objspace/taint.py ============================================================================== --- pypy/trunk/pypy/objspace/taint.py (original) +++ pypy/trunk/pypy/objspace/taint.py Tue Nov 3 11:42:42 2009 @@ -20,14 +20,11 @@ ## def getdict(self): ## return taint(self.w_obj.getdict()) -## def getdictvalue_w(self, space, attr): -## return taint(self.w_obj.getdictvalue_w(space, attr)) +## def getdictvalue(self, space, attr): +## return taint(self.w_obj.getdictvalue(space, attr)) -## def getdictvalue(self, space, w_attr): -## return taint(self.w_obj.getdictvalue(space, w_attr)) - -## def setdictvalue(self, space, w_attr, w_value): -## return self.w_obj.setdictvalue(space, w_attr, w_value) +## def setdictvalue(self, space, attr, w_value): +## return self.w_obj.setdictvalue(space, attr, w_value) ## ... From cfbolz at codespeak.net Tue Nov 3 11:44:08 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 11:44:08 +0100 (CET) Subject: [pypy-svn] r68931 - pypy/branch/shrink-multidict Message-ID: <20091103104408.3D9B2168020@codespeak.net> Author: cfbolz Date: Tue Nov 3 11:44:07 2009 New Revision: 68931 Removed: pypy/branch/shrink-multidict/ Log: kill merged branch From cfbolz at codespeak.net Tue Nov 3 14:06:19 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 14:06:19 +0100 (CET) Subject: [pypy-svn] r68932 - pypy/trunk/pypy/objspace/std Message-ID: <20091103130619.DF89A168063@codespeak.net> Author: cfbolz Date: Tue Nov 3 14:06:18 2009 New Revision: 68932 Modified: pypy/trunk/pypy/objspace/std/celldict.py Log: cleanup celldict a bit to use less isinstance calls. (Also saves a guard per call in the JIT). Modified: pypy/trunk/pypy/objspace/std/celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/celldict.py (original) +++ pypy/trunk/pypy/objspace/std/celldict.py Tue Nov 3 14:06:18 2009 @@ -150,24 +150,27 @@ self.space = space self.invalidcell = ModuleCell() self.always_invalid_cache = [] - self.neverused_dictimpl = ModuleDictImplementation(space) + self.neverused_dictcontent = {} class GlobalCacheHolder(object): def __init__(self, space): self.cache = None state = space.fromcache(State) - self.dictimpl = state.neverused_dictimpl + self.dictcontent = state.neverused_dictcontent def getcache(self, space, code, w_globals): - implementation = getimplementation(w_globals) - if self.dictimpl is implementation: + if type(w_globals) is ModuleDictImplementation: + content = w_globals.content + else: + content = None + if self.dictcontent is content: return self.cache - return self.getcache_slow(space, code, w_globals, implementation) + return self.getcache_slow(space, code, w_globals, content) getcache._always_inline_ = True - def getcache_slow(self, space, code, w_globals, implementation): + def getcache_slow(self, space, code, w_globals, content): state = space.fromcache(State) - if not isinstance(implementation, ModuleDictImplementation): + if content is None: cache = state.always_invalid_cache if len(code.co_names_w) > len(cache): cache = [state.invalidcell] * len(code.co_names_w) @@ -175,7 +178,7 @@ else: cache = [state.invalidcell] * len(code.co_names_w) self.cache = cache - self.dictimpl = implementation + self.dictcontent = content return cache getcache_slow._dont_inline_ = True @@ -210,7 +213,7 @@ LOAD_GLOBAL._always_inline_ = True def find_cell_from_dict(implementation, name): - if isinstance(implementation, ModuleDictImplementation): + if implementation is not None: return implementation.getcell(name, False) return None @@ -218,7 +221,7 @@ def load_global_fill_cache(f, nameindex): name = f.space.str_w(f.getname_w(nameindex)) implementation = getimplementation(f.w_globals) - if isinstance(implementation, ModuleDictImplementation): + if implementation is not None: cell = implementation.getcell(name, False) if cell is None: builtin_impl = getimplementation(f.get_builtin().getdict()) From pedronis at codespeak.net Tue Nov 3 14:20:32 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 3 Nov 2009 14:20:32 +0100 (CET) Subject: [pypy-svn] r68933 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091103132032.6ECA51683E2@codespeak.net> Author: pedronis Date: Tue Nov 3 14:20:31 2009 New Revision: 68933 Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt Log: my dates Modified: pypy/extradoc/sprintinfo/ddorf2009/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/people.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/people.txt Tue Nov 3 14:20:31 2009 @@ -12,6 +12,7 @@ Antonio Cuni 6/11 - 13/11 Hotel Haus Hillesheim Armin Rigo 6/11 - 13/11 Hotel Haus Hillesheim Holger Krekel 9/11 - 13/11 ? +Samuele Pedroni 5/11 - 13/11 Hotel Blaettler ==================== ============== ===================== ==================== ============== ===================== @@ -22,7 +23,6 @@ ==================== ============== ===================== Armin Rigo Michael Hudson -Samuele Pedroni Anders Chrigstroem Maciej Fijalkowski Andrew Thompson From arigo at codespeak.net Tue Nov 3 14:42:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 14:42:40 +0100 (CET) Subject: [pypy-svn] r68934 - pypy/branch/jit-removetypeptr/pypy/translator/c Message-ID: <20091103134240.7C0741683F3@codespeak.net> Author: arigo Date: Tue Nov 3 14:42:39 2009 New Revision: 68934 Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/node.py Log: Fix the comment. Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/node.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/node.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/node.py Tue Nov 3 14:42:39 2009 @@ -961,7 +961,7 @@ def enum_dependencies(self): # note: for the group used by the GC, it can grow during this phase, # which means that we might not return all members yet. This is fixed - # by get_final_dependencies() in rpython.memory.gctransform.framework + # by get_finish_tables() in rpython.memory.gctransform.framework. for member in self.obj.members: yield member._as_ptr() From arigo at codespeak.net Tue Nov 3 14:46:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 14:46:10 +0100 (CET) Subject: [pypy-svn] r68935 - in pypy/branch/jit-removetypeptr/pypy: config jit/backend/x86/test Message-ID: <20091103134610.775921683FF@codespeak.net> Author: arigo Date: Tue Nov 3 14:46:09 2009 New Revision: 68935 Modified: pypy/branch/jit-removetypeptr/pypy/config/translationoption.py pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: 'gcremovetypeptr' cannot be enabled for non-framework translations. Modified: pypy/branch/jit-removetypeptr/pypy/config/translationoption.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/config/translationoption.py (original) +++ pypy/branch/jit-removetypeptr/pypy/config/translationoption.py Tue Nov 3 14:46:09 2009 @@ -65,9 +65,12 @@ ["boehm", "ref", "framework", "none"], default="ref", cmdline=None, requires={ - "boehm": [("translation.gcrootfinder", "n/a")], - "ref": [("translation.gcrootfinder", "n/a")], - "none": [("translation.gcrootfinder", "n/a")], + "boehm": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "ref": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "none": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], }), BoolOption("gcremovetypeptr", "Remove the typeptr from every object", default=False, cmdline="--gcremovetypeptr"), Modified: pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/jit-removetypeptr/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Nov 3 14:46:09 2009 @@ -78,7 +78,8 @@ # t = TranslationContext() t.config.translation.gc = gc - t.config.translation.gcremovetypeptr = True + if gc != 'boehm': + t.config.translation.gcremovetypeptr = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) From arigo at codespeak.net Tue Nov 3 15:16:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 15:16:51 +0100 (CET) Subject: [pypy-svn] r68936 - pypy/branch/jit-removetypeptr/pypy/translator/c/gcc Message-ID: <20091103141651.1D86F168401@codespeak.net> Author: arigo Date: Tue Nov 3 15:16:49 2009 New Revision: 68936 Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py Log: A hack that appears needed for compiling with -fno-unit-at-a-time. Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py Tue Nov 3 15:16:49 2009 @@ -715,6 +715,8 @@ return self._visit_prologue() elif source == '%ebp' and target == '%esp': return self._visit_epilogue() + if source == '%esp' and self.funcname.startswith('VALGRIND_'): + return [] return self.insns_for_copy(source, target) def visit_pushl(self, line): From cfbolz at codespeak.net Tue Nov 3 15:20:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 15:20:42 +0100 (CET) Subject: [pypy-svn] r68937 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091103142042.3672A16840D@codespeak.net> Author: cfbolz Date: Tue Nov 3 15:20:41 2009 New Revision: 68937 Modified: pypy/trunk/pypy/objspace/std/sharingdict.py pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Log: don't devolve sharing dictionaries when a key is deleted Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Tue Nov 3 15:20:41 2009 @@ -126,12 +126,24 @@ w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): key = space.str_w(w_key) - if (self.structure.last_key is not None and - key == self.structure.last_key): - self.entries[self.structure.length - 1] = None - self.structure = self.structure.back_struct - return - self._as_rdict().delitem(w_key) + pos = self.structure.lookup_position(key) + if pos == -1: + raise KeyError + for i in range(pos, len(self.entries)): + self.entries[pos] = self.entries[pos + 1] + # don't make the entries list shorter, new keys might be added soon + self.entries[-1] = None + structure = self.structure + num_back = len(self.entries) - pos - 1 + keys = [None] * num_back + for i in range(num_back): + keys[i] = structure.last_key + structure = structure.back_struct + # go back the structure that contains the deleted key + structure = structure.back_struct + for i in range(num_back - 1, -1, -1): + structure = structure.get_next_structure(keys[i]) + self.structure = structure elif _is_sane_hash(space, w_key_type): raise KeyError else: Modified: pypy/trunk/pypy/objspace/std/test/test_sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Tue Nov 3 15:20:41 2009 @@ -1,6 +1,8 @@ +import py from pypy.conftest import gettestobjspace -from pypy.objspace.std.sharingdict import SharedStructure, NUM_DIGITS +from pypy.objspace.std.sharingdict import SharedStructure, NUM_DIGITS, SharedDictImplementation from pypy.interpreter import gateway +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace def instance_with_keys(structure, *keys): for key in keys: @@ -27,3 +29,19 @@ assert empty_structure.size_estimate() == 3 assert empty_structure.other_structs.get("a").size_estimate() == 6 assert empty_structure.other_structs.get("x").size_estimate() == 2 + +def test_delete(): + space = FakeSpace() + d = SharedDictImplementation(space) + d.setitem_str("a", 1) + d.setitem_str("b", 2) + d.setitem_str("c", 3) + d.delitem("b") + assert d.r_dict_content is None + assert d.entries == [1, 3, None] + assert d.structure.keys == {"a": 0, "c": 1} + assert d.getitem("a") == 1 + assert d.getitem("c") == 3 + assert d.getitem("b") is None + py.test.raises(KeyError, d.delitem, "b") + From cfbolz at codespeak.net Tue Nov 3 15:27:00 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 15:27:00 +0100 (CET) Subject: [pypy-svn] r68938 - pypy/extradoc/planning Message-ID: <20091103142700.5813E168410@codespeak.net> Author: cfbolz Date: Tue Nov 3 15:26:59 2009 New Revision: 68938 Modified: pypy/extradoc/planning/jit.txt Log: this is done Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 3 15:26:59 2009 @@ -25,7 +25,6 @@ - raising an exception tends to escape frames, due to the traceback capturing - prevent jitting really general */** calls -- sharing dict needs to cope with dels - improve test running, compile only once From fijal at codespeak.net Tue Nov 3 15:39:52 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 3 Nov 2009 15:39:52 +0100 (CET) Subject: [pypy-svn] r68939 - pypy/trunk/pypy/interpreter/test Message-ID: <20091103143952.166D716841F@codespeak.net> Author: fijal Date: Tue Nov 3 15:39:51 2009 New Revision: 68939 Modified: pypy/trunk/pypy/interpreter/test/test_executioncontext.py Log: Crashing test Modified: pypy/trunk/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/test/test_executioncontext.py Tue Nov 3 15:39:51 2009 @@ -713,7 +713,7 @@ self.leave_two_jitted_levels(ec, frame, frame2) - def test_check_escaping_jitted_with_two_differen_virtualizables(self): + def test_check_escaping_jitted_with_two_different_virtualizables(self): ec, frame, frame2 = self.enter_two_jitted_levels() frame3 = self.Frame(ec, frame) @@ -741,3 +741,17 @@ ec._unchain(frame3) assert not frame2.escaped + def test_frame_top_is_virtualizable(self): + skip("crash!") + ec, frame, frame2 = self.enter_two_jitted_levels() + frame3 = self.Frame(ec, frame2) + ec.jitted = False + ec._chain(frame3) + ec.gettopframe() + frame3.force_f_back() + ec._unchain(frame3) + assert not frame2.f_forward + ec.jitted = True + ec._unchain(frame2) + assert not frame.f_forward + assert ec.gettopframe() is frame From cfbolz at codespeak.net Tue Nov 3 16:09:32 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 16:09:32 +0100 (CET) Subject: [pypy-svn] r68940 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20091103150932.75E93168436@codespeak.net> Author: cfbolz Date: Tue Nov 3 16:09:32 2009 New Revision: 68940 Modified: pypy/trunk/pypy/interpreter/executioncontext.py pypy/trunk/pypy/interpreter/test/test_executioncontext.py Log: fix Maciek's test Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Tue Nov 3 16:09:32 2009 @@ -178,6 +178,9 @@ while frame is not None and not frame.f_back_forced: frame.f_back_some = f_back = ExecutionContext._extract_back_from_frame(frame) frame.f_back_forced = True + # now that we force the whole chain, we also have to set the + # forward links to None + frame.f_forward = None frame = f_back return orig_frame.f_back_some Modified: pypy/trunk/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/test/test_executioncontext.py Tue Nov 3 16:09:32 2009 @@ -742,7 +742,6 @@ assert not frame2.escaped def test_frame_top_is_virtualizable(self): - skip("crash!") ec, frame, frame2 = self.enter_two_jitted_levels() frame3 = self.Frame(ec, frame2) ec.jitted = False @@ -751,7 +750,10 @@ frame3.force_f_back() ec._unchain(frame3) assert not frame2.f_forward + assert ec.gettopframe() is frame2 ec.jitted = True ec._unchain(frame2) assert not frame.f_forward assert ec.gettopframe() is frame + ec._unchain(frame) + assert ec.gettopframe() is None From cfbolz at codespeak.net Tue Nov 3 16:18:04 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 16:18:04 +0100 (CET) Subject: [pypy-svn] r68941 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091103151804.43D6E168430@codespeak.net> Author: cfbolz Date: Tue Nov 3 16:18:03 2009 New Revision: 68941 Modified: pypy/trunk/pypy/objspace/std/sharingdict.py pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Log: grr, properly support deleting several attributes in a row Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Tue Nov 3 16:18:03 2009 @@ -129,12 +129,15 @@ pos = self.structure.lookup_position(key) if pos == -1: raise KeyError - for i in range(pos, len(self.entries)): - self.entries[pos] = self.entries[pos + 1] + struct_len = self.structure.length + num_back = struct_len - pos - 1 + + if num_back > 0: + for i in range(pos, struct_len): + self.entries[pos] = self.entries[pos + 1] # don't make the entries list shorter, new keys might be added soon - self.entries[-1] = None + self.entries[struct_len - 1] = None structure = self.structure - num_back = len(self.entries) - pos - 1 keys = [None] * num_back for i in range(num_back): keys[i] = structure.last_key Modified: pypy/trunk/pypy/objspace/std/test/test_sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Tue Nov 3 16:18:03 2009 @@ -45,3 +45,10 @@ assert d.getitem("b") is None py.test.raises(KeyError, d.delitem, "b") + d.delitem("c") + assert d.entries == [1, None, None] + assert d.structure.keys == {"a": 0} + + d.delitem("a") + assert d.entries == [None, None, None] + assert d.structure.keys == {} From arigo at codespeak.net Tue Nov 3 17:01:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:01:37 +0100 (CET) Subject: [pypy-svn] r68942 - pypy/branch/jit-removetypeptr/pypy/translator/c/gcc Message-ID: <20091103160137.94C6E168436@codespeak.net> Author: arigo Date: Tue Nov 3 17:01:37 2009 New Revision: 68942 Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py Log: Add comment. Modified: pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/jit-removetypeptr/pypy/translator/c/gcc/trackgcroot.py Tue Nov 3 17:01:37 2009 @@ -716,7 +716,9 @@ elif source == '%ebp' and target == '%esp': return self._visit_epilogue() if source == '%esp' and self.funcname.startswith('VALGRIND_'): - return [] + return [] # in VALGRIND_XXX functions, there is a dummy-looking + # mov %esp, %eax. Shows up only when compiling with + # gcc -fno-unit-at-a-time. return self.insns_for_copy(source, target) def visit_pushl(self, line): From arigo at codespeak.net Tue Nov 3 17:03:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:03:55 +0100 (CET) Subject: [pypy-svn] r68943 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091103160355.B8D68168436@codespeak.net> Author: arigo Date: Tue Nov 3 17:03:55 2009 New Revision: 68943 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: In VALGRIND_XXX functions, there is a dummy-looking mov %esp, %eax. Shows up only when compiling with gcc -fno-unit-at-a-time. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Tue Nov 3 17:03:55 2009 @@ -715,6 +715,10 @@ return self._visit_prologue() elif source == '%ebp' and target == '%esp': return self._visit_epilogue() + if source == '%esp' and self.funcname.startswith('VALGRIND_'): + return [] # in VALGRIND_XXX functions, there is a dummy-looking + # mov %esp, %eax. Shows up only when compiling with + # gcc -fno-unit-at-a-time. return self.insns_for_copy(source, target) def visit_pushl(self, line): From cfbolz at codespeak.net Tue Nov 3 17:08:28 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 3 Nov 2009 17:08:28 +0100 (CET) Subject: [pypy-svn] r68944 - pypy/trunk/pypy/module/__builtin__ Message-ID: <20091103160828.23A72168436@codespeak.net> Author: cfbolz Date: Tue Nov 3 17:08:27 2009 New Revision: 68944 Added: pypy/trunk/pypy/module/__builtin__/interp_inspect.py (contents, props changed) Modified: pypy/trunk/pypy/module/__builtin__/__init__.py pypy/trunk/pypy/module/__builtin__/app_inspect.py Log: Move the implementation of locals and globals to interp-level, to not force the frame chain in the case of the JIT. Modified: pypy/trunk/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/__init__.py (original) +++ pypy/trunk/pypy/module/__builtin__/__init__.py Tue Nov 3 17:08:27 2009 @@ -32,8 +32,6 @@ # redirected to functional.py, applevel version # is still needed and should stay where it is. 'sorted' : 'app_functional.sorted', - 'globals' : 'app_inspect.globals', - 'locals' : 'app_inspect.locals', 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', @@ -110,6 +108,9 @@ 'classmethod' : 'descriptor.ClassMethod', 'property' : 'descriptor.W_Property', + 'globals' : 'interp_inspect.globals', + 'locals' : 'interp_inspect.locals', + } def pick_builtin(self, w_globals): Modified: pypy/trunk/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/trunk/pypy/module/__builtin__/app_inspect.py Tue Nov 3 17:08:27 2009 @@ -5,16 +5,9 @@ import sys -def globals(): - "Return the dictionary containing the current scope's global variables." - return sys._getframe(0).f_globals - -def locals(): - """Return a dictionary containing the current scope's local variables. -Note that this may be the real dictionary of local variables, or a copy.""" - return sys._getframe(0).f_locals - def _caller_locals(): + # note: the reason why this is working is because the functions in here are + # compiled by geninterp, so they don't have a frame return sys._getframe(0).f_locals def vars(*obj): Added: pypy/trunk/pypy/module/__builtin__/interp_inspect.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/__builtin__/interp_inspect.py Tue Nov 3 17:08:27 2009 @@ -0,0 +1,12 @@ + +def globals(space): + "Return the dictionary containing the current scope's global variables." + ec = space.getexecutioncontext() + return ec.gettopframe_nohidden().w_globals + +def locals(space): + """Return a dictionary containing the current scope's local variables. +Note that this may be the real dictionary of local variables, or a copy.""" + ec = space.getexecutioncontext() + return ec.gettopframe_nohidden().getdictscope() + From arigo at codespeak.net Tue Nov 3 17:09:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:09:43 +0100 (CET) Subject: [pypy-svn] r68945 - pypy/trunk/pypy/translator/c Message-ID: <20091103160943.D47C8168436@codespeak.net> Author: arigo Date: Tue Nov 3 17:09:43 2009 New Revision: 68945 Modified: pypy/trunk/pypy/translator/c/genc.py Log: gcc uses tons of memory when translating with --gcremovetypeptr. So for now, make smaller files in that case... Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Tue Nov 3 17:09:43 2009 @@ -590,10 +590,12 @@ yield self.uniquecname(basecname), subiter() def gen_readable_parts_of_source(self, f): + split_criteria_big = SPLIT_CRITERIA if py.std.sys.platform != "win32": - split_criteria_big = SPLIT_CRITERIA * 4 - else: - split_criteria_big = SPLIT_CRITERIA + if self.database.gcpolicy.need_no_typeptr(): + pass # XXX gcc uses toooooons of memory??? + else: + split_criteria_big = SPLIT_CRITERIA * 4 if self.one_source_file: return gen_readable_parts_of_main_c_file(f, self.database, self.preimpl) From arigo at codespeak.net Tue Nov 3 17:15:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:15:34 +0100 (CET) Subject: [pypy-svn] r68946 - in pypy/trunk/pypy: config jit/backend/llsupport jit/backend/llsupport/test jit/backend/x86 jit/backend/x86/test rpython rpython/lltypesystem rpython/memory rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test translator/c Message-ID: <20091103161534.5FF13168448@codespeak.net> Author: arigo Date: Tue Nov 3 17:15:33 2009 New Revision: 68946 Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/transform.py pypy/trunk/pypy/rpython/memory/gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py pypy/trunk/pypy/translator/c/database.py pypy/trunk/pypy/translator/c/node.py Log: Merge the 'jit-removetypeptr' branch, allowing us to use --gcremovetypeptr with the JIT. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Nov 3 17:15:33 2009 @@ -65,9 +65,12 @@ ["boehm", "ref", "framework", "none"], default="ref", cmdline=None, requires={ - "boehm": [("translation.gcrootfinder", "n/a")], - "ref": [("translation.gcrootfinder", "n/a")], - "none": [("translation.gcrootfinder", "n/a")], + "boehm": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "ref": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "none": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], }), BoolOption("gcremovetypeptr", "Remove the typeptr from every object", default=False, cmdline="--gcremovetypeptr"), @@ -95,8 +98,7 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False), - ("translation.gcremovetypeptr", False)], + requires=[("translation.thread", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Tue Nov 3 17:15:33 2009 @@ -325,8 +325,7 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.layoutbuilder = framework.JITTransformerLayoutBuilder( - gcdescr.config) + self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Tue Nov 3 17:15:33 2009 @@ -28,8 +28,11 @@ else: translator = None self.gc_ll_descr = get_ll_description(gcdescr, translator) - self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, - 'typeptr', + if translator and translator.config.translation.gcremovetypeptr: + self.vtable_offset = None + else: + self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, + 'typeptr', translate_support_code) self._setup_prebuilt_error('ovf', OverflowError) self._setup_prebuilt_error('zer', ZeroDivisionError) @@ -424,8 +427,9 @@ classint = classbox.getint() descrsize = self.class_sizes[classint] res = self.gc_ll_descr.gc_malloc(descrsize) - as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) - as_array[self.vtable_offset/WORD] = classint + if self.vtable_offset is not None: + as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + as_array[self.vtable_offset/WORD] = classint return BoxPtr(res) def do_new_array(self, countbox, arraydescr): Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py Tue Nov 3 17:15:33 2009 @@ -147,19 +147,20 @@ class TestFramework: def setup_method(self, meth): - class FakeTranslator: - pass - class config: + class config_: class translation: gc = 'hybrid' gcrootfinder = 'asmgcc' gctransformer = 'framework' + gcremovetypeptr = False + class FakeTranslator: + config = config_ class FakeCPU: def cast_adr_to_int(self, adr): ptr = llmemory.cast_adr_to_ptr(adr, gc_ll_descr.WB_FUNCPTR) assert ptr._obj._callable == llop1._write_barrier_failing_case return 42 - gcdescr = get_description(config) + gcdescr = get_description(config_) translator = FakeTranslator() llop1 = FakeLLOp() gc_ll_descr = GcLLDescr_framework(gcdescr, FakeTranslator(), llop1) Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Nov 3 17:15:33 2009 @@ -518,7 +518,8 @@ self.set_vtable(eax, loc_vtable) def set_vtable(self, loc, loc_vtable): - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + if self.cpu.vtable_offset is not None: + self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -713,7 +714,24 @@ def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset - self.mc.CMP(mem(locs[0], offset), locs[1]) + if offset is not None: + self.mc.CMP(mem(locs[0], offset), locs[1]) + else: + # XXX hard-coded assumption: to go from an object to its class + # we use the following algorithm: + # - read the typeid from mem(locs[0]), i.e. at offset 0 + # - keep the lower 16 bits read there + # - multiply by 4 and use it as an offset in type_info_group. + loc = locs[1] + assert isinstance(loc, IMM32) + classptr = loc.value + # here, we have to go back from 'classptr' to the value expected + # from reading the 16 bits in the object header + type_info_group = llop.gc_get_type_info_group(llmemory.Address) + type_info_group = rffi.cast(lltype.Signed, type_info_group) + expected_typeid = (classptr - type_info_group) >> 2 + self.mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + # return self.implement_guard(addr, self.mc.JNE) def _no_const_locs(self, args): Modified: pypy/trunk/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386setup.py Tue Nov 3 17:15:33 2009 @@ -310,6 +310,11 @@ CMP = Instruction() CMP.common_modes(7) +# special mode for comparing a 16-bit operand with an immediate +CMP16 = Instruction() +CMP16.mode2(MODRM, IMM32, ['\x66', '\x81', orbyte(7<<3), modrm(1), + immediate(2,'h')]) + NOP = Instruction() NOP.mode0(['\x90']) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Tue Nov 3 17:15:33 2009 @@ -218,6 +218,8 @@ return [] # MOV [constant-address], accum if instrname == "MOV16": return [] # skipped + if instrname == "CMP16": + return [] # skipped if instrname == "LEA": if (args[1][1].__class__ != i386.MODRM or args[1][1].is_register()): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_zrpy_gc.py Tue Nov 3 17:15:33 2009 @@ -78,6 +78,8 @@ # t = TranslationContext() t.config.translation.gc = gc + if gc != 'boehm': + t.config.translation.gcremovetypeptr = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Tue Nov 3 17:15:33 2009 @@ -890,6 +890,9 @@ def op_gc_stack_bottom(self): pass # marker for trackgcroot.py + def op_gc_get_type_info_group(self): + raise NotImplementedError("gc_get_type_info_group") + def op_do_malloc_fixedsize_clear(self): raise NotImplementedError("do_malloc_fixedsize_clear") Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Tue Nov 3 17:15:33 2009 @@ -432,6 +432,7 @@ 'do_malloc_fixedsize_clear': LLOp(canunwindgc=True), 'do_malloc_varsize_clear': LLOp(canunwindgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), + 'gc_get_type_info_group': LLOp(sideeffects=False), # __________ GC operations __________ Modified: pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py Tue Nov 3 17:15:33 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder(self.GCClass, {}) + self.layoutbuilder = TypeLayoutBuilder(self.GCClass) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Nov 3 17:15:33 2009 @@ -130,12 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcremovetypeptr: - lltype2vtable = translator.rtyper.lltype2vtable - else: - lltype2vtable = {} - self.layoutbuilder = TransformerLayoutBuilder(GCClass, - lltype2vtable) + self.layoutbuilder = TransformerLayoutBuilder(translator, GCClass) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id @@ -508,11 +503,16 @@ self.write_typeid_list() return newgcdependencies - def get_final_dependencies(self): - # returns an iterator enumerating the type_info_group's members, - # to make sure that they are all followed (only a part of them - # might have been followed by a previous enum_dependencies()). - return iter(self.layoutbuilder.type_info_group.members) + def get_finish_tables(self): + # We must first make sure that the type_info_group's members + # are all followed. Do it repeatedly while new members show up. + # Once it is really done, do finish_tables(). + seen = 0 + while seen < len(self.layoutbuilder.type_info_group.members): + curtotal = len(self.layoutbuilder.type_info_group.members) + yield self.layoutbuilder.type_info_group.members[seen:curtotal] + seen = curtotal + yield self.finish_tables() def write_typeid_list(self): """write out the list of type ids together with some info""" @@ -821,6 +821,9 @@ if hasattr(self.root_walker, 'thread_die_ptr'): hop.genop("direct_call", [self.root_walker.thread_die_ptr]) + def gct_gc_get_type_info_group(self, hop): + return hop.cast_result(self.c_type_info_group) + def gct_malloc_nonmovable_varsize(self, hop): TYPE = hop.spaceop.result.concretetype if self.gcdata.gc.can_malloc_nonmovable(): @@ -964,6 +967,16 @@ class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): + def __init__(self, translator, GCClass=None): + if GCClass is None: + from pypy.rpython.memory.gc.base import choose_gc_from_config + GCClass, _ = choose_gc_from_config(translator.config) + if translator.config.translation.gcremovetypeptr: + lltype2vtable = translator.rtyper.lltype2vtable + else: + lltype2vtable = None + super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable) + def has_finalizer(self, TYPE): rtti = get_rtti(TYPE) return rtti is not None and hasattr(rtti._obj, 'destructor_funcptr') @@ -990,18 +1003,6 @@ return fptr -class JITTransformerLayoutBuilder(TransformerLayoutBuilder): - # for the JIT: currently does not support removetypeptr - def __init__(self, config): - from pypy.rpython.memory.gc.base import choose_gc_from_config - try: - assert not config.translation.gcremovetypeptr - except AttributeError: # for some tests - pass - GCClass, _ = choose_gc_from_config(config) - TransformerLayoutBuilder.__init__(self, GCClass, {}) - - def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: previous_steps = [] Modified: pypy/trunk/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/transform.py Tue Nov 3 17:15:33 2009 @@ -312,12 +312,12 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies - def get_final_dependencies(self): - pass - def finish_tables(self): pass + def get_finish_tables(self): + return self.finish_tables + def finish(self, backendopt=True): self.finish_helpers(backendopt=backendopt) self.finish_tables() Modified: pypy/trunk/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/gctypelayout.py Tue Nov 3 17:15:33 2009 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup +from pypy.rpython.lltypesystem import rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert @@ -169,7 +170,7 @@ size_of_fixed_type_info = llmemory.sizeof(GCData.TYPE_INFO) - def __init__(self, GCClass, lltype2vtable): + def __init__(self, GCClass, lltype2vtable=None): self.GCClass = GCClass self.lltype2vtable = lltype2vtable self.make_type_info_group() @@ -218,16 +219,28 @@ # store it type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id - # store the vtable of the type (if any) immediately thereafter - # (note that if gcremovetypeptr is False, lltype2vtable is empty) - vtable = self.lltype2vtable.get(TYPE, None) - if vtable is not None: - # check that if we have a vtable, we are not varsize - assert lltype.typeOf(fullinfo) == GCData.TYPE_INFO_PTR - vtable = lltype.normalizeptr(vtable) - self.type_info_group.add_member(vtable) + self.add_vtable_after_typeinfo(TYPE) return type_id + def add_vtable_after_typeinfo(self, TYPE): + # if gcremovetypeptr is False, then lltype2vtable is None and it + # means that we don't have to store the vtables in type_info_group. + if self.lltype2vtable is None: + return + # does the type have a vtable? + vtable = self.lltype2vtable.get(TYPE, None) + if vtable is not None: + # yes. check that in this case, we are not varsize + assert not TYPE._is_varsize() + vtable = lltype.normalizeptr(vtable) + self.type_info_group.add_member(vtable) + else: + # no vtable from lltype2vtable -- double-check to be sure + # that it's not a subclass of OBJECT. + while isinstance(TYPE, lltype.GcStruct): + assert TYPE is not rclass.OBJECT + _, TYPE = TYPE._first_struct() + def get_info(self, type_id): return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group._as_ptr(), Modified: pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gctypelayout.py Tue Nov 3 17:15:33 2009 @@ -37,7 +37,7 @@ for T, c in [(GC_S, 0), (GC_S2, 2), (GC_A, 0), (GC_A2, 0), (GC_S3, 2)]: assert len(offsets_to_gc_pointers(T)) == c -def test_layout_builder(lltype2vtable={}): +def test_layout_builder(lltype2vtable=None): # XXX a very minimal test layoutbuilder = TypeLayoutBuilder(FakeGC, lltype2vtable) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: @@ -67,7 +67,7 @@ layoutbuilder.size_of_fixed_type_info) def test_constfold(): - layoutbuilder = TypeLayoutBuilder(FakeGC, {}) + layoutbuilder = TypeLayoutBuilder(FakeGC) tid1 = layoutbuilder.get_type_id(GC_A) tid2 = layoutbuilder.get_type_id(GC_S3) class MockGC: Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Tue Nov 3 17:15:33 2009 @@ -709,9 +709,7 @@ if jit2gc: return jit2gc['layoutbuilder'] GCClass = cls.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) + layoutbuilder = framework.TransformerLayoutBuilder(translator, GCClass) layoutbuilder.delay_encoding() translator._jit2gc = { 'layoutbuilder': layoutbuilder, Modified: pypy/trunk/pypy/translator/c/database.py ============================================================================== --- pypy/trunk/pypy/translator/c/database.py (original) +++ pypy/trunk/pypy/translator/c/database.py Tue Nov 3 17:15:33 2009 @@ -288,10 +288,8 @@ finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) if self.gctransformer: - finish_callbacks.append(('GC transformer: tracking vtables', - self.gctransformer.get_final_dependencies)) finish_callbacks.append(('GC transformer: finished tables', - self.gctransformer.finish_tables)) + self.gctransformer.get_finish_tables())) def add_dependencies(newdependencies): for value in newdependencies: @@ -336,8 +334,18 @@ if finish_callbacks: logmsg, finish = finish_callbacks.pop(0) - newdependencies = finish() - log.database(logmsg) + if not hasattr(finish, 'next'): + newdependencies = finish() + else: + # if 'finish' is a generator, consume the next element + # and put the generator again in the queue + try: + newdependencies = finish.next() + finish_callbacks.insert(0, (None, finish)) + except StopIteration: + newdependencies = None + if logmsg: + log.database(logmsg) if newdependencies: add_dependencies(newdependencies) continue # progress - follow all dependencies again Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Tue Nov 3 17:15:33 2009 @@ -960,8 +960,8 @@ def enum_dependencies(self): # note: for the group used by the GC, it can grow during this phase, - # which means that we might not return all members yet. This is - # fixed by finish_tables() in rpython/memory/gctransform/framework.py + # which means that we might not return all members yet. This is fixed + # by get_finish_tables() in rpython.memory.gctransform.framework. for member in self.obj.members: yield member._as_ptr() From arigo at codespeak.net Tue Nov 3 17:15:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:15:49 +0100 (CET) Subject: [pypy-svn] r68947 - pypy/branch/jit-removetypeptr Message-ID: <20091103161549.EF47D168448@codespeak.net> Author: arigo Date: Tue Nov 3 17:15:49 2009 New Revision: 68947 Removed: pypy/branch/jit-removetypeptr/ Log: Remove merged branch. From arigo at codespeak.net Tue Nov 3 17:35:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 3 Nov 2009 17:35:30 +0100 (CET) Subject: [pypy-svn] r68949 - pypy/branch/gc-jit-hack Message-ID: <20091103163530.D5F6F168450@codespeak.net> Author: arigo Date: Tue Nov 3 17:35:30 2009 New Revision: 68949 Added: pypy/branch/gc-jit-hack/ - copied from r68948, pypy/trunk/ Log: A branch in which to try some lightweight custom hacking to improve the interaction between the GC and the JIT. (http://morepypy.blogspot.com/2009/11/hi-all-this-week-i-worked-on-improving.html) From magcius at codespeak.net Wed Nov 4 00:49:13 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 4 Nov 2009 00:49:13 +0100 (CET) Subject: [pypy-svn] r68951 - in pypy/branch/avm/pypy/translator: avm1 avm2 avm2/tools avm2/tools/intrinsic avm2/tools/intrinsic/adobe/utils avm2/tools/intrinsic/authoring avm2/tools/intrinsic/flash/accessibility avm2/tools/intrinsic/flash/desktop avm2/tools/intrinsic/flash/display avm2/tools/intrinsic/flash/errors avm2/tools/intrinsic/flash/events avm2/tools/intrinsic/flash/external avm2/tools/intrinsic/flash/filters avm2/tools/intrinsic/flash/geom avm2/tools/intrinsic/flash/media avm2/tools/intrinsic/flash/net avm2/tools/intrinsic/flash/printing avm2/tools/intrinsic/flash/profiler avm2/tools/intrinsic/flash/sampler avm2/tools/intrinsic/flash/system avm2/tools/intrinsic/flash/text avm2/tools/intrinsic/flash/text/engine avm2/tools/intrinsic/flash/trace avm2/tools/intrinsic/flash/ui avm2/tools/intrinsic/flash/utils avm2/tools/intrinsic/flash/xml Message-ID: <20091103234913.ADD71168440@codespeak.net> Author: magcius Date: Wed Nov 4 00:49:11 2009 New Revision: 68951 Removed: pypy/branch/avm/pypy/translator/avm2/tools/autopath.py pypy/branch/avm/pypy/translator/avm2/tools/intrgen.py pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/ArgumentError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Array.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Boolean.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Class.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Date.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/DefinitionError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Error.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/EvalError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Function.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Math.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Namespace.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Number.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Object.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/QName.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/RangeError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/ReferenceError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/RegExp.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/SecurityError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/String.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/SyntaxError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/TypeError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/URIError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/UninitializedError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/Vector.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/VerifyError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/XML.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/XMLList.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/adobe/utils/CustomActions.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/adobe/utils/ProductManager.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/adobe/utils/XMLUI.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/adobe/utils/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/arguments.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/authoring/authObject.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/accessibility/Accessibility.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/accessibility/AccessibilityImplementation.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/accessibility/AccessibilityProperties.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/desktop/Clipboard.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/desktop/ClipboardFormats.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/desktop/ClipboardTransferMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/AVM1Movie.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ActionScriptVersion.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Bitmap.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/BitmapData.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/BitmapDataChannel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/BlendMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/CapsStyle.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ColorCorrection.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ColorCorrectionSupport.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/DisplayObject.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/DisplayObjectContainer.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/FrameLabel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GradientType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Graphics.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsBitmapFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsEndFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsGradientFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsPath.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsPathCommand.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsPathWinding.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsShaderFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsSolidFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsStroke.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/GraphicsTrianglePath.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/IBitmapDrawable.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/IGraphicsData.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/IGraphicsFill.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/IGraphicsPath.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/IGraphicsStroke.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/InteractiveObject.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/InterpolationMethod.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/JointStyle.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/LineScaleMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Loader.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/LoaderInfo.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/MorphShape.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/MovieClip.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/PixelSnapping.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/SWFVersion.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Scene.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Shader.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderData.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderInput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderJob.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderParameter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderParameterType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/ShaderPrecision.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Shape.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/SimpleButton.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/SpreadMethod.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Sprite.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/Stage.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/StageAlign.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/StageDisplayState.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/StageQuality.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/StageScaleMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/display/TriangleCulling.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/EOFError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/IOError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/IllegalOperationError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/InvalidSWFError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/MemoryError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/ScriptTimeoutError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/errors/StackOverflowError.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/ActivityEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/AsyncErrorEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/ContextMenuEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/DataEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/ErrorEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/Event.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/EventDispatcher.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/EventPhase.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/FocusEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/FullScreenEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/HTTPStatusEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/IEventDispatcher.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/IMEEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/IOErrorEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/KeyboardEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/MouseEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/NetFilterEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/NetStatusEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/ProgressEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/SampleDataEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/SecurityErrorEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/ShaderEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/StatusEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/SyncEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/TextEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/TimerEvent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/WeakFunctionClosure.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/events/WeakMethodClosure.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/external/ExternalInterface.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/BevelFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/BitmapFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/BitmapFilterQuality.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/BitmapFilterType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/BlurFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/ColorMatrixFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/ConvolutionFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/DisplacementMapFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/DisplacementMapFilterMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/DropShadowFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/GlowFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/GradientBevelFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/GradientGlowFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/filters/ShaderFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/ColorTransform.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Matrix.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Matrix3D.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Orientation3D.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/PerspectiveProjection.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Point.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Rectangle.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Transform.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Utils3D.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/geom/Vector3D.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/Camera.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/ID3Info.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/Microphone.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/Sound.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/SoundChannel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/SoundCodec.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/SoundLoaderContext.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/SoundMixer.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/SoundTransform.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/Video.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/media/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/DynamicPropertyOutput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/FileFilter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/FileReference.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/FileReferenceList.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/IDynamicPropertyOutput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/IDynamicPropertyWriter.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/LocalConnection.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/NetConnection.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/NetStream.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/NetStreamInfo.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/NetStreamPlayOptions.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/NetStreamPlayTransitions.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/ObjectEncoding.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/Responder.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/SharedObject.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/SharedObjectFlushStatus.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/Socket.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLLoader.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLLoaderDataFormat.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLRequest.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLRequestHeader.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLRequestMethod.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLStream.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/URLVariables.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/XMLSocket.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/net/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/printing/PrintJob.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/printing/PrintJobOptions.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/printing/PrintJobOrientation.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/profiler/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/sampler/DeleteObjectSample.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/sampler/NewObjectSample.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/sampler/Sample.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/sampler/StackFrame.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/sampler/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/ApplicationDomain.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/Capabilities.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/FSCommand.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/IME.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/IMEConversionMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/JPEGLoaderContext.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/LoaderContext.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/Security.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/SecurityDomain.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/SecurityPanel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/System.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/system/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/AntiAliasType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/CSMSettings.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/Font.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/FontStyle.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/FontType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/GridFitType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/StaticText.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/StyleSheet.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextColorType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextDisplayMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextExtent.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextField.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextFieldAutoSize.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextFieldType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextFormat.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextFormatAlign.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextFormatDisplay.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextLineMetrics.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextRenderer.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextRun.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/TextSnapshot.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/BreakOpportunity.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/CFFHinting.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/ContentElement.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/DigitCase.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/DigitWidth.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/EastAsianJustifier.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/ElementFormat.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/FontDescription.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/FontLookup.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/FontMetrics.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/FontPosture.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/FontWeight.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/GraphicElement.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/GroupElement.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/JustificationStyle.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/Kerning.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/LigatureLevel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/LineJustification.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/RenderingMode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/SpaceJustifier.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TabAlignment.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TabStop.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextBaseline.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextBlock.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextElement.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextJustifier.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextLine.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextLineCreationResult.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextLineMirrorRegion.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextLineValidity.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TextRotation.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/text/engine/TypographicCase.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/trace/Trace.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/ContextMenu.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/ContextMenuBuiltInItems.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/ContextMenuClipboardItems.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/ContextMenuItem.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/KeyLocation.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/Keyboard.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/Mouse.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/ui/MouseCursor.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/ByteArray.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/Dictionary.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/Endian.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/IDataInput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/IDataOutput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/IExternalizable.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/ObjectInput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/ObjectOutput.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/Proxy.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/SetIntervalTimer.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/Timer.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/utils/package.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/xml/XMLDocument.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/xml/XMLNode.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/xml/XMLNodeType.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/xml/XMLParser.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/flash/xml/XMLTag.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/int.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/toplevel.as pypy/branch/avm/pypy/translator/avm2/tools/intrinsic/uint.as Modified: pypy/branch/avm/pypy/translator/avm1/tags.py pypy/branch/avm/pypy/translator/avm2/abc.py pypy/branch/avm/pypy/translator/avm2/assembler.py pypy/branch/avm/pypy/translator/avm2/avm2gen.py pypy/branch/avm/pypy/translator/avm2/constants.py pypy/branch/avm/pypy/translator/avm2/database.py pypy/branch/avm/pypy/translator/avm2/instructions.py pypy/branch/avm/pypy/translator/avm2/query.py pypy/branch/avm/pypy/translator/avm2/runtime.py pypy/branch/avm/pypy/translator/avm2/traits.py pypy/branch/avm/pypy/translator/avm2/types.py Log: More AVM2 work. Modified: pypy/branch/avm/pypy/translator/avm1/tags.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm1/tags.py (original) +++ pypy/branch/avm/pypy/translator/avm1/tags.py Wed Nov 4 00:49:11 2009 @@ -4,6 +4,7 @@ from pypy.translator.avm1.records import RecordHeader, ShapeWithStyle, Matrix, CXForm from pypy.translator.avm1.avm1 import Block from pypy.translator.avm1.util import BitStream +from pypy.translator.avm2.abc import AbcFile next_character_id = 1 @@ -40,6 +41,19 @@ def serialize_data(self): return Block.serialize(self) + +class DoABC(SwfTag, AbcFile): + + TAG_TYPE = 82 + TAG_MIN_VERSION = 9 + + def __init__(self, name="PyPy", flags=0): + AbcFile.__init__(self) + self.name = name + self.flags = flags + + def serialize_data(self): + return struct.pack(" method_name self.consts = {} # value --> AbstractConst self.delegates = {} # StaticMethod --> type_name + self.recordnames = {} # RECORD --> name self.const_count = Counter() # store statistics about constants def next_count(self): return self.unique() + def _default_record_name(self, RECORD): + trans = string.maketrans('[]<>(), :', '_________') + name = ['Record'] + # XXX: refactor this: we need a proper way to ensure unique names + for f_name, (FIELD_TYPE, f_default) in RECORD._fields.iteritems(): + type_name = FIELD_TYPE._short_name().translate(trans) + name.append(f_name) + name.append(type_name) + + return '__'.join(name) + def _default_class_name(self, INSTANCE): parts = INSTANCE._name.rsplit('.', 1) if len(parts) == 2: @@ -37,26 +50,38 @@ self.pending_node(function) return function.get_name() - # def pending_class(self, INSTANCE): - # try: - # return self.classes[INSTANCE] - # except KeyError: - # pass + def pending_record(self, RECORD): + try: + return self.recordnames[RECORD] + except KeyError: + pass + name = self._default_record_name(RECORD) + name = self.get_unique_class_name(None, name) + self.recordnames[RECORD] = name + r = Record(self, RECORD, name) + self.pending_node(r) + return name + + def pending_class(self, INSTANCE): + try: + return self.classes[INSTANCE] + except KeyError: + pass - # if isinstance(INSTANCE, dotnet.NativeInstance): - # self.classes[INSTANCE] = INSTANCE._name - # return INSTANCE._name - # else: - # namespace, name = self._default_class_name(INSTANCE) - # name = self.get_unique_class_name(namespace, name) - # if namespace is None: - # full_name = name - # else: - # full_name = '%s.%s' % (namespace, name) - # self.classes[INSTANCE] = full_name - # cls = Class(self, INSTANCE, namespace, name) - # self.pending_node(cls) - # return full_name + if isinstance(INSTANCE, runtime.NativeInstance): + self.classes[INSTANCE] = INSTANCE._name + return INSTANCE._name + else: + namespace, name = self._default_class_name(INSTANCE) + name = self.get_unique_class_name(namespace, name) + if namespace is None: + full_name = name + else: + full_name = '%s::%s' % (namespace, name) + self.classes[INSTANCE] = full_name + cls = c.Class(self, INSTANCE, namespace, name) + self.pending_node(cls) + return full_name def record_function(self, graph, name): self.functions[graph] = name @@ -70,13 +95,13 @@ i = 0 while (namespace, name) in self.classnames: name = '%s_%d' % (base_name, i) - i+= 1 + i += 1 self.classnames.add((namespace, name)) return name def class_name(self, INSTANCE): - #if INSTANCE is ootype.ROOT: - # return types.object.classname() + if INSTANCE is ootype.ROOT: + return types.types.object.classname() try: NATIVE_INSTANCE = INSTANCE._hints['NATIVE_INSTANCE'] return NATIVE_INSTANCE._name Modified: pypy/branch/avm/pypy/translator/avm2/instructions.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/instructions.py (original) +++ pypy/branch/avm/pypy/translator/avm2/instructions.py Wed Nov 4 00:49:11 2009 @@ -230,6 +230,11 @@ class _Avm2CallMNVoid(_Avm2CallMN): is_void = True +class _Avm2ApplyType(_Avm2U30Instruction): + @needs_specialized + def _set_assembler_props(self, asm): + asm.stack_depth += 1 - self.argument + class _Avm2NewArray(_Avm2U30Instruction): @needs_specialized def _set_assembler_props(self, asm): @@ -383,6 +388,7 @@ #{ Instructions that push/pop values to the stack (depends on arg) and take one U30 argument. newarray = _Avm2NewArray(0x56, 'newarray') newobject = _Avm2NewObject(0x55, 'newobject') +applytype = _Avm2ApplyType(0x53, 'applytype') #} #{ Instructions that take one U8 argument. Modified: pypy/branch/avm/pypy/translator/avm2/query.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/query.py (original) +++ pypy/branch/avm/pypy/translator/avm2/query.py Wed Nov 4 00:49:11 2009 @@ -171,8 +171,8 @@ def group_methods(self, methods, overload, meth, Meth): from pypy.translator.avm2.runtime import OverloadingResolver groups = {} - for name, args, result in methods: - groups.setdefault(name, []).append((args, result)) + for name, args, result, AS3 in methods: + groups[name] = args, result, AS3 res = {} attrs = dict(resolver=OverloadingResolver) Modified: pypy/branch/avm/pypy/translator/avm2/runtime.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/runtime.py (original) +++ pypy/branch/avm/pypy/translator/avm2/runtime.py Wed Nov 4 00:49:11 2009 @@ -6,7 +6,7 @@ # from pypy.annotation.unaryop import immutablevalue # from pypy.annotation.binaryop import _make_none_union from pypy.annotation import model as annmodel -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong +# from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from pypy.rpython.error import TyperError from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.rmodel import Repr @@ -14,7 +14,7 @@ from pypy.rpython.ootypesystem.rootype import OOInstanceRepr from pypy.rpython.ootypesystem import ootype from pypy.rpython.ootypesystem.ootype import Meth, StaticMethod -from pypy.translator.avm2 import constants +# from pypy.translator.avm2 import constants ## Annotation model @@ -234,23 +234,6 @@ #assert ARGS == self._TYPE.ARGS return self - -class _overloaded_static_meth(object): - def __init__(self, *overloadings, **attrs): - resolver = attrs.pop('resolver', OverloadingResolver) - assert not attrs - self._resolver = resolver(overloadings) - - def _set_attrs(self, cls, name): - for meth in self._resolver.overloadings: - meth._set_attrs(cls, name) - - def _get_desc(self, ARGS): - meth = self._resolver.resolve(ARGS) - assert isinstance(meth, _static_meth) - return meth._get_desc(ARGS) - - class NativeInstance(ootype.Instance): def __init__(self, namespace, name, superclass, fields={}, methods={}, _is_root=False, _hints = {}): @@ -529,7 +512,7 @@ def specialize_call(self, hop): c_type, v_length = hop.inputargs(*hop.args_r) hop.exception_cannot_occur() - return hop.genop('newvector', [c_type, v_length], hop.r_result.lowleveltype) + return hop.genop('avm2_newvector', [c_type, v_length], hop.r_result.lowleveltype) class Entry(ExtRegistryEntry): @@ -552,10 +535,7 @@ c_type, v_elems = vlist[0], vlist[1:] c_length = hop.inputconst(ootype.Signed, len(v_elems)) hop.exception_cannot_occur() - v_array = hop.genop('newvector', [c_type, c_length], hop.r_result.lowleveltype) - #for i, v_elem in enumerate(v_elems): - # c_index = hop.inputconst(ootype.Signed, i) - # hop.genop('cli_setelem', [v_array, c_index, v_elem], ootype.Void) + v_array = hop.genop('avm2_initvector', [c_type, v_elems], hop.r_result.lowleveltype) return v_array # def typeof(Class_or_type): Modified: pypy/branch/avm/pypy/translator/avm2/traits.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/traits.py (original) +++ pypy/branch/avm/pypy/translator/avm2/traits.py Wed Nov 4 00:49:11 2009 @@ -127,8 +127,8 @@ self.function = function self._function_index = None - def write_to_file(self, abc): - super(AbcFunctionTrait, self).write_to_file(abc) + def write_to_abc(self, abc): + super(AbcFunctionTrait, self).write_to_abc(abc) self._function_index = abc.methods.index_for(func) @property @@ -144,8 +144,8 @@ self.method = method self._method_index = None - def write_to_file(self, abc): - super(AbcMethodTrait, self).write_to_file(abc) + def write_to_abc(self, abc): + super(AbcMethodTrait, self).write_to_abc(abc) self._method_index = abc.methods.index_for(self.method) @property Modified: pypy/branch/avm/pypy/translator/avm2/types.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/types.py (original) +++ pypy/branch/avm/pypy/translator/avm2/types.py Wed Nov 4 00:49:11 2009 @@ -12,7 +12,11 @@ from pypy.tool.ansi_print import ansi_log -class NativeType(object): +_vec_qname = constants.QName("Vector", constants.Namespace(constants.PACKAGE_NAMESPACE, "__AS3__.vec")) +_str_qname = constants.QName("String") +_arr_qname = constants.QName("Array") + +class Avm2Type(object): def typename(self): raise NotImplementedError @@ -29,125 +33,85 @@ return self.typename() != other.typename() -class NativePrimitiveType(NativeType): +class Avm2PrimitiveType(Avm2Type): def __init__(self, name): self.name = name def typename(self): return self.name - -# class NativeReferenceType(NativeType): -# prefix = 'class ' + def multiname(self): + return constants.QName(self.typename()) -# def typename(self): -# return self.prefix + self.classname() -# def classname(self): -# raise NotImplementedError - -class NativeClassType(NativeReferenceType): - def __init__(self, assembly, name): - self.name = name +class Avm2NamespacedType(Avm2Type): + nstype = constants.TYPE_NAMESPACE_Namespace + + def __init__(self, name, namespace=''): + if '::' in name and namespace == '': + self.ns, self.name = name.rsplit('::', 1) + else: + self.name = name + self.ns = namespace + + def typename(self): + return "%s::%s" % (self.ns, self.name) + + def classname(self): + raise NotImplementedError - def avm2constant(self): - constants. + def mutliname(self): + return constants.QName(self.name, constants.Namespace(self.nstype, self.ns)) -class CliValueType(NativeClassType): - prefix = 'valuetype ' +class Avm2PackagedType(Avm2NamespacedType): + nstype = constants.TYPE_NAMESPACE_PackageNamespace -class NativeGenericType(NativeReferenceType): - def __init__(self, assembly, name, numparam): - self.assembly = assembly - self.name = name - self.numparam = numparam - - def classname(self): - paramtypes = [self.paramtype(i) for i in range(self.numparam)] - thistype = self.specialize(*paramtypes) - return thistype.classname() - - def specialize(self, *types): - assert len(types) == self.numparam - return NativeSpecializedType(self, types) - - def paramtype(self, num): - assert 0 <= num < self.numparam - return NativePrimitiveType('!%d' % num) - -class NativeSpecializedType(NativeReferenceType): - def __init__(self, generic_type, arg_types): - self.generic_type = generic_type - self.arg_types = arg_types - - def avm2constant(self): - name = self.generic_type.name - numparam = self.generic_type.numparam - arglist = ', '.join([arg.typename() for arg in self.arg_types]) - -class CliArrayType(CliType): +class Avm2ArrayType(Avm2Type): def __init__(self, itemtype): self.itemtype = itemtype - def typename(self): - return '%s[]' % self.itemtype.typename() - + def multiname(self): + return constants.TypeName(_vec_qname, itemtype) -T = CliPrimitiveType +T = Avm2PrimitiveType +N = Avm2PackagedType class types: - void = T('void') - int32 = T('int32') - uint32 = T('unsigned int32') - int64 = T('int64') - uint64 = T('unsigned int64') - bool = T('bool') - float64 = T('float64') - char = T('char') - string = T('string') - - weakref = CliClassType('pypylib', 'pypy.runtime.WeakReference') - type = CliClassType('mscorlib', 'System.Type') - object = CliClassType('mscorlib', 'System.Object') - list = CliGenericType('pypylib', 'pypy.runtime.List', 1) - list_of_void = CliClassType('pypylib', 'pypy.runtime.ListOfVoid') - dict = CliGenericType('pypylib', 'pypy.runtime.Dict', 2) - dict_void_void = CliClassType('pypylib', 'pypy.runtime.DictVoidVoid') - dict_items_iterator = CliGenericType('pypylib', 'pypy.runtime.DictItemsIterator', 2) - string_builder = CliClassType('pypylib', 'pypy.runtime.StringBuilder') + void = T('void') + int = T('int') + uint = T('uint') + bool = T('Boolean') + float = T('Number') + string = T('String') + + # weakref = CliClassType('pypylib', 'pypy.runtime.WeakReference') + type = T('Class') + object = T('Object') + list = N('List', 'pypy.lib') + dict = N('Dict', 'pypy.lib') + sb = N('StringBuilder', 'pypy.lib') del T -WEAKREF = types.weakref.classname() -PYPY_DICT_OF_VOID = '[pypylib]pypy.runtime.DictOfVoid`2<%s, int32>' - - _lltype_to_cts = { ootype.Void: types.void, - ootype.Signed: types.int32, - ootype.Unsigned: types.uint32, - ootype.SignedLongLong: types.int64, - ootype.UnsignedLongLong: types.uint64, + ootype.Signed: types.int, + ootype.Unsigned: types.uint, + ootype.SignedLongLong: types.int, + ootype.UnsignedLongLong: types.uint, ootype.Bool: types.bool, - ootype.Float: types.float64, - ootype.Char: types.char, - ootype.UniChar: types.char, + ootype.Float: types.float, + ootype.Char: types.string, + ootype.UniChar: types.string, ootype.Class: types.type, ootype.String: types.string, - ootype.StringBuilder: types.string_builder, + ootype.StringBuilder: types.sb, ootype.Unicode: types.string, - ootype.UnicodeBuilder: types.string_builder, - ootype.WeakReference: types.weakref, + ootype.UnicodeBuilder: types.sb, # maps generic types to their ordinal ootype.List.SELFTYPE_T: types.list, - ootype.List.ITEMTYPE_T: types.list.paramtype(0), ootype.Dict.SELFTYPE_T: types.dict, - ootype.Dict.KEYTYPE_T: types.dict.paramtype(0), - ootype.Dict.VALUETYPE_T: types.dict.paramtype(1), - ootype.DictItemsIterator.SELFTYPE_T: types.dict_items_iterator, - ootype.DictItemsIterator.KEYTYPE_T: types.dict_items_iterator.paramtype(0), - ootype.DictItemsIterator.VALUETYPE_T: types.dict_items_iterator.paramtype(1), - } +} def _get_from_dict(d, key, error): @@ -160,7 +124,7 @@ else: assert False, error -class CTS(object): +class Avm2TypeSystem(object): def __init__(self, db): self.db = db @@ -171,41 +135,24 @@ elif isinstance(t, lltype.Ptr) and isinstance(t.TO, lltype.OpaqueType): return types.object elif isinstance(t, ootype.Instance): - if getattr(t, '_is_value_type', False): - cls = CliValueType - else: - cls = CliClassType NATIVE_INSTANCE = t._hints.get('NATIVE_INSTANCE', None) if NATIVE_INSTANCE: - return cls(None, NATIVE_INSTANCE._name) + return Avm2NamespacedClass(NATIVE_INSTANCE._name) else: name = self.db.pending_class(t) - return cls(None, name) + return Avm2NamespacedClass(name) elif isinstance(t, ootype.Record): name = self.db.pending_record(t) - return CliClassType(None, name) + return Avm2NamespacedClass(name) elif isinstance(t, ootype.StaticMethod): delegate = self.db.record_delegate(t) - return CliClassType(None, delegate) - elif isinstance(t, ootype.Array): - item_type = self.lltype_to_cts(t.ITEM) - if item_type == types.void: # special case: Array of Void - return types.list_of_void - return CliArrayType(item_type) - elif isinstance(t, ootype.List): + return Avm2NamespacedClass(delegate) + elif isinstance(t, (ootype.Array, ootype.List)): item_type = self.lltype_to_cts(t.ITEM) - if item_type == types.void: # special case: List of Void - return types.list_of_void return types.list.specialize(item_type) elif isinstance(t, ootype.Dict): key_type = self.lltype_to_cts(t._KEYTYPE) value_type = self.lltype_to_cts(t._VALUETYPE) - if value_type == types.void: # special cases: Dict with voids - if key_type == types.void: - return types.dict_void_void - else: - # XXX - return CliClassType(None, PYPY_DICT_OF_VOID % key_type) return types.dict.specialize(key_type, value_type) elif isinstance(t, ootype.DictItemsIterator): key_type = self.lltype_to_cts(t._KEYTYPE) @@ -224,89 +171,89 @@ def llconst_to_cts(self, const): return self.lltype_to_cts(const.concretetype), const.value - def ctor_name(self, t): - return 'instance void %s::.ctor()' % self.lltype_to_cts(t) + # def ctor_name(self, t): + # return 'instance void %s::.ctor()' % self.lltype_to_cts(t) - def graph_to_signature(self, graph, is_method = False, func_name = None): - ret_type, ret_var = self.llvar_to_cts(graph.getreturnvar()) - func_name = func_name or graph.name - func_name = self.escape_name(func_name) - namespace = getattr(graph.func, '_namespace_', None) - if namespace: - func_name = '%s::%s' % (namespace, func_name) - - args = [arg for arg in graph.getargs() if arg.concretetype is not ootype.Void] - if is_method: - args = args[1:] - - arg_types = [self.lltype_to_cts(arg.concretetype).typename() for arg in args] - arg_list = ', '.join(arg_types) - - return '%s %s(%s)' % (ret_type, func_name, arg_list) - - def op_to_signature(self, op, func_name): - ret_type, ret_var = self.llvar_to_cts(op.result) - func_name = self.escape_name(func_name) - - args = [arg for arg in op.args[1:] - if arg.concretetype is not ootype.Void] - - arg_types = [self.lltype_to_cts(arg.concretetype).typename() for arg in args] - arg_list = ', '.join(arg_types) - - return '%s %s(%s)' % (ret_type, func_name, arg_list) - - - def method_signature(self, TYPE, name_or_desc): - # TODO: use callvirt only when strictly necessary - if isinstance(TYPE, ootype.Instance): - if isinstance(name_or_desc, ootype._overloaded_meth_desc): - name = name_or_desc.name - METH = name_or_desc.TYPE - virtual = True - else: - name = name_or_desc - owner, meth = TYPE._lookup(name) - METH = meth._TYPE - virtual = getattr(meth, '_virtual', True) - class_name = self.db.class_name(TYPE) - full_name = 'class %s::%s' % (class_name, self.escape_name(name)) - returntype = self.lltype_to_cts(METH.RESULT) - arg_types = [self.lltype_to_cts(ARG).typename() for ARG in METH.ARGS if ARG is not ootype.Void] - arg_list = ', '.join(arg_types) - return '%s %s(%s)' % (returntype, full_name, arg_list), virtual - - elif isinstance(TYPE, (ootype.BuiltinType, ootype.StaticMethod)): - assert isinstance(name_or_desc, str) - name = name_or_desc - if isinstance(TYPE, ootype.StaticMethod): - METH = TYPE - else: - METH = oopspec.get_method(TYPE, name) - class_name = self.lltype_to_cts(TYPE) - if isinstance(TYPE, ootype.Dict): - KEY = TYPE._KEYTYPE - VALUE = TYPE._VALUETYPE - name = name_or_desc - if KEY is ootype.Void and VALUE is ootype.Void and name == 'll_get_items_iterator': - # ugly, ugly special case - ret_type = types.dict_items_iterator.specialize(types.int32, types.int32) - elif VALUE is ootype.Void and METH.RESULT is ootype.Dict.VALUETYPE_T: - ret_type = types.void - else: - ret_type = self.lltype_to_cts(METH.RESULT) - ret_type = dict_of_void_ll_copy_hack(TYPE, ret_type) - else: - ret_type = self.lltype_to_cts(METH.RESULT) - generic_types = getattr(TYPE, '_generic_types', {}) - arg_types = [self.lltype_to_cts(arg).typename() for arg in METH.ARGS if - arg is not ootype.Void and \ - generic_types.get(arg, arg) is not ootype.Void] - arg_list = ', '.join(arg_types) - return '%s %s::%s(%s)' % (ret_type, class_name, name, arg_list), False + # def graph_to_signature(self, graph, is_method = False, func_name = None): + # ret_type, ret_var = self.llvar_to_cts(graph.getreturnvar()) + # func_name = func_name or graph.name + # func_name = self.escape_name(func_name) + # namespace = getattr(graph.func, '_namespace_', None) + # if namespace: + # func_name = '%s::%s' % (namespace, func_name) + + # args = [arg for arg in graph.getargs() if arg.concretetype is not ootype.Void] + # if is_method: + # args = args[1:] + + # arg_types = [self.lltype_to_cts(arg.concretetype).typename() for arg in args] + # arg_list = ', '.join(arg_types) + + # return '%s %s(%s)' % (ret_type, func_name, arg_list) + + # def op_to_signature(self, op, func_name): + # ret_type, ret_var = self.llvar_to_cts(op.result) + # func_name = self.escape_name(func_name) + + # args = [arg for arg in op.args[1:] + # if arg.concretetype is not ootype.Void] + + # arg_types = [self.lltype_to_cts(arg.concretetype).typename() for arg in args] + # arg_list = ', '.join(arg_types) + + # return '%s %s(%s)' % (ret_type, func_name, arg_list) + + + # def method_signature(self, TYPE, name_or_desc): + # # TODO: use callvirt only when strictly necessary + # if isinstance(TYPE, ootype.Instance): + # if isinstance(name_or_desc, ootype._overloaded_meth_desc): + # name = name_or_desc.name + # METH = name_or_desc.TYPE + # virtual = True + # else: + # name = name_or_desc + # owner, meth = TYPE._lookup(name) + # METH = meth._TYPE + # virtual = getattr(meth, '_virtual', True) + # class_name = self.db.class_name(TYPE) + # full_name = 'class %s::%s' % (class_name, self.escape_name(name)) + # returntype = self.lltype_to_cts(METH.RESULT) + # arg_types = [self.lltype_to_cts(ARG).typename() for ARG in METH.ARGS if ARG is not ootype.Void] + # arg_list = ', '.join(arg_types) + # return '%s %s(%s)' % (returntype, full_name, arg_list), virtual + + # elif isinstance(TYPE, (ootype.BuiltinType, ootype.StaticMethod)): + # assert isinstance(name_or_desc, str) + # name = name_or_desc + # if isinstance(TYPE, ootype.StaticMethod): + # METH = TYPE + # else: + # METH = oopspec.get_method(TYPE, name) + # class_name = self.lltype_to_cts(TYPE) + # if isinstance(TYPE, ootype.Dict): + # KEY = TYPE._KEYTYPE + # VALUE = TYPE._VALUETYPE + # name = name_or_desc + # if KEY is ootype.Void and VALUE is ootype.Void and name == 'll_get_items_iterator': + # # ugly, ugly special case + # ret_type = types.dict_items_iterator.specialize(types.int32, types.int32) + # elif VALUE is ootype.Void and METH.RESULT is ootype.Dict.VALUETYPE_T: + # ret_type = types.void + # else: + # ret_type = self.lltype_to_cts(METH.RESULT) + # ret_type = dict_of_void_ll_copy_hack(TYPE, ret_type) + # else: + # ret_type = self.lltype_to_cts(METH.RESULT) + # generic_types = getattr(TYPE, '_generic_types', {}) + # arg_types = [self.lltype_to_cts(arg).typename() for arg in METH.ARGS if + # arg is not ootype.Void and \ + # generic_types.get(arg, arg) is not ootype.Void] + # arg_list = ', '.join(arg_types) + # return '%s %s::%s(%s)' % (ret_type, class_name, name, arg_list), False - else: - assert False + # else: + # assert False def dict_of_void_ll_copy_hack(TYPE, ret_type): # XXX: ugly hack to make the ll_copy signature correct when From magcius at codespeak.net Wed Nov 4 00:51:45 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 4 Nov 2009 00:51:45 +0100 (CET) Subject: [pypy-svn] r68952 - in pypy/branch/avm/pypy/translator/avm2: . intrinsic intrinsic/src intrinsic/src/adobe intrinsic/src/adobe/utils intrinsic/src/authoring intrinsic/src/flash intrinsic/src/flash/accessibility intrinsic/src/flash/desktop intrinsic/src/flash/display intrinsic/src/flash/errors intrinsic/src/flash/events intrinsic/src/flash/external intrinsic/src/flash/filters intrinsic/src/flash/geom intrinsic/src/flash/media intrinsic/src/flash/net intrinsic/src/flash/printing intrinsic/src/flash/profiler intrinsic/src/flash/sampler intrinsic/src/flash/system intrinsic/src/flash/text intrinsic/src/flash/text/engine intrinsic/src/flash/trace intrinsic/src/flash/ui intrinsic/src/flash/utils intrinsic/src/flash/xml test Message-ID: <20091103235145.6DA10168446@codespeak.net> Author: magcius Date: Wed Nov 4 00:51:35 2009 New Revision: 68952 Added: pypy/branch/avm/pypy/translator/avm2/class_.py pypy/branch/avm/pypy/translator/avm2/constant.py pypy/branch/avm/pypy/translator/avm2/function.py pypy/branch/avm/pypy/translator/avm2/genavm.py pypy/branch/avm/pypy/translator/avm2/intrinsic/ pypy/branch/avm/pypy/translator/avm2/intrinsic/autopath.py pypy/branch/avm/pypy/translator/avm2/intrinsic/intrgen.py pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ArgumentError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Array.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Boolean.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Class.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Date.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/DefinitionError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Error.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/EvalError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Function.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Math.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Namespace.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Number.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Object.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/QName.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RangeError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ReferenceError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RegExp.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SecurityError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/String.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SyntaxError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/TypeError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/URIError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/UninitializedError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Vector.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/VerifyError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XML.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XMLList.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/CustomActions.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/ProductManager.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/XMLUI.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/arguments.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/authoring/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/authoring/authObject.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/Accessibility.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityImplementation.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityProperties.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/Clipboard.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardFormats.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardTransferMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/AVM1Movie.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ActionScriptVersion.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Bitmap.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapData.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapDataChannel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BlendMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/CapsStyle.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrection.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrectionSupport.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObject.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObjectContainer.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/FrameLabel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GradientType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Graphics.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsBitmapFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsEndFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsGradientFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPath.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathCommand.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathWinding.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsShaderFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsSolidFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsStroke.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsTrianglePath.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IBitmapDrawable.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsData.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsFill.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsPath.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsStroke.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InteractiveObject.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InterpolationMethod.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/JointStyle.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LineScaleMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Loader.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LoaderInfo.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MorphShape.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MovieClip.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/PixelSnapping.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SWFVersion.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Scene.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shader.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderData.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderInput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderJob.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameterType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderPrecision.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shape.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SimpleButton.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SpreadMethod.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Sprite.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Stage.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageAlign.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageDisplayState.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageQuality.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageScaleMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/TriangleCulling.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/EOFError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IOError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IllegalOperationError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/InvalidSWFError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/MemoryError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/ScriptTimeoutError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/StackOverflowError.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ActivityEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/AsyncErrorEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ContextMenuEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/DataEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ErrorEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/Event.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventDispatcher.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventPhase.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FocusEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FullScreenEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/HTTPStatusEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IEventDispatcher.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IMEEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IOErrorEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/KeyboardEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/MouseEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetFilterEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetStatusEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ProgressEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SampleDataEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SecurityErrorEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ShaderEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/StatusEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SyncEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TextEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TimerEvent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakFunctionClosure.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakMethodClosure.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/external/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/external/ExternalInterface.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BevelFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterQuality.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BlurFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ColorMatrixFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ConvolutionFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilterMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DropShadowFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GlowFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientBevelFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientGlowFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ShaderFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/ColorTransform.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix3D.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Orientation3D.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/PerspectiveProjection.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Point.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Rectangle.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Transform.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Utils3D.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Vector3D.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Camera.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/ID3Info.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Microphone.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Sound.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundChannel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundCodec.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundLoaderContext.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundMixer.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundTransform.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Video.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/DynamicPropertyOutput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileFilter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReference.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReferenceList.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyOutput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyWriter.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/LocalConnection.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetConnection.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStream.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamInfo.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayOptions.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayTransitions.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/ObjectEncoding.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Responder.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObject.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObjectFlushStatus.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Socket.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoader.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoaderDataFormat.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequest.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestHeader.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestMethod.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLStream.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLVariables.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/XMLSocket.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJob.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOptions.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOrientation.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/profiler/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/profiler/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/DeleteObjectSample.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/NewObjectSample.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/Sample.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/StackFrame.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/ApplicationDomain.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Capabilities.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/FSCommand.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IME.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IMEConversionMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/JPEGLoaderContext.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/LoaderContext.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Security.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityDomain.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityPanel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/System.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/AntiAliasType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/CSMSettings.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/Font.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontStyle.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/GridFitType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StaticText.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StyleSheet.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextColorType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextDisplayMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextExtent.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextField.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldAutoSize.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormat.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatAlign.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatDisplay.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextLineMetrics.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRenderer.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRun.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextSnapshot.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/BreakOpportunity.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/CFFHinting.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ContentElement.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitCase.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitWidth.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/EastAsianJustifier.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ElementFormat.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontDescription.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontLookup.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontMetrics.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontPosture.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontWeight.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GraphicElement.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GroupElement.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/JustificationStyle.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/Kerning.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LigatureLevel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LineJustification.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/RenderingMode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/SpaceJustifier.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabAlignment.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabStop.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBaseline.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBlock.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextElement.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextJustifier.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLine.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineCreationResult.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineMirrorRegion.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineValidity.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextRotation.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TypographicCase.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/trace/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/trace/Trace.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenu.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuBuiltInItems.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuClipboardItems.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuItem.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/KeyLocation.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Keyboard.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Mouse.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/MouseCursor.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ByteArray.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Dictionary.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Endian.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataInput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataOutput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IExternalizable.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectInput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectOutput.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Proxy.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/SetIntervalTimer.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Timer.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/package.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLDocument.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNode.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNodeType.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLParser.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLTag.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/int.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/toplevel.as pypy/branch/avm/pypy/translator/avm2/intrinsic/src/uint.as pypy/branch/avm/pypy/translator/avm2/metavm.py pypy/branch/avm/pypy/translator/avm2/opcodes.py pypy/branch/avm/pypy/translator/avm2/test/ pypy/branch/avm/pypy/translator/avm2/test/__init__.py pypy/branch/avm/pypy/translator/avm2/test/autopath.py pypy/branch/avm/pypy/translator/avm2/test/bootstrap.py pypy/branch/avm/pypy/translator/avm2/test/browsertest.py pypy/branch/avm/pypy/translator/avm2/test/harness.py pypy/branch/avm/pypy/translator/avm2/test/mylib.py pypy/branch/avm/pypy/translator/avm2/test/runtest.py pypy/branch/avm/pypy/translator/avm2/test/runtest2.py pypy/branch/avm/pypy/translator/avm2/test/simpletest.py pypy/branch/avm/pypy/translator/avm2/test/test_backendopt.py pypy/branch/avm/pypy/translator/avm2/test/test_bool.py pypy/branch/avm/pypy/translator/avm2/test/test_builtin.py pypy/branch/avm/pypy/translator/avm2/test/test_carbonpython.py pypy/branch/avm/pypy/translator/avm2/test/test_cast.py pypy/branch/avm/pypy/translator/avm2/test/test_class.py pypy/branch/avm/pypy/translator/avm2/test/test_constant.py pypy/branch/avm/pypy/translator/avm2/test/test_cts.py pypy/branch/avm/pypy/translator/avm2/test/test_dict.py pypy/branch/avm/pypy/translator/avm2/test/test_dotnet.py pypy/branch/avm/pypy/translator/avm2/test/test_exception.py pypy/branch/avm/pypy/translator/avm2/test/test_float.py pypy/branch/avm/pypy/translator/avm2/test/test_harness.py pypy/branch/avm/pypy/translator/avm2/test/test_int.py pypy/branch/avm/pypy/translator/avm2/test/test_list.py pypy/branch/avm/pypy/translator/avm2/test/test_objectmodel.py pypy/branch/avm/pypy/translator/avm2/test/test_oo.py pypy/branch/avm/pypy/translator/avm2/test/test_op.py pypy/branch/avm/pypy/translator/avm2/test/test_overflow.py pypy/branch/avm/pypy/translator/avm2/test/test_pbc.py pypy/branch/avm/pypy/translator/avm2/test/test_primitive.py pypy/branch/avm/pypy/translator/avm2/test/test_query.py pypy/branch/avm/pypy/translator/avm2/test/test_range.py pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py pypy/branch/avm/pypy/translator/avm2/test/test_snippet.py pypy/branch/avm/pypy/translator/avm2/test/test_streamio.py pypy/branch/avm/pypy/translator/avm2/test/test_string.py pypy/branch/avm/pypy/translator/avm2/test/test_tuple.py pypy/branch/avm/pypy/translator/avm2/test/test_unicode.py pypy/branch/avm/pypy/translator/avm2/test/test_weakref.py Log: And new files Added: pypy/branch/avm/pypy/translator/avm2/class_.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/class_.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,149 @@ +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.node import Node +from pypy.translator.cli.cts import CTS +from pypy.translator.oosupport.constant import push_constant +from pypy.translator.cli.ilgenerator import CLIBaseGenerator + +try: + set +except NameError: + from sets import Set as set + +class Class(Node): + def __init__(self, db, INSTANCE, namespace, name): + self.db = db + self.cts = db.genoo.TypeSystem(db) + self.INSTANCE = INSTANCE + self.namespace = namespace + self.name = name + + def dependencies(self): + if not self.is_root(self.INSTANCE): + self.db.pending_class(self.INSTANCE._superclass) + + def __hash__(self): + return hash(self.INSTANCE) + + def __eq__(self, other): + return self.INSTANCE == other.INSTANCE + + def __ne__(self, other): + return not self == other + + def is_root(INSTANCE): + return INSTANCE._superclass is None + is_root = staticmethod(is_root) + + def get_name(self): + return self.name + + def __repr__(self): + return '' % self.name + + def get_base_class(self): + base_class = self.INSTANCE._superclass + if self.is_root(base_class): + return 'Object' + else: + return self.db.class_name(base_class) + + def is_abstract(self): + return False # XXX + + # if INSTANCE has an abstract method, the class is abstract + method_names = set() + for m_name, m_meth in self.INSTANCE._methods.iteritems(): + if not hasattr(m_meth, 'graph'): + return True + method_names.add(m_name) + + # if superclasses have abstract methods not overriden by + # INSTANCE, the class is abstract + abstract_method_names = set() + cls = self.INSTANCE._superclass + while cls is not None: + abstract_method_names.update(cls._methods.keys()) + cls = cls._superclass + not_overriden = abstract_method_names.difference(method_names) + if not_overriden: + return True + + return False + + def render(self, ilasm): + if self.is_root(self.INSTANCE): + return + + self.ilasm = ilasm + self.gen = CLIBaseGenerator(self.db, ilasm) + + if self.namespace: + ilasm.begin_namespace(self.namespace) + + ilasm.begin_class(self.name, self.get_base_class(), abstract=self.is_abstract()) + for f_name, (f_type, f_default) in self.INSTANCE._fields.iteritems(): + cts_type = self.cts.lltype_to_cts(f_type) + f_name = self.cts.escape_name(f_name) + if cts_type != CTS.types.void: + ilasm.field(f_name, cts_type) + + self._ctor() + self._toString() + + for m_name, m_meth in self.INSTANCE._methods.iteritems(): + if hasattr(m_meth, 'graph'): + # if the first argument's type is not a supertype of + # this class it means that this method this method is + # not really used by the class: don't render it, else + # there would be a type mismatch. + args = m_meth.graph.getargs() + SELF = args[0].concretetype + if not ootype.isSubclass(self.INSTANCE, SELF): + continue + f = self.db.genoo.Function(self.db, m_meth.graph, m_name, is_method = True) + f.render(ilasm) + else: + # abstract method + METH = m_meth._TYPE + arglist = [(self.cts.lltype_to_cts(ARG), 'v%d' % i) + for i, ARG in enumerate(METH.ARGS) + if ARG is not ootype.Void] + returntype = self.cts.lltype_to_cts(METH.RESULT) + ilasm.begin_function(m_name, arglist, returntype, False, 'virtual') #, 'abstract') + ilasm.add_comment('abstract method') + if isinstance(METH.RESULT, ootype.OOType): + ilasm.opcode('ldnull') + else: + push_constant(self.db, METH.RESULT, 0, self.gen) + ilasm.opcode('ret') + ilasm.end_function() + + def _ctor(self): + self.ilasm.begin_function('.ctor', [], 'void', False, 'specialname', 'rtspecialname', 'instance') + self.ilasm.opcode('ldarg.0') + self.ilasm.call('instance void %s::.ctor()' % self.get_base_class()) + # set default values for fields + default_values = self.INSTANCE._fields.copy() + default_values.update(self.INSTANCE._overridden_defaults) + for f_name, (F_TYPE, f_default) in default_values.iteritems(): + if getattr(F_TYPE, '_is_value_type', False): + continue # we can't set it to null + INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) + cts_type = self.cts.lltype_to_cts(F_TYPE) + f_name = self.cts.escape_name(f_name) + if cts_type != CTS.types.void: + self.ilasm.opcode('ldarg.0') + push_constant(self.db, F_TYPE, f_default, self.gen) + class_name = self.db.class_name(INSTANCE_DEF) + self.ilasm.set_field((cts_type, class_name, f_name)) + + self.ilasm.opcode('ret') + self.ilasm.end_function() + + def _toString(self): + self.ilasm.begin_function('ToString', [], 'string', False, 'virtual', 'instance', 'default') + self.ilasm.opcode('ldarg.0') + self.ilasm.call('string class [pypylib]pypy.test.Result::InstanceToPython(object)') + self.ilasm.ret() + self.ilasm.end_function() + Added: pypy/branch/avm/pypy/translator/avm2/constant.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/constant.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,465 @@ +""" +___________________________________________________________________________ +CLI Constants + +This module extends the oosupport/constant.py to be specific to the +CLI. Most of the code in this file is in the constant generators, which +determine how constants are stored and loaded (static fields, lazy +initialization, etc), but some constant classes have been overloaded or +extended to allow for special handling. + +The CLI implementation is broken into three sections: + +* Constant Generators: different generators implementing different + techniques for loading constants (Static fields, singleton fields, etc) + +* Mixins: mixins are used to add a few CLI-specific methods to each + constant class. Basically, any time I wanted to extend a base class + (such as AbstractConst or DictConst), I created a mixin, and then + mixed it in to each sub-class of that base-class. + +* Subclasses: here are the CLI specific classes. Eventually, these + probably wouldn't need to exist at all (the JVM doesn't have any, + for example), or could simply have empty bodies and exist to + combine a mixin and the generic base class. For now, though, they + contain the create_pointer() and initialize_data() routines. +""" + +from pypy.translator.oosupport.constant import \ + push_constant, WeakRefConst, StaticMethodConst, CustomDictConst, \ + ListConst, ClassConst, InstanceConst, RecordConst, DictConst, \ + BaseConstantGenerator, AbstractConst, ArrayConst +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.comparer import EqualityComparer +from pypy.translator.avm2 import constants, types +from pypy.rpython.lltypesystem import lltype + +CONST_CLASS = types.Avm2PackagedType('pypy.runtime::CONSTANTS') + +DEBUG_CONST_INIT = False +DEBUG_CONST_INIT_VERBOSE = False +SERIALIZE = False + +# ______________________________________________________________________ +# Constant Generators +# +# Different generators implementing different techniques for loading +# constants (Static fields, singleton fields, etc) + +class Avm2BaseConstGenerator(BaseConstantGenerator): + """ + Base of all CLI constant generators. It implements the oosupport + constant generator in terms of the CLI interface. + """ + + def __init__(self, db): + BaseConstantGenerator.__init__(self, db) + self.cts = db.genoo.TypeSystem(db) + + def _begin_gen_constants(self, gen, all_constants): + pass + + def _end_gen_constants(self, gen, numsteps): + assert gen.ilasm is self.ilasm + self.end_class() + + def begin_class(self): + self.ilasm.begin_namespace(CONST_NAMESPACE) + self.ilasm.begin_class(CONST_CLASSNAME, beforefieldinit=True) + + def end_class(self): + self.ilasm.end_class() + self.ilasm.end_namespace() + + def _declare_const(self, gen, const): + self.ilasm.field(const.name, const.get_type(), static=True) + + def downcast_constant(self, gen, const, EXPECTED_TYPE): + type = self.cts.lltype_to_cts(EXPECTED_TYPE) + gen.ilasm.opcode('castclass', type) + + def _get_key_for_const(self, value): + if isinstance(value, ootype._view) and isinstance(value._inst, ootype._record): + return value._inst + return BaseConstantGenerator._get_key_for_const(self, value) + + def _create_complex_const(self, value): + from pypy.translator.cli.dotnet import _fieldinfo + + if isinstance(value, _fieldinfo): + uniq = self.db.unique() + return CLIFieldInfoConst(self.db, value.llvalue, uniq) + elif isinstance(value, ootype._view) and isinstance(value._inst, ootype._record): + self.db.cts.lltype_to_cts(value._inst._TYPE) # record the type of the record + return self.record_const(value._inst) + else: + return BaseConstantGenerator._create_complex_const(self, value) + +class FieldConstGenerator(CLIBaseConstGenerator): + pass + +class StaticFieldConstGenerator(FieldConstGenerator): + + # _________________________________________________________________ + # OOSupport interface + + def push_constant(self, gen, const): + type_ = const.get_type() + gen.ilasm.load_static_constant(type_, CONST_NAMESPACE, CONST_CLASSNAME, const.name) + + def _push_constant_during_init(self, gen, const): + full_name = '%s::%s' % (CONST_CLASS, const.name) + gen.ilasm.opcode('ldsfld %s %s' % (const.get_type(), full_name)) + + def _store_constant(self, gen, const): + type_ = const.get_type() + gen.ilasm.store_static_constant(type_, CONST_NAMESPACE, CONST_CLASSNAME, const.name) + + # _________________________________________________________________ + # CLI interface + + def _declare_step(self, gen, stepnum): + gen.ilasm.begin_function( + 'step%d' % stepnum, [], 'void', False, 'static') + + def _close_step(self, gen, stepnum): + gen.ilasm.ret() + gen.ilasm.end_function() + + def _end_gen_constants(self, gen, numsteps): + + self.ilasm.begin_function('.cctor', [], 'void', False, 'static', + 'specialname', 'rtspecialname', 'default') + self.ilasm.stderr('CONST: initialization starts', DEBUG_CONST_INIT) + for i in range(numsteps): + self.ilasm.stderr('CONST: step %d of %d' % (i, numsteps), + DEBUG_CONST_INIT) + step_name = 'step%d' % i + self.ilasm.call('void %s::%s()' % (CONST_CLASS, step_name)) + self.ilasm.stderr('CONST: initialization completed', DEBUG_CONST_INIT) + self.ilasm.ret() + self.ilasm.end_function() + + super(StaticFieldConstGenerator, self)._end_gen_constants( + gen, numsteps) + +class InstanceFieldConstGenerator(FieldConstGenerator): + + # _________________________________________________________________ + # OOSupport interface + + def push_constant(self, gen, const): + # load the singleton instance + gen.ilasm.opcode('ldsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) + gen.ilasm.opcode('ldfld %s %s::%s' % (const.get_type(), CONST_CLASS, const.name)) + + def _push_constant_during_init(self, gen, const): + # during initialization, we load the 'this' pointer from our + # argument rather than the singleton argument + gen.ilasm.opcode('ldarg.0') + gen.ilasm.opcode('ldfld %s %s::%s' % (const.get_type(), CONST_CLASS, const.name)) + + def _pre_store_constant(self, gen, const): + gen.ilasm.opcode('ldarg.0') + + def _store_constant(self, gen, const): + gen.ilasm.set_field((const.get_type(), CONST_CLASS, const.name)) + + # _________________________________________________________________ + # CLI interface + + def _declare_const(self, gen, all_constants): + gen.ilasm.field(const.name, const.get_type(), static=False) + + def _declare_step(self, gen, stepnum): + gen.ilasm.begin_function('step%d' % stepnum, [], 'void', False) + + def _close_step(self, gen, stepnum): + gen.ilasm.ret() + gen.ilasm.end_function() + + def _end_gen_constants(self, gen, numsteps): + + ilasm = gen.ilasm + + ilasm.begin_function('.ctor', [], 'void', False, 'specialname', 'rtspecialname', 'instance') + ilasm.opcode('ldarg.0') + ilasm.call('instance void object::.ctor()') + + ilasm.opcode('ldarg.0') + ilasm.opcode('stsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) + + for i in range(numsteps): + step_name = 'step%d' % i + ilasm.opcode('ldarg.0') + ilasm.call('instance void %s::%s()' % (CONST_CLASS, step_name)) + ilasm.ret() + ilasm.end_function() + + # declare&init the Singleton containing the constants + ilasm.field('Singleton', 'class %s' % CONST_CLASS, static=True) + ilasm.begin_function('.cctor', [], 'void', False, 'static', 'specialname', 'rtspecialname', 'default') + if SERIALIZE: + self._serialize_ctor() + else: + self._plain_ctor() + ilasm.end_function() + + super(StaticFieldConstGenerator, self)._end_gen_constants(gen, numsteps) + + def _plain_ctor(self): + self.ilasm.new('instance void class %s::.ctor()' % CONST_CLASS) + self.ilasm.pop() + self.ilasm.ret() + + def _serialize_ctor(self): + self.ilasm.opcode('ldstr "constants.dat"') + self.ilasm.call('object [pypylib]pypy.runtime.Utils::Deserialize(string)') + self.ilasm.opcode('dup') + self.ilasm.opcode('brfalse initialize') + self.ilasm.stderr('Constants deserialized successfully') + self.ilasm.opcode('stsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) + self.ilasm.ret() + self.ilasm.label('initialize') + self.ilasm.pop() + self.ilasm.stderr('Cannot deserialize constants... initialize them!') + self.ilasm.new('instance void class %s::.ctor()' % CONST_CLASS) + self.ilasm.opcode('ldstr "constants.dat"') + self.ilasm.call('void [pypylib]pypy.runtime.Utils::Serialize(object, string)') + self.ilasm.ret() + +class LazyConstGenerator(StaticFieldConstGenerator): + def push_constant(self, ilasm, const): + getter_name = '%s::%s' % (CONST_CLASS, 'get_%s' % const.name) + ilasm.call('%s %s()' % (const.get_type(), getter_name)) + + def _create_pointers(self, gen, all_constants): + # overload to do nothing since we handle everything in lazy fashion + pass + + def _initialize_data(self, gen, all_constants): + # overload to do nothing since we handle everything in lazy fashion + pass + + def _declare_const(self, gen, const): + # Declare the field + super(LazyConstGenerator, self)._declare_const(gen, const) + + # Create the method for accessing the field + getter_name = 'get_%s' % const.name + type_ = const.get_type() + self.ilasm.begin_function(getter_name, [], type_, False, 'static') + self.ilasm.load_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) + # if it's already initialized, just return it + self.ilasm.opcode('dup') + self.ilasm.opcode('brfalse', 'initialize') + self.ilasm.opcode('ret') + # else, initialize! + self.ilasm.label('initialize') + self.ilasm.opcode('pop') # discard the null value we know is on the stack + const.instantiate(ilasm) + self.ilasm.opcode('dup') # two dups because const.init pops the value at the end + self.ilasm.opcode('dup') + self.ilasm.store_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) + const.init(ilasm) + self.ilasm.opcode('ret') + self.ilasm.end_function() + +# ______________________________________________________________________ +# Mixins +# +# Mixins are used to add a few CLI-specific methods to each constant +# class. Basically, any time I wanted to extend a base class (such as +# AbstractConst or DictConst), I created a mixin, and then mixed it in +# to each sub-class of that base-class. Kind of awkward. + +class CLIBaseConstMixin(object): + """ A mix-in with a few extra methods the CLI backend uses """ + + def get_type(self): + """ Returns the CLI type for this constant's representation """ + return self.cts.lltype_to_cts(self.value._TYPE) + + def push_inline(self, gen, TYPE): + """ Overload the oosupport version so that we use the CLI opcode + for pushing NULL """ + assert self.is_null() + gen.ilasm.opcode('ldnull') + +class CLIDictMixin(CLIBaseConstMixin): + def _check_for_void_dict(self, gen): + KEYTYPE = self.value._TYPE._KEYTYPE + keytype = self.cts.lltype_to_cts(KEYTYPE) + keytype_T = self.cts.lltype_to_cts(self.value._TYPE.KEYTYPE_T) + VALUETYPE = self.value._TYPE._VALUETYPE + valuetype = self.cts.lltype_to_cts(VALUETYPE) + valuetype_T = self.cts.lltype_to_cts(self.value._TYPE.VALUETYPE_T) + if VALUETYPE is ootype.Void: + gen.add_comment(' CLI Dictionary w/ void value') + class_name = PYPY_DICT_OF_VOID % keytype + for key in self.value._dict: + gen.ilasm.opcode('dup') + push_constant(self.db, KEYTYPE, key, gen) + meth = 'void class %s::ll_set(%s)' % (class_name, keytype_T) + gen.ilasm.call_method(meth, False) + return True + return False + + def initialize_data(self, constgen, gen): + # special case: dict of void, ignore the values + if self._check_for_void_dict(gen): + return + return super(CLIDictMixin, self).initialize_data(constgen, gen) + +# ______________________________________________________________________ +# Constant Classes +# +# Here we overload a few methods, and mix in the base classes above. +# Note that the mix-ins go first so that they overload methods where +# required. +# +# Eventually, these probably wouldn't need to exist at all (the JVM +# doesn't have any, for example), or could simply have empty bodies +# and exist to combine a mixin and the generic base class. For now, +# though, they contain the create_pointer() and initialize_data() +# routines. In order to get rid of them, we would need to implement +# the generator interface in the CLI. + +class CLIRecordConst(CLIBaseConstMixin, RecordConst): + def create_pointer(self, gen): + self.db.const_count.inc('Record') + super(CLIRecordConst, self).create_pointer(gen) + +class CLIInstanceConst(CLIBaseConstMixin, InstanceConst): + def create_pointer(self, gen): + self.db.const_count.inc('Instance') + self.db.const_count.inc('Instance', self.OOTYPE()) + super(CLIInstanceConst, self).create_pointer(gen) + + +class CLIClassConst(CLIBaseConstMixin, ClassConst): + def is_inline(self): + return True + + def push_inline(self, gen, EXPECTED_TYPE): + if not self.is_null(): + if hasattr(self.value, '_FUNC'): + FUNC = self.value._FUNC + classname = self.db.record_delegate(FUNC) + else: + INSTANCE = self.value._INSTANCE + classname = self.db.class_name(INSTANCE) + gen.ilasm.opcode('ldtoken', classname) + gen.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + return + super(CLIClassConst, self).push_inline(gen, EXPECTED_TYPE) + +class CLIListConst(CLIBaseConstMixin, ListConst): + + def _do_not_initialize(self): + # Check if it is a list of all zeroes: + try: + if self.value._list == [0] * len(self.value._list): + return True + except: + pass + return super(CLIListConst, self)._do_not_initialize() + + def create_pointer(self, gen): + self.db.const_count.inc('List') + self.db.const_count.inc('List', self.value._TYPE.ITEM) + self.db.const_count.inc('List', len(self.value._list)) + super(CLIListConst, self).create_pointer(gen) + + +class CLIArrayConst(CLIBaseConstMixin, ArrayConst): + + def _do_not_initialize(self): + # Check if it is an array of all zeroes: + try: + if self.value._list == [0] * len(self.value._list): + return True + except: + pass + return super(CLIArrayConst, self)._do_not_initialize() + + def _setitem(self, SELFTYPE, gen): + gen.array_setitem(SELFTYPE) + + +class CLIDictConst(CLIDictMixin, DictConst): + def create_pointer(self, gen): + self.db.const_count.inc('Dict') + self.db.const_count.inc('Dict', self.value._TYPE._KEYTYPE, self.value._TYPE._VALUETYPE) + super(CLIDictConst, self).create_pointer(gen) + +class CLICustomDictConst(CLIDictMixin, CustomDictConst): + def record_dependencies(self): + if not self.value: + return + eq = self.value._dict.key_eq + hash = self.value._dict.key_hash + self.comparer = EqualityComparer(self.db, self.value._TYPE._KEYTYPE, eq, hash) + self.db.pending_node(self.comparer) + super(CLICustomDictConst, self).record_dependencies() + + def create_pointer(self, gen): + assert not self.is_null() + gen.ilasm.new(self.comparer.get_ctor()) + class_name = self.get_type() + gen.ilasm.new('instance void %s::.ctor(class ' + '[mscorlib]System.Collections.Generic.IEqualityComparer`1)' + % class_name) + self.db.const_count.inc('CustomDict') + self.db.const_count.inc('CustomDict', self.value._TYPE._KEYTYPE, self.value._TYPE._VALUETYPE) + +class CLIStaticMethodConst(CLIBaseConstMixin, StaticMethodConst): + def create_pointer(self, gen): + assert not self.is_null() + signature = self.cts.graph_to_signature(self.value.graph) + gen.ilasm.opcode('ldnull') + gen.ilasm.opcode('ldftn', signature) + gen.ilasm.new('instance void class %s::.ctor(object, native int)' % self.delegate_type) + self.db.const_count.inc('StaticMethod') + + def initialize_data(self, constgen, gen): + return + + +class CLIWeakRefConst(CLIBaseConstMixin, WeakRefConst): + def create_pointer(self, gen): + gen.ilasm.new('instance void %s::.ctor()' % self.get_type()) + self.db.const_count.inc('WeakRef') + + def get_type(self, include_class=True): + return 'class ' + WEAKREF + + def initialize_data(self, constgen, gen): + if self.value is not None: + push_constant(self.db, self.value._TYPE, self.value, gen) + gen.ilasm.call_method('void %s::ll_set(object)' % self.get_type(), True) + return True + + +class CLIFieldInfoConst(AbstractConst): + def __init__(self, db, llvalue, count): + AbstractConst.__init__(self, db, llvalue, count) + self.name = 'FieldInfo__%d' % count + + def create_pointer(self, generator): + constgen = generator.db.constant_generator + const = constgen.record_const(self.value) + generator.ilasm.opcode('ldtoken', CONST_CLASS) + generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + generator.ilasm.opcode('ldstr', '"%s"' % const.name) + generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) + + def get_type(self): + from pypy.translator.cli.cts import CliClassType + return CliClassType('mscorlib', 'System.Reflection.FieldInfo') + + def initialize_data(self, constgen, gen): + pass + + def record_dependencies(self): + self.db.constant_generator.record_const(self.value) Added: pypy/branch/avm/pypy/translator/avm2/function.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/function.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,183 @@ +from functools import partial + +from pypy.objspace.flow import model as flowmodel +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem.lltype import Void +from pypy.translator.oosupport.function import Function as OOFunction +from pypy.translator.cli.node import Node +from pypy.translator.avm2 import constants, types + +class Function(OOFunction, Node): + + auto_propagate_exceptions = True + + def __init__(self, *args, **kwargs): + OOFunction.__init__(self, *args, **kwargs) + + if hasattr(self.db.genoo, 'exceptiontransformer'): + self.auto_propagate_exceptions = False + + namespace = getattr(self.graph.func, '_namespace_', None) + if namespace: + if '.' in namespace: + self.namespace, self.classname = namespace.rsplit('.', 1) + else: + self.namespace = None + self.classname = namespace + else: + self.namespace = None + self.classname = None + + def _create_generator(self, ilasm): + ilasm.db = self.db + return ilasm + + def record_ll_meta_exc(self, ll_meta_exc): + # record the type only if it doesn't belong to a native_class + ll_exc = ll_meta_exc._INSTANCE + NATIVE_INSTANCE = ll_exc._hints.get('NATIVE_INSTANCE', None) + if NATIVE_INSTANCE is None: + OOFunction.record_ll_meta_exc(self, ll_meta_exc) + + def begin_render(self): + self._set_args() + self._set_locals() + if self.is_method: + self.generator.enter_method(self.name, args[1:]) + elif self.classname: + self.generator.enter_static_method(self.name, args) + else: + self.generator.enter_function(self.name, args) + + def end_render(self): + if self.generator.scope.islabel: + self.generator.exit_scope() + self.generator.exit_context() + + def render_return_block(self, block): + return_var = block.inputargs[0] + if return_var.concretetype is Void: + self.generator.emit('returnvoid') + else: + self.generator.load(return_var) + self.generator.return_stmt() + + def set_label(self, label): + return self.generator.set_label(label) + + # def _render_op(self, op): + # #instr_list = self.db.genoo.opcodes.get(op.opname, None) + # #instr_list.render(self.generator, op) + # super(Function, self)._render_op(op) + + def _setup_link(self, link): + target = link.target + linkvars = [] + for to_load, to_store in zip(link.args, target.inputargs): + if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: + continue + if to_load.concretetype is ootype.Void: + continue + linkvars.append((to_load, to_store)) + + # after SSI_to_SSA it can happen to have to_load = [a, b] and + # to_store = [b, c]. If we store each variable sequentially, + # 'b' would be overwritten before being read. To solve, we + # first load all the values on the stack, then store in the + # appropriate places. + + if self._trace_enabled(): + self._trace('link', writeline=True) + for to_load, to_store in linkvars: + self._trace_value('%s <-- %s' % (to_store, to_load), to_load) + self._trace('', writeline=True) + + for to_load, to_store in linkvars: + self.generator.load(to_store) + self.generator.load(to_load) + self.generator.set_variable() + + + # def begin_try(self, cond): + # if cond: + # self.ilasm.begin_try() + + # def end_try(self, target_label, cond): + # if cond: + # self.ilasm.leave(target_label) + # self.ilasm.end_try() + # else: + # self.ilasm.branch(target_label) + + # def begin_catch(self, llexitcase): + # ll_meta_exc = llexitcase + # ll_exc = ll_meta_exc._INSTANCE + # cts_exc = self.cts.lltype_to_cts(ll_exc) + # self.ilasm.begin_catch(cts_exc.classname()) + + # def end_catch(self, target_label): + # self.ilasm.leave(target_label) + # self.ilasm.end_catch() + + # def render_raise_block(self, block): + # exc = block.inputargs[1] + # self.load(exc) + # self.ilasm.opcode('throw') + + # def store_exception_and_link(self, link): + # if self._is_raise_block(link.target): + # # the exception value is on the stack, use it as the 2nd target arg + # assert len(link.args) == 2 + # assert len(link.target.inputargs) == 2 + # self.store(link.target.inputargs[1]) + # else: + # # the exception value is on the stack, store it in the proper place + # if isinstance(link.last_exception, flowmodel.Variable): + # self.ilasm.opcode('dup') + # self.store(link.last_exc_value) + # self.ilasm.call_method( + # 'class [mscorlib]System.Type object::GetType()', + # virtual=True) + # self.store(link.last_exception) + # else: + # self.store(link.last_exc_value) + # self._setup_link(link) + + # def render_numeric_switch(self, block): + # if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): + # # TODO: it could be faster to check is the values fit in + # # 32bit, and perform a cast in that case + # self.render_numeric_switch_naive(block) + # return + + # cases, min_case, max_case, default = self._collect_switch_cases(block) + # is_sparse = self._is_sparse_switch(cases, min_case, max_case) + + # naive = (min_case < 0) or is_sparse + # if naive: + # self.render_numeric_switch_naive(block) + # return + + # targets = [] + # for i in xrange(max_case+1): + # link, lbl = cases.get(i, default) + # targets.append(lbl) + # self.generator.load(block.exitswitch) + # self.ilasm.switch(targets) + # self.render_switch_case(*default) + # for link, lbl in cases.itervalues(): + # self.render_switch_case(link, lbl) + + # def call_oostring(self, ARGTYPE): + # if isinstance(ARGTYPE, ootype.Instance): + # argtype = self.cts.types.object + # else: + # argtype = self.cts.lltype_to_cts(ARGTYPE) + # self.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) + + # def call_oounicode(self, ARGTYPE): + # argtype = self.cts.lltype_to_cts(ARGTYPE) + # self.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + + # Those parts of the generator interface that are function + # specific Added: pypy/branch/avm/pypy/translator/avm2/genavm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/genavm.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,38 @@ + +import py +from pypy.translator.oosupport.genoo import GenOO +from pypy.translator.avm2.avm1gen import Avm2ilasm +from pypy.translator.avm2.constant import AVM1ConstGenerator +from pypy.translator.avm2.database import LowLevelDatabase +from pypy.translator.avm2.function import Function +from pypy.translator.avm2.opcodes import opcodes +from pypy.translator.avm2.types import AVM1TypeSystem + +class GenAVM2(GenOO): + + opcodes = opcodes + Function = Function + Database = LowLevelDatabase + TypeSystem = AVM1TypeSystem + + ConstantGenerator = AVM1ConstGenerator + + def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): + GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) + self.const_stat = str(tmpdir.join('const_stat')) + self.ilasm = None + + def create_assembler(self): + return AVM1Gen() + + def generate_source(self): + if self.ilasm is None: + self.ilasm = self.create_assembler() + self.fix_names() + self.gen_entrypoint() + self.gen_pendings() + self.db.gen_constants(self.ilasm) + + # Don't do treebuilding stuff + def stack_optimization(self): + pass Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/autopath.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/autopath.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,135 @@ +""" +self cloning, automatic path configuration + +copy this into any subdirectory of pypy from which scripts need +to be run, typically all of the test subdirs. +The idea is that any such script simply issues + + import autopath + +and this will make sure that the parent directory containing "pypy" +is in sys.path. + +If you modify the master "autopath.py" version (in pypy/tool/autopath.py) +you can directly run it which will copy itself on all autopath.py files +it finds under the pypy root directory. + +This module always provides these attributes: + + pypydir pypy root directory path + this_dir directory where this autopath.py resides + +""" + +def __dirinfo(part): + """ return (partdir, this_dir) and insert parent of partdir + into sys.path. If the parent directories don't have the part + an EnvironmentError is raised.""" + + import sys, os + try: + head = this_dir = os.path.realpath(os.path.dirname(__file__)) + except NameError: + head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + + error = None + while head: + partdir = head + head, tail = os.path.split(head) + if tail == part: + # check if "../py/__init__.py" exists + checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) + break + else: + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) + + pypy_root = os.path.join(head, '') + try: + sys.path.remove(head) + except ValueError: + pass + sys.path.insert(0, head) + + munged = {} + for name, mod in sys.modules.items(): + if '.' in name: + continue + fn = getattr(mod, '__file__', None) + if not isinstance(fn, str): + continue + newname = os.path.splitext(os.path.basename(fn))[0] + if not newname.startswith(part + '.'): + continue + path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') + if path.startswith(pypy_root) and newname != part: + modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) + if newname != '__init__': + modpaths.append(newname) + modpath = '.'.join(modpaths) + if modpath not in sys.modules: + munged[modpath] = mod + + for name, mod in munged.iteritems(): + if name not in sys.modules: + sys.modules[name] = mod + if '.' in name: + prename = name[:name.rfind('.')] + postname = name[len(prename)+1:] + if prename not in sys.modules: + __import__(prename) + if not hasattr(sys.modules[prename], postname): + setattr(sys.modules[prename], postname, mod) + + return partdir, this_dir + +def __clone(): + """ clone master version of autopath.py into all subdirs """ + from os.path import join, walk + if not this_dir.endswith(join('pypy','tool')): + raise EnvironmentError("can only clone master version " + "'%s'" % join(pypydir, 'tool',_myname)) + + + def sync_walker(arg, dirname, fnames): + if _myname in fnames: + fn = join(dirname, _myname) + f = open(fn, 'rwb+') + try: + if f.read() == arg: + print "checkok", fn + else: + print "syncing", fn + f = open(fn, 'w') + f.write(arg) + finally: + f.close() + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) + +_myname = 'autopath.py' + +# set guaranteed attributes + +pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) + +if __name__ == '__main__': + __clone() Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/intrgen.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/intrgen.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,161 @@ + +import autopath +import os +import os.path + +intrinsic_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "src")) + +from pypy.translator.avm2.query import ClassDesc + +def get_ootype(t, resolved={}): + if "=" in t: + return get_ootype(t.split("=")[0].strip(), resolved) + if t == "*": + return "" + elif t == "void": + return "ootype.Void" + elif t == "int": + return "ootype.SignedLongLong" + elif t == "uint": + return "ootype.UnsignedLongLong" + elif t == "Boolean": + return "ootype.Bool" + elif t == "Number": + return "ootype.Float" + elif t == "String": + return "ootype.String" + elif t == "Array": + return "ootype.List" + elif t.startswith("Vector.<"): + return get_ootype(t[len("Vector.<"):-1]) + "[]" + elif t in ("Object", "Dictionary"): + return "ootype.Dict" + return resolved.get(t, t) + +def parse_file(file): + + FullName = "" + ShortName = "" + BaseType = "" + Package = "" + Resolved = {} + Methods = [] + Fields = [] + StaticMethods = [] + StaticFields = [] + + for line in file: + line = line.strip().strip(";") + + if line.startswith("package "): + Package = line[len("package "):].strip("{") + + elif line.startswith("import "): + resolvedname = line[len("import "):] + ns, name = resolvedname.rsplit(".", 1) + Resolved[name] = resolvedname + + elif line.startswith("public class "): + line = line[len("public class "):].split() + ShortName = line[0] + if Package: + FullName = "%s.%s" % (Package, ShortName) + else: + FullName = ShortName + + if line == "extends": + BaseType = get_ootype(line[2], Resolved) + + elif line.startswith(("public function ", "public static function ")): + + prop = Methods + line = line[len("public "):] + + if "static" in line: + line = line[len("static function "):] + prop = StaticMethods + else: + line = line[len("function "):] + + linearr = line.split() + + name = linearr[0] + if name == ShortName: + name = "!CONSTRUCTOR!" + + arglist = line[line.find("(")+1:] + if name == "!CONSTRUCTOR!": + rettype = None + else: + rettype = get_ootype(arglist[arglist.rfind(":")+2:], Resolved) + + args = [arg for arg in arglist[:arglist.find(")")].split(",")] + args = [(get_ootype(arg.strip().split(":")[1], Resolved) if ":" in arg else ("*args" if "..." in arg else arg.strip())) for arg in args] + + if name in ("get", "set"): + + if prop == StaticMethods: + prop = StaticFields + else: + prop = Fields + + pname = linearr[1] + if name == "set": + type = args[0] + elif name == "get": + type = rettype + + if (pname, type) not in prop: + prop.append((pname, type)) + else: + prop.append((name, args, rettype)) + + elif line.startswith("public static "): + line = line.split() + if ":" in line: + StaticFields.append((line[3], line[5])) + else: + StaticFields.append(tuple(line[3].split(":"))) + + desc = ClassDesc() + desc.FullName = FullName + desc.BaseType = BaseType + desc.Package = Package + desc.Resolved = Resolved + desc.Methods = Methods + desc.Fields = Fields + desc.StaticMethods = StaticMethods + desc.StaticFields = StaticFields + + return desc + +def print_desc(desc): + print + print "desc = ClassDesc()" + print "desc.FullName = %r" % desc.FullName + print "desc.BaseType = %r" % desc.BaseType + print_tuples("Methods", desc.Methods) + print_tuples("StaticMethods", desc.StaticMethods) + print_tuples("Fields", desc.Fields) + print_tuples("StaticFields", desc.StaticFields) + print "types['%s'] = desc" % desc.FullName + print "del desc" + +def print_tuples(varname, L): + if len(L): + print "desc.%s = [" % varname + for t in L: + print " %s," % repr(t) + print "]" + else: + print "desc.%s = []" % varname + +print "# This file has been autogenerated by intrgen.py -- DO NOT EDIT" +print +print "from pypy.translator.avm2.query import ClassDesc" +print +print "types = {}" + +for path, dirs, files in os.walk(intrinsic_dir): + for filename in files: + print_desc(parse_file(open(os.path.join(path, filename)))) Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ArgumentError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ArgumentError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// The ArgumentError class represents an error that occurs when the arguments supplied in a function do not match the arguments defined for that function. + public class ArgumentError extends Error + { + public static const length : int; + + /// Creates an ArgumentError object. + public function ArgumentError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Array.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Array.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,79 @@ +package +{ + /// The Array class lets you access and manipulate arrays. + public class Array extends Object + { + /// Specifies case-insensitive sorting for the Array class sorting methods. + public static const CASEINSENSITIVE : uint; + /// Specifies descending sorting for the Array class sorting methods. + public static const DESCENDING : uint; + /// A non-negative integer specifying the number of elements in the array. + public static const length : int; + /// Specifies numeric (instead of character-string) sorting for the Array class sorting methods. + public static const NUMERIC : uint; + /// Specifies that a sort returns an array that consists of array indices. + public static const RETURNINDEXEDARRAY : uint; + /// Specifies the unique sorting requirement for the Array class sorting methods. + public static const UNIQUESORT : uint; + + public function get length () : uint; + public function set length (newLength:uint) : void; + + /// Lets you create an array that contains the specified elements. + public function Array (...rest); + + /// Concatenates the elements specified in the parameters. + public function concat (...rest) : Array; + + /// Executes a test function on each item in the array until an item is reached that returns false for the specified function. + public function every (callback:Function, thisObject:* = null) : Boolean; + + /// Executes a test function on each item in the array and constructs a new array for all items that return true for the specified function. + public function filter (callback:Function, thisObject:* = null) : Array; + + /// Executes a function on each item in the array. + public function forEach (callback:Function, thisObject:* = null) : void; + + /// Searches for an item in an array by using strict equality (===) and returns the index position of the item. + public function indexOf (searchElement:*, fromIndex:* = 0) : int; + + /// Converts the elements in an array to strings. + public function join (sep:* = null) : String; + + /// Searches for an item in an array, working backward from the last item, and returns the index position of the matching item using strict equality (===). + public function lastIndexOf (searchElement:*, fromIndex:* = 2147483647) : int; + + /// Executes a function on each item in an array, and constructs a new array of items corresponding to the results of the function on each item in the original array. + public function map (callback:Function, thisObject:* = null) : Array; + + /// Removes the last element from an array and returns the value of that element. + public function pop () : *; + + /// Adds one or more elements to the end of an array and returns the new length of the array. + public function push (...rest) : uint; + + /// Reverses the array in place. + public function reverse () : Array; + + /// Removes the first element from an array and returns that element. + public function shift () : *; + + /// Returns a new array that consists of a range of elements from the original array. + public function slice (startIndex:* = 0, endIndex:* = 16777215) : Array; + + /// Executes a test function on each item in the array until an item is reached that returns true. + public function some (callback:Function, thisObject:* = null) : Boolean; + + /// Sorts the elements in an array. + public function sort (...rest) : *; + + /// Sorts the elements in an array according to one or more fields in the array. + public function sortOn (names:*, options:* = null) : *; + + /// Adds elements to and removes elements from an array. + public function splice (startIndex:int, deleteCount:uint, ...rest) : *; + + /// Adds one or more elements to the beginning of an array and returns the new length of the array. + public function unshift (...rest) : uint; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Boolean.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Boolean.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package +{ + /// A Boolean object is a data type that can have one of two values, either true or false, used for logical operations. + public class Boolean extends Object + { + public static const length : int; + + /// Creates a Boolean object with the specified value. + public function Boolean (value:* = null); + + /// Returns the string representation ("true" or "false") of the Boolean object. + public function toString () : String; + + /// Returns true if the value of the specified Boolean object is true; false otherwise. + public function valueOf () : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Class.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Class.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + public class Class extends Object + { + public static const length : int; + + public function get prototype () : *; + + public function Class (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Date.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Date.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,209 @@ +package +{ + /// The Date class represents date and time information. + public class Date extends Object + { + public static const length : int; + + /// The day of the month (an integer from 1 to 31) specified by a Date object according to local time. + public function get date () : Number; + public function set date (value:Number) : void; + + /// The day of the month (an integer from 1 to 31) of a Date object according to universal time (UTC). + public function get dateUTC () : Number; + public function set dateUTC (value:Number) : void; + + /// The day of the week (0 for Sunday, 1 for Monday, and so on) specified by this Date according to local time. + public function get day () : Number; + + /// The day of the week (0 for Sunday, 1 for Monday, and so on) of this Date according to universal time (UTC). + public function get dayUTC () : Number; + + /// The full year (a four-digit number, such as 2000) of a Date object according to local time. + public function get fullYear () : Number; + public function set fullYear (value:Number) : void; + + /// The four-digit year of a Date object according to universal time (UTC). + public function get fullYearUTC () : Number; + public function set fullYearUTC (value:Number) : void; + + /// The hour (an integer from 0 to 23) of the day portion of a Date object according to local time. + public function get hours () : Number; + public function set hours (value:Number) : void; + + /// The hour (an integer from 0 to 23) of the day of a Date object according to universal time (UTC). + public function get hoursUTC () : Number; + public function set hoursUTC (value:Number) : void; + + /// The milliseconds (an integer from 0 to 999) portion of a Date object according to local time. + public function get milliseconds () : Number; + public function set milliseconds (value:Number) : void; + + /// The milliseconds (an integer from 0 to 999) portion of a Date object according to universal time (UTC). + public function get millisecondsUTC () : Number; + public function set millisecondsUTC (value:Number) : void; + + /// The minutes (an integer from 0 to 59) portion of a Date object according to local time. + public function get minutes () : Number; + public function set minutes (value:Number) : void; + + /// The minutes (an integer from 0 to 59) portion of a Date object according to universal time (UTC). + public function get minutesUTC () : Number; + public function set minutesUTC (value:Number) : void; + + /// The month (0 for January, 1 for February, and so on) portion of a Date object according to local time. + public function get month () : Number; + public function set month (value:Number) : void; + + /// The month (0 [January] to 11 [December]) portion of a Date object according to universal time (UTC). + public function get monthUTC () : Number; + public function set monthUTC (value:Number) : void; + + /// The seconds (an integer from 0 to 59) portion of a Date object according to local time. + public function get seconds () : Number; + public function set seconds (value:Number) : void; + + /// The seconds (an integer from 0 to 59) portion of a Date object according to universal time (UTC). + public function get secondsUTC () : Number; + public function set secondsUTC (value:Number) : void; + + /// The number of milliseconds since midnight January 1, 1970, universal time, for a Date object. + public function get time () : Number; + public function set time (value:Number) : void; + + /// The difference, in minutes, between universal time (UTC) and the computer's local time. + public function get timezoneOffset () : Number; + + /// Constructs a new Date object that holds the specified date and time. + public function Date (year:* = null, month:* = null, date:* = null, hours:* = null, minutes:* = null, seconds:* = null, ms:* = null); + + /// Returns the day of the month (an integer from 1 to 31) specified by a Date object according to local time. + public function getDate () : Number; + + /// Returns the day of the week (0 for Sunday, 1 for Monday, and so on) specified by this Date according to local time. + public function getDay () : Number; + + /// Returns the full year (a four-digit number, such as 2000) of a Date object according to local time. + public function getFullYear () : Number; + + /// Returns the hour (an integer from 0 to 23) of the day portion of a Date object according to local time. + public function getHours () : Number; + + /// Returns the milliseconds (an integer from 0 to 999) portion of a Date object according to local time. + public function getMilliseconds () : Number; + + /// Returns the minutes (an integer from 0 to 59) portion of a Date object according to local time. + public function getMinutes () : Number; + + /// Returns the month (0 for January, 1 for February, and so on) portion of this Date according to local time. + public function getMonth () : Number; + + /// Returns the seconds (an integer from 0 to 59) portion of a Date object according to local time. + public function getSeconds () : Number; + + /// Returns the number of milliseconds since midnight January 1, 1970, universal time, for a Date object. + public function getTime () : Number; + + /// Returns the difference, in minutes, between universal time (UTC) and the computer's local time. + public function getTimezoneOffset () : Number; + + /// Returns the day of the month (an integer from 1 to 31) of a Date object, according to universal time (UTC). + public function getUTCDate () : Number; + + /// Returns the day of the week (0 for Sunday, 1 for Monday, and so on) of this Date according to universal time (UTC). + public function getUTCDay () : Number; + + /// Returns the four-digit year of a Date object according to universal time (UTC). + public function getUTCFullYear () : Number; + + /// Returns the hour (an integer from 0 to 23) of the day of a Date object according to universal time (UTC). + public function getUTCHours () : Number; + + /// Returns the milliseconds (an integer from 0 to 999) portion of a Date object according to universal time (UTC). + public function getUTCMilliseconds () : Number; + + /// Returns the minutes (an integer from 0 to 59) portion of a Date object according to universal time (UTC). + public function getUTCMinutes () : Number; + + /// Returns the month (0 [January] to 11 [December]) portion of a Date object according to universal time (UTC). + public function getUTCMonth () : Number; + + /// Returns the seconds (an integer from 0 to 59) portion of a Date object according to universal time (UTC). + public function getUTCSeconds () : Number; + + /// Converts a string representing a date into a number equaling the number of milliseconds elapsed since January 1, 1970, UTC. + public static function parse (s:*) : Number; + + /// Sets the day of the month, according to local time, and returns the new time in milliseconds. + public function setDate (date:* = null) : Number; + + /// Sets the year, according to local time, and returns the new time in milliseconds. + public function setFullYear (year:* = null, month:* = null, date:* = null) : Number; + + /// Sets the hour, according to local time, and returns the new time in milliseconds. + public function setHours (hour:* = null, min:* = null, sec:* = null, ms:* = null) : Number; + + /// Sets the milliseconds, according to local time, and returns the new time in milliseconds. + public function setMilliseconds (ms:* = null) : Number; + + /// Sets the minutes, according to local time, and returns the new time in milliseconds. + public function setMinutes (min:* = null, sec:* = null, ms:* = null) : Number; + + /// Sets the month and optionally the day of the month, according to local time, and returns the new time in milliseconds. + public function setMonth (month:* = null, date:* = null) : Number; + + /// Sets the seconds, according to local time, and returns the new time in milliseconds. + public function setSeconds (sec:* = null, ms:* = null) : Number; + + /// Sets the date in milliseconds since midnight on January 1, 1970, and returns the new time in milliseconds. + public function setTime (t:* = null) : Number; + + /// Sets the day of the month, in universal time (UTC), and returns the new time in milliseconds. + public function setUTCDate (date:* = null) : Number; + + /// Sets the year, in universal time (UTC), and returns the new time in milliseconds. + public function setUTCFullYear (year:* = null, month:* = null, date:* = null) : Number; + + /// Sets the hour, in universal time (UTC), and returns the new time in milliseconds. + public function setUTCHours (hour:* = null, min:* = null, sec:* = null, ms:* = null) : Number; + + /// Sets the milliseconds, in universal time (UTC), and returns the new time in milliseconds. + public function setUTCMilliseconds (ms:* = null) : Number; + + /// Sets the minutes, in universal time (UTC), and returns the new time in milliseconds. + public function setUTCMinutes (min:* = null, sec:* = null, ms:* = null) : Number; + + /// Sets the month, and optionally the day, in universal time(UTC) and returns the new time in milliseconds. + public function setUTCMonth (month:* = null, date:* = null) : Number; + + /// Sets the seconds, and optionally the milliseconds, in universal time (UTC) and returns the new time in milliseconds. + public function setUTCSeconds (sec:* = null, ms:* = null) : Number; + + /// Returns a string representation of the day and date only, and does not include the time or timezone. + public function toDateString () : String; + + /// Returns a String representation of the day and date only, and does not include the time or timezone. + public function toLocaleDateString () : String; + + /// Returns a String representation of the day, date, time, given in local time. + public function toLocaleString () : String; + + /// Returns a String representation of the time only, and does not include the day, date, year, or timezone. + public function toLocaleTimeString () : String; + + /// Returns a String representation of the day, date, time, and timezone. + public function toString () : String; + + /// Returns a String representation of the time and timezone only, and does not include the day and date. + public function toTimeString () : String; + + /// Returns a String representation of the day, date, and time in universal time (UTC). + public function toUTCString () : String; + + /// Returns the number of milliseconds between midnight on January 1, 1970, universal time, and the time specified in the parameters. + public static function UTC (year:*, month:*, date:* = 1, hours:* = 0, minutes:* = 0, seconds:* = 0, ms:* = 0, ...rest) : Number; + + /// Returns the number of milliseconds since midnight January 1, 1970, universal time, for a Date object. + public function valueOf () : Number; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/DefinitionError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/DefinitionError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// The DefinitionError class represents an error that occurs when user code attempts to define an identifier that is already defined. + public class DefinitionError extends Error + { + public static const length : int; + + /// Creates a new DefinitionError object. + public function DefinitionError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Error.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Error.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,25 @@ +package +{ + /// The Error class contains information about an error that occurred in a script. + public class Error extends Object + { + public static const length : int; + /// Contains the message associated with the Error object. + public var message : *; + /// Contains the name of the Error object. + public var name : *; + + /// Contains the reference number associated with the specific error message. + public function get errorID () : int; + + /// Creates a new Error instance with the specified error message. + public function Error (message:* = "", id:* = 0); + + public static function getErrorMessage (index:int) : String; + + /// Returns the call stack for an error in a readable form. + public function getStackTrace () : String; + + public static function throwError (type:Class, index:uint, ...rest) : *; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/EvalError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/EvalError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// The EvalError class represents an error that occurs when user code calls the eval() function or attempts to use the new operator with the Function object. + public class EvalError extends Error + { + public static const length : int; + + /// Creates a new EvalError object. + public function EvalError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Function.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Function.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package +{ + /// A function is the basic unit of code that can be invoked in ActionScript. + public class Function extends Object + { + public static const length : int; + + public function get length () : int; + + public function get prototype () : *; + public function set prototype (p:*) : void; + + /// Specifies the object instance on which the Function is called. + public function apply (thisArg:* = null, argArray:* = null) : *; + + /// Invokes this Function. + public function call (thisArg:* = null, ...rest) : *; + + public function Function (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Math.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Math.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,79 @@ +package +{ + /// The Math class contains methods and constants that represent common mathematical functions and values. + public class Math extends Object + { + /// A mathematical constant for the base of natural logarithms, expressed as e. + public static const E : Number; + /// A mathematical constant for the natural logarithm of 10, expressed as loge10, with an approximate value of 2.302585092994046. + public static const LN10 : Number; + /// A mathematical constant for the natural logarithm of 2, expressed as loge2, with an approximate value of 0.6931471805599453. + public static const LN2 : Number; + /// A mathematical constant for the base-10 logarithm of the constant e (Math.E), expressed as log10e, with an approximate value of 0.4342944819032518. + public static const LOG10E : Number; + /// A mathematical constant for the base-2 logarithm of the constant e, expressed as log2e, with an approximate value of 1.442695040888963387. + public static const LOG2E : Number; + /// A mathematical constant for the ratio of the circumference of a circle to its diameter, expressed as pi, with a value of 3.141592653589793. + public static const PI : Number; + /// A mathematical constant for the square root of one-half, with an approximate value of 0.7071067811865476. + public static const SQRT1_2 : Number; + /// A mathematical constant for the square root of 2, with an approximate value of 1.4142135623730951. + public static const SQRT2 : Number; + + /// Returns the absolute value of the specified Number. + public static function abs (x:Number) : Number; + + /// Returns the arc cosine, in radians, of the specified Number. + public static function acos (x:Number) : Number; + + /// Returns the value, in radians, of the arc sine of the specified Number parameter. + public static function asin (x:Number) : Number; + + /// Returns the angle, in radians, whose tangent is specified by parameter val. + public static function atan (x:Number) : Number; + + /// Returns the angle of the point y/x in radians, when measured counterclockwise from a circle's x axis. + public static function atan2 (y:Number, x:Number) : Number; + + /// Returns the ceiling of the specified number or expression. + public static function ceil (x:Number) : Number; + + /// Returns the cosine of the specified angle. + public static function cos (x:Number) : Number; + + /// Returns the value of the base of the natural logarithm (e), to the power of the exponent specified in the parameter val. + public static function exp (x:Number) : Number; + + /// Returns the floor of the number or expression specified in the parameter val. + public static function floor (x:Number) : Number; + + /// Returns the natural logarithm of parameter val. + public static function log (x:Number) : Number; + + public function Math (); + + /// Evaluates parameters val1 and val2 and returns the larger value. + public static function max (x:Number = null, y:Number = null, ...rest) : Number; + + /// Evaluates parameters val1 and val2 and returns the smaller value. + public static function min (x:Number = null, y:Number = null, ...rest) : Number; + + /// Returns val1 to the power of val2. + public static function pow (x:Number, y:Number) : Number; + + /// Returns a pseudo-random number n, where 0 <= n < 1. + public static function random () : Number; + + /// Returns the value of parameter val rounded up or down to the nearest integer. + public static function round (x:Number) : Number; + + /// Returns the sine of the specified angle. + public static function sin (x:Number) : Number; + + /// Returns the square root of the specified number. + public static function sqrt (x:Number) : Number; + + /// Returns the tangent of the specified angle. + public static function tan (x:Number) : Number; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Namespace.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Namespace.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package +{ + /// The Namespace class contains methods and properties for defining and working with namespaces. + public class Namespace extends Object + { + public static const length : *; + + /// The prefix of the namespace. + public function get prefix () : *; + + /// The Uniform Resource Identifier (URI) of the namespace. + public function get uri () : String; + + /// Creates a Namespace object, given the prefixValue and uriValue. + public function Namespace (prefix:* = null, uri:* = null); + + /// Equivalent to the Namespace.uri property. + public function toString () : String; + + /// Equivalent to the Namespace.uri property. + public function valueOf () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Number.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Number.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,36 @@ +package +{ + /// A data type representing an IEEE-754 double-precision floating-point number. + public class Number extends Object + { + public static const length : int; + /// The largest representable number (double-precision IEEE-754). + public static const MAX_VALUE : Number; + /// The smallest representable non-negative, non-zero, number (double-precision IEEE-754). + public static const MIN_VALUE : Number; + /// The IEEE-754 value representing Not a Number (NaN). + public static const NaN : Number; + /// Specifies the IEEE-754 value representing negative infinity. + public static const NEGATIVE_INFINITY : Number; + /// Specifies the IEEE-754 value representing positive infinity. + public static const POSITIVE_INFINITY : Number; + + /// Creates a Number with the specified value. + public function Number (value:* = 0); + + /// Returns a string representation of the number in exponential notation. + public function toExponential (p:* = 0) : String; + + /// Returns a string representation of the number in fixed-point notation. + public function toFixed (p:* = 0) : String; + + /// Returns a string representation of the number either in exponential notation or in fixed-point notation. + public function toPrecision (p:* = 0) : String; + + /// Returns the string representation of this Number using the specified radix parameter as the numeric base. + public function toString (radix:* = 10) : String; + + /// Returns the primitive value type of the specified Number object. + public function valueOf () : Number; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Object.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Object.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package +{ + /// The Object class is at the root of the ActionScript class hierarchy. + public class Object extends * + { + /// Indicates whether an object has a specified property defined. + public function hasOwnProperty (V:* = null) : Boolean; + + /// Indicates whether an instance of the Object class is in the prototype chain of the object specified as the parameter. + public function isPrototypeOf (V:* = null) : Boolean; + + /// Creates an Object object and stores a reference to the object's constructor method in the object's constructor property. + public function Object (); + + /// Indicates whether the specified property exists and is enumerable. + public function propertyIsEnumerable (V:* = null) : Boolean; + + public function toString () : String; + + public function valueOf () : Object; + + public function setPropertyIsEnumerable (name:String, isEnum:Boolean = true) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/QName.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/QName.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package +{ + /// QName objects represent qualified names of XML elements and attributes. + public class QName extends Object + { + public static const length : *; + + /// The local name of the QName object. + public function get localName () : String; + + /// The Uniform Resource Identifier (URI) of the QName object. + public function get uri () : *; + + /// Creates a QName object that is a copy of another QName object. + public function QName (namespace:* = null, name:* = null); + + /// Returns a string composed of the URI, and the local name for the QName object, separated by "::". + public function toString () : String; + + /// Returns the QName object. + public function valueOf () : QName; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RangeError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RangeError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// A RangeError exception is thrown when a numeric value is outside the acceptable range. + public class RangeError extends Error + { + public static const length : int; + + /// Creates a new RangeError object. + public function RangeError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ReferenceError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/ReferenceError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// A ReferenceError exception is thrown when a reference to an undefined property is attempted on a sealed (nondynamic) object. + public class ReferenceError extends Error + { + public static const length : int; + + /// Creates a new ReferenceError object. + public function ReferenceError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RegExp.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/RegExp.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +package +{ + /// The RegExp class lets you work with regular expressions, which are patterns that you can use to perform searches in strings and to replace text in strings. + public class RegExp extends Object + { + public static const length : int; + + /// Specifies whether the dot character (.) in a regular expression pattern matches new-line characters. + public function get dotall () : Boolean; + + /// Specifies whether to use extended mode for the regular expression. + public function get extended () : Boolean; + + /// Specifies whether to use global matching for the regular expression. + public function get global () : Boolean; + + /// Specifies whether the regular expression ignores case sensitivity. + public function get ignoreCase () : Boolean; + + /// Specifies the index position in the string at which to start the next search. + public function get lastIndex () : int; + public function set lastIndex (i:int) : void; + + /// Specifies whether the m (multiline) flag is set. + public function get multiline () : Boolean; + + /// Specifies the pattern portion of the regular expression. + public function get source () : String; + + /// Performs a search for the regular expression on the given string str. + public function exec (s:String = "") : *; + + /// Lets you construct a regular expression from two strings. + public function RegExp (pattern:* = null, options:* = null); + + /// Tests for the match of the regular expression in the given string str. + public function test (s:String = "") : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SecurityError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SecurityError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// The SecurityError exception is thrown when some type of security violation takes place. + public class SecurityError extends Error + { + public static const length : int; + + /// Creates a new SecurityError object. + public function SecurityError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/String.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/String.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,73 @@ +package +{ + /// The String class is a data type that represents a string of characters. + public class String extends Object + { + /// An integer specifying the number of characters in the specified String object. + public static const length : int; + + public function get length () : int; + + /// Returns the character in the position specified by the index parameter. + public function charAt (i:Number = 0) : String; + + /// Returns the numeric Unicode character code of the character at the specified index. + public function charCodeAt (i:Number = 0) : Number; + + /// Appends the supplied arguments to the end of the String object, converting them to strings if necessary, and returns the resulting string. + public function concat (...rest) : String; + + /// Returns a string comprising the characters represented by the Unicode character codes in the parameters. + public static function fromCharCode (...rest) : String; + + /// Searches the string and returns the position of the first occurrence of val found at or after startIndex within the calling string. + public function indexOf (s:String = undefined, i:Number = 0) : int; + + /// Searches the string from right to left and returns the index of the last occurrence of val found before startIndex. + public function lastIndexOf (s:String = undefined, i:Number = 2147483647) : int; + + /// Compares the sort order of two or more strings and returns the result of the comparison as an integer. + public function localeCompare (other:String = null) : int; + + /// Matches the specifed pattern against the string. + public function match (p:* = null) : Array; + + /// Matches the specifed pattern against the string and returns a new string in which the first match of pattern is replaced with the content specified by repl. + public function replace (p:* = null, repl:* = null) : String; + + /// Searches for the specifed pattern and returns the index of the first matching substring. + public function search (p:* = null) : int; + + /// Returns a string that includes the startIndex character and all characters up to, but not including, the endIndex character. + public function slice (start:Number = 0, end:Number = 2147483647) : String; + + /// Splits a String object into an array of substrings by dividing it wherever the specified delimiter parameter occurs. + public function split (delim:* = null, limit:* = 4294967295) : Array; + + /// Creates a new String object initialized to the specified string. + public function String (value:* = ""); + + /// Returns a substring consisting of the characters that start at the specified startIndex and with a length specified by len. + public function substr (start:Number = 0, len:Number = 2147483647) : String; + + /// Returns a string consisting of the character specified by startIndex and all characters up to endIndex - 1. + public function substring (start:Number = 0, end:Number = 2147483647) : String; + + /// Returns a copy of this string, with all uppercase characters converted to lowercase. + public function toLocaleLowerCase () : String; + + /// Returns a copy of this string, with all lowercase characters converted to uppercase. + public function toLocaleUpperCase () : String; + + /// Returns a copy of this string, with all uppercase characters converted to lowercase. + public function toLowerCase () : String; + + public function toString () : String; + + /// Returns a copy of this string, with all lowercase characters converted to uppercase. + public function toUpperCase () : String; + + /// Returns the primitive value of a String instance. + public function valueOf () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SyntaxError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/SyntaxError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// A SyntaxError exception is thrown when a parsing error occurs, for one of the following reasons:. + public class SyntaxError extends Error + { + public static const length : int; + + /// Creates a new SyntaxError object. + public function SyntaxError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/TypeError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/TypeError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// A TypeError exception is thrown when the actual type of an operand is different from the expected type. + public class TypeError extends Error + { + public static const length : int; + + /// Creates a new TypeError object. + public function TypeError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/URIError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/URIError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// A URIError exception is thrown when one of the global URI handling functions is used in a way that is incompatible with its definition. + public class URIError extends Error + { + public static const length : int; + + /// Creates a new URIError object. + public function URIError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/UninitializedError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/UninitializedError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package +{ + public class UninitializedError extends Error + { + public static const length : int; + + public function UninitializedError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Vector.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/Vector.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,75 @@ +package +{ + /// The Vector class lets you access and manipulate a vector - an array whose elements all have the same data type. + public class Vector. + { + /// The range of valid indices available in the Vector. + public var length:int; + + /// Indicates whether the length property of the Vector can be changed. + public var fixed:Boolean; + + /// [FP10] Creates a Vector with the specified base type. + public function Vector (length:uint = 0, fixed:Boolean = false); + + /// [FP10] Concatenates the elements specified in the parameters. + public function concat (...args) : Vector.; + + /// [FP10] Executes a test function on each item in the Vector until an item is reached that returns false for the specified function. + public function every (callback:Function, thisObject:Object = null) : Boolean; + + /// [FP10] Executes a test function on each item in the Vector and returns a new Vector containing all items that return true for the specified function. + public function filter (callback:Function, thisObject:Object = null) : Vector.; + + /// [FP10] Executes a function on each item in the Vector. + public function forEach (callback:Function, thisObject:Object = null) : void; + + /// [FP10] Searches for an item in the Vector and returns the index position of the item. + public function indexOf (searchElement:T, fromIndex:int = 0) : int; + + /// [FP10] Converts the elements in the Vector to strings. + public function join (sep:String = ",") : String; + + /// [FP10] Searches for an item in the Vector, working backward from the specified index position, and returns the index position of the matching item. + public function lastIndexOf (searchElement:T, fromIndex:int = 0x7fffffff) : int; + + /// [FP10] Executes a function on each item in the Vector, and returns a new Vector of items corresponding to the results of calling the function on each item in this Vector. + public function map (callback:Function, thisObject:Object = null) : Vector.; + + /// [FP10] Removes the last element from the Vector and returns that element. + public function pop () : T; + + /// [FP10] Adds one or more elements to the end of the Vector and returns the new length of the Vector. + public function push (...args) : uint; + + /// [FP10] Reverses the order of the elements in the Vector. + public function reverse () : Vector.; + + /// [FP10] Removes the first element from the Vector and returns that element. + public function shift () : T; + + /// [FP10] Returns a new Vector that consists of a range of elements from the original Vector. + public function slice (startIndex:int = 0, endIndex:int = 16777215) : Vector.; + + /// [FP10] Executes a test function on each item in the Vector until an item is reached that returns true. + public function some (callback:Function, thisObject:Object = null) : Boolean; + + /// [FP10] Sorts the elements in the Vector. + public function sort (compareFunction:Function) : Vector.; + + /// [FP10] Adds elements to and removes elements from the Vector. + public function splice (startIndex:int, deleteCount:uint, ...items) : Vector.; + + /// [FP10] Returns a string that represents the elements in the Vector. + public function toString () : String; + + /// [FP10] Returns a string that represents the elements in the specified Vector. + public function toLocaleString () : String; + + /// [FP10] Adds one or more elements to the beginning of the Vector and returns the new length of the Vector. + public function unshift (...args) : uint; + + } + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/VerifyError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/VerifyError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package +{ + /// The VerifyError class represents an error that occurs when a malformed or corrupted SWF file is encountered. + public class VerifyError extends Error + { + public static const length : int; + + /// Creates a new VerifyError object. + public function VerifyError (message:* = "", id:* = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XML.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XML.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,161 @@ +package +{ + /// The XML class contains methods and properties for working with XML objects. + public class XML extends Object + { + /// For XML objects, this method always returns the integer 1. + public static const length : *; + + /// Determines whether XML comments are ignored when XML objects parse the source XML data. + public static function get ignoreComments () : Boolean; + public static function set ignoreComments (newIgnore:Boolean) : void; + + /// Determines whether XML processing instructions are ignored when XML objects parse the source XML data. + public static function get ignoreProcessingInstructions () : Boolean; + public static function set ignoreProcessingInstructions (newIgnore:Boolean) : void; + + /// Determines whether white space characters at the beginning and end of text nodes are ignored during parsing. + public static function get ignoreWhitespace () : Boolean; + public static function set ignoreWhitespace (newIgnore:Boolean) : void; + + /// Determines the amount of indentation applied by the toString() and toXMLString() methods when the XML.prettyPrinting property is set to true. + public static function get prettyIndent () : int; + public static function set prettyIndent (newIndent:int) : void; + + /// Determines whether the toString() and toXMLString() methods normalize white space characters between some tags. + public static function get prettyPrinting () : Boolean; + public static function set prettyPrinting (newPretty:Boolean) : void; + + /// Adds a namespace to the set of in-scope namespaces for the XML object. + public function addNamespace (ns:*) : XML; + + /// Appends the given child to the end of the XML object's properties. + public function appendChild (child:*) : XML; + + /// Returns the XML value of the attribute that has the name matching the attributeName parameter. + public function attribute (arg:*) : XMLList; + + /// Returns a list of attribute values for the given XML object. + public function attributes () : XMLList; + + /// Lists the children of an XML object. + public function child (propertyName:*) : XMLList; + + /// Identifies the zero-indexed position of this XML object within the context of its parent. + public function childIndex () : int; + + /// Lists the children of the XML object in the sequence in which they appear. + public function children () : XMLList; + + /// Lists the properties of the XML object that contain XML comments. + public function comments () : XMLList; + + /// Compares the XML object against the given value parameter. + public function contains (value:*) : Boolean; + + /// Returns a copy of the given XML object. + public function copy () : XML; + + /// Returns an object with the following properties set to the default values: ignoreComments, ignoreProcessingInstructions, ignoreWhitespace, prettyIndent, and prettyPrinting. + public static function defaultSettings () : Object; + + /// Returns all descendants (children, grandchildren, great-grandchildren, and so on) of the XML object that have the given name parameter. + public function descendants (name:* = "*") : XMLList; + + /// Lists the elements of an XML object. + public function elements (name:* = "*") : XMLList; + + /// Checks to see whether the XML object contains complex content. + public function hasComplexContent () : Boolean; + + /// Checks to see whether the object has the property specified by the p parameter. + public function hasOwnProperty (P:* = null) : Boolean; + + /// Checks to see whether the XML object contains simple content. + public function hasSimpleContent () : Boolean; + + /// Lists the namespaces for the XML object, based on the object's parent. + public function inScopeNamespaces () : Array; + + /// Inserts the given child2 parameter after the child1 parameter in this XML object and returns the resulting object. + public function insertChildAfter (child1:*, child2:*) : *; + + /// Inserts the given child2 parameter before the child1 parameter in this XML object and returns the resulting object. + public function insertChildBefore (child1:*, child2:*) : *; + + public function length () : int; + + /// Gives the local name portion of the qualified name of the XML object. + public function localName () : Object; + + /// Gives the qualified name for the XML object. + public function name () : Object; + + /// If no parameter is provided, gives the namespace associated with the qualified name of this XML object. + public function namespace (prefix:* = null) : *; + + /// Lists namespace declarations associated with the XML object in the context of its parent. + public function namespaceDeclarations () : Array; + + /// Specifies the type of node: text, comment, processing-instruction, attribute, or element. + public function nodeKind () : String; + + /// For the XML object and all descendant XML objects, merges adjacent text nodes and eliminates empty text nodes. + public function normalize () : XML; + + public function notification () : Function; + + /// Returns the parent of the XML object. + public function parent () : *; + + /// Inserts a copy of the provided child object into the XML element before any existing XML properties for that element. + public function prependChild (value:*) : XML; + + /// If a name parameter is provided, lists all the children of the XML object that contain processing instructions with that name. + public function processingInstructions (name:* = "*") : XMLList; + + /// Checks whether the property p is in the set of properties that can be iterated in a for..in statement applied to the XML object. + public function propertyIsEnumerable (P:* = null) : Boolean; + + /// Removes the given namespace for this object and all descendants. + public function removeNamespace (ns:*) : XML; + + /// Replaces the properties specified by the propertyName parameter with the given value parameter. + public function replace (propertyName:*, value:*) : XML; + + /// Replaces the child properties of the XML object with the specified set of XML properties, provided in the value parameter. + public function setChildren (value:*) : XML; + + /// Changes the local name of the XML object to the given name parameter. + public function setLocalName (name:*) : void; + + /// Sets the name of the XML object to the given qualified name or attribute name. + public function setName (name:*) : void; + + /// Sets the namespace associated with the XML object. + public function setNamespace (ns:*) : void; + + public function setNotification (f:Function) : *; + + /// Sets values for the following XML properties: ignoreComments, ignoreProcessingInstructions, ignoreWhitespace, prettyIndent, and prettyPrinting. + public static function setSettings (o:Object = null) : void; + + /// Retrieves the following properties: ignoreComments, ignoreProcessingInstructions, ignoreWhitespace, prettyIndent, and prettyPrinting. + public static function settings () : Object; + + /// Returns an XMLList object of all XML properties of the XML object that represent XML text nodes. + public function text () : XMLList; + + /// Returns a string representation of the XML object. + public function toString () : String; + + /// Returns a string representation of the XML object. + public function toXMLString () : String; + + /// Returns the XML object. + public function valueOf () : XML; + + /// Creates a new XML object. + public function XML (value:* = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XMLList.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/XMLList.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,110 @@ +package +{ + /// The XMLList class contains methods for working with one or more XML elements. + public class XMLList extends Object + { + /// Returns the number of properties in the XMLList object. + public static const length : *; + + public function addNamespace (ns:*) : XML; + + public function appendChild (child:*) : XML; + + /// Calls the attribute() method of each XML object and returns an XMLList object of the results. + public function attribute (arg:*) : XMLList; + + /// Calls the attributes() method of each XML object and returns an XMLList object of attributes for each XML object. + public function attributes () : XMLList; + + /// Calls the child() method of each XML object and returns an XMLList object that contains the results in order. + public function child (propertyName:*) : XMLList; + + public function childIndex () : int; + + /// Calls the children() method of each XML object and returns an XMLList object that contains the results. + public function children () : XMLList; + + /// Calls the comments() method of each XML object and returns an XMLList of comments. + public function comments () : XMLList; + + /// Checks whether the XMLList object contains an XML object that is equal to the given value parameter. + public function contains (value:*) : Boolean; + + /// Returns a copy of the given XMLList object. + public function copy () : XMLList; + + /// Returns all descendants (children, grandchildren, great-grandchildren, and so on) of the XML object that have the given name parameter. + public function descendants (name:* = "*") : XMLList; + + /// Calls the elements() method of each XML object. + public function elements (name:* = "*") : XMLList; + + /// Checks whether the XMLList object contains complex content. + public function hasComplexContent () : Boolean; + + /// Checks for the property specified by p. + public function hasOwnProperty (P:* = null) : Boolean; + + /// Checks whether the XMLList object contains simple content. + public function hasSimpleContent () : Boolean; + + public function inScopeNamespaces () : Array; + + public function insertChildAfter (child1:*, child2:*) : *; + + public function insertChildBefore (child1:*, child2:*) : *; + + public function length () : int; + + public function localName () : Object; + + public function name () : Object; + + public function namespace (prefix:* = null) : *; + + public function namespaceDeclarations () : Array; + + public function nodeKind () : String; + + /// Merges adjacent text nodes and eliminates empty text nodes for each of the following: all text nodes in the XMLList, all the XML objects contained in the XMLList, and the descendants of all the XML objects in the XMLList. + public function normalize () : XMLList; + + /// Returns the parent of the XMLList object if all items in the XMLList object have the same parent. + public function parent () : *; + + public function prependChild (value:*) : XML; + + /// If a name parameter is provided, lists all the children of the XMLList object that contain processing instructions with that name. + public function processingInstructions (name:* = "*") : XMLList; + + /// Checks whether the property p is in the set of properties that can be iterated in a for..in statement applied to the XMLList object. + public function propertyIsEnumerable (P:* = null) : Boolean; + + public function removeNamespace (ns:*) : XML; + + public function replace (propertyName:*, value:*) : XML; + + public function setChildren (value:*) : XML; + + public function setLocalName (name:*) : void; + + public function setName (name:*) : void; + + public function setNamespace (ns:*) : void; + + /// Calls the text() method of each XML object and returns an XMLList object that contains the results. + public function text () : XMLList; + + /// Returns a string representation of all the XML objects in an XMLList object. + public function toString () : String; + + /// Returns a string representation of all the XML objects in an XMLList object. + public function toXMLString () : String; + + /// Returns the XMLList object. + public function valueOf () : XMLList; + + /// Creates a new XMLList object. + public function XMLList (value:* = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/CustomActions.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/CustomActions.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package adobe.utils +{ + /// The methods of the CustomActions class allow a SWF file playing in the Flash authoring tool to manage any custom actions that are registered with the authoring tool. + public class CustomActions extends Object + { + /// Returns an Array object containing the names of all the custom actions that are registered with the Flash authoring tool. + public static function get actionsList () : Array; + + public function CustomActions (); + + /// Reads the contents of the custom action XML definition file named name. + public static function getActions (name:String) : String; + + /// Installs a new custom action XML definition file indicated by the name parameter. + public static function installActions (name:String, data:String) : void; + + /// Removes the Custom Actions XML definition file named name. + public static function uninstallActions (name:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/ProductManager.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/ProductManager.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package adobe.utils +{ + import flash.events.EventDispatcher; + + public class ProductManager extends EventDispatcher + { + public function get installed () : Boolean; + + public function get installedVersion () : String; + + public function get running () : Boolean; + + public function download (caption:String = null, fileName:String = null, pathElements:Array = null) : Boolean; + + public function launch (parameters:String = null) : Boolean; + + public function ProductManager (name:String); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/XMLUI.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/XMLUI.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package adobe.utils +{ + /// The XMLUI class enables communication with SWF files that are used as custom user interfaces for the Flash authoring tool's extensibility features. + public class XMLUI extends Object + { + /// Makes the current XMLUI dialog box close with an "accept" state. + public static function accept () : void; + + /// Makes the current XMLUI dialog box close with a "cancel" state. + public static function cancel () : void; + + /// Retrieves the value of the specified property of the current XMLUI dialog box. + public static function getProperty (name:String) : String; + + /// Modifies the value of the specified property of the current XMLUI dialog. + public static function setProperty (name:String, value:String) : void; + + public function XMLUI (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/adobe/utils/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,10 @@ +package adobe.utils +{ + /// Lets you issue Flash JavaScript API (JSAPI) commands from ActionScript. + public function MMExecute (name:String) : String; + + /// [FP10] Notifies an application hosting a SWF command that a command is done and instructs the application to commit or discard the changes submitted by the MMExecute() command. + public function MMEndCommand (endStatus:Boolean, notifyString:String) : void; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/arguments.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/arguments.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +?package +{ + /// Context of the executing function. + public class arguments + { + /// A reference to the currently executing function. + public static var callee:Function; + + /// The number of arguments passed to the function. + public static var length:Number; + } + +} \ No newline at end of file Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/authoring/authObject.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/authoring/authObject.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,85 @@ +package authoring +{ + import authoring.authObject; + import flash.geom.Rectangle; + import flash.geom.ColorTransform; + import flash.geom.Point; + import flash.geom.Matrix3D; + import flash.geom.Matrix; + + public class authObject extends Object + { + public function get FirstChild () : authObject; + + public function get Key () : uint; + + public function get NextSibling () : authObject; + + public static function get offScreenSurfaceRenderingEnabled () : Boolean; + public static function set offScreenSurfaceRenderingEnabled (value:Boolean) : void; + + public function get SwfKey () : uint; + + public function get Type () : uint; + + public function authObject (key:uint); + + public function BlendingMode () : String; + + public function Bounds (flags:uint, minFrame:int = -16000, maxFrame:int = 16000) : Rectangle; + + public function CacheAsBitmap () : Boolean; + + public function CenterPoint () : Point; + + public function ColorXForm () : ColorTransform; + + public function EndPosition () : int; + + public function Filters () : Array; + + public function FrameForFrameNumber (frameNum:int) : authObject; + + public function FrameOffset () : int; + + public function FrameType () : uint; + + public function HasEmptyPath () : Boolean; + + public function HasShapeSelection () : Boolean; + + public function IsFloater () : Boolean; + + public function IsPrimitive () : Boolean; + + public function IsSelected () : Boolean; + + public function IsVisible (inThumbnailPreview:Boolean) : Boolean; + + public function LivePreviewSize () : Point; + + public function Locked () : Boolean; + + public function MotionPath () : authObject; + + public function ObjMatrix () : Matrix; + + public function OutlineColor () : uint; + + public function OutlineMode () : Boolean; + + public function RegistrationPoint () : Point; + + public function Scale9Grid () : Rectangle; + + public function StartPosition () : int; + + public function SymbolBehavior () : int; + + public function SymbolMode () : int; + + public function ThreeDMatrix () : Matrix3D; + + public function ThreeDTranslationHandlePoints () : Array; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/Accessibility.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/Accessibility.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.accessibility +{ + import flash.display.DisplayObject; + + /// The Accessibility class manages communication with screen readers. + public class Accessibility extends Object + { + /// Indicates whether a screen reader is currently active and the player is communicating with it. + public static function get active () : Boolean; + + public function Accessibility (); + + public static function sendEvent (source:DisplayObject, childID:uint, eventType:uint, nonHTML:Boolean = false) : void; + + /// Tells Flash Player to apply any accessibility changes made by using the DisplayObject.accessibilityProperties property. + public static function updateProperties () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityImplementation.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityImplementation.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,36 @@ +package flash.accessibility +{ + import flash.geom.Rectangle; + + public class AccessibilityImplementation extends Object + { + public var errno : uint; + public var stub : Boolean; + + public function accDoDefaultAction (childID:uint) : void; + + public function AccessibilityImplementation (); + + public function accLocation (childID:uint) : *; + + public function accSelect (operation:uint, childID:uint) : void; + + public function get_accDefaultAction (childID:uint) : String; + + public function get_accFocus () : uint; + + public function get_accName (childID:uint) : String; + + public function get_accRole (childID:uint) : uint; + + public function get_accSelection () : Array; + + public function get_accState (childID:uint) : uint; + + public function get_accValue (childID:uint) : String; + + public function getChildIDArray () : Array; + + public function isLabeledBy (labelBounds:Rectangle) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityProperties.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/accessibility/AccessibilityProperties.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +package flash.accessibility +{ + /// The AccessibilityProperties class lets you control the presentation of Flash objects to accessibility aids, such as screen readers. + public class AccessibilityProperties extends Object + { + /// Provides a description for this display object in the accessible presentation. + public var description : String; + /// If true, causes Flash Player to exclude child objects within this display object from the accessible presentation. + public var forceSimple : Boolean; + /// Provides a name for this display object in the accessible presentation. + public var name : String; + /// If true, disables the Flash Player default auto-labeling system. + public var noAutoLabeling : Boolean; + /// Indicates a keyboard shortcut associated with this display object. + public var shortcut : String; + /// If true, excludes this display object from accessible presentation. + public var silent : Boolean; + + /// Creates a new AccessibilityProperties object. + public function AccessibilityProperties (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/Clipboard.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/Clipboard.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,37 @@ +package flash.desktop +{ + import flash.utils.Dictionary; + import flash.desktop.Clipboard; + import flash.utils.ByteArray; + + /// The Clipboard class provides a container for transferring data and objects through the clipboard and through drag-and-drop operations (AIR only). + public class Clipboard extends Object + { + /// An array of strings containing the names of the data formats available in this Clipboard object. + public function get formats () : Array; + + /// The operating system clipboard. + public static function get generalClipboard () : Clipboard; + + /// Deletes all data representations from this Clipboard object. + public function clear () : void; + + /// Deletes the data representation for the specified format. + public function clearData (format:String) : void; + + /// Creates an empty Clipboard object. + public function Clipboard (); + + /// Gets the clipboard data if data in the specified format is present. + public function getData (format:String, transferMode:String = "originalPreferred") : Object; + + /// Checks whether data in the specified format exists in this Clipboard object. + public function hasFormat (format:String) : Boolean; + + /// Adds a representation of the information to be transferred in the specified data format. + public function setData (format:String, data:Object, serializable:Boolean = true) : Boolean; + + /// Adds a reference to a handler function that produces the data for the specified format on demand. + public function setDataHandler (format:String, handler:Function, serializable:Boolean = true) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardFormats.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardFormats.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.desktop +{ + /// Defines constants for the names of the standard data formats used with the Clipboard class. + public class ClipboardFormats extends Object + { + /// HTML data. + public static const HTML_FORMAT : String; + /// Rich Text Format data. + public static const RICH_TEXT_FORMAT : String; + /// String data. + public static const TEXT_FORMAT : String; + + public function ClipboardFormats (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardTransferMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/desktop/ClipboardTransferMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.desktop +{ + /// Defines constants for the modes used as values of the transferMode parameter of the Clipboard.getData() method. + public class ClipboardTransferMode extends Object + { + /// The Clipboard object should only return a copy. + public static const CLONE_ONLY : String; + /// The Clipboard object should return a copy if available and a reference if not. + public static const CLONE_PREFERRED : String; + /// The Clipboard object should only return a reference. + public static const ORIGINAL_ONLY : String; + /// The Clipboard object should return a reference if available and a copy if not. + public static const ORIGINAL_PREFERRED : String; + + public function ClipboardTransferMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/AVM1Movie.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/AVM1Movie.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + import flash.utils.ByteArray; + + public class AVM1Movie extends DisplayObject + { + public function addCallback (functionName:String, closure:Function) : void; + + public function AVM1Movie (); + + public function call (functionName:String, ...rest) : *; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ActionScriptVersion.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ActionScriptVersion.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// The ActionScriptVersion class is an enumeration of constant values that indicate the language version of a loaded SWF file. + public class ActionScriptVersion extends Object + { + /// ActionScript language version 2.0 and earlier. + public static const ACTIONSCRIPT2 : uint; + /// ActionScript language version 3.0. + public static const ACTIONSCRIPT3 : uint; + + public function ActionScriptVersion (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Bitmap.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Bitmap.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.display +{ + import flash.display.BitmapData; + + /// The Bitmap class represents display objects that represent bitmap images. + public class Bitmap extends DisplayObject + { + /// The BitmapData object being referenced. + public function get bitmapData () : BitmapData; + public function set bitmapData (value:BitmapData) : void; + + /// Controls whether or not the Bitmap object is snapped to the nearest pixel. + public function get pixelSnapping () : String; + public function set pixelSnapping (value:String) : void; + + /// Controls whether or not the bitmap is smoothed when scaled. + public function get smoothing () : Boolean; + public function set smoothing (value:Boolean) : void; + + /// Initializes a Bitmap object to refer to the specified BitmapData object. + public function Bitmap (bitmapData:BitmapData = null, pixelSnapping:String = "auto", smoothing:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapData.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapData.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,122 @@ +package flash.display +{ + import flash.display.BitmapData; + import flash.geom.Rectangle; + import flash.geom.Point; + import flash.filters.BitmapFilter; + import flash.geom.ColorTransform; + import flash.display.IBitmapDrawable; + import flash.geom.Matrix; + import flash.utils.ByteArray; + + /// The BitmapData class lets you work with the data (pixels) of a Bitmap object. + public class BitmapData extends Object implements IBitmapDrawable + { + /// The height of the bitmap image in pixels. + public function get height () : int; + + /// The rectangle that defines the size and location of the bitmap image. + public function get rect () : Rectangle; + + /// Defines whether the bitmap image supports per-pixel transparency. + public function get transparent () : Boolean; + + /// The width of the bitmap image in pixels. + public function get width () : int; + + /// Takes a source image and a filter object and generates the filtered image. + public function applyFilter (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, filter:BitmapFilter) : void; + + /// Creates a BitmapData object with a specified width and height. + public function BitmapData (width:int, height:int, transparent:Boolean = true, fillColor:uint = 4294967295); + + /// Returns a new BitmapData object with an exact copy of the original bitmap. + public function clone () : BitmapData; + + /// Adjusts the color values in a specified area of a bitmap image by using a ColorTransform object. + public function colorTransform (rect:Rectangle, colorTransform:ColorTransform) : void; + + /// Compares two BitmapData objects. + public function compare (otherBitmapData:BitmapData) : Object; + + /// Transfers data from one channel of another BitmapData object or the current BitmapData object into a channel of the current BitmapData object. + public function copyChannel (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, sourceChannel:uint, destChannel:uint) : void; + + /// Provides a fast routine to perform pixel manipulation between images with no stretching, rotation, or color effects. + public function copyPixels (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, alphaBitmapData:BitmapData = null, alphaPoint:Point = null, mergeAlpha:Boolean = false) : void; + + /// Frees memory that is used to store the BitmapData object. + public function dispose () : void; + + /// Draws the source display object onto the bitmap image, using the Flash Player vector renderer. + public function draw (source:IBitmapDrawable, matrix:Matrix = null, colorTransform:ColorTransform = null, blendMode:String = null, clipRect:Rectangle = null, smoothing:Boolean = false) : void; + + /// Fills a rectangular area of pixels with a specified ARGB color. + public function fillRect (rect:Rectangle, color:uint) : void; + + /// Performs a flood fill operation on an image starting at a (x, y) coordinate. + public function floodFill (x:int, y:int, color:uint) : void; + + /// Determines the destination rectangle that will be affected by the applyFilter() call. + public function generateFilterRect (sourceRect:Rectangle, filter:BitmapFilter) : Rectangle; + + /// Determines a rectangular region that either fully encloses all pixels of a specified color within the bitmap image (if the findColor parameter is set to true) or fully encloses all pixels that do not include the specified color (if the findColor parameter is set to false). + public function getColorBoundsRect (mask:uint, color:uint, findColor:Boolean = true) : Rectangle; + + /// Returns an integer representing a RGB pixel value from a BitmapData object at a specific point. + public function getPixel (x:int, y:int) : uint; + + /// Returns an ARGB color value that contains alpha channel data and RGB data. + public function getPixel32 (x:int, y:int) : uint; + + /// Generates a byte array from a rectangular region of pixel data. + public function getPixels (rect:Rectangle) : ByteArray; + + /// Generates a vector array from a rectangular region of pixel data. + public function getVector (rect:Rectangle) : Vector.; + + /// Computes a 256 value binary number histogram of a BitmapData object. + public function histogram (hRect:Rectangle = null) : Vector.>; + + /// Performs pixel-level hit detection between one bitmap image and a point, rectangle, or other bitmap image. + public function hitTest (firstPoint:Point, firstAlphaThreshold:uint, secondObject:Object, secondBitmapDataPoint:Point = null, secondAlphaThreshold:uint = 1) : Boolean; + + /// Locks an image so that any objects that reference the BitmapData object, such as Bitmap objects, are not updated when this BitmapData object changes. + public function lock () : void; + + /// Performs per-channel blending from a source image to a destination image. + public function merge (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, redMultiplier:uint, greenMultiplier:uint, blueMultiplier:uint, alphaMultiplier:uint) : void; + + /// Fills an image with pixels representing random noise. + public function noise (randomSeed:int, low:uint = 0, high:uint = 255, channelOptions:uint = 7, grayScale:Boolean = false) : void; + + /// Remaps the color channel values in an image that has up to four arrays of color palette data, one for each channel. + public function paletteMap (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, redArray:Array = null, greenArray:Array = null, blueArray:Array = null, alphaArray:Array = null) : void; + + /// Generates a Perlin noise image. + public function perlinNoise (baseX:Number, baseY:Number, numOctaves:uint, randomSeed:int, stitch:Boolean, fractalNoise:Boolean, channelOptions:uint = 7, grayScale:Boolean = false, offsets:Array = null) : void; + + /// Performs a pixel dissolve either from a source image to a destination image or by using the same image. + public function pixelDissolve (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, randomSeed:int = 0, numPixels:int = 0, fillColor:uint = 0) : int; + + /// Scrolls an image by a certain (x, y) pixel amount. + public function scroll (x:int, y:int) : void; + + /// Sets a single pixel of a BitmapData object. + public function setPixel (x:int, y:int, color:uint) : void; + + /// Sets the color and alpha transparency values of a single pixel of a BitmapData object. + public function setPixel32 (x:int, y:int, color:uint) : void; + + /// Converts a byte array into a rectangular region of pixel data. + public function setPixels (rect:Rectangle, inputByteArray:ByteArray) : void; + + public function setVector (rect:Rectangle, inputVector:Vector.) : void; + + /// Tests pixel values in an image against a specified threshold and sets pixels that pass the test to new color values. + public function threshold (sourceBitmapData:BitmapData, sourceRect:Rectangle, destPoint:Point, operation:String, threshold:uint, color:uint = 0, mask:uint = 4294967295, copySource:Boolean = false) : uint; + + /// Unlocks an image so that any objects that reference the BitmapData object, such as Bitmap objects, are updated when this BitmapData object changes. + public function unlock (changeRect:Rectangle = null) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapDataChannel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BitmapDataChannel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + /// The BitmapDataChannel class is an enumeration of constant values that indicate which channel to use: red, blue, green, or alpha transparency. + public class BitmapDataChannel extends Object + { + /// The alpha channel. + public static const ALPHA : uint; + /// The blue channel. + public static const BLUE : uint; + /// The green channel. + public static const GREEN : uint; + /// The red channel. + public static const RED : uint; + + public function BitmapDataChannel (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BlendMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/BlendMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +package flash.display +{ + /// A class that provides constant values for visual blend mode effects. + public class BlendMode extends Object + { + /// Adds the values of the constituent colors of the display object to the colors of its background, applying a ceiling of 0xFF. + public static const ADD : String; + /// Applies the alpha value of each pixel of the display object to the background. + public static const ALPHA : String; + /// Selects the darker of the constituent colors of the display object and the colors of the background (the colors with the smaller values). + public static const DARKEN : String; + /// Compares the constituent colors of the display object with the colors of its background, and subtracts the darker of the values of the two constituent colors from the lighter value. + public static const DIFFERENCE : String; + /// Erases the background based on the alpha value of the display object. + public static const ERASE : String; + /// Adjusts the color of each pixel based on the darkness of the display object. + public static const HARDLIGHT : String; + /// Inverts the background. + public static const INVERT : String; + /// Forces the creation of a transparency group for the display object. + public static const LAYER : String; + /// Selects the lighter of the constituent colors of the display object and the colors of the background (the colors with the larger values). + public static const LIGHTEN : String; + /// Multiplies the values of the display object constituent colors by the constituent colors of the background color, and normalizes by dividing by 0xFF, resulting in darker colors. + public static const MULTIPLY : String; + /// The display object appears in front of the background. + public static const NORMAL : String; + /// Adjusts the color of each pixel based on the darkness of the background. + public static const OVERLAY : String; + /// Multiplies the complement (inverse) of the display object color by the complement of the background color, resulting in a bleaching effect. + public static const SCREEN : String; + /// Uses a shader to define the blend between objects. + public static const SHADER : String; + /// Subtracts the values of the constituent colors in the display object from the values of the backgroundcolor, applying a floor of 0. + public static const SUBTRACT : String; + + public function BlendMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/CapsStyle.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/CapsStyle.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The CapsStyle class is an enumeration of constant values that specify the caps style to use in drawing lines. + public class CapsStyle extends Object + { + /// Used to specify no caps in the caps parameter of the flash.display.Graphics.lineStyle() method. + public static const NONE : String; + /// Used to specify round caps in the caps parameter of the flash.display.Graphics.lineStyle() method. + public static const ROUND : String; + /// Used to specify square caps in the caps parameter of the flash.display.Graphics.lineStyle() method. + public static const SQUARE : String; + + public function CapsStyle (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrection.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrection.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The ColorCorrection class provides values for the flash.display.Stage.colorCorrection property. + public class ColorCorrection extends Object + { + /// Uses the host's default color correction. + public static const DEFAULT : String; + /// Turns off color correction regardless of the player host environment. + public static const OFF : String; + /// Turns on color correction regardless of the player host environment, if available. + public static const ON : String; + + public function ColorCorrection (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrectionSupport.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ColorCorrectionSupport.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The ColorCorrectionSupport class provides values for the flash.display.Stage.colorCorrectionSupport property. + public class ColorCorrectionSupport extends Object + { + /// Color correction is supported, but off by default. + public static const DEFAULT_OFF : String; + /// Color correction is supported, and on by default. + public static const DEFAULT_ON : String; + /// Color correction is not supported by the host environment. + public static const UNSUPPORTED : String; + + public function ColorCorrectionSupport (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObject.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObject.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,210 @@ +package flash.display +{ + import flash.events.EventDispatcher; + import flash.display.IBitmapDrawable; + import flash.display.DisplayObject; + import flash.geom.Point; + import flash.geom.Rectangle; + import flash.accessibility.AccessibilityProperties; + import flash.display.Shader; + import flash.display.DisplayObjectContainer; + import flash.geom.Vector3D; + import flash.geom.Transform; + import flash.display.LoaderInfo; + import flash.display.Stage; + + /** + * [broadcast event] Dispatched when the display list is about to be updated and rendered. + * @eventType flash.events.Event.RENDER + */ + [Event(name="render", type="flash.events.Event")] + + /** + * Dispatched when a display object is about to be removed from the display list, either directly or through the removal of a sub tree in which the display object is contained. + * @eventType flash.events.Event.REMOVED_FROM_STAGE + */ + [Event(name="removedFromStage", type="flash.events.Event")] + + /** + * Dispatched when a display object is about to be removed from the display list. + * @eventType flash.events.Event.REMOVED + */ + [Event(name="removed", type="flash.events.Event")] + + /** + * [broadcast event] Dispatched when the playhead is exiting the current frame. + * @eventType flash.events.Event.EXIT_FRAME + */ + [Event(name="exitFrame", type="flash.events.Event")] + + /** + * [broadcast event] Dispatched after the constructors of frame display objects have run but before frame scripts have run. + * @eventType flash.events.Event.FRAME_CONSTRUCTED + */ + [Event(name="frameConstructed", type="flash.events.Event")] + + /** + * [broadcast event] Dispatched when the playhead is entering a new frame. + * @eventType flash.events.Event.ENTER_FRAME + */ + [Event(name="enterFrame", type="flash.events.Event")] + + /** + * Dispatched when a display object is added to the on stage display list, either directly or through the addition of a sub tree in which the display object is contained. + * @eventType flash.events.Event.ADDED_TO_STAGE + */ + [Event(name="addedToStage", type="flash.events.Event")] + + /** + * Dispatched when a display object is added to the display list. + * @eventType flash.events.Event.ADDED + */ + [Event(name="added", type="flash.events.Event")] + + /// The DisplayObject class is the base class for all objects that can be placed on the display list. + public class DisplayObject extends EventDispatcher implements IBitmapDrawable + { + /// The current accessibility options for this display object. + public function get accessibilityProperties () : AccessibilityProperties; + public function set accessibilityProperties (value:AccessibilityProperties) : void; + + /// Indicates the alpha transparency value of the object specified. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// A value from the BlendMode class that specifies which blend mode to use. + public function get blendMode () : String; + public function set blendMode (value:String) : void; + + /// Sets a shader that is used for blending the foreground and background. + public function set blendShader (value:Shader) : void; + + /// If set to true, Flash Player caches an internal bitmap representation of the display object. + public function get cacheAsBitmap () : Boolean; + public function set cacheAsBitmap (value:Boolean) : void; + + /// An indexed array that contains each filter object currently associated with the display object. + public function get filters () : Array; + public function set filters (value:Array) : void; + + /// Indicates the height of the display object, in pixels. + public function get height () : Number; + public function set height (value:Number) : void; + + /// Returns a LoaderInfo object containing information about loading the file to which this display object belongs. + public function get loaderInfo () : LoaderInfo; + + /// The calling display object is masked by the specified mask object. + public function get mask () : DisplayObject; + public function set mask (value:DisplayObject) : void; + + /// Indicates the x coordinate of the mouse position, in pixels. + public function get mouseX () : Number; + + /// Indicates the y coordinate of the mouse position, in pixels. + public function get mouseY () : Number; + + /// Indicates the instance name of the DisplayObject. + public function get name () : String; + public function set name (value:String) : void; + + /// Specifies whether the display object is opaque with a certain background color. + public function get opaqueBackground () : Object; + public function set opaqueBackground (value:Object) : void; + + /// Indicates the DisplayObjectContainer object that contains this display object. + public function get parent () : DisplayObjectContainer; + + /// For a display object in a loaded SWF file, the root property is the top-most display object in the portion of the display list's tree structure represented by that SWF file. + public function get root () : DisplayObject; + + /// Indicates the rotation of the DisplayObject instance, in degrees, from its original orientation. + public function get rotation () : Number; + public function set rotation (value:Number) : void; + + /// Indicates the x-axis rotation of the DisplayObject instance, in degrees, from its original orientation relative to the 3D parent container. + public function get rotationX () : Number; + public function set rotationX (value:Number) : void; + + /// Indicates the y-axis rotation of the DisplayObject instance, in degrees, from its original orientation relative to the 3D parent container. + public function get rotationY () : Number; + public function set rotationY (value:Number) : void; + + /// Indicates the z-axis rotation of the DisplayObject instance, in degrees, from its original orientation relative to the 3D parent container. + public function get rotationZ () : Number; + public function set rotationZ (value:Number) : void; + + /// The current scaling grid that is in effect. + public function get scale9Grid () : Rectangle; + public function set scale9Grid (innerRectangle:Rectangle) : void; + + /// Indicates the horizontal scale (percentage) of the object as applied from the registration point. + public function get scaleX () : Number; + public function set scaleX (value:Number) : void; + + /// Indicates the vertical scale (percentage) of an object as applied from the registration point of the object. + public function get scaleY () : Number; + public function set scaleY (value:Number) : void; + + /// Indicates the depth scale (percentage) of an object as applied from the registration point of the object. + public function get scaleZ () : Number; + public function set scaleZ (value:Number) : void; + + /// The scroll rectangle bounds of the display object. + public function get scrollRect () : Rectangle; + public function set scrollRect (value:Rectangle) : void; + + /// The Stage of the display object. + public function get stage () : Stage; + + /// An object with properties pertaining to a display object's matrix, color transform, and pixel bounds. + public function get transform () : Transform; + public function set transform (value:Transform) : void; + + /// Indicates the width of the display object, in pixels. + public function get width () : Number; + public function set width (value:Number) : void; + + /// Whether or not the display object is visible. + public function get visible () : Boolean; + public function set visible (value:Boolean) : void; + + /// Indicates the x coordinate of the DisplayObject instance relative to the local coordinates of the parent DisplayObjectContainer. + public function get x () : Number; + public function set x (value:Number) : void; + + /// Indicates the y coordinate of the DisplayObject instance relative to the local coordinates of the parent DisplayObjectContainer. + public function get y () : Number; + public function set y (value:Number) : void; + + /// Indicates the z coordinate position along the z-axis of the DisplayObject instance relative to the 3D parent container. + public function get z () : Number; + public function set z (value:Number) : void; + + public function DisplayObject (); + + /// Returns a rectangle that defines the area of the display object relative to the coordinate system of the targetCoordinateSpace object. + public function getBounds (targetCoordinateSpace:DisplayObject) : Rectangle; + + /// Returns a rectangle that defines the boundary of the display object, based on the coordinate system defined by the targetCoordinateSpace parameter, excluding any strokes on shapes. + public function getRect (targetCoordinateSpace:DisplayObject) : Rectangle; + + /// Converts the point object from Stage (global) coordinates to the display object's (local) coordinates. + public function globalToLocal (point:Point) : Point; + + /// Converts a two-dimensional point from the Stage (global) coordinates to a three-dimensional display object's (local) coordinates. + public function globalToLocal3D (point:Point) : Vector3D; + + /// Evaluates the display object to see if it overlaps or intersects with the display object passed as a parameter. + public function hitTestObject (obj:DisplayObject) : Boolean; + + /// Evaluates the display object to see if it overlaps or intersects with a point specified by x and y. + public function hitTestPoint (x:Number, y:Number, shapeFlag:Boolean = false) : Boolean; + + /// Converts a three-dimensional point of the three-dimensional display object's (local) coordinates to a two-dimensional point in the Stage (global) coordinates. + public function local3DToGlobal (point3d:Vector3D) : Point; + + /// Converts the point object from the display object's (local) coordinates to the Stage (global) coordinates. + public function localToGlobal (point:Point) : Point; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObjectContainer.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/DisplayObjectContainer.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,66 @@ +package flash.display +{ + import flash.display.DisplayObject; + import flash.text.TextSnapshot; + import flash.geom.Point; + + /// The DisplayObjectContainer class is the base class for all objects that can serve as display object containers on the display list. + public class DisplayObjectContainer extends InteractiveObject + { + /// Determines whether or not the children of the object are mouse enabled. + public function get mouseChildren () : Boolean; + public function set mouseChildren (enable:Boolean) : void; + + /// Returns the number of children of this object. + public function get numChildren () : int; + + /// Determines whether the children of the object are tab enabled. + public function get tabChildren () : Boolean; + public function set tabChildren (enable:Boolean) : void; + + /// Returns a TextSnapshot object for this DisplayObjectContainer instance. + public function get textSnapshot () : TextSnapshot; + + /// Adds a child object to this DisplayObjectContainer instance. + public function addChild (child:DisplayObject) : DisplayObject; + + /// Adds a child object to this DisplayObjectContainer instance. + public function addChildAt (child:DisplayObject, index:int) : DisplayObject; + + /// Indicates whether the security restrictions would cause any display objects to be omitted from the list returned by calling the DisplayObjectContainer.getObjectsUnderPoint() method with the specified point point. + public function areInaccessibleObjectsUnderPoint (point:Point) : Boolean; + + /// Determines whether the specified display object is a child of the DisplayObjectContainer instance or the instance itself. + public function contains (child:DisplayObject) : Boolean; + + /// Calling the new DisplayObjectContainer() constructor throws an ArgumentError exception. + public function DisplayObjectContainer (); + + /// Returns the child display object instance that exists at the specified index. + public function getChildAt (index:int) : DisplayObject; + + /// Returns the child display object that exists with the specified name. + public function getChildByName (name:String) : DisplayObject; + + /// Returns the index number of a child DisplayObject instance. + public function getChildIndex (child:DisplayObject) : int; + + /// Returns an array of objects that lie under the specified point and are children (or grandchildren, and so on) of this DisplayObjectContainer instance. + public function getObjectsUnderPoint (point:Point) : Array; + + /// Removes a child display object from the DisplayObjectContainer instance. + public function removeChild (child:DisplayObject) : DisplayObject; + + /// Removes a child display object, at the specified index position, from the DisplayObjectContainer instance. + public function removeChildAt (index:int) : DisplayObject; + + /// Changes the index number of an existing child. + public function setChildIndex (child:DisplayObject, index:int) : void; + + /// Swaps the z-order (front-to-back order) of the two specified child objects. + public function swapChildren (child1:DisplayObject, child2:DisplayObject) : void; + + /// Swaps the z-order (front-to-back order) of the child objects at the two specified index positions in the child list. + public function swapChildrenAt (index1:int, index2:int) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/FrameLabel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/FrameLabel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.display +{ + /// The FrameLabel object contains properties that specify a frame number and the corresponding label name. + public class FrameLabel extends Object + { + /// The frame number containing the label. + public function get frame () : int; + + /// The name of the label. + public function get name () : String; + + public function FrameLabel (name:String, frame:int); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GradientType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GradientType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// The GradientType class provides values for the type parameter in the beginGradientFill() and lineGradientStyle() methods of the flash.display.Graphics class. + public class GradientType extends Object + { + /// Value used to specify a linear gradient fill. + public static const LINEAR : String; + /// Value used to specify a radial gradient fill. + public static const RADIAL : String; + + public function GradientType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Graphics.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Graphics.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,82 @@ +package flash.display +{ + import flash.display.Shader; + import flash.geom.Matrix; + import flash.display.BitmapData; + import flash.display.IGraphicsStroke; + import flash.display.IGraphicsPath; + import flash.display.IGraphicsFill; + import flash.display.IGraphicsData; + import flash.display.Graphics; + + /// The Graphics class contains a set of methods that you can use to create a vector shape. + public class Graphics extends Object + { + /// Begins a bitmap filled shape. + public function beginBitmapFill (bitmap:BitmapData, matrix:Matrix = null, repeat:Boolean = true, smooth:Boolean = false) : void; + + /// Specifies a single-color fill. + public function beginFill (color:uint, alpha:Number = 1) : void; + + /// Specifies a gradient fill. + public function beginGradientFill (type:String, colors:Array, alphas:Array, ratios:Array, matrix:Matrix = null, spreadMethod:String = "pad", interpolationMethod:String = "rgb", focalPointRatio:Number = 0) : void; + + /// Specifies a shader fill. + public function beginShaderFill (shader:Shader, matrix:Matrix = null) : void; + + /// Clears the graphics that were drawn to this Graphics object, and resets fill and line style settings. + public function clear () : void; + + /// Copies all of drawing commands from the source Graphics object into the calling Graphics object. + public function copyFrom (sourceGraphics:Graphics) : void; + + /// Draws a curve from the current drawing position to (anchorX, anchorY) using the control point specified by (controlX, controlY). + public function curveTo (controlX:Number, controlY:Number, anchorX:Number, anchorY:Number) : void; + + /// Draws a circle. + public function drawCircle (x:Number, y:Number, radius:Number) : void; + + /// Draws an ellipse. + public function drawEllipse (x:Number, y:Number, width:Number, height:Number) : void; + + /// Submits a series of IGraphicsData instances for drawing. + public function drawGraphicsData (graphicsData:Vector.) : void; + + /// Submits a series of commands for drawing. + public function drawPath (commands:Vector., data:Vector., winding:String = "evenOdd") : void; + + /// Draws a round rectangle. + public function drawRect (x:Number, y:Number, width:Number, height:Number) : void; + + /// Draws a round rectangle. + public function drawRoundRect (x:Number, y:Number, width:Number, height:Number, ellipseWidth:Number, ellipseHeight:Number = null) : void; + + public function drawRoundRectComplex (x:Number, y:Number, width:Number, height:Number, topLeftRadius:Number, topRightRadius:Number, bottomLeftRadius:Number, bottomRightRadius:Number) : void; + + /// Renders a set of triangles, typically to distort bitmaps and give them a three-dimensional appearance. + public function drawTriangles (vertices:Vector., indices:Vector. = null, uvtData:Vector. = null, culling:String = "none") : void; + + /// Applies a fill to the lines and curves. + public function endFill () : void; + + public function Graphics (); + + /// Specifies a bitmap to use for the line stroke when drawing lines. + public function lineBitmapStyle (bitmap:BitmapData, matrix:Matrix = null, repeat:Boolean = true, smooth:Boolean = false) : void; + + /// Specifies a gradient to use for the stroke when drawing lines. + public function lineGradientStyle (type:String, colors:Array, alphas:Array, ratios:Array, matrix:Matrix = null, spreadMethod:String = "pad", interpolationMethod:String = "rgb", focalPointRatio:Number = 0) : void; + + /// Specifies a shader to use for the line stroke when drawing lines. + public function lineShaderStyle (shader:Shader, matrix:Matrix = null) : void; + + /// Specifies a line style that Flash uses for drawing lines. + public function lineStyle (thickness:Number = null, color:uint = 0, alpha:Number = 1, pixelHinting:Boolean = false, scaleMode:String = "normal", caps:String = null, joints:String = null, miterLimit:Number = 3) : void; + + /// Draws a line from the current drawing position to (x, y). + public function lineTo (x:Number, y:Number) : void; + + /// Moves the current drawing position to (x, y). + public function moveTo (x:Number, y:Number) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsBitmapFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsBitmapFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package flash.display +{ + import flash.geom.Matrix; + import flash.display.BitmapData; + + /// Defines a bitmap fill. + public class GraphicsBitmapFill extends Object implements IGraphicsFill, IGraphicsData + { + /// A transparent or opaque bitmap image. + public var bitmapData : BitmapData; + /// A matrix object (of the flash.geom.Matrix class) that defines transformations on the bitmap. + public var matrix : Matrix; + /// Specifies whether to repeat the bitmap image in a tiled pattern. + public var repeat : Boolean; + /// Specifies whether to apply a smoothing algorithm to the bitmap image. + public var smooth : Boolean; + + /// Creates a new GraphicsBitmapFill object. + public function GraphicsBitmapFill (bitmapData:BitmapData = null, matrix:Matrix = null, repeat:Boolean = true, smooth:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsEndFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsEndFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.display +{ + /// Indicates the end of a graphics fill. + public class GraphicsEndFill extends Object implements IGraphicsFill, IGraphicsData + { + /// Creates an object to use with the Graphics.drawGraphicsData() method to end the fill, explicitly. + public function GraphicsEndFill (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsGradientFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsGradientFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,34 @@ +package flash.display +{ + import flash.geom.Matrix; + + /// Defines a gradient fill. + public class GraphicsGradientFill extends Object implements IGraphicsFill, IGraphicsData + { + /// An array of alpha values for the corresponding colors in the colors array. + public var alphas : Array; + /// An array of RGB hexadecimal color values to use in the gradient. + public var colors : Array; + /// A number that controls the location of the focal point of the gradient. + public var focalPointRatio : Number; + /// A transformation matrix as defined by the Matrix class. + public var matrix : Matrix; + /// An array of color distribution ratios. + public var ratios : Array; + + /// A value from the InterpolationMethod class that specifies which value to use. + public function get interpolationMethod () : String; + public function set interpolationMethod (value:String) : void; + + /// A value from the SpreadMethod class that specifies which spread method to use. + public function get spreadMethod () : String; + public function set spreadMethod (value:String) : void; + + /// A value from the GradientType class that specifies which gradient type to use. + public function get type () : String; + public function set type (value:String) : void; + + /// Creates a new GraphicsGradientFill object. + public function GraphicsGradientFill (type:String = "linear", colors:Array = null, alphas:Array = null, ratios:Array = null, matrix:* = null, spreadMethod:* = "pad", interpolationMethod:String = "rgb", focalPointRatio:Number = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPath.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPath.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,33 @@ +package flash.display +{ + /// A collection of drawing commands and the coordinate parameters for those commands. + public class GraphicsPath extends Object implements IGraphicsPath, IGraphicsData + { + /// The Vector of drawing commands as integers representing the path. + public var commands : Vector.; + /// The Vector of Numbers containing the parameters used with the drawing commands. + public var data : Vector.; + + /// Specifies the winding rule using a value defined in the GraphicsPathWinding class. + public function get winding () : String; + public function set winding (value:String) : void; + + /// Adds a new "curveTo" command to the commands vector and new coordinates to the data vector. + public function curveTo (controlX:Number, controlY:Number, anchorX:Number, anchorY:Number) : void; + + /// Creates a new GraphicsPath object. + public function GraphicsPath (commands:Vector. = null, data:Vector. = null, winding:String = "evenOdd"); + + /// Adds a new "lineTo" command to the commands vector and new coordinates to the data vector. + public function lineTo (x:Number, y:Number) : void; + + /// Adds a new "moveTo" command to the commands vector and new coordinates to the data vector. + public function moveTo (x:Number, y:Number) : void; + + /// Adds a new "wideLineTo" command to the commands vector and new coordinates to the data vector. + public function wideLineTo (x:Number, y:Number) : void; + + /// Adds a new "wideMoveTo" command to the commands vector and new coordinates to the data vector. + public function wideMoveTo (x:Number, y:Number) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathCommand.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathCommand.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package flash.display +{ + /// Defines the values to use for specifying path-drawing commands. + public class GraphicsPathCommand extends Object + { + /// Specifies a drawing command that draws a curve from the current drawing position to the x- and y-coordinates specified in the data vector, using a control point. + public static const CURVE_TO : int; + /// Specifies a drawing command that draws a line from the current drawing position to the x- and y-coordinates specified in the data vector. + public static const LINE_TO : int; + /// Specifies a drawing command that moves the current drawing position to the x- and y-coordinates specified in the data vector. + public static const MOVE_TO : int; + /// Represents the default "do nothing" command. + public static const NO_OP : int; + /// Specifies a "line to" drawing command, but uses two sets of coordinates (four values) instead of one set. + public static const WIDE_LINE_TO : int; + /// Specifies a "move to" drawing command, but uses two sets of coordinates (four values) instead of one set. + public static const WIDE_MOVE_TO : int; + + public function GraphicsPathCommand (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathWinding.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsPathWinding.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// The GraphicsPathWinding class provides values for the flash.display.GraphicsPath.winding property and the flash.display.Graphics.drawPath() methodto determine the direction to draw a path. + public class GraphicsPathWinding extends Object + { + /// Establishes the even-odd winding type. + public static const EVEN_ODD : String; + /// Establishes the non-zero winding type. + public static const NON_ZERO : String; + + public function GraphicsPathWinding (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsShaderFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsShaderFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + import flash.geom.Matrix; + import flash.display.Shader; + + /// Defines a shader fill. + public class GraphicsShaderFill extends Object implements IGraphicsFill, IGraphicsData + { + /// A matrix object (of the flash.geom.Matrix class), which you can use to define transformations on the shader. + public var matrix : Matrix; + /// The shader to use for the fill. + public var shader : Shader; + + /// Creates a new GraphicsShaderFill object. + public function GraphicsShaderFill (shader:Shader = null, matrix:Matrix = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsSolidFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsSolidFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.display +{ + /// Defines a solid fill. + public class GraphicsSolidFill extends Object implements IGraphicsFill, IGraphicsData + { + /// Indicates the alpha transparency value of the fill. + public var alpha : Number; + /// The color of the fill. + public var color : uint; + + /// Creates a new GraphicsSolidFill object. + public function GraphicsSolidFill (color:uint = 0, alpha:Number = 1); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsStroke.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsStroke.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,32 @@ +package flash.display +{ + import flash.display.IGraphicsFill; + + /// Defines a line style or stroke. + public class GraphicsStroke extends Object implements IGraphicsStroke, IGraphicsData + { + /// Specifies the instance containing data for filling a stroke. + public var fill : IGraphicsFill; + /// Indicates the limit at which a miter is cut off. + public var miterLimit : Number; + /// Specifies whether to hint strokes to full pixels. + public var pixelHinting : Boolean; + /// Indicates the thickness of the line in points; valid values are 0-255. + public var thickness : Number; + + /// Specifies the type of caps at the end of lines. + public function get caps () : String; + public function set caps (value:String) : void; + + /// Specifies the type of joint appearance used at angles. + public function get joints () : String; + public function set joints (value:String) : void; + + /// Specifies the stroke thickness scaling. + public function get scaleMode () : String; + public function set scaleMode (value:String) : void; + + /// Creates a new GraphicsStroke object. + public function GraphicsStroke (thickness:Number = NaN, pixelHinting:Boolean = false, scaleMode:String = "normal", caps:String = "none", joints:String = "round", miterLimit:Number = 3, fill:IGraphicsFill = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsTrianglePath.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/GraphicsTrianglePath.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.display +{ + /// Defines an ordered set of triangles that can be rendered using either (u,v) fill coordinates or a normal fill. + public class GraphicsTrianglePath extends Object implements IGraphicsPath, IGraphicsData + { + /// A Vector of integers or indexes, where every three indexes define a triangle. + public var indices : Vector.; + /// A Vector of normalized coordinates used to apply texture mapping. + public var uvtData : Vector.; + /// A Vector of Numbers where each pair of numbers is treated as a point (an x, y pair). + public var vertices : Vector.; + + /// Specifies whether to render triangles that face in a given direction. + public function get culling () : String; + public function set culling (value:String) : void; + + /// Creates a new GraphicsTrianglePath object. + public function GraphicsTrianglePath (vertices:Vector. = null, indices:Vector. = null, uvtData:Vector. = null, culling:String = "none"); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IBitmapDrawable.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IBitmapDrawable.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.display +{ + public interface IBitmapDrawable + { + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsData.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsData.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.display +{ + public interface IGraphicsData + { + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsFill.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsFill.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.display +{ + public interface IGraphicsFill + { + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsPath.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsPath.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.display +{ + public interface IGraphicsPath + { + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsStroke.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/IGraphicsStroke.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.display +{ + public interface IGraphicsStroke + { + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InteractiveObject.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InteractiveObject.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,183 @@ +package flash.display +{ + import flash.accessibility.AccessibilityImplementation; + import flash.ui.ContextMenu; + + /** + * Dispatched when the value of the object's tabIndex property changes. + * @eventType flash.events.Event.TAB_INDEX_CHANGE + */ + [Event(name="tabIndexChange", type="flash.events.Event")] + + /** + * Dispatched when the object's tabEnabled flag changes. + * @eventType flash.events.Event.TAB_ENABLED_CHANGE + */ + [Event(name="tabEnabledChange", type="flash.events.Event")] + + /** + * Dispatched when the value of the object's tabChildren flag changes. + * @eventType flash.events.Event.TAB_CHILDREN_CHANGE + */ + [Event(name="tabChildrenChange", type="flash.events.Event")] + + /** + * Dispatched when the user releases a key. + * @eventType flash.events.KeyboardEvent.KEY_UP + */ + [Event(name="keyUp", type="flash.events.KeyboardEvent")] + + /** + * Dispatched when the user presses a key. + * @eventType flash.events.KeyboardEvent.KEY_DOWN + */ + [Event(name="keyDown", type="flash.events.KeyboardEvent")] + + /** + * Dispatched when the user moves a pointing device over an InteractiveObject instance. + * @eventType flash.events.MouseEvent.ROLL_OVER + */ + [Event(name="rollOver", type="flash.events.MouseEvent")] + + /** + * Dispatched when the user moves a pointing device away from an InteractiveObject instance. + * @eventType flash.events.MouseEvent.ROLL_OUT + */ + [Event(name="rollOut", type="flash.events.MouseEvent")] + + /** + * Dispatched when a mouse wheel is spun over an InteractiveObject instance in the Flash Player window. + * @eventType flash.events.MouseEvent.MOUSE_WHEEL + */ + [Event(name="mouseWheel", type="flash.events.MouseEvent")] + + /** + * Dispatched when a user releases the pointing device button over an InteractiveObject instance in the Flash Player window. + * @eventType flash.events.MouseEvent.MOUSE_UP + */ + [Event(name="mouseUp", type="flash.events.MouseEvent")] + + /** + * Dispatched when the user moves a pointing device over an InteractiveObject instance in the Flash Player window. + * @eventType flash.events.MouseEvent.MOUSE_OVER + */ + [Event(name="mouseOver", type="flash.events.MouseEvent")] + + /** + * Dispatched when the user moves a pointing device away from an InteractiveObject instance. + * @eventType flash.events.MouseEvent.MOUSE_OUT + */ + [Event(name="mouseOut", type="flash.events.MouseEvent")] + + /** + * Dispatched when a user moves the pointing device while it is over an InteractiveObject. + * @eventType flash.events.MouseEvent.MOUSE_MOVE + */ + [Event(name="mouseMove", type="flash.events.MouseEvent")] + + /** + * Dispatched when a user presses the pointing device button over an InteractiveObject instance in the Flash Player window. + * @eventType flash.events.MouseEvent.MOUSE_DOWN + */ + [Event(name="mouseDown", type="flash.events.MouseEvent")] + + /** + * Dispatched when a user presses and releases the main button of a pointing device twice in rapid succession over the same InteractiveObject when that object's doubleClickEnabled flag is set to true. + * @eventType flash.events.MouseEvent.DOUBLE_CLICK + */ + [Event(name="doubleClick", type="flash.events.MouseEvent")] + + /** + * Dispatched when a user presses and releases the main button of the user's pointing device over the same InteractiveObject. + * @eventType flash.events.MouseEvent.CLICK + */ + [Event(name="click", type="flash.events.MouseEvent")] + + /** + * Dispatched when the user attempts to change focus by using a pointer device. + * @eventType flash.events.FocusEvent.MOUSE_FOCUS_CHANGE + */ + [Event(name="mouseFocusChange", type="flash.events.FocusEvent")] + + /** + * Dispatched when the user attempts to change focus by using keyboard navigation. + * @eventType flash.events.FocusEvent.KEY_FOCUS_CHANGE + */ + [Event(name="keyFocusChange", type="flash.events.FocusEvent")] + + /** + * Dispatched after a display object loses focus. + * @eventType flash.events.FocusEvent.FOCUS_OUT + */ + [Event(name="focusOut", type="flash.events.FocusEvent")] + + /** + * Dispatched after a display object gains focus. + * @eventType flash.events.FocusEvent.FOCUS_IN + */ + [Event(name="focusIn", type="flash.events.FocusEvent")] + + /** + * Dispatched when the user activates the platform specific accelerator key combination for a select all operation or selects 'Select All' from the text context menu. + * @eventType flash.events.Event.SELECT_ALL + */ + [Event(name="selectAll", type="flash.events.Event")] + + /** + * Dispatched when the user activates the platform specific accelerator key combination for a paste operation or selects 'Paste' from the text context menu. + * @eventType flash.events.Event.PASTE + */ + [Event(name="paste", type="flash.events.Event")] + + /** + * Dispatched when the user activates the platform specific accelerator key combination for a cut operation or selects 'Cut' from the text context menu. + * @eventType flash.events.Event.CUT + */ + [Event(name="cut", type="flash.events.Event")] + + /** + * Dispatched when the user activates the platform specific accelerator key combination for a copy operation or selects 'Copy' from the text context menu. + * @eventType flash.events.Event.COPY + */ + [Event(name="copy", type="flash.events.Event")] + + /** + * Dispatched when the user selects 'Clear' (or 'Delete') from the text context menu. + * @eventType flash.events.Event.CLEAR + */ + [Event(name="clear", type="flash.events.Event")] + + /// The InteractiveObject class is the abstract base class for all display objects with which the user can interact, using the mouse and keyboard. + public class InteractiveObject extends DisplayObject + { + public function get accessibilityImplementation () : AccessibilityImplementation; + public function set accessibilityImplementation (value:AccessibilityImplementation) : void; + + /// Specifies the context menu associated with this object. + public function get contextMenu () : ContextMenu; + public function set contextMenu (cm:ContextMenu) : void; + + /// Specifies whether the object receives doubleClick events. + public function get doubleClickEnabled () : Boolean; + public function set doubleClickEnabled (enabled:Boolean) : void; + + /// Specifies whether this object displays a focus rectangle. + public function get focusRect () : Object; + public function set focusRect (focusRect:Object) : void; + + /// Specifies whether this object receives mouse messages. + public function get mouseEnabled () : Boolean; + public function set mouseEnabled (enabled:Boolean) : void; + + /// Specifies whether this object is in the tab order. + public function get tabEnabled () : Boolean; + public function set tabEnabled (enabled:Boolean) : void; + + /// Specifies the tab ordering of objects in a SWF file. + public function get tabIndex () : int; + public function set tabIndex (index:int) : void; + + /// Calling the new InteractiveObject() constructor throws an ArgumentError exception. + public function InteractiveObject (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InterpolationMethod.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/InterpolationMethod.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// The InterpolationMethod class provides values for the interpolationMethod parameter in the Graphics.beginGradientFill() and Graphics.lineGradientStyle() methods. + public class InterpolationMethod extends Object + { + /// Specifies that the linear RGB interpolation method should be used. + public static const LINEAR_RGB : String; + /// Specifies that the RGB interpolation method should be used. + public static const RGB : String; + + public function InterpolationMethod (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/JointStyle.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/JointStyle.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The JointStyle class is an enumeration of constant values that specify the joint style to use in drawing lines. + public class JointStyle extends Object + { + /// Specifies beveled joints in the joints parameter of the flash.display.Graphics.lineStyle() method. + public static const BEVEL : String; + /// Specifies mitered joints in the joints parameter of the flash.display.Graphics.lineStyle() method. + public static const MITER : String; + /// Specifies round joints in the joints parameter of the flash.display.Graphics.lineStyle() method. + public static const ROUND : String; + + public function JointStyle (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LineScaleMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LineScaleMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + /// The LineScaleMode class provides values for the scaleMode parameter in the Graphics.lineStyle() method. + public class LineScaleMode extends Object + { + /// With this setting used as the scaleMode parameter of the lineStyle() method, the thickness of the line scales only vertically. + public static const HORIZONTAL : String; + /// With this setting used as the scaleMode parameter of the lineStyle() method, the thickness of the line never scales. + public static const NONE : String; + /// With this setting used as the scaleMode parameter of the lineStyle() method, the thickness of the line always scales when the object is scaled (the default). + public static const NORMAL : String; + /// With this setting used as the scaleMode parameter of the lineStyle() method, the thickness of the line scales only horizontally. + public static const VERTICAL : String; + + public function LineScaleMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Loader.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Loader.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,48 @@ +package flash.display +{ + import flash.display.LoaderInfo; + import flash.net.URLRequest; + import flash.system.ApplicationDomain; + import flash.system.SecurityDomain; + import flash.system.LoaderContext; + import flash.display.DisplayObject; + import flash.utils.ByteArray; + + /// The Loader class is used to load SWF files or image (JPG, PNG, or GIF) files. + public class Loader extends DisplayObjectContainer + { + /// Contains the root display object of the SWF file or image (JPG, PNG, or GIF) file that was loaded by using the load() or loadBytes() methods. + public function get content () : DisplayObject; + + /// Returns a LoaderInfo object corresponding to the object being loaded. + public function get contentLoaderInfo () : LoaderInfo; + + public function addChild (child:DisplayObject) : DisplayObject; + + public function addChildAt (child:DisplayObject, index:int) : DisplayObject; + + /// Cancels a load() method operation that is currently in progress for the Loader instance. + public function close () : void; + + /// Loads a SWF file or image file into a DisplayObject that is a child of this Loader instance. + public function load (request:URLRequest, context:LoaderContext = null) : void; + + /// Loads from binary data stored in a ByteArray object. + public function loadBytes (bytes:ByteArray, context:LoaderContext = null) : void; + + /// Creates a Loader object that you can use to load files, such as SWF, JPEG, GIF, or PNG files. + public function Loader (); + + public function removeChild (child:DisplayObject) : DisplayObject; + + public function removeChildAt (index:int) : DisplayObject; + + public function setChildIndex (child:DisplayObject, index:int) : void; + + /// Removes a child of this Loader object that was loaded by using the load() method. + public function unload () : void; + + /// Attempts to unload child SWF file contents and stops the execution of commands from loaded SWF files. + public function unloadAndStop (gc:Boolean = true) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LoaderInfo.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/LoaderInfo.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,120 @@ +package flash.display +{ + import flash.events.EventDispatcher; + import flash.display.LoaderInfo; + import flash.events.Event; + import flash.utils.ByteArray; + import flash.system.ApplicationDomain; + import flash.display.Loader; + import flash.display.DisplayObject; + + /** + * Dispatched when a network request is made over HTTP and Flash Player can detect the HTTP status code. + * @eventType flash.events.HTTPStatusEvent.HTTP_STATUS + */ + [Event(name="httpStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched by a LoaderInfo object whenever a loaded object is removed by using the unload() method of the Loader object, or when a second load is performed by the same Loader object and the original content is removed prior to the load beginning. + * @eventType flash.events.Event.UNLOAD + */ + [Event(name="unload", type="flash.events.Event")] + + /** + * Dispatched when data is received as the download operation progresses. + * @eventType flash.events.ProgressEvent.PROGRESS + */ + [Event(name="progress", type="flash.events.ProgressEvent")] + + /** + * Dispatched when a load operation starts. + * @eventType flash.events.Event.OPEN + */ + [Event(name="open", type="flash.events.Event")] + + /** + * Dispatched when an input or output error occurs that causes a load operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when the properties and methods of a loaded SWF file are accessible and ready for use. + * @eventType flash.events.Event.INIT + */ + [Event(name="init", type="flash.events.Event")] + + /** + * Dispatched when data has loaded successfully. + * @eventType flash.events.Event.COMPLETE + */ + [Event(name="complete", type="flash.events.Event")] + + /// The LoaderInfo class provides information about a loaded SWF file or a loaded image file (JPEG, GIF, or PNG). + public class LoaderInfo extends EventDispatcher + { + /// The ActionScript version of the loaded SWF file. + public function get actionScriptVersion () : uint; + + /// When an external SWF file is loaded, all ActionScript 3.0 definitions contained in the loaded class are stored in the applicationDomain property. + public function get applicationDomain () : ApplicationDomain; + + /// The bytes associated with a LoaderInfo object. + public function get bytes () : ByteArray; + + /// The number of bytes that are loaded for the media. + public function get bytesLoaded () : uint; + + /// The number of compressed bytes in the entire media file. + public function get bytesTotal () : uint; + + /// Expresses the trust relationship from content (child) to the Loader (parent). + public function get childAllowsParent () : Boolean; + + /// The loaded object associated with this LoaderInfo object. + public function get content () : DisplayObject; + + /// The MIME type of the loaded file. + public function get contentType () : String; + + /// The nominal frame rate, in frames per second, of the loaded SWF file. + public function get frameRate () : Number; + + /// The nominal height of the loaded file. + public function get height () : int; + + /// The Loader object associated with this LoaderInfo object. + public function get loader () : Loader; + + /// The URL of the SWF file that initiated the loading of the media described by this LoaderInfo object. + public function get loaderURL () : String; + + /// An object that contains name-value pairs that represent the parameters provided to the loaded SWF file. + public function get parameters () : Object; + + /// Expresses the trust relationship from Loader (parent) to the content (child). + public function get parentAllowsChild () : Boolean; + + /// Expresses the domain relationship between the loader and the content: true if they have the same origin domain; false otherwise. + public function get sameDomain () : Boolean; + + /// An EventDispatcher instance that can be used to exchange events across security boundaries. + public function get sharedEvents () : EventDispatcher; + + /// The file format version of the loaded SWF file. + public function get swfVersion () : uint; + + /// The URL of the media being loaded. + public function get url () : String; + + /// The nominal width of the loaded content. + public function get width () : int; + + public function dispatchEvent (event:Event) : Boolean; + + /// Returns the LoaderInfo object associated with a SWF file defined as an object. + public static function getLoaderInfoByDefinition (object:Object) : LoaderInfo; + + public function LoaderInfo (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MorphShape.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MorphShape.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.display +{ + public class MorphShape extends DisplayObject + { + public function MorphShape (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MovieClip.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/MovieClip.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,70 @@ +package flash.display +{ + import flash.display.Scene; + + /// The MovieClip class inherits from the following classes: Sprite, DisplayObjectContainer, InteractiveObject, DisplayObject, and EventDispatcher. + public class MovieClip extends Sprite + { + /// Specifies the number of the frame in which the playhead is located in the timeline of the MovieClip instance. + public function get currentFrame () : int; + + /// The label at the current frame in the timeline of the MovieClip instance. + public function get currentFrameLabel () : String; + + /// The current label in which the playhead is located in the timeline of the MovieClip instance. + public function get currentLabel () : String; + + /// Returns an array of FrameLabel objects from the current scene. + public function get currentLabels () : Array; + + /// The current scene in which the playhead is located in the timeline of the MovieClip instance. + public function get currentScene () : Scene; + + /// A Boolean value that indicates whether a movie clip is enabled. + public function get enabled () : Boolean; + public function set enabled (value:Boolean) : void; + + /// The number of frames that are loaded from a streaming SWF file. + public function get framesLoaded () : int; + + /// An array of Scene objects, each listing the name, the number of frames, and the frame labels for a scene in the MovieClip instance. + public function get scenes () : Array; + + /// The total number of frames in the MovieClip instance. + public function get totalFrames () : int; + + /// Indicates whether other display objects that are SimpleButton or MovieClip objects can receive mouse release events. + public function get trackAsMenu () : Boolean; + public function set trackAsMenu (value:Boolean) : void; + + /// [Undocumented] Takes a collection of frame (zero-based) - method pairs that associates a method with a frame on the timeline. + public function addFrameScript (frame:int, method:Function) : void; + + /// Starts playing the SWF file at the specified frame. + public function gotoAndPlay (frame:Object, scene:String = null) : void; + + /// Brings the playhead to the specified frame of the movie clip and stops it there. + public function gotoAndStop (frame:Object, scene:String = null) : void; + + /// Creates a new MovieClip instance. + public function MovieClip (); + + /// Sends the playhead to the next frame and stops it. + public function nextFrame () : void; + + /// Moves the playhead to the next scene of the MovieClip instance. + public function nextScene () : void; + + /// Moves the playhead in the timeline of the movie clip. + public function play () : void; + + /// Sends the playhead to the previous frame and stops it. + public function prevFrame () : void; + + /// Moves the playhead to the previous scene of the MovieClip instance. + public function prevScene () : void; + + /// Stops the playhead in the movie clip. + public function stop () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/PixelSnapping.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/PixelSnapping.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The PixelSnapping class is an enumeration of constant values for setting the pixel snapping options by using the pixelSnapping property of a Bitmap object. + public class PixelSnapping extends Object + { + /// A constant value used in the pixelSnapping property of a Bitmap object to specify that the bitmap image is always snapped to the nearest pixel, independent of any transformation. + public static const ALWAYS : String; + /// A constant value used in the pixelSnapping property of a Bitmap object to specify that the bitmap image is snapped to the nearest pixel if it is drawn with no rotation or skew and it is drawn at a scale factor of 99.9% to 100.1%. + public static const AUTO : String; + /// A constant value used in the pixelSnapping property of a Bitmap object to specify that no pixel snapping occurs. + public static const NEVER : String; + + public function PixelSnapping (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SWFVersion.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SWFVersion.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,29 @@ +package flash.display +{ + /// The SWFVersion class is an enumeration of constant values that indicate the file format version of a loaded SWF file. + public class SWFVersion extends Object + { + /// SWF file format version 1.0. + public static const FLASH1 : uint; + /// SWF file format version 10.0. + public static const FLASH10 : uint; + /// SWF file format version 2.0. + public static const FLASH2 : uint; + /// SWF file format version 3.0. + public static const FLASH3 : uint; + /// SWF file format version 4.0. + public static const FLASH4 : uint; + /// SWF file format version 5.0. + public static const FLASH5 : uint; + /// SWF file format version 6.0. + public static const FLASH6 : uint; + /// SWF file format version 7.0. + public static const FLASH7 : uint; + /// SWF file format version 8.0. + public static const FLASH8 : uint; + /// SWF file format version 9.0. + public static const FLASH9 : uint; + + public function SWFVersion (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Scene.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Scene.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + /// The Scene class includes properties for identifying the name, labels, and number of frames in a scene. + public class Scene extends Object + { + /// An array of FrameLabel objects for the scene. + public function get labels () : Array; + + /// The name of the scene. + public function get name () : String; + + /// The number of frames in the scene. + public function get numFrames () : int; + + public function Scene (name:String, labels:Array, numFrames:int); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shader.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shader.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.display +{ + import flash.utils.ByteArray; + import flash.display.ShaderData; + + /// A Shader instance represents a pixel shader in ActionScript. + public class Shader extends Object + { + /// The raw shader bytecode for this Shader instance. + public function set byteCode (code:ByteArray) : void; + + /// Provides access to parameters, input images, and metadata for the Shader instance. + public function get data () : ShaderData; + public function set data (p:ShaderData) : void; + + /// The precision of math operations performed by the shader. + public function get precisionHint () : String; + public function set precisionHint (p:String) : void; + + /// Creates a new Shader instance. + public function Shader (code:ByteArray = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderData.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderData.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package flash.display +{ + import flash.utils.ByteArray; + + /// A ShaderData object contains properties representing any parameters and inputs for a shader kernel, as well as properties containing any metadata specified for the shader. + public class ShaderData extends Object + { + /// Creates a ShaderData instance. + public function ShaderData (byteCode:ByteArray); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderInput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderInput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,27 @@ +package flash.display +{ + /// A ShaderInput instance represents a single input image for a shader kernel. + public class ShaderInput extends Object + { + /// The number of channels that a shader input expects. + public function get channels () : int; + + /// The height of the shader input. + public function get height () : int; + public function set height (value:int) : void; + + /// The zero-based index of the input in the shader, indicating the order of the input definitions in the shader. + public function get index () : int; + + /// The input data that is used when the shader executes. + public function get input () : Object; + public function set input (input:Object) : void; + + /// The width of the shader input. + public function get width () : int; + public function set width (value:int) : void; + + /// Creates a ShaderInput instance. + public function ShaderInput (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderJob.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderJob.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,43 @@ +package flash.display +{ + import flash.events.EventDispatcher; + import flash.display.Shader; + + /** + * Dispatched when the ShaderJob finishes processing the data using the shader. + * @eventType flash.events.ShaderEvent.COMPLETE + */ + [Event(name="complete", type="flash.events.ShaderEvent")] + + /// A ShaderJob instance is used to execute a shader operation in the background. + public class ShaderJob extends EventDispatcher + { + /// The height of the result data in the target if it is a ByteArray or Vector. instance. + public function get height () : int; + public function set height (v:int) : void; + + /// The progress of a running shader. + public function get progress () : Number; + + /// The shader that's used for the operation. + public function get shader () : Shader; + public function set shader (s:Shader) : void; + + /// The object into which the result of the shader operation is written. + public function get target () : Object; + public function set target (s:Object) : void; + + /// The width of the result data in the target if it is a ByteArray or Vector. instance. + public function get width () : int; + public function set width (v:int) : void; + + /// Cancels the currently running shader operation. + public function cancel () : void; + + /// A ShaderJob instance is used to execute a shader operation in the background. + public function ShaderJob (shader:Shader = null, target:Object = null, width:int = 0, height:int = 0); + + /// Starts a background shader operation. + public function start (waitForCompletion:Boolean = false) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.display +{ + /// A ShaderParameter instance represents a single input parameter of a shader kernel. + public class ShaderParameter extends Object + { + /// The zero-based index of the parameter. + public function get index () : int; + + /// The data type of the parameter as defined in the shader. + public function get type () : String; + + /// The value or values that are passed in as the parameter value to the shader. + public function get value () : Array; + public function set value (v:Array) : void; + + /// Creates a ShaderParameter instance. + public function ShaderParameter (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameterType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderParameterType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +package flash.display +{ + /// This class defines the constants that represent the possible values for the ShaderParameter class's type property. + public class ShaderParameterType extends Object + { + /// Indicates that the shader parameter is defined as a bool value, equivalent to a single Boolean instance in ActionScript. + public static const BOOL : String; + /// Indicates that the shader parameter is defined as a bool2 value, equivalent to an Array of two Boolean instances in ActionScript. + public static const BOOL2 : String; + /// Indicates that the shader parameter is defined as a bool3 value, equivalent to an Array of three Boolean instances in ActionScript. + public static const BOOL3 : String; + /// Indicates that the shader parameter is defined as a bool4 value, equivalent to an Array of four Boolean instances in ActionScript. + public static const BOOL4 : String; + /// Indicates that the shader parameter is defined as a float value, equivalent to a single Number instance in ActionScript. + public static const FLOAT : String; + /// Indicates that the shader parameter is defined as a float2 value, equivalent to an Array of two Number instances in ActionScript. + public static const FLOAT2 : String; + /// Indicates that the shader parameter is defined as a float3 value, equivalent to an Array of three Number instances in ActionScript. + public static const FLOAT3 : String; + /// Indicates that the shader parameter is defined as a float4 value, equivalent to an Array of four Number instances in ActionScript. + public static const FLOAT4 : String; + /// Indicates that the shader parameter is defined as an int value, equivalent to a single int or uint instance in ActionScript. + public static const INT : String; + /// Indicates that the shader parameter is defined as an int2 value, equivalent to an Array of two int or uint instances in ActionScript. + public static const INT2 : String; + /// Indicates that the shader parameter is defined as an int3 value, equivalent to an Array of three int or uint instances in ActionScript. + public static const INT3 : String; + /// Indicates that the shader parameter is defined as an int4 value, equivalent to an Array of four int or uint instances in ActionScript. + public static const INT4 : String; + /// Indicates that the shader parameter is defined as a float2x2 value, equivalent to a 2-by-2 matrix. + public static const MATRIX2X2 : String; + /// Indicates that the shader parameter is defined as a float3x3 value, equivalent to a 3-by-3 matrix. + public static const MATRIX3X3 : String; + /// Indicates that the shader parameter is defined as a float4x4 value, equivalent to a 4-by-4 matrix. + public static const MATRIX4X4 : String; + + public function ShaderParameterType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderPrecision.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/ShaderPrecision.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// This class defines the constants that represent the possible values for the Shader class's precisionHint property. + public class ShaderPrecision extends Object + { + /// Represents fast precision mode. + public static const FAST : String; + /// Represents full precision mode. + public static const FULL : String; + + public function ShaderPrecision (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shape.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Shape.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.display +{ + import flash.display.Graphics; + + /// The Shape class is used to create lightweight shapes by using the ActionScript drawing application program interface (API). + public class Shape extends DisplayObject + { + /// Specifies the Graphics object belonging to this Shape object, where vector drawing commands can occur. + public function get graphics () : Graphics; + + /// Creates a new Shape object. + public function Shape (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SimpleButton.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SimpleButton.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,44 @@ +package flash.display +{ + import flash.display.DisplayObject; + import flash.media.SoundTransform; + + /// The SimpleButton class lets you control all instances of button symbols in a SWF file. + public class SimpleButton extends InteractiveObject + { + /// Specifies a display object that is used as the visual object for the button "Down" state - the state that the button is in when the user clicks the hitTestState object. + public function get downState () : DisplayObject; + public function set downState (value:DisplayObject) : void; + + /// A Boolean value that specifies whether a button is enabled. + public function get enabled () : Boolean; + public function set enabled (value:Boolean) : void; + + /// Specifies a display object that is used as the hit testing object for the button. + public function get hitTestState () : DisplayObject; + public function set hitTestState (value:DisplayObject) : void; + + /// Specifies a display object that is used as the visual object for the button over state - the state that the button is in when the mouse is positioned over the button. + public function get overState () : DisplayObject; + public function set overState (value:DisplayObject) : void; + + /// The SoundTransform object assigned to this button. + public function get soundTransform () : SoundTransform; + public function set soundTransform (sndTransform:SoundTransform) : void; + + /// Indicates whether other display objects that are SimpleButton or MovieClip objects can receive mouse release events. + public function get trackAsMenu () : Boolean; + public function set trackAsMenu (value:Boolean) : void; + + /// Specifies a display object that is used as the visual object for the button up state - the state that the button is in when the mouse is not positioned over the button. + public function get upState () : DisplayObject; + public function set upState (value:DisplayObject) : void; + + /// A Boolean value that, when set to true, indicates whether Flash Player displays the hand cursor when the mouse rolls over a button. + public function get useHandCursor () : Boolean; + public function set useHandCursor (value:Boolean) : void; + + /// Creates a new SimpleButton instance. + public function SimpleButton (upState:DisplayObject = null, overState:DisplayObject = null, downState:DisplayObject = null, hitTestState:DisplayObject = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SpreadMethod.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/SpreadMethod.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// The SpreadMethod class provides values for the spreadMethod parameter in the beginGradientFill() and lineGradientStyle() methods of the Graphics class. + public class SpreadMethod extends Object + { + /// Specifies that the gradient use the pad spread method. + public static const PAD : String; + /// Specifies that the gradient use the reflect spread method. + public static const REFLECT : String; + /// Specifies that the gradient use the repeat spread method. + public static const REPEAT : String; + + public function SpreadMethod (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Sprite.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Sprite.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,45 @@ +package flash.display +{ + import flash.display.DisplayObject; + import flash.media.SoundTransform; + import flash.display.Sprite; + import flash.display.Graphics; + import flash.geom.Rectangle; + + /// The Sprite class is a basic display list building block: a display list node that can display graphics and can also contain children. + public class Sprite extends DisplayObjectContainer + { + /// Specifies the button mode of this sprite. + public function get buttonMode () : Boolean; + public function set buttonMode (value:Boolean) : void; + + /// Specifies the display object over which the sprite is being dragged, or on which the sprite was dropped. + public function get dropTarget () : DisplayObject; + + /// Specifies the Graphics object that belongs to this sprite where vector drawing commands can occur. + public function get graphics () : Graphics; + + /// Designates another sprite to serve as the hit area for a sprite. + public function get hitArea () : Sprite; + public function set hitArea (value:Sprite) : void; + + /// Controls sound within this sprite. + public function get soundTransform () : SoundTransform; + public function set soundTransform (sndTransform:SoundTransform) : void; + + /// A Boolean value that indicates whether the pointing hand (hand cursor) appears when the mouse rolls over a sprite in which the buttonMode property is set to true. + public function get useHandCursor () : Boolean; + public function set useHandCursor (value:Boolean) : void; + + /// Creates a new Sprite instance. + public function Sprite (); + + /// Lets the user drag the specified sprite. + public function startDrag (lockCenter:Boolean = false, bounds:Rectangle = null) : void; + + /// Ends the startDrag() method. + public function stopDrag () : void; + + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Stage.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/Stage.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,204 @@ +package flash.display +{ + import flash.display.DisplayObject; + import flash.geom.Rectangle; + import flash.text.TextSnapshot; + import flash.display.InteractiveObject; + import flash.accessibility.AccessibilityImplementation; + import flash.accessibility.AccessibilityProperties; + import flash.events.Event; + import flash.ui.ContextMenu; + import flash.geom.Transform; + + /** + * Dispatched when the Stage object enters, or leaves, full-screen mode. + * @eventType flash.events.FullScreenEvent.FULL_SCREEN + */ + [Event(name="fullScreen", type="flash.events.FullScreenEvent")] + + /** + * Dispatched when the scaleMode property of the Stage object is set to StageScaleMode.NO_SCALE and the SWF file is resized. + * @eventType flash.events.Event.RESIZE + */ + [Event(name="resize", type="flash.events.Event")] + + /** + * Dispatched by the Stage object when the mouse pointer moves out of the stage area. + * @eventType flash.events.Event.MOUSE_LEAVE + */ + [Event(name="mouseLeave", type="flash.events.Event")] + + /// The Stage class represents the main drawing area. + public class Stage extends DisplayObjectContainer + { + public function set accessibilityImplementation (value:AccessibilityImplementation) : void; + + public function set accessibilityProperties (value:AccessibilityProperties) : void; + + /// A value from the StageAlign class that specifies the alignment of the stage in Flash Player or the browser. + public function get align () : String; + public function set align (value:String) : void; + + public function set alpha (value:Number) : void; + + public function set blendMode (value:String) : void; + + public function set cacheAsBitmap (value:Boolean) : void; + + /// Controls Flash Player color correction for displays. + public function get colorCorrection () : String; + public function set colorCorrection (value:String) : void; + + /// Specifies whether Flash Player is running on an operating system that supports color correction and whether the color profile of the main (primary) monitor can be read and understood by Flash Player. + public function get colorCorrectionSupport () : String; + + public function set contextMenu (value:ContextMenu) : void; + + /// A value from the StageDisplayState class that specifies which display state to use. + public function get displayState () : String; + public function set displayState (value:String) : void; + + public function set filters (value:Array) : void; + + /// The interactive object with keyboard focus; or null if focus is not set or if the focused object belongs to a security sandbox to which the calling object does not have access. + public function get focus () : InteractiveObject; + public function set focus (newFocus:InteractiveObject) : void; + + public function set focusRect (value:Object) : void; + + /// Gets and sets the frame rate of the stage. + public function get frameRate () : Number; + public function set frameRate (value:Number) : void; + + /// Returns the height of the monitor that will be used when going to full screen size, if that state is entered immediately. + public function get fullScreenHeight () : uint; + + /// Sets Flash Player to scale a specific region of the stage to full-screen mode. + public function get fullScreenSourceRect () : Rectangle; + public function set fullScreenSourceRect (value:Rectangle) : void; + + /// Returns the width of the monitor that will be used when going to full screen size, if that state is entered immediately. + public function get fullScreenWidth () : uint; + + /// Indicates the height of the display object, in pixels. + public function get height () : Number; + public function set height (value:Number) : void; + + public function set mask (value:DisplayObject) : void; + + /// Determines whether or not the children of the object are mouse enabled. + public function get mouseChildren () : Boolean; + public function set mouseChildren (value:Boolean) : void; + + public function set mouseEnabled (value:Boolean) : void; + + public function set name (value:String) : void; + + /// Returns the number of children of this object. + public function get numChildren () : int; + + public function set opaqueBackground (value:Object) : void; + + /// A value from the StageQuality class that specifies which rendering quality is used. + public function get quality () : String; + public function set quality (value:String) : void; + + public function set rotation (value:Number) : void; + + public function set rotationX (value:Number) : void; + + public function set rotationY (value:Number) : void; + + public function set rotationZ (value:Number) : void; + + public function set scale9Grid (value:Rectangle) : void; + + /// A value from the StageScaleMode class that specifies which scale mode to use. + public function get scaleMode () : String; + public function set scaleMode (value:String) : void; + + public function set scaleX (value:Number) : void; + + public function set scaleY (value:Number) : void; + + public function set scaleZ (value:Number) : void; + + public function set scrollRect (value:Rectangle) : void; + + /// Specifies whether to show or hide the default items in the Flash Player context menu. + public function get showDefaultContextMenu () : Boolean; + public function set showDefaultContextMenu (value:Boolean) : void; + + /// Specifies whether or not objects display a glowing border when they have focus. + public function get stageFocusRect () : Boolean; + public function set stageFocusRect (on:Boolean) : void; + + /// The current height, in pixels, of the Stage. + public function get stageHeight () : int; + public function set stageHeight (value:int) : void; + + /// Specifies the current width, in pixels, of the Stage. + public function get stageWidth () : int; + public function set stageWidth (value:int) : void; + + /// Determines whether the children of the object are tab enabled. + public function get tabChildren () : Boolean; + public function set tabChildren (value:Boolean) : void; + + public function set tabEnabled (value:Boolean) : void; + + public function set tabIndex (value:int) : void; + + /// Returns a TextSnapshot object for this DisplayObjectContainer instance. + public function get textSnapshot () : TextSnapshot; + + public function set transform (value:Transform) : void; + + /// Indicates the width of the display object, in pixels. + public function get width () : Number; + public function set width (value:Number) : void; + + public function set visible (value:Boolean) : void; + + public function set x (value:Number) : void; + + public function set y (value:Number) : void; + + public function set z (value:Number) : void; + + /// Adds a child DisplayObject instance to this DisplayObjectContainer instance. + public function addChild (child:DisplayObject) : DisplayObject; + + /// Adds a child DisplayObject instance to this DisplayObjectContainer instance. + public function addChildAt (child:DisplayObject, index:int) : DisplayObject; + + /// Registers an event listener object with an EventDispatcher object so that the listener receives notification of an event. + public function addEventListener (type:String, listener:Function, useCapture:Boolean = false, priority:int = 0, useWeakReference:Boolean = false) : void; + + /// Dispatches an event into the event flow. + public function dispatchEvent (event:Event) : Boolean; + + /// Checks whether the EventDispatcher object has any listeners registered for a specific type of event. + public function hasEventListener (type:String) : Boolean; + + /// Signals Flash Player to update properties of display objects on the next opportunity it has to refresh the Stage. + public function invalidate () : void; + + /// Determines whether the Stage.focus property would return null for security reasons. + public function isFocusInaccessible () : Boolean; + + /// Removes a child DisplayObject from the specified index position in the child list of the DisplayObjectContainer. + public function removeChildAt (index:int) : DisplayObject; + + /// Changes the position of an existing child in the display object container. + public function setChildIndex (child:DisplayObject, index:int) : void; + + public function Stage (); + + /// Swaps the z-order (front-to-back order) of the child objects at the two specified index positions in the child list. + public function swapChildrenAt (index1:int, index2:int) : void; + + /// Checks whether an event listener is registered with this EventDispatcher object or any of its ancestors for the specified event type. + public function willTrigger (type:String) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageAlign.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageAlign.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,25 @@ +package flash.display +{ + /// The StageAlign class provides constant values to use for the Stage.align property. + public class StageAlign extends Object + { + /// Specifies that the Stage is aligned at the bottom. + public static const BOTTOM : String; + /// Specifies that the Stage is aligned in the bottom-left corner. + public static const BOTTOM_LEFT : String; + /// Specifies that the Stage is aligned in the bottom-right corner. + public static const BOTTOM_RIGHT : String; + /// Specifies that the Stage is aligned on the left. + public static const LEFT : String; + /// Specifies that the Stage is aligned to the right. + public static const RIGHT : String; + /// Specifies that the Stage is aligned at the top. + public static const TOP : String; + /// Specifies that the Stage is aligned in the top-left corner. + public static const TOP_LEFT : String; + /// Specifies that the Stage is aligned in the top-right corner. + public static const TOP_RIGHT : String; + + public function StageAlign (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageDisplayState.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageDisplayState.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.display +{ + /// The StageDisplayState class provides values for the Stage.displayState property. + public class StageDisplayState extends Object + { + /// Specifies that the Stage is in full-screen mode. + public static const FULL_SCREEN : String; + /// Specifies that the Stage is in normal mode. + public static const NORMAL : String; + + public function StageDisplayState (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageQuality.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageQuality.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + /// The StageQuality class provides values for the Stage.quality property. + public class StageQuality extends Object + { + /// Specifies very high rendering quality: graphics are anti-aliased using a 4 x 4 pixel grid and bitmaps are always smoothed. + public static const BEST : String; + /// Specifies high rendering quality: graphics are anti-aliased using a 4 x 4 pixel grid, and bitmaps are smoothed if the movie is static. + public static const HIGH : String; + /// Specifies low rendering quality: graphics are not anti-aliased, and bitmaps are not smoothed. + public static const LOW : String; + /// Specifies medium rendering quality: graphics are anti-aliased using a 2 x 2 pixel grid, but bitmaps are not smoothed. + public static const MEDIUM : String; + + public function StageQuality (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageScaleMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/StageScaleMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.display +{ + /// The StageScaleMode class provides values for the Stage.scaleMode property. + public class StageScaleMode extends Object + { + /// Specifies that the entire application be visible in the specified area without trying to preserve the original aspect ratio. + public static const EXACT_FIT : String; + /// Specifies that the entire application fill the specified area, without distortion but possibly with some cropping, while maintaining the original aspect ratio of the application. + public static const NO_BORDER : String; + /// Specifies that the size of the application be fixed, so that it remains unchanged even as the size of the player window changes. + public static const NO_SCALE : String; + /// Specifies that the entire application be visible in the specified area without distortion while maintaining the original aspect ratio of the application. + public static const SHOW_ALL : String; + + public function StageScaleMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/TriangleCulling.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/display/TriangleCulling.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.display +{ + /// Defines codes for culling algorithms that determine which triangles not to render when drawing triangle paths. + public class TriangleCulling extends Object + { + /// Specifies culling of all triangles facing toward the current view point. + public static const NEGATIVE : String; + /// Specifies no culling. + public static const NONE : String; + /// Specifies culling of all triangles facing away from the current view point. + public static const POSITIVE : String; + + public function TriangleCulling (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/EOFError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/EOFError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// An EOFError exception is thrown when you attempt to read past the end of the available data. + public class EOFError extends IOError + { + /// Creates a new EOFError object. + public function EOFError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IOError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IOError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// The IOError exception is thrown when some type of input or output failure occurs. + public class IOError extends Error + { + /// Creates a new IOError object. + public function IOError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IllegalOperationError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/IllegalOperationError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// The IllegalOperationError exception is thrown when a method is not implemented or the implementation doesn't cover the current usage. + public class IllegalOperationError extends Error + { + /// Creates a new IllegalOperationError object. + public function IllegalOperationError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/InvalidSWFError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/InvalidSWFError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.errors +{ + public class InvalidSWFError extends Error + { + public function InvalidSWFError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/MemoryError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/MemoryError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// The MemoryError exception is thrown when a memory allocation request fails. + public class MemoryError extends Error + { + /// Creates a new MemoryError object. + public function MemoryError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/ScriptTimeoutError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/ScriptTimeoutError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// The ScriptTimeoutError exception is thrown when the script timeout interval is reached. + public class ScriptTimeoutError extends Error + { + /// Creates a new ScriptTimeoutError object. + public function ScriptTimeoutError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/StackOverflowError.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/errors/StackOverflowError.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.errors +{ + /// ActionScript throws a StackOverflowError exception when the stack available to the script is exhausted. + public class StackOverflowError extends Error + { + /// Creates a new StackOverflowError object. + public function StackOverflowError (message:String = "", id:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ActivityEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ActivityEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package flash.events +{ + import flash.events.Event; + + /// A Camera or Microphone object dispatches an ActivityEvent object whenever a camera or microphone reports that it has become active or inactive. + public class ActivityEvent extends Event + { + /// The ActivityEvent.ACTIVITY constant defines the value of the type property of an activity event object. + public static const ACTIVITY : String = "activity"; + + /// Indicates whether the device is activating (true) or deactivating (false). + public function get activating () : Boolean; + public function set activating (value:Boolean) : void; + + /// Constructor for ActivityEvent objects. + public function ActivityEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, activating:Boolean = false); + + /// Creates a copy of an ActivityEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Returns a string that contains all the properties of the ActivityEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/AsyncErrorEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/AsyncErrorEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches an AsyncErrorEvent when an exception is thrown from native asynchronous code, which could be from, for example, LocalConnection, NetConnection, SharedObject, or NetStream. + public class AsyncErrorEvent extends ErrorEvent + { + /// The AsyncErrorEvent.ASYNC_ERROR constant defines the value of the type property of an asyncError event object. + public static const ASYNC_ERROR : String = "asyncError"; + /// The exception that was thrown. + public var error : Error; + + /// Constructor for AsyncErrorEvent objects. + public function AsyncErrorEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = "", error:Error = null); + + /// Creates a copy of the AsyncErrorEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Returns a string that contains all the properties of the AsyncErrorEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ContextMenuEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ContextMenuEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,35 @@ +package flash.events +{ + import flash.display.InteractiveObject; + import flash.events.Event; + + /// Flash Player dispatches ContextMenuEvent objects when a user generates or interacts with the context menu. + public class ContextMenuEvent extends Event + { + /// Defines the value of the type property of a menuItemSelect event object. + public static const MENU_ITEM_SELECT : String = "menuItemSelect"; + /// Defines the value of the type property of a menuSelect event object. + public static const MENU_SELECT : String = "menuSelect"; + + /// The display list object to which the menu is attached. + public function get contextMenuOwner () : InteractiveObject; + public function set contextMenuOwner (value:InteractiveObject) : void; + + /// Indicates whether the mouseTarget property was set to null for security reasons. + public function get isMouseTargetInaccessible () : Boolean; + public function set isMouseTargetInaccessible (value:Boolean) : void; + + /// The display list object on which the user right-clicked to display the context menu. + public function get mouseTarget () : InteractiveObject; + public function set mouseTarget (value:InteractiveObject) : void; + + /// Creates a copy of the ContextMenuEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for ContextMenuEvent objects. + public function ContextMenuEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, mouseTarget:InteractiveObject = null, contextMenuOwner:InteractiveObject = null); + + /// Returns a string that contains all the properties of the ContextMenuEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/DataEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/DataEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,26 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches DataEvent objects when raw data has completed loading into Flash Player. + public class DataEvent extends TextEvent + { + /// Defines the value of the type property of a data event object. + public static const DATA : String = "data"; + /// Defines the value of the type property of an uploadCompleteData event object. + public static const UPLOAD_COMPLETE_DATA : String = "uploadCompleteData"; + + /// The raw data loaded into Flash Player. + public function get data () : String; + public function set data (value:String) : void; + + /// Creates a copy of the DataEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for DataEvent objects. + public function DataEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, data:String = ""); + + /// Returns a string that contains all the properties of the DataEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ErrorEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ErrorEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches ErrorEvent objects when an error causes a network operation to fail. + public class ErrorEvent extends TextEvent + { + /// Defines the value of the type property of an error event object. + public static const ERROR : String = "error"; + + /// Creates a copy of the ErrorEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for ErrorEvent objects. + public function ErrorEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = ""); + + /// Returns a string that contains all the properties of the ErrorEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/Event.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/Event.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,117 @@ +package flash.events +{ + import flash.events.Event; + + /// The Event class is used as the base class for the creation of Event objects, which are passed as parameters to event listeners when an event occurs. + public class Event extends Object + { + /// The Event.ACTIVATE constant defines the value of the type property of an activate event object. + public static const ACTIVATE : String = "activate"; + /// The Event.ADDED constant defines the value of the type property of an added event object. + public static const ADDED : String = "added"; + /// The Event.ADDED_TO_STAGE constant defines the value of the type property of an addedToStage event object. + public static const ADDED_TO_STAGE : String = "addedToStage"; + /// The Event.CANCEL constant defines the value of the type property of a cancel event object. + public static const CANCEL : String = "cancel"; + /// The Event.CHANGE constant defines the value of the type property of a change event object. + public static const CHANGE : String = "change"; + /// Defines the value of the type property of a clear event object. + public static const CLEAR : String = "clear"; + /// The Event.CLOSE constant defines the value of the type property of a close event object. + public static const CLOSE : String = "close"; + /// The Event.COMPLETE constant defines the value of the type property of a complete event object. + public static const COMPLETE : String = "complete"; + /// The Event.CONNECT constant defines the value of the type property of a connect event object. + public static const CONNECT : String = "connect"; + /// Defines the value of the type property of a copy event object. + public static const COPY : String = "copy"; + /// Defines the value of the type property of a cut event object. + public static const CUT : String = "cut"; + /// The Event.DEACTIVATE constant defines the value of the type property of a deactivate event object. + public static const DEACTIVATE : String = "deactivate"; + /// The Event.ENTER_FRAME constant defines the value of the type property of an enterFrame event object. + public static const ENTER_FRAME : String = "enterFrame"; + /// Defines the value of the type property of an exitFrame event object. + public static const EXIT_FRAME : String = "exitFrame"; + /// Defines the value of the type property of an frameConstructed event object. + public static const FRAME_CONSTRUCTED : String = "frameConstructed"; + /// The Event.FULL_SCREEN constant defines the value of the type property of a fullScreen event object. + public static const FULLSCREEN : String = "fullscreen"; + /// The Event.ID3 constant defines the value of the type property of an id3 event object. + public static const ID3 : String = "id3"; + /// The Event.INIT constant defines the value of the type property of an init event object. + public static const INIT : String = "init"; + /// The Event.MOUSE_LEAVE constant defines the value of the type property of a mouseLeave event object. + public static const MOUSE_LEAVE : String = "mouseLeave"; + /// The Event.OPEN constant defines the value of the type property of an open event object. + public static const OPEN : String = "open"; + /// Defines the value of the type property of a paste event object. + public static const PASTE : String = "paste"; + /// The Event.REMOVED constant defines the value of the type property of a removed event object. + public static const REMOVED : String = "removed"; + /// The Event.REMOVED_FROM_STAGE constant defines the value of the type property of a removedFromStage event object. + public static const REMOVED_FROM_STAGE : String = "removedFromStage"; + /// The Event.RENDER constant defines the value of the type property of a render event object. + public static const RENDER : String = "render"; + /// The Event.RESIZE constant defines the value of the type property of a resize event object. + public static const RESIZE : String = "resize"; + /// The Event.SCROLL constant defines the value of the type property of a scroll event object. + public static const SCROLL : String = "scroll"; + /// The Event.SELECT constant defines the value of the type property of a select event object. + public static const SELECT : String = "select"; + /// Defines the value of the type property of a selectAll event object. + public static const SELECT_ALL : String = "selectAll"; + /// The Event.SOUND_COMPLETE constant defines the value of the type property of a soundComplete event object. + public static const SOUND_COMPLETE : String = "soundComplete"; + /// The Event.TAB_CHILDREN_CHANGE constant defines the value of the type property of a tabChildrenChange event object. + public static const TAB_CHILDREN_CHANGE : String = "tabChildrenChange"; + /// The Event.TAB_ENABLED_CHANGE constant defines the value of the type property of a tabEnabledChange event object. + public static const TAB_ENABLED_CHANGE : String = "tabEnabledChange"; + /// The Event.TAB_INDEX_CHANGE constant defines the value of the type property of a tabIndexChange event object. + public static const TAB_INDEX_CHANGE : String = "tabIndexChange"; + /// The Event.UNLOAD constant defines the value of the type property of an unload event object. + public static const UNLOAD : String = "unload"; + + /// Indicates whether an event is a bubbling event. + public function get bubbles () : Boolean; + + /// Indicates whether the behavior associated with the event can be prevented. + public function get cancelable () : Boolean; + + /// The object that is actively processing the Event object with an event listener. + public function get currentTarget () : Object; + + /// The current phase in the event flow. + public function get eventPhase () : uint; + + /// The event target. + public function get target () : Object; + + /// The type of event. + public function get type () : String; + + /// Duplicates an instance of an Event subclass. + public function clone () : Event; + + /// Used to create new Event object. + public function Event (type:String, bubbles:Boolean = false, cancelable:Boolean = false); + + /// A utility function for implementing the toString() method in custom ActionScript 3.0 Event classes. + public function formatToString (className:String, ...rest) : String; + + /// Checks whether the preventDefault() method has been called on the event. + public function isDefaultPrevented () : Boolean; + + /// Cancels an event's default behavior if that behavior can be canceled. + public function preventDefault () : void; + + /// Prevents processing of any event listeners in the current node and any subsequent nodes in the event flow. + public function stopImmediatePropagation () : void; + + /// Prevents processing of any event listeners in nodes subsequent to the current node in the event flow. + public function stopPropagation () : void; + + /// Returns a string containing all the properties of the Event object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventDispatcher.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventDispatcher.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,41 @@ +package flash.events +{ + import flash.events.Event; + import flash.events.IEventDispatcher; + + /** + * [broadcast event] Dispatched when Flash Player loses operating system focus and is becoming inactive. + * @eventType flash.events.Event.DEACTIVATE + */ + [Event(name="deactivate", type="flash.events.Event")] + + /** + * [broadcast event] Dispatched when Flash Player gains operating system focus and becomes active. + * @eventType flash.events.Event.ACTIVATE + */ + [Event(name="activate", type="flash.events.Event")] + + /// The EventDispatcher class implements the IEventDispatcher interface and is the base class for the DisplayObject class. + public class EventDispatcher extends Object implements IEventDispatcher + { + /// Registers an event listener object with an EventDispatcher object so that the listener receives notification of an event. + public function addEventListener (type:String, listener:Function, useCapture:Boolean = false, priority:int = 0, useWeakReference:Boolean = false) : void; + + /// Dispatches an event into the event flow. + public function dispatchEvent (event:Event) : Boolean; + + /// Aggregates an instance of the EventDispatcher class. + public function EventDispatcher (target:IEventDispatcher = null); + + /// Checks whether the EventDispatcher object has any listeners registered for a specific type of event. + public function hasEventListener (type:String) : Boolean; + + /// Removes a listener from the EventDispatcher object. + public function removeEventListener (type:String, listener:Function, useCapture:Boolean = false) : void; + + public function toString () : String; + + /// Checks whether an event listener is registered with this EventDispatcher object or any of its ancestors for the specified event type. + public function willTrigger (type:String) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventPhase.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/EventPhase.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.events +{ + /// The EventPhase class provides values for the eventPhase property of the Event class. + public class EventPhase extends Object + { + /// The target phase, which is the second phase of the event flow. + public static const AT_TARGET : uint; + /// The bubbling phase, which is the third phase of the event flow. + public static const BUBBLING_PHASE : uint; + /// The capturing phase, which is the first phase of the event flow. + public static const CAPTURING_PHASE : uint; + + public function EventPhase (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FocusEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FocusEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,43 @@ +package flash.events +{ + import flash.display.InteractiveObject; + import flash.events.Event; + + /// Flash Player dispatches FocusEvent objects when the user changes the focus from one object in the display list to another. + public class FocusEvent extends Event + { + /// Defines the value of the type property of a focusIn event object. + public static const FOCUS_IN : String = "focusIn"; + /// Defines the value of the type property of a focusOut event object. + public static const FOCUS_OUT : String = "focusOut"; + /// Defines the value of the type property of a keyFocusChange event object. + public static const KEY_FOCUS_CHANGE : String = "keyFocusChange"; + /// Defines the value of the type property of a mouseFocusChange event object. + public static const MOUSE_FOCUS_CHANGE : String = "mouseFocusChange"; + + /// Indicates whether the relatedObject property was set to null for security reasons. + public function get isRelatedObjectInaccessible () : Boolean; + public function set isRelatedObjectInaccessible (value:Boolean) : void; + + /// The key code value of the key pressed to trigger a keyFocusChange event. + public function get keyCode () : uint; + public function set keyCode (value:uint) : void; + + /// A reference to the complementary InteractiveObject instance that is affected by the change in focus. + public function get relatedObject () : InteractiveObject; + public function set relatedObject (value:InteractiveObject) : void; + + /// Indicates whether the Shift key modifier is activated, in which case the value is true. + public function get shiftKey () : Boolean; + public function set shiftKey (value:Boolean) : void; + + /// Creates a copy of the FocusEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for FocusEvent objects. + public function FocusEvent (type:String, bubbles:Boolean = true, cancelable:Boolean = false, relatedObject:InteractiveObject = null, shiftKey:Boolean = false, keyCode:uint = 0); + + /// Returns a string that contains all the properties of the FocusEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FullScreenEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/FullScreenEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches a FullScreenEvent object whenever the Stage enters or leaves full-screen display mode. + public class FullScreenEvent extends ActivityEvent + { + /// The FullScreenEvent.FULL_SCREEN constant defines the value of the type property of a fullScreen event object. + public static const FULL_SCREEN : String = "fullScreen"; + + /// Indicates whether the Stage object is in full-screen mode (true) or not (false). + public function get fullScreen () : Boolean; + + /// Creates a copy of a FullScreenEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for FullScreenEvent objects. + public function FullScreenEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, fullScreen:Boolean = false); + + /// Returns a string that contains all the properties of the FullScreenEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/HTTPStatusEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/HTTPStatusEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.events +{ + import flash.events.Event; + + /// The application dispatches HTTPStatusEvent objects when a network request returns an HTTPstatus code. + public class HTTPStatusEvent extends Event + { + /// The HTTPStatusEvent.HTTP_STATUS constant defines the value of the type property of a httpStatus event object. + public static const HTTP_STATUS : String = "httpStatus"; + + /// The HTTP status code returned by the server. + public function get status () : int; + + /// Creates a copy of the HTTPStatusEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for HTTPStatusEvent objects. + public function HTTPStatusEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, status:int = 0); + + /// Returns a string that contains all the properties of the HTTPStatusEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IEventDispatcher.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IEventDispatcher.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.events +{ + import flash.events.Event; + + /// The IEventDispatcher interface defines methods for adding or removing event listeners, checks whether specific types of event listeners are registered, and dispatches events. + public interface IEventDispatcher + { + /// Registers an event listener object with an EventDispatcher object so that the listener receives notification of an event. + public function addEventListener (type:String, listener:Function, useCapture:Boolean = false, priority:int = 0, useWeakReference:Boolean = false) : void; + + /// Dispatches an event into the event flow. + public function dispatchEvent (event:Event) : Boolean; + + /// Checks whether the EventDispatcher object has any listeners registered for a specific type of event. + public function hasEventListener (type:String) : Boolean; + + /// Removes a listener from the EventDispatcher object. + public function removeEventListener (type:String, listener:Function, useCapture:Boolean = false) : void; + + /// Checks whether an event listener is registered with this EventDispatcher object or any of its ancestors for the specified event type. + public function willTrigger (type:String) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IMEEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IMEEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches IMEEvent objects when a user enters text using an input method editor (IME). + public class IMEEvent extends TextEvent + { + /// Defines the value of the type property of an imeComposition event object. + public static const IME_COMPOSITION : String = "imeComposition"; + + /// Creates a copy of the IMEEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for IMEEvent objects. + public function IMEEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = ""); + + /// Returns a string that contains all the properties of the IMEEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IOErrorEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/IOErrorEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches an IOErrorEvent object when an error causes a send or load operation to fail. + public class IOErrorEvent extends ErrorEvent + { + public static const DISK_ERROR : String = "diskError"; + /// Defines the value of the type property of an ioError event object. + public static const IO_ERROR : String = "ioError"; + public static const NETWORK_ERROR : String = "networkError"; + public static const VERIFY_ERROR : String = "verifyError"; + + /// Creates a copy of the IOErrorEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for IOErrorEvent objects. + public function IOErrorEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = ""); + + /// Returns a string that contains all the properties of the IOErrorEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/KeyboardEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/KeyboardEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,49 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches KeyboardEvent objects in response to user input through a keyboard. + public class KeyboardEvent extends Event + { + /// Defines the value of the type property of a keyDown event object. + public static const KEY_DOWN : String = "keyDown"; + /// Defines the value of the type property of a keyUp event object. + public static const KEY_UP : String = "keyUp"; + + /// Indicates whether the Alt key is active (true) or inactive (false). + public function get altKey () : Boolean; + public function set altKey (value:Boolean) : void; + + /// Contains the character code value of the key pressed or released. + public function get charCode () : uint; + public function set charCode (value:uint) : void; + + /// Indicates whether the Control key is active (true) or inactive (false). + public function get ctrlKey () : Boolean; + public function set ctrlKey (value:Boolean) : void; + + /// The key code value of the key pressed or released. + public function get keyCode () : uint; + public function set keyCode (value:uint) : void; + + /// Indicates the location of the key on the keyboard. + public function get keyLocation () : uint; + public function set keyLocation (value:uint) : void; + + /// Indicates whether the Shift key modifier is active (true) or inactive (false). + public function get shiftKey () : Boolean; + public function set shiftKey (value:Boolean) : void; + + /// Creates a copy of the KeyboardEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for KeyboardEvent objects. + public function KeyboardEvent (type:String, bubbles:Boolean = true, cancelable:Boolean = false, charCode:uint = 0, keyCode:uint = 0, keyLocation:uint = 0, ctrlKey:Boolean = false, altKey:Boolean = false, shiftKey:Boolean = false); + + /// Returns a string that contains all the properties of the KeyboardEvent object. + public function toString () : String; + + /// Instructs Flash Player to render after processing of this event completes, if the display list has been modified + public function updateAfterEvent () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/MouseEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/MouseEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,84 @@ +package flash.events +{ + import flash.display.InteractiveObject; + import flash.events.Event; + + /// Flash Player dispatches MouseEvent objects into the event flow whenever mouse events occur. + public class MouseEvent extends Event + { + /// Defines the value of the type property of a click event object. + public static const CLICK : String = "click"; + /// Defines the value of the type property of a doubleClick event object. + public static const DOUBLE_CLICK : String = "doubleClick"; + /// Defines the value of the type property of a mouseDown event object. + public static const MOUSE_DOWN : String = "mouseDown"; + /// Defines the value of the type property of a mouseMove event object. + public static const MOUSE_MOVE : String = "mouseMove"; + /// Defines the value of the type property of a mouseOut event object. + public static const MOUSE_OUT : String = "mouseOut"; + /// Defines the value of the type property of a mouseOver event object. + public static const MOUSE_OVER : String = "mouseOver"; + /// Defines the value of the type property of a mouseUp event object. + public static const MOUSE_UP : String = "mouseUp"; + /// Defines the value of the type property of a mouseWheel event object. + public static const MOUSE_WHEEL : String = "mouseWheel"; + /// Defines the value of the type property of a rollOut event object. + public static const ROLL_OUT : String = "rollOut"; + /// Defines the value of the type property of a rollOver event object. + public static const ROLL_OVER : String = "rollOver"; + + /// Indicates whether the Alt key is active (true) or inactive (false). + public function get altKey () : Boolean; + public function set altKey (value:Boolean) : void; + + /// Indicates whether the primary mouse button is pressed (true) or not (false). + public function get buttonDown () : Boolean; + public function set buttonDown (value:Boolean) : void; + + /// Indicates whether the Control key is active (true) or inactive (false). + public function get ctrlKey () : Boolean; + public function set ctrlKey (value:Boolean) : void; + + /// Indicates how many lines should be scrolled for each unit the user rotates the mouse wheel. + public function get delta () : int; + public function set delta (value:int) : void; + + /// Indicates whether the relatedObject property was set to null for security reasons. + public function get isRelatedObjectInaccessible () : Boolean; + public function set isRelatedObjectInaccessible (value:Boolean) : void; + + /// The horizontal coordinate at which the event occurred relative to the containing sprite. + public function get localX () : Number; + public function set localX (value:Number) : void; + + /// The vertical coordinate at which the event occurred relative to the containing sprite. + public function get localY () : Number; + public function set localY (value:Number) : void; + + /// A reference to a display list object that is related to the event. + public function get relatedObject () : InteractiveObject; + public function set relatedObject (value:InteractiveObject) : void; + + /// Indicates whether the Shift key is active (true) or inactive (false). + public function get shiftKey () : Boolean; + public function set shiftKey (value:Boolean) : void; + + /// The horizontal coordinate at which the event occurred in global Stage coordinates. + public function get stageX () : Number; + + /// The vertical coordinate at which the event occurred in global Stage coordinates. + public function get stageY () : Number; + + /// Creates a copy of the MouseEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for MouseEvent objects. + public function MouseEvent (type:String, bubbles:Boolean = true, cancelable:Boolean = false, localX:Number = null, localY:Number = null, relatedObject:InteractiveObject = null, ctrlKey:Boolean = false, altKey:Boolean = false, shiftKey:Boolean = false, buttonDown:Boolean = false, delta:int = 0); + + /// Returns a string that contains all the properties of the MouseEvent object. + public function toString () : String; + + /// Instructs Flash Player to render after processing of this event completes, if the display list has been modified. + public function updateAfterEvent () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetFilterEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetFilterEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.events +{ + import flash.utils.ByteArray; + import flash.events.Event; + + public class NetFilterEvent extends Event + { + public var data : ByteArray; + public var header : ByteArray; + + public function clone () : Event; + + public function NetFilterEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, header:ByteArray = null, data:ByteArray = null); + + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetStatusEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/NetStatusEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches NetStatusEvent objects when a NetConnection, NetStream, orSharedObject object reports its status. + public class NetStatusEvent extends Event + { + /// Defines the value of the type property of a netStatus event object. + public static const NET_STATUS : String = "netStatus"; + + /// An object with properties that describe the object's status or error condition. + public function get info () : Object; + public function set info (value:Object) : void; + + /// Creates a copy of the NetStatusEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for NetStatusEvent objects. + public function NetStatusEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, info:Object = null); + + /// Returns a string that contains all the properties of the NetStatusEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ProgressEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ProgressEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches ProgressEvent objects when a load operation has begun or a socket has received data. + public class ProgressEvent extends Event + { + /// Defines the value of the type property of a progress event object. + public static const PROGRESS : String = "progress"; + /// Defines the value of the type property of a socketData event object. + public static const SOCKET_DATA : String = "socketData"; + + /// The number of items or bytes loaded when the listener processes the event. + public function get bytesLoaded () : uint; + public function set bytesLoaded (value:uint) : void; + + /// The total number of items or bytes that will be loaded if the loading process succeeds. + public function get bytesTotal () : uint; + public function set bytesTotal (value:uint) : void; + + /// Creates a copy of the ProgressEvent object and sets each property's value to match that of the original. + public function clone () : Event; + + /// Constructor for ProgressEvent objects. + public function ProgressEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, bytesLoaded:uint = 0, bytesTotal:uint = 0); + + /// Returns a string that contains all the properties of the ProgressEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SampleDataEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SampleDataEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,28 @@ +package flash.events +{ + import flash.utils.ByteArray; + import flash.events.Event; + + /// Dispatched when the player requests new audio data. + public class SampleDataEvent extends Event + { + public static const SAMPLE_DATA : String = "sampleData"; + + /// The data in the audio stream. + public function get data () : ByteArray; + public function set data (thedata:ByteArray) : void; + + /// The position of the data in the audio stream. + public function get position () : Number; + public function set position (theposition:Number) : void; + + /// Creates a copy of the SampleDataEvent object and sets each property's value to match that of the original. + public function clone () : Event; + + /// Creates an event object that contains information about audio data events. + public function SampleDataEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, theposition:Number = 0, thedata:ByteArray = null); + + /// Returns a string that contains all the properties of the SampleDataEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SecurityErrorEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SecurityErrorEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches SecurityErrorEvent objects to report the occurrence of a security error. + public class SecurityErrorEvent extends ErrorEvent + { + /// The SecurityErrorEvent.SECURITY_ERROR constant defines the value of the type property of a securityError event object. + public static const SECURITY_ERROR : String = "securityError"; + + /// Creates a copy of the SecurityErrorEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for SecurityErrorEvent objects. + public function SecurityErrorEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = ""); + + /// Returns a string that contains all the properties of the SecurityErrorEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ShaderEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/ShaderEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,34 @@ +package flash.events +{ + import flash.utils.ByteArray; + import flash.display.BitmapData; + import flash.events.Event; + + /// A ShaderEvent is dispatched when a shader operation launched from a ShaderJob finishes. + public class ShaderEvent extends Event + { + /// Defines the value of the type property of a complete event object. + public static const COMPLETE : String = "complete"; + + /// The BitmapData object that was passed to the ShaderJob.start() method. + public function get bitmapData () : BitmapData; + public function set bitmapData (bmpData:BitmapData) : void; + + /// The ByteArray object that was passed to the ShaderJob.start() method. + public function get byteArray () : ByteArray; + public function set byteArray (bArray:ByteArray) : void; + + /// The Vector. object that was passed to the ShaderJob.start() method. + public function get vector () : Vector.; + public function set vector (v:Vector.) : void; + + /// Creates a copy of the ShaderEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Creates a ShaderEvent object to pass to event listeners. + public function ShaderEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, bitmap:BitmapData = null, array:ByteArray = null, vector:Vector. = null); + + /// Returns a string that contains all the properties of the ShaderEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/StatusEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/StatusEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,28 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches StatusEvent objects when a device, such as a camera or microphone, or an object such as a LocalConnection object reports its status. + public class StatusEvent extends Event + { + /// Defines the value of the type property of a status event object. + public static const STATUS : String = "status"; + + /// A description of the object's status. + public function get code () : String; + public function set code (value:String) : void; + + /// The category of the message, such as "status", "warning" or "error". + public function get level () : String; + public function set level (value:String) : void; + + /// Creates a copy of the StatusEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for StatusEvent objects. + public function StatusEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, code:String = "", level:String = ""); + + /// Returns a string that contains all the properties of the StatusEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SyncEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/SyncEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches SyncEvent objects when a remote SharedObject instance has been updated by the server. + public class SyncEvent extends Event + { + /// Defines the value of the type property of a sync event object. + public static const SYNC : String = "sync"; + + /// An array of objects; each object contains properties that describe the changed members of a remote shared object. + public function get changeList () : Array; + public function set changeList (value:Array) : void; + + /// Creates a copy of the SyncEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for SyncEvent objects. + public function SyncEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, changeList:Array = null); + + /// Returns a string that contains all the properties of the SyncEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TextEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TextEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,26 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches TextEvent objects when a user enters text in a text field or clicks a hyperlink in an HTML-enabled text field. + public class TextEvent extends Event + { + /// Defines the value of the type property of a link event object. + public static const LINK : String = "link"; + /// Defines the value of the type property of a textInput event object. + public static const TEXT_INPUT : String = "textInput"; + + /// For a textInput event, the character or sequence of characters entered by the user. + public function get text () : String; + public function set text (value:String) : void; + + /// Creates a copy of the TextEvent object and sets the value of each property to match that of the original. + public function clone () : Event; + + /// Constructor for TextEvent objects. + public function TextEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false, text:String = ""); + + /// Returns a string that contains all the properties of the TextEvent object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TimerEvent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/TimerEvent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,25 @@ +package flash.events +{ + import flash.events.Event; + + /// Flash Player dispatches TimerEvent objects whenever a Timer object reaches the interval specified by the Timer.delay property. + public class TimerEvent extends Event + { + /// Defines the value of the type property of a timer event object. + public static const TIMER : String = "timer"; + /// Defines the value of the type property of a timerComplete event object. + public static const TIMER_COMPLETE : String = "timerComplete"; + + /// Creates a copy of the TimerEvent object and sets each property's value to match that of the original. + public function clone () : Event; + + /// Constructor for TimerEvent objects. + public function TimerEvent (type:String, bubbles:Boolean = false, cancelable:Boolean = false); + + /// Returns a string that contains all the properties of the TimerEvent object. + public function toString () : String; + + /// Instructs Flash Player to render after processing of this event completes, if the display list has been modified. + public function updateAfterEvent () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakFunctionClosure.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakFunctionClosure.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.events +{ + public class WeakFunctionClosure extends Object + { + public function WeakFunctionClosure (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakMethodClosure.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/events/WeakMethodClosure.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.events +{ + public class WeakMethodClosure extends Object + { + public function WeakMethodClosure (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/external/ExternalInterface.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/external/ExternalInterface.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.external +{ + /// The ExternalInterface class is the External API, an application programming interface that enables straightforward communication between ActionScript and the Flash Player container - for example, an HTML page with JavaScript. + public class ExternalInterface extends Object + { + /// Indicates whether the external interface should attempt to pass ActionScript exceptions to the current browser and JavaScript exceptions to Flash Player. + public static var marshallExceptions : Boolean; + + /// Indicates whether this player is in a container that offers an external interface. + public static function get available () : Boolean; + + /// Returns the id attribute of the object tag in Internet Explorer, or the name attribute of the embed tag in Netscape. + public static function get objectID () : String; + + /// Registers an ActionScript method as callable from the container. + public static function addCallback (functionName:String, closure:Function) : void; + + /// Calls a function in the container. + public static function call (functionName:String, ...rest) : *; + + public function ExternalInterface (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BevelFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BevelFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,62 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The BevelFilter class lets you add a bevel effect to display objects. + public class BevelFilter extends BitmapFilter + { + /// The angle of the bevel. + public function get angle () : Number; + public function set angle (value:Number) : void; + + /// The amount of horizontal blur, in pixels. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur, in pixels. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// The offset distance of the bevel. + public function get distance () : Number; + public function set distance (value:Number) : void; + + /// The alpha transparency value of the highlight color. + public function get highlightAlpha () : Number; + public function set highlightAlpha (value:Number) : void; + + /// The highlight color of the bevel. + public function get highlightColor () : uint; + public function set highlightColor (value:uint) : void; + + /// Applies a knockout effect (true), which effectively makes the object's fill transparent and reveals the background color of the document. + public function get knockout () : Boolean; + public function set knockout (value:Boolean) : void; + + /// The number of times to apply the filter. + public function get quality () : int; + public function set quality (value:int) : void; + + /// The alpha transparency value of the shadow color. + public function get shadowAlpha () : Number; + public function set shadowAlpha (value:Number) : void; + + /// The shadow color of the bevel. + public function get shadowColor () : uint; + public function set shadowColor (value:uint) : void; + + /// The strength of the imprint or spread. + public function get strength () : Number; + public function set strength (value:Number) : void; + + /// The placement of the bevel on the object. + public function get type () : String; + public function set type (value:String) : void; + + /// Initializes a new BevelFilter instance with the specified parameters. + public function BevelFilter (distance:Number = 4, angle:Number = 45, highlightColor:uint = 16777215, highlightAlpha:Number = 1, shadowColor:uint = 0, shadowAlpha:Number = 1, blurX:Number = 4, blurY:Number = 4, strength:Number = 1, quality:int = 1, type:String = "inner", knockout:Boolean = false); + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The BitmapFilter class is the base class for all image filter effects. + public class BitmapFilter extends Object + { + public function BitmapFilter (); + + /// A copy of the BitmapFilter object. + public function clone () : BitmapFilter; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterQuality.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterQuality.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.filters +{ + /// The BitmapFilterQuality class contains values to set the rendering quality of a BitmapFilter object. + public class BitmapFilterQuality extends Object + { + /// Defines the high quality filter setting. + public static const HIGH : int; + /// Defines the low quality filter setting. + public static const LOW : int; + /// Defines the medium quality filter setting. + public static const MEDIUM : int; + + public function BitmapFilterQuality (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BitmapFilterType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.filters +{ + /// The BitmapFilterType class contains values to set the type of a BitmapFilter. + public class BitmapFilterType extends Object + { + /// Defines the setting that applies a filter to the entire area of an object. + public static const FULL : String; + /// Defines the setting that applies a filter to the inner area of an object. + public static const INNER : String; + /// Defines the setting that applies a filter to the outer area of an object. + public static const OUTER : String; + + public function BitmapFilterType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BlurFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/BlurFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,26 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The BlurFilter class lets you apply a blur visual effect to display objects. + public class BlurFilter extends BitmapFilter + { + /// The amount of horizontal blur. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// The number of times to perform the blur. + public function get quality () : int; + public function set quality (value:int) : void; + + /// Initializes the filter. + public function BlurFilter (blurX:Number = 4, blurY:Number = 4, quality:int = 1); + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ColorMatrixFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ColorMatrixFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The ColorMatrixFilter class lets you apply a 4 x 5 matrix transformation on the RGBA color and alpha valuesof every pixel in the input image to produce a result with a new set of RGBA color and alpha values. + public class ColorMatrixFilter extends BitmapFilter + { + /// An array of 20 items for 4 x 5 color transform. + public function get matrix () : Array; + public function set matrix (value:Array) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes a new ColorMatrixFilter instance. + public function ColorMatrixFilter (matrix:Array = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ConvolutionFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ConvolutionFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,50 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The ConvolutionFilter class applies a matrix convolution filter effect. + public class ConvolutionFilter extends BitmapFilter + { + /// The alpha transparency value of the substitute color. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// The amount of bias to add to the result of the matrix transformation. + public function get bias () : Number; + public function set bias (value:Number) : void; + + /// Indicates whether the image should be clamped. + public function get clamp () : Boolean; + public function set clamp (value:Boolean) : void; + + /// The hexadecimal color to substitute for pixels that are off the source image. + public function get color () : uint; + public function set color (value:uint) : void; + + /// The divisor used during matrix transformation. + public function get divisor () : Number; + public function set divisor (value:Number) : void; + + /// An array of values used for matrix transformation. + public function get matrix () : Array; + public function set matrix (value:Array) : void; + + /// The x dimension of the matrix (the number of columns in the matrix). + public function get matrixX () : Number; + public function set matrixX (value:Number) : void; + + /// The y dimension of the matrix (the number of rows in the matrix). + public function get matrixY () : Number; + public function set matrixY (value:Number) : void; + + /// Indicates if the alpha channel is preserved without the filter effect or if the convolution filter is applied to the alpha channel as well as the color channels. + public function get preserveAlpha () : Boolean; + public function set preserveAlpha (value:Boolean) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes a ConvolutionFilter instance with the specified parameters. + public function ConvolutionFilter (matrixX:Number = 0, matrixY:Number = 0, matrix:Array = null, divisor:Number = 1, bias:Number = 0, preserveAlpha:Boolean = true, clamp:Boolean = true, color:uint = 0, alpha:Number = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,52 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + import flash.geom.Point; + import flash.display.BitmapData; + + /// The DisplacementMapFilter class uses the pixel values from the specified BitmapData object (called the displacement map image) to perform a displacement of an object. + public class DisplacementMapFilter extends BitmapFilter + { + /// Specifies the alpha transparency value to use for out-of-bounds displacements. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// Specifies what color to use for out-of-bounds displacements. + public function get color () : uint; + public function set color (value:uint) : void; + + /// Describes which color channel to use in the map image to displace the x result. + public function get componentX () : uint; + public function set componentX (value:uint) : void; + + /// Describes which color channel to use in the map image to displace the y result. + public function get componentY () : uint; + public function set componentY (value:uint) : void; + + /// A BitmapData object containing the displacement map data. + public function get mapBitmap () : BitmapData; + public function set mapBitmap (value:BitmapData) : void; + + /// A value that contains the offset of the upper-left corner of the target display object from the upper-left corner of the map image. + public function get mapPoint () : Point; + public function set mapPoint (value:Point) : void; + + /// The mode for the filter. + public function get mode () : String; + public function set mode (value:String) : void; + + /// The multiplier to use to scale the x displacement result from the map calculation. + public function get scaleX () : Number; + public function set scaleX (value:Number) : void; + + /// The multiplier to use to scale the y displacement result from the map calculation. + public function get scaleY () : Number; + public function set scaleY (value:Number) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes a DisplacementMapFilter instance. + public function DisplacementMapFilter (mapBitmap:BitmapData = null, mapPoint:Point = null, componentX:uint = 0, componentY:uint = 0, scaleX:Number = 0, scaleY:Number = 0, mode:String = "wrap", color:uint = 0, alpha:Number = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilterMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DisplacementMapFilterMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.filters +{ + /// The DisplacementMapFilterMode class provides values for the mode propertyof the DisplacementMapFilter class. + public class DisplacementMapFilterMode extends Object + { + /// Clamps the displacement value to the edge of the source image. + public static const CLAMP : String; + /// If the displacement value is outside the image, substitutes the values in the color and alpha properties. + public static const COLOR : String; + /// If the displacement value is out of range, ignores the displacement and uses the source pixel. + public static const IGNORE : String; + /// Wraps the displacement value to the other side of the source image. + public static const WRAP : String; + + public function DisplacementMapFilterMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DropShadowFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/DropShadowFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,58 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The DropShadowFilter class lets you add a drop shadow to display objects. + public class DropShadowFilter extends BitmapFilter + { + /// The alpha transparency value for the shadow color. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// The angle of the shadow. + public function get angle () : Number; + public function set angle (value:Number) : void; + + /// The amount of horizontal blur. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// The color of the shadow. + public function get color () : uint; + public function set color (value:uint) : void; + + /// The offset distance for the shadow, in pixels. + public function get distance () : Number; + public function set distance (value:Number) : void; + + /// Indicates whether or not the object is hidden. + public function get hideObject () : Boolean; + public function set hideObject (value:Boolean) : void; + + /// Indicates whether or not the shadow is an inner shadow. + public function get inner () : Boolean; + public function set inner (value:Boolean) : void; + + /// Applies a knockout effect (true), which effectively makes the object's fill transparent and reveals the background color of the document. + public function get knockout () : Boolean; + public function set knockout (value:Boolean) : void; + + /// The number of times to apply the filter. + public function get quality () : int; + public function set quality (value:int) : void; + + /// The strength of the imprint or spread. + public function get strength () : Number; + public function set strength (value:Number) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Creates a new DropShadowFilter instance with the specified parameters. + public function DropShadowFilter (distance:Number = 4, angle:Number = 45, color:uint = 0, alpha:Number = 1, blurX:Number = 4, blurY:Number = 4, strength:Number = 1, quality:int = 1, inner:Boolean = false, knockout:Boolean = false, hideObject:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GlowFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GlowFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,46 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The GlowFilter class lets you apply a glow effect to display objects. + public class GlowFilter extends BitmapFilter + { + /// The alpha transparency value for the color. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// The amount of horizontal blur. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// The color of the glow. + public function get color () : uint; + public function set color (value:uint) : void; + + /// Specifies whether the glow is an inner glow. + public function get inner () : Boolean; + public function set inner (value:Boolean) : void; + + /// Specifies whether the object has a knockout effect. + public function get knockout () : Boolean; + public function set knockout (value:Boolean) : void; + + /// The number of times to apply the filter. + public function get quality () : int; + public function set quality (value:int) : void; + + /// The strength of the imprint or spread. + public function get strength () : Number; + public function set strength (value:Number) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes a new GlowFilter instance with the specified parameters. + public function GlowFilter (color:uint = 16711680, alpha:Number = 1, blurX:Number = 6, blurY:Number = 6, strength:Number = 2, quality:int = 1, inner:Boolean = false, knockout:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientBevelFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientBevelFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,58 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The GradientBevelFilter class lets you apply a gradient bevel effect to display objects. + public class GradientBevelFilter extends BitmapFilter + { + /// An array of alpha transparency values for the corresponding colors in the colors array. + public function get alphas () : Array; + public function set alphas (value:Array) : void; + + /// The angle, in degrees. + public function get angle () : Number; + public function set angle (value:Number) : void; + + /// The amount of horizontal blur. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// An array of RGB hexadecimal color values to use in the gradient. + public function get colors () : Array; + public function set colors (value:Array) : void; + + /// The offset distance. + public function get distance () : Number; + public function set distance (value:Number) : void; + + /// Specifies whether the object has a knockout effect. + public function get knockout () : Boolean; + public function set knockout (value:Boolean) : void; + + /// The number of times to apply the filter. + public function get quality () : int; + public function set quality (value:int) : void; + + /// An array of color distribution ratios for the corresponding colors in the colors array. + public function get ratios () : Array; + public function set ratios (value:Array) : void; + + /// The strength of the imprint or spread. + public function get strength () : Number; + public function set strength (value:Number) : void; + + /// The placement of the bevel effect. + public function get type () : String; + public function set type (value:String) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes the filter with the specified parameters. + public function GradientBevelFilter (distance:Number = 4, angle:Number = 45, colors:Array = null, alphas:Array = null, ratios:Array = null, blurX:Number = 4, blurY:Number = 4, strength:Number = 1, quality:int = 1, type:String = "inner", knockout:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientGlowFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/GradientGlowFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,58 @@ +package flash.filters +{ + import flash.filters.BitmapFilter; + + /// The GradientGlowFilter class lets you apply a gradient glow effect to display objects. + public class GradientGlowFilter extends BitmapFilter + { + /// An array of alpha transparency values for the corresponding colors in the colors array. + public function get alphas () : Array; + public function set alphas (value:Array) : void; + + /// The angle, in degrees. + public function get angle () : Number; + public function set angle (value:Number) : void; + + /// The amount of horizontal blur. + public function get blurX () : Number; + public function set blurX (value:Number) : void; + + /// The amount of vertical blur. + public function get blurY () : Number; + public function set blurY (value:Number) : void; + + /// An array of colors that defines a gradient. + public function get colors () : Array; + public function set colors (value:Array) : void; + + /// The offset distance of the glow. + public function get distance () : Number; + public function set distance (value:Number) : void; + + /// Specifies whether the object has a knockout effect. + public function get knockout () : Boolean; + public function set knockout (value:Boolean) : void; + + /// The number of times to apply the filter. + public function get quality () : int; + public function set quality (value:int) : void; + + /// An array of color distribution ratios for the corresponding colors in the colors array. + public function get ratios () : Array; + public function set ratios (value:Array) : void; + + /// The strength of the imprint or spread. + public function get strength () : Number; + public function set strength (value:Number) : void; + + /// The placement of the filter effect. + public function get type () : String; + public function set type (value:String) : void; + + /// Returns a copy of this filter object. + public function clone () : BitmapFilter; + + /// Initializes the filter with the specified parameters. + public function GradientGlowFilter (distance:Number = 4, angle:Number = 45, colors:Array = null, alphas:Array = null, ratios:Array = null, blurX:Number = 4, blurY:Number = 4, strength:Number = 1, quality:int = 1, type:String = "inner", knockout:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ShaderFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/filters/ShaderFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,32 @@ +package flash.filters +{ + import flash.geom.Rectangle; + import flash.display.Shader; + + /// The ShaderFilter class applies a filter by executing a shader on the object being filtered. + public class ShaderFilter extends BitmapFilter + { + /// The growth in pixels on the bottom side of the target object. + public function get bottomExtension () : int; + public function set bottomExtension (v:int) : void; + + /// The growth in pixels on the left side of the target object. + public function get leftExtension () : int; + public function set leftExtension (v:int) : void; + + /// The growth in pixels on the right side of the target object. + public function get rightExtension () : int; + public function set rightExtension (v:int) : void; + + /// The shader to use for this filter. + public function get shader () : Shader; + public function set shader (shader:Shader) : void; + + /// The growth in pixels on the top side of the target object. + public function get topExtension () : int; + public function set topExtension (v:int) : void; + + /// Creates a new shader filter. + public function ShaderFilter (shader:Shader = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/ColorTransform.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/ColorTransform.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,38 @@ +package flash.geom +{ + import flash.geom.ColorTransform; + + /// The ColorTransform class lets you adjust the color values in a display object. + public class ColorTransform extends Object + { + /// A decimal value that is multiplied with the alpha transparency channel value. + public var alphaMultiplier : Number; + /// A number from -255 to 255 that is added to the alpha transparency channel value after it has been multiplied by the alphaMultiplier value. + public var alphaOffset : Number; + /// A decimal value that is multiplied with the blue channel value. + public var blueMultiplier : Number; + /// A number from -255 to 255 that is added to the blue channel value after it has been multiplied by the blueMultiplier value. + public var blueOffset : Number; + /// A decimal value that is multiplied with the green channel value. + public var greenMultiplier : Number; + /// A number from -255 to 255 that is added to the green channel value after it has been multiplied by the greenMultiplier value. + public var greenOffset : Number; + /// A decimal value that is multiplied with the red channel value. + public var redMultiplier : Number; + /// A number from -255 to 255 that is added to the red channel value after it has been multiplied by the redMultiplier value. + public var redOffset : Number; + + /// The RGB color value for a ColorTransform object. + public function get color () : uint; + public function set color (newColor:uint) : void; + + /// Creates a ColorTransform object for a display object. + public function ColorTransform (redMultiplier:Number = 1, greenMultiplier:Number = 1, blueMultiplier:Number = 1, alphaMultiplier:Number = 1, redOffset:Number = 0, greenOffset:Number = 0, blueOffset:Number = 0, alphaOffset:Number = 0); + + /// Concatenates the ColorTranform object specified by the second parameter with the current ColorTransform object and sets the current object as the result, which is an additive combination of the two color transformations. + public function concat (second:ColorTransform) : void; + + /// Formats and returns a string that describes all of the properties of the ColorTransform object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,61 @@ +package flash.geom +{ + import flash.geom.Matrix; + import flash.geom.Point; + + /// The Matrix class represents a transformation matrix that determines how to map points from one coordinate space to another. + public class Matrix extends Object + { + /// The value that affects the positioning of pixels along the x axis when scaling or rotating an image. + public var a : Number; + /// The value that affects the positioning of pixels along the y axis when rotating or skewing an image. + public var b : Number; + /// The value that affects the positioning of pixels along the x axis when rotating or skewing an image. + public var c : Number; + /// The value that affects the positioning of pixels along the y axis when scaling or rotating an image. + public var d : Number; + /// The distance by which to translate each point along the x axis. + public var tx : Number; + /// The distance by which to translate each point along the y axis. + public var ty : Number; + + /// Returns a new Matrix object that is a copy of the current matrix. + public function clone () : Matrix; + + /// Concatenates a matrix with the current matrix, effectively combining the geometric effects of the two. + public function concat (m:Matrix) : void; + + /// Creates a Matrix with scaling, rotation, and translation values. + public function createBox (scaleX:Number, scaleY:Number, rotation:Number = 0, tx:Number = 0, ty:Number = 0) : void; + + /// Creates the specific style of matrix expected by the beginGradientFill() method of the Graphics class. + public function createGradientBox (width:Number, height:Number, rotation:Number = 0, tx:Number = 0, ty:Number = 0) : void; + + /// Given a point in the pretransform coordinate space, returns the coordinates of that point after the transformation occurs. + public function deltaTransformPoint (point:Point) : Point; + + /// Sets each matrix property to a value that causes a null transformation. + public function identity () : void; + + /// Performs the opposite transformation of the original matrix. + public function invert () : void; + + /// Creates a new two-dimensional Matrix object. + public function Matrix (a:Number = 1, b:Number = 0, c:Number = 0, d:Number = 1, tx:Number = 0, ty:Number = 0); + + /// Applies a rotation transformation to the Matrix object. + public function rotate (angle:Number) : void; + + /// Applies a scaling transformation to the matrix. + public function scale (sx:Number, sy:Number) : void; + + /// Returns a text value listing the properties of this Matrix object. + public function toString () : String; + + /// Returns the result of a geometric transformation to a Point object. + public function transformPoint (point:Point) : Point; + + /// A transformation that moves an object along the x and y axes. + public function translate (dx:Number, dy:Number) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix3D.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Matrix3D.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,83 @@ +package flash.geom +{ + import flash.geom.Matrix3D; + import flash.geom.Vector3D; + + /// The Matrix3D class represents a transformation matrix that determines the position and orientation of the three-dimensional display object. + public class Matrix3D extends Object + { + /// A Number that determines whether a matrix is invertible. + public function get determinant () : Number; + + /// A Vector3D object that holds the position, the three-dimensional coordinate (x,y,z) of a display object within the transformation's frame of reference. + public function get position () : Vector3D; + public function set position (pos:Vector3D) : void; + + /// A Vector of 16 Numbers, where every four elements can be a row or a column of a 4x4 matrix. + public function get rawData () : Vector.; + public function set rawData (v:Vector.) : void; + + /// Appends the matrix by multiplying another Matrix3D object by the current Matrix3D object. + public function append (lhs:Matrix3D) : void; + + /// Appends an incremental rotation to a Matrix3D object. + public function appendRotation (degrees:Number, axis:Vector3D, pivotPoint:Vector3D = null) : void; + + /// Appends an incremental scale change along the x, y, and z axes to a Matrix3D object. + public function appendScale (xScale:Number, yScale:Number, zScale:Number) : void; + + /// Appends an incremental translation, a repositioning along the x, y, and z axes, to a Matrix3D object. + public function appendTranslation (x:Number, y:Number, z:Number) : void; + + /// Returns a new Matrix3D object that is an exact copy of the current Matrix3D object. + public function clone () : Matrix3D; + + /// Returns the transformation matrix's translation, rotation, and scale settings as a Vector of three Vector3D objects. + public function decompose (orientationStyle:String = "eulerAngles") : Vector.; + + /// Uses the transformation matrix without its translation elements to transform a Vector3D object from one space coordinate to another. + public function deltaTransformVector (v:Vector3D) : Vector3D; + + /// Converts the current matrix to an identity or unit matrix. + public function identity () : void; + + /// Interpolates a display object a percent point closer to a target display object. + public static function interpolate (thisMat:Matrix3D, toMat:Matrix3D, percent:Number) : Matrix3D; + + /// Interpolates the display object's matrix a percent closer to a target's matrix. + public function interpolateTo (toMat:Matrix3D, percent:Number) : void; + + /// Inverts the current matrix. + public function invert () : Boolean; + + /// Creates a Matrix3D object. + public function Matrix3D (v:Vector. = null); + + /// Rotates the display object so that it faces a specified position. + public function pointAt (pos:Vector3D, at:Vector3D = null, up:Vector3D = null) : void; + + /// Prepends a matrix by multiplying the current Matrix3D object by another Matrix3D object. + public function prepend (rhs:Matrix3D) : void; + + /// Prepends an incremental rotation to a Matrix3D object. + public function prependRotation (degrees:Number, axis:Vector3D, pivotPoint:Vector3D = null) : void; + + /// Prepends an incremental scale change along the x, y, and z axes to a Matrix3D object. + public function prependScale (xScale:Number, yScale:Number, zScale:Number) : void; + + /// Prepends an incremental translation, a repositioning along the x, y, and z axes, to a Matrix3D object. + public function prependTranslation (x:Number, y:Number, z:Number) : void; + + /// Sets the transformation matrix's translation, rotation, and scale settings. + public function recompose (components:Vector., orientationStyle:String = "eulerAngles") : Boolean; + + /// Uses the transformation matrix to transform a Vector3D object from one space coordinate to another. + public function transformVector (v:Vector3D) : Vector3D; + + /// Uses the transformation matrix to transform a Vector of Numbers from one coordinate space to another. + public function transformVectors (vin:Vector., vout:Vector.) : void; + + /// Converts the current Matrix3D object to a matrix where the rows and columns are swapped. + public function transpose () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Orientation3D.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Orientation3D.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.geom +{ + /// The Orientation3D class is an enumeration of constant values for representing the orientation styleof a Matrix3D object. + public class Orientation3D extends Object + { + /// The axis angle orientation uses a combination of an axis and an angle to determine the orientation. + public static const AXIS_ANGLE : String; + /// Euler angles, the default orientation for decompose() and recompose() methods, defines the orientation with three separate angles of rotation for each axis. + public static const EULER_ANGLES : String; + /// The quaternion orientation uses complex numbers. + public static const QUATERNION : String; + + public function Orientation3D (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/PerspectiveProjection.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/PerspectiveProjection.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,27 @@ +package flash.geom +{ + import flash.geom.Point; + import flash.geom.Matrix3D; + + /// The PerspectiveProjection class provides an easy way to assign or modify the perspective transformations of a display object and all of its children. + public class PerspectiveProjection extends Object + { + /// Specifies an angle, as a degree between 0 and 180, for the field of view in three dimensions. + public function get fieldOfView () : Number; + public function set fieldOfView (fieldOfViewAngleInDegrees:Number) : void; + + /// The distance between the eye or the viewpoint's origin (0,0,0) and the display object located in the z axis. + public function get focalLength () : Number; + public function set focalLength (value:Number) : void; + + /// A two-dimensional point representing the center of the projection, the vanishing point for the display object. + public function get projectionCenter () : Point; + public function set projectionCenter (p:Point) : void; + + /// Creates an instance of a PerspectiveProjection object. + public function PerspectiveProjection (); + + /// Returns the underlying Matrix3D object of the display object. + public function toMatrix3D () : Matrix3D; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Point.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Point.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,49 @@ +package flash.geom +{ + import flash.geom.Point; + + /// The Point object represents a location in a two-dimensional coordinate system, where x represents the horizontal axis and y represents the vertical axis. + public class Point extends Object + { + /// The horizontal coordinate of the point. + public var x : Number; + /// The vertical coordinate of the point. + public var y : Number; + + /// The length of the line segment from (0,0) to this point. + public function get length () : Number; + + /// Adds the coordinates of another point to the coordinates of this point to create a new point. + public function add (v:Point) : Point; + + /// Creates a copy of the Point object. + public function clone () : Point; + + /// Returns the distance between pt1 and pt2. + public static function distance (pt1:Point, pt2:Point) : Number; + + /// Determines whether two points are equal. + public function equals (toCompare:Point) : Boolean; + + /// Determines a point between two specified points. + public static function interpolate (pt1:Point, pt2:Point, f:Number) : Point; + + /// Scales the line segment between (0,0) and the current point to a set length. + public function normalize (thickness:Number) : void; + + /// Offsets the Point object by the specified amount. + public function offset (dx:Number, dy:Number) : void; + + /// Creates a new point. + public function Point (x:Number = 0, y:Number = 0); + + /// Converts a pair of polar coordinates to a Cartesian point coordinate. + public static function polar (len:Number, angle:Number) : Point; + + /// Subtracts the coordinates of another point from the coordinates of this point to create a new point. + public function subtract (v:Point) : Point; + + /// Returns a string that contains the values of the x and y coordinates. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Rectangle.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Rectangle.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,94 @@ +package flash.geom +{ + import flash.geom.Point; + import flash.geom.Rectangle; + + /// A Rectangle object is an area defined by its position, as indicated by its top-left corner point (x, y) and by its width and its height. + public class Rectangle extends Object + { + /// The height of the rectangle, in pixels. + public var height : Number; + /// The width of the rectangle, in pixels. + public var width : Number; + /// The x coordinate of the top-left corner of the rectangle. + public var x : Number; + /// The y coordinate of the top-left corner of the rectangle. + public var y : Number; + + /// The sum of the y and height properties. + public function get bottom () : Number; + public function set bottom (value:Number) : void; + + /// The location of the Rectangle object's bottom-right corner, determined by the values of the right and bottom properties. + public function get bottomRight () : Point; + public function set bottomRight (value:Point) : void; + + /// The x coordinate of the top-left corner of the rectangle. + public function get left () : Number; + public function set left (value:Number) : void; + + /// The sum of the x and width properties. + public function get right () : Number; + public function set right (value:Number) : void; + + /// The size of the Rectangle object, expressed as a Point object with the values of the width and height properties. + public function get size () : Point; + public function set size (value:Point) : void; + + /// The y coordinate of the top-left corner of the rectangle. + public function get top () : Number; + public function set top (value:Number) : void; + + /// The location of the Rectangle object's top-left corner, determined by the x and y coordinates of the point. + public function get topLeft () : Point; + public function set topLeft (value:Point) : void; + + /// Returns a copy of this Rectangle object. + public function clone () : Rectangle; + + /// Determines if the specified point is contained within the rectangular region. + public function contains (x:Number, y:Number) : Boolean; + + /// Determines if the specified point is contained within the rectangular region defined by this Rectangle object using a Point object as a parameter. + public function containsPoint (point:Point) : Boolean; + + /// Determines if the Rectangle object specified by the rect parameter is contained within this Rectangle object. + public function containsRect (rect:Rectangle) : Boolean; + + /// Determines if the object specified in the toCompare parameter is equal to this Rectangle object. + public function equals (toCompare:Rectangle) : Boolean; + + /// Increases the size of the Rectangle object by the specified amounts, in pixels. + public function inflate (dx:Number, dy:Number) : void; + + /// Increases the size of the Rectangle object using a Point object as a parameter. + public function inflatePoint (point:Point) : void; + + /// Returns the area of intersection. + public function intersection (toIntersect:Rectangle) : Rectangle; + + /// Determines if the object specified in the toIntersect parameter intersects with this Rectangle object. + public function intersects (toIntersect:Rectangle) : Boolean; + + /// Determines whether or not this Rectangle object is empty. + public function isEmpty () : Boolean; + + /// Adjusts the location of the Rectangle object. + public function offset (dx:Number, dy:Number) : void; + + /// Adjusts the location of the Rectangle object using a Point object as a parameter. + public function offsetPoint (point:Point) : void; + + /// Creates a new Rectangle object with the top-left corner specified by the x and y parameters and with the specified width and height. + public function Rectangle (x:Number = 0, y:Number = 0, width:Number = 0, height:Number = 0); + + /// Sets all properties to 0. + public function setEmpty () : void; + + /// Builds and returns a string that lists the horizontal and vertical positions and the width and height of the Rectangle object. + public function toString () : String; + + /// Adds two rectangles together to create a new Rectangle object. + public function union (toUnion:Rectangle) : Rectangle; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Transform.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Transform.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,43 @@ +package flash.geom +{ + import flash.geom.Matrix; + import flash.geom.Matrix3D; + import flash.geom.ColorTransform; + import flash.geom.PerspectiveProjection; + import flash.display.DisplayObject; + import flash.geom.Rectangle; + + /// The Transform class provides access to color adjustment properties and two- or three-dimensional transformation objects that can be applied to a display object. + public class Transform extends Object + { + /// A ColorTransform object containing values that universally adjust the colors in the display object. + public function get colorTransform () : ColorTransform; + public function set colorTransform (value:ColorTransform) : void; + + /// A ColorTransform object representing the combined color transformations applied to the display object and all of its parent objects, back to the root level. + public function get concatenatedColorTransform () : ColorTransform; + + /// A Matrix object representing the combined transformation matrixes of the display object and all of its parent objects, back to the root level. + public function get concatenatedMatrix () : Matrix; + + /// A Matrix object containing values that alter the scaling, rotation, and translation of the display object. + public function get matrix () : Matrix; + public function set matrix (value:Matrix) : void; + + /// Provides access to the Matrix3D object of a three-dimensional display object. + public function get matrix3D () : Matrix3D; + public function set matrix3D (m:Matrix3D) : void; + + /// Provides access to the PerspectiveProjection object of a three-dimensional display object. + public function get perspectiveProjection () : PerspectiveProjection; + public function set perspectiveProjection (pm:PerspectiveProjection) : void; + + /// A Rectangle object that defines the bounding rectangle of the display object on the stage. + public function get pixelBounds () : Rectangle; + + /// Returns a Matrix3D object, which can transform the space of a specified display object in relation to the current display object's space. + public function getRelativeMatrix3D (relativeTo:DisplayObject) : Matrix3D; + + public function Transform (displayObject:DisplayObject); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Utils3D.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Utils3D.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.geom +{ + import flash.geom.Matrix3D; + import flash.geom.Vector3D; + + /// The Utils3D class contains static methods that simplify the implementation of certain three-dimensional matrix operations. + public class Utils3D extends Object + { + /// Interpolates the orientation of an object toward a position. + public static function pointTowards (percent:Number, mat:Matrix3D, pos:Vector3D, at:Vector3D = null, up:Vector3D = null) : Matrix3D; + + /// Using a projection Matrix3D object, projects a Vector3D object from one space coordinate to another. + public static function projectVector (m:Matrix3D, v:Vector3D) : Vector3D; + + /// Projects a Vector of three-dimensional space coordinates to a Vector of two-dimensional space coordinates. + public static function projectVectors (m:Matrix3D, verts:Vector., projectedVerts:Vector., uvts:Vector.) : void; + + public function Utils3D (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Vector3D.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/geom/Vector3D.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,80 @@ +package flash.geom +{ + import flash.geom.Vector3D; + + /// The Vector3D class represents a point or a location in the three-dimensional space using the Cartesian coordinates x, y, and z. + public class Vector3D extends Object + { + /// The fourth element of a Vector3D object (in addition to the x, y, and z properties) can hold data such as the angle of rotation. + public var w : Number; + /// The first element of a Vector3D object, such as the x coordinate of a point in the three-dimensional space. + public var x : Number; + /// The x axis defined as a Vector3D object with coordinates (1,0,0). + public static const X_AXIS : Vector3D; + /// The second element of a Vector3D object, such as the y coordinate of a point in the three-dimensional space. + public var y : Number; + /// The y axis defined as a Vector3D object with coordinates (0,1,0). + public static const Y_AXIS : Vector3D; + /// The third element of a Vector3D object, such as the z coordinate of a point in three-dimensional space. + public var z : Number; + /// The z axis defined as a Vector3D object with coordinates (0,0,1). + public static const Z_AXIS : Vector3D; + + /// The length, magnitude, of the current Vector3D object from the origin (0,0,0) to the object's x, y, and z coordinates. + public function get length () : Number; + + /// The square of the length of the current Vector3D object, calculated using the x, y, and z properties. + public function get lengthSquared () : Number; + + /// Adds the current Vector3D object to another in order to create a new Vector3D object. + public function add (a:Vector3D) : Vector3D; + + /// Returns the angle in radians between two vectors. + public static function angleBetween (a:Vector3D, b:Vector3D) : Number; + + /// Returns a new Vector3D object that is an exact copy of the current Vector3D object. + public function clone () : Vector3D; + + /// Returns a new Vector3D object that is perpendicular (at a right angle) to the current Vector3D and another Vector3D object. + public function crossProduct (a:Vector3D) : Vector3D; + + /// Decrements the current Vector3D object by another Vector3D object. + public function decrementBy (a:Vector3D) : void; + + /// Returns the distance between two Vector3D objects. + public static function distance (pt1:Vector3D, pt2:Vector3D) : Number; + + /// Returns the dot product of current and another Vector3D object. + public function dotProduct (a:Vector3D) : Number; + + /// Determines whether two Vector3D objects are equal by comparing the x, y, and z elements of the current Vector3D object with a specified Vector3D object. + public function equals (toCompare:Vector3D, allFour:Boolean = false) : Boolean; + + /// Increments the current Vector3D object by another Vector3D object. + public function incrementBy (a:Vector3D) : void; + + /// Compares the elements of the current Vector3D object with the elements of a specified Vector3D object to determine whether they are nearly equal. + public function nearEquals (toCompare:Vector3D, tolerance:Number, allFour:Boolean = false) : Boolean; + + /// Sets the current Vector3D object to its inverse. + public function negate () : void; + + /// Converts a Vector3D object to a unit vector by dividing the first three elements (x, y, z) by the length of the vector. + public function normalize () : Number; + + /// Divides the value of the x, y, and z properties of the current Vector3D object by the value of its w property. + public function project () : void; + + /// Scales the current Vector3D object by a scalar, a magnitude. + public function scaleBy (s:Number) : void; + + /// Subtracts the current Vector3D from another Vector3D object in order to create a new Vector3D object. + public function subtract (a:Vector3D) : Vector3D; + + /// Returns a string representation of the current Vector3D object. + public function toString () : String; + + /// Creates an instance of a Vector3D object. + public function Vector3D (x:Number = 0, y:Number = 0, z:Number = 0, w:Number = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Camera.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Camera.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,88 @@ +package flash.media +{ + import flash.events.EventDispatcher; + import flash.media.Camera; + + /** + * Dispatched when a camera reports its status. + * @eventType flash.events.StatusEvent.STATUS + */ + [Event(name="status", type="flash.events.StatusEvent")] + + /** + * Dispatched when a camera begins or ends a session. + * @eventType flash.events.ActivityEvent.ACTIVITY + */ + [Event(name="activity", type="flash.events.ActivityEvent")] + + /// Use the Camera class to capture video from a camera attached to a computer running Flash Player. + public class Camera extends EventDispatcher + { + /// The amount of motion the camera is detecting. + public function get activityLevel () : Number; + + /// The maximum amount of bandwidth the current outgoing video feed can use, in bytes. + public function get bandwidth () : int; + + /// The rate at which the camera is capturing data, in frames per second. + public function get currentFPS () : Number; + + /// The maximum rate at which the camera can capture data, in frames per second. + public function get fps () : Number; + + /// The current capture height, in pixels. + public function get height () : int; + + /// A zero-based integer that specifies the index of the camera, as reflected in the array returned by the names property. + public function get index () : int; + + /// The number of video frames transmitted in full (called keyframes) instead of being interpolated by the video compression algorithm. + public function get keyFrameInterval () : int; + + /// Indicates whether a local view of what the camera is capturing is compressed and decompressed (true), as it would be for live transmission using Flash Media Server, or uncompressed (false). + public function get loopback () : Boolean; + + /// The amount of motion required to invoke the activity event. + public function get motionLevel () : int; + + /// The number of milliseconds between the time the camera stops detecting motion and the time the activity event is invoked. + public function get motionTimeout () : int; + + /// A Boolean value indicating whether the user has denied access to the camera (true) or allowed access (false) in the Flash Player Privacy dialog box. + public function get muted () : Boolean; + + /// The name of the current camera, as returned by the camera hardware. + public function get name () : String; + + /// An array of strings indicating the names of all available cameras without displaying the Flash Player Privacy dialog box. + public static function get names () : Array; + + /// The required level of picture quality, as determined by the amount of compression being applied to each video frame. + public function get quality () : int; + + /// The current capture width, in pixels. + public function get width () : int; + + public function Camera (); + + /// Returns a reference to a Camera object for capturing video. + public static function getCamera (name:String = null) : Camera; + + public function setCursor (value:Boolean) : void; + + /// Specifies which video frames are transmitted in full (called keyframes) instead of being interpolated by the video compression algorithm. + public function setKeyFrameInterval (keyFrameInterval:int) : void; + + /// Specifies whether to use a compressed video stream for a local view of the camera. + public function setLoopback (compress:Boolean = false) : void; + + /// Sets the camera capture mode to the native mode that best meets the specified requirements. + public function setMode (width:int, height:int, fps:Number, favorArea:Boolean = true) : void; + + /// Specifies how much motion is required to dispatch the activity event. + public function setMotionLevel (motionLevel:int, timeout:int = 2000) : void; + + /// Sets the maximum amount of bandwidth per second or the required picture quality of the current outgoing video feed. + public function setQuality (bandwidth:int, quality:int) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/ID3Info.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/ID3Info.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.media +{ + /// The ID3Info class contains properties that reflect ID3 metadata. + public class ID3Info extends Object + { + /// The name of the album; corresponds to the ID3 2.0 tag TALB. + public var album : String; + /// The name of the artist; corresponds to the ID3 2.0 tag TPE1. + public var artist : String; + /// A comment about the recording; corresponds to the ID3 2.0 tag COMM. + public var comment : String; + /// The genre of the song; corresponds to the ID3 2.0 tag TCON. + public var genre : String; + /// The name of the song; corresponds to the ID3 2.0 tag TIT2. + public var songName : String; + /// The track number; corresponds to the ID3 2.0 tag TRCK. + public var track : String; + /// The year of the recording; corresponds to the ID3 2.0 tag TYER. + public var year : String; + + public function ID3Info (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Microphone.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Microphone.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,84 @@ +package flash.media +{ + import flash.events.EventDispatcher; + import flash.media.Microphone; + import flash.media.SoundTransform; + + /** + * Dispatched when a microphone reports its status. + * @eventType flash.events.StatusEvent.STATUS + */ + [Event(name="status", type="flash.events.StatusEvent")] + + /** + * Dispatched when a microphone begins or ends a session. + * @eventType flash.events.ActivityEvent.ACTIVITY + */ + [Event(name="activity", type="flash.events.ActivityEvent")] + + /// Use the Microphone class to capture audio from a microphone attached to a computer running Flash Player. + public class Microphone extends EventDispatcher + { + /// The amount of sound the microphone is detecting. + public function get activityLevel () : Number; + + /// The codec to use for compressing audio. + public function get codec () : String; + public function set codec (codec:String) : void; + + /// The encoded speech quality when using the Speex codec. + public function get encodeQuality () : int; + public function set encodeQuality (quality:int) : void; + + /// Number of Speex speech frames transmitted in a packet (message). + public function get framesPerPacket () : int; + public function set framesPerPacket (frames:int) : void; + + /// The microphone gain--that is, the amount by which the microphone multiplies the signal before transmitting it. + public function get gain () : Number; + public function set gain (gain:Number) : void; + + /// The index of the microphone, as reflected in the array returned by Microphone.names. + public function get index () : int; + + /// Specifies whether the user has denied access to the microphone (true) or allowed access (false). + public function get muted () : Boolean; + + /// The name of the current sound capture device, as returned by the sound capture hardware. + public function get name () : String; + + /// An array of strings containing the names of all available sound capture devices. + public static function get names () : Array; + + /// The rate at which the microphone captures sound, in kHz. + public function get rate () : int; + public function set rate (rate:int) : void; + + /// The amount of sound required to activate the microphone and dispatch the activity event. + public function get silenceLevel () : Number; + + /// The number of milliseconds between the time the microphone stops detecting sound and the time the activity event is dispatched. + public function get silenceTimeout () : int; + + /// Controls the sound of this microphone object when it is in loopback mode. + public function get soundTransform () : SoundTransform; + public function set soundTransform (sndTransform:SoundTransform) : void; + + /// Set to true if echo suppression is enabled; false otherwise. + public function get useEchoSuppression () : Boolean; + + /// Returns a reference to a Microphone object for capturing audio. + public static function getMicrophone (index:int = -1) : Microphone; + + public function Microphone (); + + /// Routes audio captured by a microphone to the local speakers. + public function setLoopBack (state:Boolean = true) : void; + + /// Sets the minimum input level that should be considered sound and (optionally) the amount of silent time signifying that silence has actually begun. + public function setSilenceLevel (silenceLevel:Number, timeout:int = -1) : void; + + /// Specifies whether to use the echo suppression feature of the audio codec. + public function setUseEchoSuppression (useEchoSuppression:Boolean) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Sound.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Sound.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,83 @@ +package flash.media +{ + import flash.events.EventDispatcher; + import flash.utils.ByteArray; + import flash.net.URLRequest; + import flash.media.SoundLoaderContext; + import flash.media.SoundChannel; + import flash.media.SoundTransform; + import flash.media.ID3Info; + + /** + * Dispatched when data is received as a load operation progresses. + * @eventType flash.events.ProgressEvent.PROGRESS + */ + [Event(name="progress", type="flash.events.ProgressEvent")] + + /** + * Dispatched when a load operation starts. + * @eventType flash.events.Event.OPEN + */ + [Event(name="open", type="flash.events.Event")] + + /** + * Dispatched when an input/output error occurs that causes a load operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched by a Sound object when ID3 data is available for an MP3 sound. + * @eventType flash.events.Event.ID3 + */ + [Event(name="id3", type="flash.events.Event")] + + /** + * Dispatched when data has loaded successfully. + * @eventType flash.events.Event.COMPLETE + */ + [Event(name="complete", type="flash.events.Event")] + + /** + * Dispatched when the player requests new audio data. + * @eventType flash.events.SampleDataEvent.SAMPLE_DATA + */ + [Event(name="sampleData", type="flash.events.SampleDataEvent")] + + /// The Sound class lets you work with sound in an application. + public class Sound extends EventDispatcher + { + /// Returns the currently available number of bytes in this sound object. + public function get bytesLoaded () : uint; + + /// Returns the total number of bytes in this sound object. + public function get bytesTotal () : int; + + /// Provides access to the metadata that is part of an MP3 file. + public function get id3 () : ID3Info; + + /// Returns the buffering state of external MP3 files. + public function get isBuffering () : Boolean; + + /// The length of the current sound in milliseconds. + public function get length () : Number; + + /// The URL from which this sound was loaded. + public function get url () : String; + + /// Closes the stream, causing any download of data to cease. + public function close () : void; + + /// Extracts raw sound data from a Sound object. + public function extract (target:ByteArray, length:Number, startPosition:Number = -1) : Number; + + /// Initiates loading of an external MP3 file from the specified URL. + public function load (stream:URLRequest, context:SoundLoaderContext = null) : void; + + /// Generates a new SoundChannel object to play back the sound. + public function play (startTime:Number = 0, loops:int = 0, sndTransform:SoundTransform = null) : SoundChannel; + + /// Creates a new Sound object. + public function Sound (stream:URLRequest = null, context:SoundLoaderContext = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundChannel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundChannel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,33 @@ +package flash.media +{ + import flash.events.EventDispatcher; + import flash.media.SoundTransform; + + /** + * Dispatched when a sound has finished playing. + * @eventType flash.events.Event.SOUND_COMPLETE + */ + [Event(name="soundComplete", type="flash.events.Event")] + + /// The SoundChannel class controls a sound in an application. + public class SoundChannel extends EventDispatcher + { + /// The current amplitude (volume) of the left channel, from 0 (silent) to 1 (full amplitude). + public function get leftPeak () : Number; + + /// When the sound is playing, the position property indicates the current point that is being played in the sound file. + public function get position () : Number; + + /// The current amplitude (volume) of the right channel, from 0 (silent) to 1 (full amplitude). + public function get rightPeak () : Number; + + /// The SoundTransform object assigned to the sound channel. + public function get soundTransform () : SoundTransform; + public function set soundTransform (sndTransform:SoundTransform) : void; + + public function SoundChannel (); + + /// Stops the sound playing in the channel. + public function stop () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundCodec.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundCodec.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.media +{ + /// The SoundCodec class is an enumeration of constant values used in setting the codec propertyof the Microphone class. + public class SoundCodec extends Object + { + /// Specifies that the Nellymoser codec be used for compressing audio. + public static const NELLYMOSER : String; + /// Specifies that the Speex codec be used for compressing audio. + public static const SPEEX : String; + + public function SoundCodec (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundLoaderContext.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundLoaderContext.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.media +{ + /// The SoundLoaderContext class provides security checks for SWF files that load sound. + public class SoundLoaderContext extends Object + { + /// The number of milliseconds to preload a streaming sound into a buffer before the sound starts to stream. + public var bufferTime : Number; + /// Specifies whether Flash Player should try to download a URL policy file from the loaded sound's server before beginning to load the sound. + public var checkPolicyFile : Boolean; + + /// Creates a new sound loader context object. + public function SoundLoaderContext (bufferTime:Number = 1000, checkPolicyFile:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundMixer.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundMixer.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,28 @@ +package flash.media +{ + import flash.media.SoundTransform; + import flash.utils.ByteArray; + + /// The SoundMixer class contains static properties and methods for global sound controlin the SWF file. + public class SoundMixer extends Object + { + /// The number of seconds to preload an embedded streaming sound into a buffer before it starts to stream. + public static function get bufferTime () : int; + public static function set bufferTime (bufferTime:int) : void; + + /// The SoundTransform object that controls global sound properties. + public static function get soundTransform () : SoundTransform; + public static function set soundTransform (sndTransform:SoundTransform) : void; + + /// Determines whether any sounds are not accessible due to security restrictions. + public static function areSoundsInaccessible () : Boolean; + + /// Takes a snapshot of the current sound wave and places it into the specified ByteArray object. + public static function computeSpectrum (outputArray:ByteArray, FFTMode:Boolean = false, stretchFactor:int = 0) : void; + + public function SoundMixer (); + + /// Stops all sounds currently playing. + public static function stopAll () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundTransform.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/SoundTransform.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,33 @@ +package flash.media +{ + /// The SoundTransform class contains properties for volume and panning. + public class SoundTransform extends Object + { + /// A value, from 0 (none) to 1 (all), specifying how much of the left input is played in the left speaker. + public function get leftToLeft () : Number; + public function set leftToLeft (leftToLeft:Number) : void; + + /// A value, from 0 (none) to 1 (all), specifying how much of the left input is played in the right speaker. + public function get leftToRight () : Number; + public function set leftToRight (leftToRight:Number) : void; + + /// The left-to-right panning of the sound, ranging from -1 (full pan left) to 1 (full pan right). + public function get pan () : Number; + public function set pan (panning:Number) : void; + + /// A value, from 0 (none) to 1 (all), specifying how much of the right input is played in the left speaker. + public function get rightToLeft () : Number; + public function set rightToLeft (rightToLeft:Number) : void; + + /// A value, from 0 (none) to 1 (all), specifying how much of the right input is played in the right speaker. + public function get rightToRight () : Number; + public function set rightToRight (rightToRight:Number) : void; + + /// The volume, ranging from 0 (silent) to 1 (full volume). + public function get volume () : Number; + public function set volume (volume:Number) : void; + + /// Creates a SoundTransform object. + public function SoundTransform (vol:Number = 1, panning:Number = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Video.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/Video.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,36 @@ +package flash.media +{ + import flash.display.DisplayObject; + import flash.media.Camera; + import flash.net.NetStream; + + /// The Video class displays live or recorded video in an application without embedding the video in your SWF file. + public class Video extends DisplayObject + { + /// Indicates the type of filter applied to decoded video as part of post-processing. + public function get deblocking () : int; + public function set deblocking (value:int) : void; + + /// Specifies whether the video should be smoothed (interpolated) when it is scaled. + public function get smoothing () : Boolean; + public function set smoothing (value:Boolean) : void; + + /// An integer specifying the height of the video stream, in pixels. + public function get videoHeight () : int; + + /// An integer specifying the width of the video stream, in pixels. + public function get videoWidth () : int; + + /// Specifies a video stream from a camera to be displayed within the boundaries of the Video object in the application. + public function attachCamera (camera:Camera) : void; + + /// Specifies a video stream to be displayed within the boundaries of the Video object in the application. + public function attachNetStream (netStream:NetStream) : void; + + /// Clears the image currently displayed in the Video object (not the video stream). + public function clear () : void; + + /// Creates a new Video instance. + public function Video (width:int = 320, height:int = 240); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/media/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +package flash.media +{ + /// Forces a rescan of the microphones and cameras on the system. + public function scanHardware():void; + +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/DynamicPropertyOutput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/DynamicPropertyOutput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.net +{ + public class DynamicPropertyOutput extends Object implements IDynamicPropertyOutput + { + public function DynamicPropertyOutput (); + + public function writeDynamicProperty (name:String, value:*) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileFilter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileFilter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package flash.net +{ + /// The FileFilter class is used to indicate what files on the user's system are shown in the file-browsing dialog box that is displayed when FileReference.browse() or FileReferenceList.browse() is called. + public class FileFilter extends Object + { + /// The description string for the filter. + public function get description () : String; + public function set description (value:String) : void; + + /// A list of file extensions. + public function get extension () : String; + public function set extension (value:String) : void; + + /// A list of Macintosh file types. + public function get macType () : String; + public function set macType (value:String) : void; + + /// Creates a new FileFilter instance. + public function FileFilter (description:String, extension:String, macType:String = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReference.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReference.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,112 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.net.URLRequest; + import flash.utils.ByteArray; + + /** + * Dispatched after data is received from the server after a successful upload. + * @eventType flash.events.DataEvent.UPLOAD_COMPLETE_DATA + */ + [Event(name="uploadCompleteData", type="flash.events.DataEvent")] + + /** + * Dispatched if a call to the upload() or uploadUnencoded() method attempts to access data over HTTP and Adobe AIR is able to detect and return the status code for the request. + * @eventType flash.events.HTTPStatusEvent.HTTP_RESPONSE_STATUS + */ + [Event(name="httpResponseStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched when an upload fails and an HTTP status code is available to describe the failure. + * @eventType flash.events.HTTPStatusEvent.HTTP_STATUS + */ + [Event(name="httpStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched when the user selects a file for upload or download from the file-browsing dialog box. + * @eventType flash.events.Event.SELECT + */ + [Event(name="select", type="flash.events.Event")] + + /** + * Dispatched when a call to the FileReference.upload() or FileReference.download() method tries to upload a file to a server or get a file from a server that is outside the caller's security sandbox. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched periodically during the file upload or download operation. + * @eventType flash.events.ProgressEvent.PROGRESS + */ + [Event(name="progress", type="flash.events.ProgressEvent")] + + /** + * Dispatched when an upload or download operation starts. + * @eventType flash.events.Event.OPEN + */ + [Event(name="open", type="flash.events.Event")] + + /** + * Dispatched when the upload or download fails. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when download is complete or when upload generates an HTTP status code of 200. + * @eventType flash.events.Event.COMPLETE + */ + [Event(name="complete", type="flash.events.Event")] + + /** + * Dispatched when a file upload or download is canceled through the file-browsing dialog box by the user. + * @eventType flash.events.Event.CANCEL + */ + [Event(name="cancel", type="flash.events.Event")] + + /// The FileReference class provides a means to upload and download files between a user's computer and a server. + public class FileReference extends EventDispatcher + { + /// The creation date of the file on the local disk. + public function get creationDate () : Date; + + /// The Macintosh creator type of the file, which is only used in Mac OS versions prior to Mac OS X. + public function get creator () : String; + + /// The ByteArray representing the loaded file after a successful call to load(). + public function get data () : ByteArray; + + /// The date that the file on the local disk was last modified. + public function get modificationDate () : Date; + + /// The name of the file on the local disk. + public function get name () : String; + + /// The size of the file on the local disk in bytes. + public function get size () : uint; + + /// The file type. + public function get type () : String; + + /// Displays a file-browsing dialog box that lets the user select a file to upload. + public function browse (typeFilter:Array = null) : Boolean; + + /// Cancels any ongoing upload or download. + public function cancel () : void; + + /// Opens a dialog box that lets the user download a file from a remote server. + public function download (request:URLRequest, defaultFileName:String = null) : void; + + /// Creates a new FileReference object. + public function FileReference (); + + /// Starts the load of a local file. + public function load () : void; + + /// Opens a dialog box that lets the user save a file to the local filesystem. + public function save (data:*, defaultFileName:String = null) : void; + + /// Starts the upload of a file to a remote server. + public function upload (request:URLRequest, uploadDataFieldName:String = "Filedata", testUpload:Boolean = false) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReferenceList.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/FileReferenceList.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,29 @@ +package flash.net +{ + import flash.events.EventDispatcher; + + /** + * Dispatched when the user selects one or more files to upload from the file-browsing dialog box. + * @eventType flash.events.Event.SELECT + */ + [Event(name="select", type="flash.events.Event")] + + /** + * Dispatched when the user dismisses the file-browsing dialog box. + * @eventType flash.events.Event.CANCEL + */ + [Event(name="cancel", type="flash.events.Event")] + + /// The FileReferenceList class provides a means to let users select one or more files for uploading. + public class FileReferenceList extends EventDispatcher + { + /// An array of FileReference objects. + public function get fileList () : Array; + + /// Displays a file-browsing dialog box that lets the user select local files to upload. + public function browse (typeFilter:Array = null) : Boolean; + + /// Creates a new FileReferenceList object. + public function FileReferenceList (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyOutput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyOutput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.net +{ + /// This interface controls the serialization of dynamic properties of dynamic objects. + public interface IDynamicPropertyOutput + { + /// Adds a dynamic property to the binary output of a serialized object. + public function writeDynamicProperty (name:String, value:*) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyWriter.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/IDynamicPropertyWriter.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +package flash.net +{ + import flash.net.IDynamicPropertyOutput; + + /// This interface is used with the IDynamicPropertyOutput interface to control the serialization of dynamic properties of dynamic objects. + public interface IDynamicPropertyWriter + { + /// Writes the name and value of an IDynamicPropertyOutput object to an object with dynamic properties. + public function writeDynamicProperties (obj:Object, output:IDynamicPropertyOutput) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/LocalConnection.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/LocalConnection.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,51 @@ +package flash.net +{ + import flash.events.EventDispatcher; + + /** + * Dispatched when a LocalConnection object reports its status. + * @eventType flash.events.StatusEvent.STATUS + */ + [Event(name="status", type="flash.events.StatusEvent")] + + /** + * Dispatched if a call to LocalConnection.send() attempts to send data to a different security sandbox. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched when an exception is thrown asynchronously -- that is, from native asynchronous code. + * @eventType flash.events.AsyncErrorEvent.ASYNC_ERROR + */ + [Event(name="asyncError", type="flash.events.AsyncErrorEvent")] + + /// The LocalConnection class lets you create a LocalConnection object that can invoke a method in another LocalConnection object, either within a single SWF file or between multiple SWF files. + public class LocalConnection extends EventDispatcher + { + /// Indicates the object on which callback methods are invoked. + public function get client () : Object; + public function set client (client:Object) : void; + + /// A string representing the domain of the location of the current SWF file. + public function get domain () : String; + + /// Specifies one or more domains that can send LocalConnection calls to this LocalConnection instance. + public function allowDomain (...rest) : void; + + /// Specifies one or more domains that can send LocalConnection calls to this LocalConnection object. + public function allowInsecureDomain (...rest) : void; + + /// Closes (disconnects) a LocalConnection object. + public function close () : void; + + /// Prepares a LocalConnection object to receive commands from a send() command (called the sending LocalConnection object). + public function connect (connectionName:String) : void; + + /// Creates a LocalConnection object. + public function LocalConnection (); + + /// Invokes the method named methodName on a connection opened with the connect(connectionName) method (the receiving LocalConnection object). + public function send (connectionName:String, methodName:String, ...rest) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetConnection.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetConnection.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,98 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.net.Responder; + + /** + * Dispatched when a NetConnection object is reporting its status or error condition. + * @eventType flash.events.NetStatusEvent.NET_STATUS + */ + [Event(name="netStatus", type="flash.events.NetStatusEvent")] + + /** + * Dispatched if a call to NetConnection.call() attempts to connect to a server outside the caller's security sandbox. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched when an input or output error occurs that causes a network operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when an exception is thrown asynchronously -- that is, from native asynchronous code. + * @eventType flash.events.AsyncErrorEvent.ASYNC_ERROR + */ + [Event(name="asyncError", type="flash.events.AsyncErrorEvent")] + + /// The NetConnection class creates a bidirectional connection between Flash Player and a Flash Media Server application or between Flash Player and an application server running Flash Remoting. + public class NetConnection extends EventDispatcher + { + /// Indicates the object on which callback methods should be invoked. + public function get client () : Object; + public function set client (object:Object) : void; + + /// Indicates whether Flash Player is connected to a server through a persistent RTMP connection (true) or not (false). + public function get connected () : Boolean; + + /// The proxy type used to make a successful NetConnection.connect() call to Flash Media Server: "none", "HTTP", "HTTPS", or "CONNECT". + public function get connectedProxyType () : String; + + /// The default object encoding for NetConnection objects created in the SWF file. + public static function get defaultObjectEncoding () : uint; + public static function set defaultObjectEncoding (version:uint) : void; + + /// The identifier of the Flash Media Server instance to which this Flash Player or Adobe AIR instance is connected. + public function get farID () : String; + + /// A value chosen substantially by Flash Media Server, unique to this connection. + public function get farNonce () : String; + + /// The total number of inbound and outbound peer connections that this instance of Flash Player or Adobe AIR allows. + public function get maxPeerConnections () : uint; + public function set maxPeerConnections (maxPeers:uint) : void; + + /// The identifier of this Flash Player or Adobe AIR instance for this NetConnection instance. + public function get nearID () : String; + + /// A value chosen substantially by this Flash Player or Adobe AIR instance, unique to this connection. + public function get nearNonce () : String; + + /// The object encoding for this NetConnection instance. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// The protocol used to establish the connection. + public function get protocol () : String; + + /// Determines which fallback methods are tried if an initial connection attempt to the server fails. + public function get proxyType () : String; + public function set proxyType (ptype:String) : void; + + /// An object that holds all of the peer subscriber NetStream objects that are not associated with publishing NetStream objects. + public function get unconnectedPeerStreams () : Array; + + /// The URI passed to the NetConnection.connect() method. + public function get uri () : String; + + /// Indicates whether a secure connection was made using native Transport Layer Security (TLS) rather than HTTPS. + public function get usingTLS () : Boolean; + + /// Adds a context header to the Action Message Format (AMF) packet structure. + public function addHeader (operation:String, mustUnderstand:Boolean = false, param:Object = null) : void; + + /// Invokes a command or method on Flash Media Server or on an application server running Flash Remoting. + public function call (command:String, responder:Responder, ...rest) : void; + + /// Closes the connection that was opened locally or to the server and dispatches a netStatus event with a code property of NetConnection.Connect.Closed. + public function close () : void; + + /// Creates a bidirectional connection between Flash Player and a Flash Media Server application. + public function connect (command:String, ...rest) : void; + + /// Creates a NetConnection object. + public function NetConnection (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStream.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStream.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,185 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.media.SoundTransform; + import flash.media.Camera; + import flash.media.Microphone; + import flash.net.NetConnection; + import flash.net.NetStream; + import flash.net.Responder; + import flash.net.NetStreamInfo; + import flash.net.NetStreamPlayOptions; + + /** + * Establishes a listener to respond when a NetStream object has completely played a stream. + * @eventType flash.events. + */ + [Event(name="onPlayStatus", type="flash.events")] + + /** + * Establishes a listener to respond when an embedded cue point is reached while playing a video file. + * @eventType flash.events. + */ + [Event(name="onCuePoint", type="flash.events")] + + /** + * Establishes a listener to respond when Flash Player receives text data embedded in a media file that is playing. + * @eventType flash.events. + */ + [Event(name="onTextData", type="flash.events")] + + /** + * Establishes a listener to respond when Flash Player receives image data as a byte array embedded in a media file that is playing. + * @eventType flash.events. + */ + [Event(name="onImageData", type="flash.events")] + + /** + * Establishes a listener to respond when Flash Player receives descriptive information embedded in the video being played. + * @eventType flash.events. + */ + [Event(name="onMetaData", type="flash.events")] + + /** + * Establishes a listener to respond when Flash Player receives information specific to Adobe Extensible Metadata Platform (XMP) embedded in the video being played. + * @eventType flash.events. + */ + [Event(name="onXMPData", type="flash.events")] + + /** + * Dispatched when a NetStream object is reporting its status or error condition. + * @eventType flash.events.NetStatusEvent.NET_STATUS + */ + [Event(name="netStatus", type="flash.events.NetStatusEvent")] + + /** + * Dispatched when an input or output error occurs that causes a network operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when an exception is thrown asynchronously -- that is, from native asynchronous code. + * @eventType flash.events.AsyncErrorEvent.ASYNC_ERROR + */ + [Event(name="asyncError", type="flash.events.AsyncErrorEvent")] + + /// The NetStream class opens a one-way streaming connection between Flash Player and Flash Media Server, or between Flash Player and the local file system. + public class NetStream extends EventDispatcher + { + /// A static object used as a parameter to the constructor for a NetStream instance. + public static const CONNECT_TO_FMS : String; + /// Creates a peer-to-peer publisher connection. + public static const DIRECT_CONNECTIONS : String; + + public function get audioCodec () : uint; + + /// The number of seconds of data currently in the buffer. + public function get bufferLength () : Number; + + /// Specifies how long to buffer messages before starting to display the stream. + public function get bufferTime () : Number; + public function set bufferTime (bufferTime:Number) : void; + + /// The number of bytes of data that have been loaded into Flash Player. + public function get bytesLoaded () : uint; + + /// The total size in bytes of the file being loaded into Flash Player. + public function get bytesTotal () : uint; + + /// Specifies whether Flash Player should try to download a URL policy file from the loaded video file's server before beginning to load the video file. + public function get checkPolicyFile () : Boolean; + public function set checkPolicyFile (state:Boolean) : void; + + /// Specifies the object on which callback methods are invoked to handle streaming or FLV file data. + public function get client () : Object; + public function set client (object:Object) : void; + + /// The number of frames per second being displayed. + public function get currentFPS () : Number; + + public function get decodedFrames () : uint; + + /// The identifier of the far end that is connected to this NetStream instance. + public function get farID () : String; + + /// A value chosen substantially by the other end of this stream, unique to this connection. + public function get farNonce () : String; + + /// Returns a NetStreamInfo object whose properties contain statistics about the quality of service. + public function get info () : NetStreamInfo; + + /// The number of seconds of data in the subscribing stream's buffer in live (unbuffered) mode. + public function get liveDelay () : Number; + + /// Specifies how long to buffer messages during pause mode. + public function get maxPauseBufferTime () : Number; + public function set maxPauseBufferTime (pauseBufferTime:Number) : void; + + /// A value chosen substantially by this end of the stream, unique to this connection. + public function get nearNonce () : String; + + /// The object encoding (AMF version) for this NetStream object. + public function get objectEncoding () : uint; + + /// An object that holds all of the subscribing NetStream instances that are listening to this publishing NetStream instance. + public function get peerStreams () : Array; + + /// Controls sound in this NetStream object. + public function get soundTransform () : SoundTransform; + public function set soundTransform (sndTransform:SoundTransform) : void; + + /// The position of the playhead, in seconds. + public function get time () : Number; + + public function get videoCodec () : uint; + + /// Specifies an audio stream sent over the NetStream object, from a Microphone object passed as the source. + public function attachAudio (microphone:Microphone) : void; + + /// Starts capturing video from a camera, or stops capturing if theCamera is set to null. + public function attachCamera (theCamera:Camera, snapshotMilliseconds:int = -1) : void; + + /// Stops playing all data on the stream, sets the time property to 0, and makes the stream available for another use. + public function close () : void; + + /// Creates a stream that can be used for playing video files through the specified NetConnection object. + public function NetStream (connection:NetConnection, peerID:String = "connectToFMS"); + + /// Invoked when a peer-publishing stream matches a peer-subscribing stream. + public function onPeerConnect (subscriber:NetStream) : Boolean; + + /// Pauses playback of a video stream. + public function pause () : void; + + /// Begins playback of video files. + public function play (...rest) : void; + + /// Begins playback of media files, with several options for playback. + public function play2 (param:NetStreamPlayOptions) : void; + + /// Sends streaming audio, video, and text messages from a client to Flash Media Server, optionally recording the stream during transmission. + public function publish (name:String = null, type:String = null) : void; + + /// Specifies whether incoming audio plays on the stream. + public function receiveAudio (flag:Boolean) : void; + + /// Specifies whether incoming video will play on the stream. + public function receiveVideo (flag:Boolean) : void; + + /// Specifies the frame rate for incoming video. + public function receiveVideoFPS (FPS:Number) : void; + + /// Resumes playback of a video stream that is paused. + public function resume () : void; + + /// Seeks the keyframe (also called an I-frame in the video industry) closest to the specified location. + public function seek (offset:Number) : void; + + /// Sends a message on a published stream to all subscribing clients. + public function send (handlerName:String, ...rest) : void; + + /// Pauses or resumes playback of a stream. + public function togglePause () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamInfo.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamInfo.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,69 @@ +package flash.net +{ + /// The NetStreamInfo class specifies the various Quality of Service (QOS) statistics related to a NetStream object and the underlying streaming buffer for audio, video, and data. + public class NetStreamInfo extends Object + { + /// Provides the NetStream audio buffer size in bytes. + public function get audioBufferByteLength () : Number; + + /// Provides NetStream audio buffer size in seconds. + public function get audioBufferLength () : Number; + + /// Specifies the total number of audio bytes that have arrived in the queue, regardless of how many have been played or flushed. + public function get audioByteCount () : Number; + + /// Specifies the rate at which the NetStream audio buffer is filled in bytes per second. + public function get audioBytesPerSecond () : Number; + + /// Specifies the audio loss for the NetStream session. + public function get audioLossRate () : Number; + + /// Specifies the total number of bytes that have arrived into the queue, regardless of how many have been played or flushed. + public function get byteCount () : Number; + + /// Specifies the rate at which the NetStream buffer is filled in bytes per second. + public function get currentBytesPerSecond () : Number; + + /// Provides the NetStream data buffer size in bytes. + public function get dataBufferByteLength () : Number; + + /// Provides NetStream data buffer size in seconds. + public function get dataBufferLength () : Number; + + /// Specifies the total number of bytes of data messages that have arrived in the queue, regardless of how many have been played or flushed. + public function get dataByteCount () : Number; + + /// Specifies the rate at which the NetStream data buffer is filled in bytes per second. + public function get dataBytesPerSecond () : Number; + + /// Returns the number of video frames dropped in the current NetStream playback session. + public function get droppedFrames () : Number; + + /// Specifies the maximum rate at which the NetStream buffer is filled in bytes per second. + public function get maxBytesPerSecond () : Number; + + /// Returns the stream playback rate in bytes per second. + public function get playbackBytesPerSecond () : Number; + + /// Specifies the Smooth Round Trip Time for the NetStream session. + public function get SRTT () : Number; + + /// Provides the NetStream video buffer size in bytes. + public function get videoBufferByteLength () : Number; + + /// Provides NetStream video buffer size in seconds. + public function get videoBufferLength () : Number; + + /// Specifies the total number of video bytes that have arrived in the queue, regardless of how many have been played or flushed. + public function get videoByteCount () : Number; + + /// Specifies the rate at which the NetStream video buffer is filled in bytes per second. + public function get videoBytesPerSecond () : Number; + + /// For internal use only; not recommended for use. + public function NetStreamInfo (curBPS:Number, byteCount:Number, maxBPS:Number, audioBPS:Number, audioByteCount:Number, videoBPS:Number, videoByteCount:Number, dataBPS:Number, dataByteCount:Number, playbackBPS:Number, droppedFrames:Number, audioBufferByteLength:Number, videoBufferByteLength:Number, dataBufferByteLength:Number, audioBufferLength:Number, videoBufferLength:Number, dataBufferLength:Number, srtt:Number, audioLossRate:Number); + + /// Returns a text value listing the properties of this NetStreamInfo object. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayOptions.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayOptions.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +package flash.net +{ + import flash.events.EventDispatcher; + + /// The NetStreamPlayOptions class specifies the various options that can be passed to the NetStream.play2() method. + public class NetStreamPlayOptions extends EventDispatcher + { + /// The duration of playback, in seconds, for the stream specified in streamName. + public var len : Number; + /// The name of the old stream or the stream to transition from. + public var oldStreamName : String; + /// The start time, in seconds, for streamName. + public var start : Number; + /// The name of the new stream to transition to or to play. + public var streamName : String; + /// The mode in which streamName is played or transitioned to. + public var transition : String; + + /// Creates a NetStreamPlayOptions object to specify the options that are passed to the NetStream.play2() method. + public function NetStreamPlayOptions (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayTransitions.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/NetStreamPlayTransitions.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,20 @@ +package flash.net +{ + /// The NetStreamPlayTransitions class specifies the valid strings that you can use with the NetStreamPlayOptions.transition property. + public class NetStreamPlayTransitions extends Object + { + /// Adds the stream to a playlist. + public static const APPEND : *; + /// Clears any previous play calls and plays the specified stream immediately. + public static const RESET : *; + /// Stops playing the streams in a playlist. + public static const STOP : *; + /// Replaces a content stream with a different content stream and maintains the rest of the playlist. + public static const SWAP : *; + /// Switches from playing one stream to another stream, typically with streams of the same content. + public static const SWITCH : *; + + /// The NetStreamPlayTransitions class specifies the valid strings that you can use with the NetStreamPlayOptions.transition property. + public function NetStreamPlayTransitions (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/ObjectEncoding.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/ObjectEncoding.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package flash.net +{ + import flash.net.IDynamicPropertyWriter; + + /// The ObjectEncoding class allows classes that serialize objects (such as NetStream, NetConnection, SharedObject, and ByteArray) to work with prior versions of ActionScript. + public class ObjectEncoding extends Object + { + /// Specifies that objects are serialized using the Action Message Format for ActionScript 1.0 and 2.0. + public static const AMF0 : uint; + /// Specifies that objects are serialized using the Action Message Format for ActionScript 3.0. + public static const AMF3 : uint; + /// Specifies the default (latest) format for the current player. + public static const DEFAULT : uint; + + /// Allows greater control over the serialization of dynamic properties of dynamic objects. + public static function get dynamicPropertyWriter () : IDynamicPropertyWriter; + public static function set dynamicPropertyWriter (object:IDynamicPropertyWriter) : void; + + public function ObjectEncoding (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Responder.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Responder.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.net +{ + /// The Responder class provides an object that is used in NetConnection.call() to handle return values from the server related to the success or failure of specific operations. + public class Responder extends Object + { + /// Creates a new Responder object. + public function Responder (result:Function, status:Function = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObject.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObject.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,82 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.net.SharedObject; + import flash.net.NetConnection; + + /** + * Dispatched when a remote shared object has been updated by the server. + * @eventType flash.events.SyncEvent.SYNC + */ + [Event(name="sync", type="flash.events.SyncEvent")] + + /** + * Dispatched when a SharedObject instance is reporting its status or error condition. + * @eventType flash.events.NetStatusEvent.NET_STATUS + */ + [Event(name="netStatus", type="flash.events.NetStatusEvent")] + + /** + * Dispatched when an exception is thrown asynchronously -- that is, from native asynchronous code. + * @eventType flash.events.AsyncErrorEvent.ASYNC_ERROR + */ + [Event(name="asyncError", type="flash.events.AsyncErrorEvent")] + + /// The SharedObject class is used to read and store limited amounts of data on a user's computer or on a server. + public class SharedObject extends EventDispatcher + { + /// Indicates the object on which callback methods are invoked. + public function get client () : Object; + public function set client (object:Object) : void; + + /// The collection of attributes assigned to the data property of the object; these attributes can be shared and stored. + public function get data () : Object; + + /// The default object encoding (AMF version) for all local shared objects created in the SWF file. + public static function get defaultObjectEncoding () : uint; + public static function set defaultObjectEncoding (version:uint) : void; + + /// Specifies the number of times per second that a client's changes to a shared object are sent to the server. + public function set fps (updatesPerSecond:Number) : void; + + /// The object encoding (AMF version) for this shared object. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// The current size of the shared object, in bytes. + public function get size () : uint; + + /// For local shared objects, purges all of the data and deletes the shared object from the disk. + public function clear () : void; + + /// Closes the connection between a remote shared object and the server. + public function close () : void; + + /// Connects to a remote shared object on a server through a specified NetConnection object. + public function connect (myConnection:NetConnection, params:String = null) : void; + + public static function deleteAll (url:String) : int; + + /// Immediately writes a locally persistent shared object to a local file. + public function flush (minDiskSpace:int = 0) : String; + + public static function getDiskUsage (url:String) : int; + + /// Returns a reference to a locally persistent shared object that is only available to the current client. + public static function getLocal (name:String, localPath:String = null, secure:Boolean = false) : SharedObject; + + /// Returns a reference to a shared object on Flash Media Server that multiple clients can access. + public static function getRemote (name:String, remotePath:String = null, persistence:Object = false, secure:Boolean = false) : SharedObject; + + /// Broadcasts a message to all clients connected to a remote shared object, including the client that sent the message. + public function send (...rest) : void; + + /// Indicates to the server that the value of a property in the shared object has changed. + public function setDirty (propertyName:String) : void; + + /// Updates the value of a property in a shared object and indicates to the server that the value of the property has changed. + public function setProperty (propertyName:String, value:Object = null) : void; + + public function SharedObject (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObjectFlushStatus.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/SharedObjectFlushStatus.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.net +{ + /// The SharedObjectFlushStatus class provides values for the code returned from a call to the SharedObject.flush() method. + public class SharedObjectFlushStatus extends Object + { + /// Indicates that the flush completed successfully. + public static const FLUSHED : String; + /// Indicates that the user is being prompted to increase disk space for the shared object before the flush can occur. + public static const PENDING : String; + + public function SharedObjectFlushStatus (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Socket.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/Socket.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,152 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.events.TimerEvent; + import flash.events.SecurityErrorEvent; + import flash.utils.IDataInput; + import flash.utils.IDataOutput; + import flash.utils.Timer; + import flash.utils.ByteArray; + + /** + * Dispatched if a call to Socket.connect() attempts to connect either to a server outside the caller's security sandbox or to a port lower than 1024. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched when a socket has received data. + * @eventType flash.events.ProgressEvent.SOCKET_DATA + */ + [Event(name="socketData", type="flash.events.ProgressEvent")] + + /** + * Dispatched when an input/output error occurs that causes a send or load operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when a network connection has been established. + * @eventType flash.events.Event.CONNECT + */ + [Event(name="connect", type="flash.events.Event")] + + /** + * Dispatched when the server closes the socket connection. + * @eventType flash.events.Event.CLOSE + */ + [Event(name="close", type="flash.events.Event")] + + /// The Socket class enables ActionScript code to make socket connections and to read and write raw binary data. + public class Socket extends EventDispatcher implements IDataInput, IDataOutput + { + /// The number of bytes of data available for reading in the input buffer. + public function get bytesAvailable () : uint; + + /// Indicates whether this Socket object is currently connected. + public function get connected () : Boolean; + + /// Indicates the byte order for the data; possible values are constants from the flash.utils.Endian class, Endian.BIG_ENDIAN or Endian.LITTLE_ENDIAN. + public function get endian () : String; + public function set endian (type:String) : void; + + /// Controls the version of AMF used when writing or reading an object. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// Indicates the number of milliseconds to wait for a connection. + public function get timeout () : uint; + public function set timeout (value:uint) : void; + + /// Closes the socket. + public function close () : void; + + /// Connects the socket to the specified host and port. + public function connect (host:String, port:int) : void; + + /// Flushes any accumulated data in the socket's output buffer. + public function flush () : void; + + /// Reads a Boolean value from the socket. + public function readBoolean () : Boolean; + + /// Reads a signed byte from the socket. + public function readByte () : int; + + /// Reads the number of data bytes specified by the length parameter from the socket. + public function readBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Reads an IEEE 754 double-precision floating-point number from the socket. + public function readDouble () : Number; + + /// Reads an IEEE 754 single-precision floating-point number from the socket. + public function readFloat () : Number; + + /// Reads a signed 32-bit integer from the socket. + public function readInt () : int; + + /// Reads a multibyte string from the byte stream, using the specified character set. + public function readMultiByte (length:uint, charSet:String) : String; + + /// Reads an object from the socket, encoded in AMF serialized format. + public function readObject () : *; + + /// Reads a signed 16-bit integer from the socket. + public function readShort () : int; + + /// Reads an unsigned byte from the socket. + public function readUnsignedByte () : uint; + + /// Reads an unsigned 32-bit integer from the socket. + public function readUnsignedInt () : uint; + + /// Reads an unsigned 16-bit integer from the socket. + public function readUnsignedShort () : uint; + + /// Reads a UTF-8 string from the socket. + public function readUTF () : String; + + /// Reads the number of UTF-8 data bytes specified by the length parameter from the socket, and returns a string. + public function readUTFBytes (length:uint) : String; + + /// Creates a new Socket object. + public function Socket (host:String = null, port:int = 0); + + /// Writes a Boolean value to the socket. + public function writeBoolean (value:Boolean) : void; + + /// Writes a byte to the socket. + public function writeByte (value:int) : void; + + /// Writes a sequence of bytes from the specified byte array. + public function writeBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Writes an IEEE 754 double-precision floating-point number to the socket. + public function writeDouble (value:Number) : void; + + /// Writes an IEEE 754 single-precision floating-point number to the socket. + public function writeFloat (value:Number) : void; + + /// Writes a 32-bit signed integer to the socket. + public function writeInt (value:int) : void; + + /// Writes a multibyte string from the byte stream, using the specified character set. + public function writeMultiByte (value:String, charSet:String) : void; + + /// Write an object to the socket in AMF serialized format. + public function writeObject (object:*) : void; + + /// Writes a 16-bit integer to the socket. + public function writeShort (value:int) : void; + + /// Writes a 32-bit unsigned integer to the socket. + public function writeUnsignedInt (value:uint) : void; + + /// Writes the following data to the socket: a 16-bit unsigned integer, which indicates the length of the specified UTF-8 string in bytes, followed by the string itself. + public function writeUTF (value:String) : void; + + /// Writes a UTF-8 string to the socket. + public function writeUTFBytes (value:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoader.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoader.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,66 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.net.URLStream; + import flash.net.URLRequest; + import flash.events.ProgressEvent; + import flash.events.Event; + + /** + * Dispatched if a call to URLLoader.load() attempts to access data over HTTP and the current Flash Player environment is able to detect and return the status code for the request. + * @eventType flash.events.HTTPStatusEvent.HTTP_STATUS + */ + [Event(name="httpStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched if a call to URLLoader.load() attempts to load data from a server outside the security sandbox. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched if a call to URLLoader.load() results in a fatal error that terminates the download. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched when data is received as the download operation progresses. + * @eventType flash.events.ProgressEvent.PROGRESS + */ + [Event(name="progress", type="flash.events.ProgressEvent")] + + /** + * Dispatched after all the received data is decoded and placed in the data property of the URLLoader object. + * @eventType flash.events.Event.COMPLETE + */ + [Event(name="complete", type="flash.events.Event")] + + /** + * Dispatched when the download operation commences following a call to the URLLoader.load() method. + * @eventType flash.events.Event.OPEN + */ + [Event(name="open", type="flash.events.Event")] + + /// The URLLoader class downloads data from a URL as text, binary data, or URL-encoded variables. + public class URLLoader extends EventDispatcher + { + /// Indicates the number of bytes that have been loaded thus far during the load operation. + public var bytesLoaded : uint; + /// Indicates the total number of bytes in the downloaded data. + public var bytesTotal : uint; + /// The data received from the load operation. + public var data : *; + /// Controls whether the downloaded data is received as text (URLLoaderDataFormat.TEXT), raw binary data (URLLoaderDataFormat.BINARY), or URL-encoded variables (URLLoaderDataFormat.VARIABLES). + public var dataFormat : String; + + /// Closes the load operation in progress. + public function close () : void; + + /// Sends and loads data from the specified URL. + public function load (request:URLRequest) : void; + + /// Creates a URLLoader object. + public function URLLoader (request:URLRequest = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoaderDataFormat.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLLoaderDataFormat.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.net +{ + /// The URLLoaderDataFormat class provides values that specify how downloaded data is received. + public class URLLoaderDataFormat extends Object + { + /// Specifies that downloaded data is received as raw binary data. + public static const BINARY : String; + /// Specifies that downloaded data is received as text. + public static const TEXT : String; + /// Specifies that downloaded data is received as URL-encoded variables. + public static const VARIABLES : String; + + public function URLLoaderDataFormat (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequest.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequest.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,33 @@ +package flash.net +{ + /// The URLRequest class captures all of the information in a single HTTP request. + public class URLRequest extends Object + { + /// The MIME content type of the content in the the data property. + public function get contentType () : String; + public function set contentType (value:String) : void; + + /// An object containing data to be transmitted with the URL request. + public function get data () : Object; + public function set data (value:Object) : void; + + /// A string that uniquely identifies the signed Adobe platform component to be stored to (or retrieved from) the Flash Player cache. + public function get digest () : String; + public function set digest (value:String) : void; + + /// Controls the HTTP form submission method. + public function get method () : String; + public function set method (value:String) : void; + + /// The array of HTTP request headers to be appended to the HTTP request. + public function get requestHeaders () : Array; + public function set requestHeaders (value:Array) : void; + + /// The URL to be requested. + public function get url () : String; + public function set url (value:String) : void; + + /// Creates a URLRequest object. + public function URLRequest (url:String = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestHeader.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestHeader.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.net +{ + /// A URLRequestHeader object encapsulates a single HTTP request header and consists of a name/value pair. + public class URLRequestHeader extends Object + { + /// An HTTP request header name (such as Content-Type or SOAPAction). + public var name : String; + /// The value associated with the name property (such as text/plain). + public var value : String; + + /// Creates a new URLRequestHeader object that encapsulates a single HTTP request header. + public function URLRequestHeader (name:String = "", value:String = ""); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestMethod.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLRequestMethod.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.net +{ + /// The URLRequestMethod class provides values that specify whether the URLRequest object should use the POST method or the GET method when sending data to a server. + public class URLRequestMethod extends Object + { + /// Specifies that the URLRequest object is a GET. + public static const GET : String; + /// Specifies that the URLRequest object is a POST. + public static const POST : String; + + public function URLRequestMethod (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLStream.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLStream.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,116 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.net.URLRequest; + import flash.utils.ByteArray; + + /** + * Dispatched when data is received as the download operation progresses. + * @eventType flash.events.ProgressEvent.PROGRESS + */ + [Event(name="progress", type="flash.events.ProgressEvent")] + + /** + * Dispatched when a load operation starts. + * @eventType flash.events.Event.OPEN + */ + [Event(name="open", type="flash.events.Event")] + + /** + * Dispatched when an input/output error occurs that causes a load operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched if a call to the URLStream.load() method attempts to access data over HTTP and Adobe AIR is able to detect and return the status code for the request. + * @eventType flash.events.HTTPStatusEvent.HTTP_RESPONSE_STATUS + */ + [Event(name="httpResponseStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched if a call to URLStream.load() attempts to access data over HTTP, and Flash Player or or Adobe AIR is able to detect and return the status code for the request. + * @eventType flash.events.HTTPStatusEvent.HTTP_STATUS + */ + [Event(name="httpStatus", type="flash.events.HTTPStatusEvent")] + + /** + * Dispatched if a call to URLStream.load() attempts to load data from a server outside the security sandbox. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched when data has loaded successfully. + * @eventType flash.events.Event.COMPLETE + */ + [Event(name="complete", type="flash.events.Event")] + + /// The URLStream class provides low-level access to downloading URLs. + public class URLStream extends EventDispatcher implements IDataInput + { + /// Returns the number of bytes of data available for reading in the input buffer. + public function get bytesAvailable () : uint; + + /// Indicates whether this URLStream object is currently connected. + public function get connected () : Boolean; + + /// Indicates the byte order for the data. + public function get endian () : String; + public function set endian (type:String) : void; + + /// Controls the version of Action Message Format (AMF) used when writing or reading an object. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// Immediately closes the stream and cancels the download operation. + public function close () : void; + + /// Begins downloading the URL specified in the request parameter. + public function load (request:URLRequest) : void; + + /// Reads a Boolean value from the stream. + public function readBoolean () : Boolean; + + /// Reads a signed byte from the stream. + public function readByte () : int; + + /// Reads length bytes of data from the stream. + public function readBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Reads an IEEE 754 double-precision floating-point number from the stream. + public function readDouble () : Number; + + /// Reads an IEEE 754 single-precision floating-point number from the stream. + public function readFloat () : Number; + + /// Reads a signed 32-bit integer from the stream. + public function readInt () : int; + + /// Reads a multibyte string of specified length from the byte stream using the specified character set. + public function readMultiByte (length:uint, charSet:String) : String; + + /// Reads an object from the socket, encoded in Action Message Format (AMF). + public function readObject () : *; + + /// Reads a signed 16-bit integer from the stream. + public function readShort () : int; + + /// Reads an unsigned byte from the stream. + public function readUnsignedByte () : uint; + + /// Reads an unsigned 32-bit integer from the stream. + public function readUnsignedInt () : uint; + + /// Reads an unsigned 16-bit integer from the stream. + public function readUnsignedShort () : uint; + + /// Reads a UTF-8 string from the stream. + public function readUTF () : String; + + /// Reads a sequence of length UTF-8 bytes from the stream, and returns a string. + public function readUTFBytes (length:uint) : String; + + public function URLStream (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLVariables.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/URLVariables.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.net +{ + /// The URLVariables class allows you to transfer variables between an application and a server. + public class URLVariables extends Object + { + /// Converts the variable string to properties of the specified URLVariables object. + public function decode (source:String) : void; + + /// Returns a string containing all enumerable variables, in the MIME content encoding application/x-www-form-urlencoded. + public function toString () : String; + + /// Creates a new URLVariables object. + public function URLVariables (source:String = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/XMLSocket.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/XMLSocket.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,61 @@ +package flash.net +{ + import flash.events.EventDispatcher; + import flash.utils.ByteArray; + import flash.net.Socket; + import flash.events.ProgressEvent; + import flash.events.Event; + + /** + * Dispatched if a call to the XMLSocket.connect() method attempts to connect either to a server outside the caller's security sandbox or to a port lower than 1024. + * @eventType flash.events.SecurityErrorEvent.SECURITY_ERROR + */ + [Event(name="securityError", type="flash.events.SecurityErrorEvent")] + + /** + * Dispatched when an input/output error occurs that causes a send or receive operation to fail. + * @eventType flash.events.IOErrorEvent.IO_ERROR + */ + [Event(name="ioError", type="flash.events.IOErrorEvent")] + + /** + * Dispatched after raw data is sent or received. + * @eventType flash.events.DataEvent.DATA + */ + [Event(name="data", type="flash.events.DataEvent")] + + /** + * Dispatched after a successful call to the XMLSocket.connect() method. + * @eventType flash.events.Event.CONNECT + */ + [Event(name="connect", type="flash.events.Event")] + + /** + * Dispatched when the server closes the socket connection. + * @eventType flash.events.Event.CLOSE + */ + [Event(name="close", type="flash.events.Event")] + + /// The XMLSocket class implements client sockets that let the computer that is running Flash Player communicate with a server computer identified by an IP address or domain name. + public class XMLSocket extends EventDispatcher + { + /// Indicates whether this XMLSocket object is currently connected. + public function get connected () : Boolean; + + /// Indicates the number of milliseconds to wait for a connection. + public function get timeout () : int; + public function set timeout (value:int) : void; + + /// Closes the connection specified by the XMLSocket object. + public function close () : void; + + /// Establishes a connection to the specified Internet host using the specified TCP port. + public function connect (host:String, port:int) : void; + + /// Converts the XML object or data specified in the object parameter to a string and transmits it to the server, followed by a zero (0) byte. + public function send (object:*) : void; + + /// Creates a new XMLSocket object. + public function XMLSocket (host:String = null, port:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/net/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.net +{ + import flash.net.URLRequest; + + /// Preserves the class (type) of an object when the object is encoded in Action Message Format (AMF). + public function registerClassAlias (aliasName:String, classObject:Class) : void; + + /// Looks up a class that previously had an alias registered through a call to the registerClassAlias() method. + public function getClassByAlias (aliasName:String) : Class; + + /// Opens or replaces a window in the application that contains the Flash Player container (usually a browser). + public function navigateToURL (request:URLRequest, window:String = null) : void; + + /// Sends a URL request to a server, but ignores any response. + public function sendToURL (request:URLRequest) : void; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJob.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJob.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,38 @@ +package flash.printing +{ + import flash.events.EventDispatcher; + import flash.geom.Rectangle; + import flash.display.Sprite; + import flash.printing.PrintJobOptions; + + /// The PrintJob class lets you create content and print it to one or more pages. + public class PrintJob extends EventDispatcher + { + /// The image orientation for printing. + public function get orientation () : String; + + /// The height of the actual printable area on the page, in points. + public function get pageHeight () : int; + + /// The width of the actual printable area on the page, in points. + public function get pageWidth () : int; + + /// The overall paper height, in points. + public function get paperHeight () : int; + + /// The overall paper width, in points. + public function get paperWidth () : int; + + /// Sends the specified Sprite object as a single page to the print spooler. + public function addPage (sprite:Sprite, printArea:Rectangle = null, options:PrintJobOptions = null, frameNum:int = 0) : void; + + /// Creates a PrintJob object that you can use to print one or more pages. + public function PrintJob (); + + /// Sends spooled pages to the printer after PrintJob.start() and PrintJob.addPage() have been successful. + public function send () : void; + + /// Displays the operating system's Print dialog box, starts spooling, and sets the PrintJob read-only property values. + public function start () : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOptions.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOptions.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,12 @@ +package flash.printing +{ + /// The PrintJobOptions class contains properties to use with the options parameter of the PrintJob.addPage() method. + public class PrintJobOptions extends Object + { + /// Specifies whether the content in the print job is printed as a bitmap or as a vector. + public var printAsBitmap : Boolean; + + /// Creates a new PrintJobOptions object. + public function PrintJobOptions (printAsBitmap:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOrientation.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/printing/PrintJobOrientation.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.printing +{ + /// This class provides values that are used by the PrintJob.orientation property for the image position of a printed page. + public class PrintJobOrientation extends Object + { + /// The landscape (horizontal) image orientation for printing. + public static const LANDSCAPE : String; + /// The portrait (vertical) image orientation for printing. + public static const PORTRAIT : String; + + public function PrintJobOrientation (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/profiler/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/profiler/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.profiler +{ + /// Shows or hides redraw regions. + public function showRedrawRegions (on:Boolean, color:uint = 0xFF0000) : void; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/DeleteObjectSample.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/DeleteObjectSample.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.sampler +{ + /// The DeleteObjectSample class represents objects that are created within a getSamples() stream; each DeleteObjectSample object corresponds to a NewObjectSample object. + public class DeleteObjectSample extends Sample + { + /// The unique identification number that matches up with a NewObjectSample's identification number. + public const id : Number; + /// The size of the DeleteObjectSample object before it is deleted. + public const size : Number; + + public function DeleteObjectSample (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/NewObjectSample.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/NewObjectSample.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,16 @@ +package flash.sampler +{ + /// The NewObjectSample class represents objects that are created within a getSamples() stream. + public class NewObjectSample extends Sample + { + /// The unique identification number that matches up with a DeleteObjectSample's identification number. + public const id : Number; + /// The Class object corresponding to the object created within a getSamples() stream. + public const type : Class; + + /// The NewObjectSample object if it still exists. + public function get object () : *; + + public function NewObjectSample (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/Sample.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/Sample.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.sampler +{ + /// The Sample class creates objects that hold memory analysis information over distinct durations. + public class Sample extends Object + { + /// Contains information about the methods executed by Flash Player over a specified period of time. + public const stack : Array; + /// The microseconds that define the duration of the Sample instance. + public const time : Number; + + public function Sample (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/StackFrame.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/StackFrame.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.sampler +{ + /// The StackFrame class provides access to the properties of a data block containing a function. + public class StackFrame extends Object + { + /// The file name of the SWF file being debugged. + public const file : String; + /// The line number for the function in the SWF file being debugged. + public const line : uint; + /// The function name in the stack frame. + public const name : String; + + public function StackFrame (); + + /// Converts the StackFrame to a string of its properties. + public function toString () : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/sampler/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,40 @@ +package flash.sampler +{ + /// Clears the current set of Sample objects. + public function clearSamples () : void; + + /// Begins the process of collecting memory usage Sample objects. + public function startSampling () : void; + + /// Ends the process of collecting memory usage Sample objects and frees resources dedicated to the sampling process. + public function stopSampling () : void; + + /// Stops the sampling process momentarily. + public function pauseSampling () : void; + + /// Returns the size in memory of a specified object when used with the Flash Player 9.0.115.0 or later debugger version. + public function getSize (o:*) : Number; + + /// Returns an object containing all members of a specified object, including private members. + public function getMemberNames (o:Object, instanceNames:Boolean = false) : Object; + + /// Returns an object of memory usage Sample instances from the last sampling session. + public function getSamples () : Object; + + /// Returns the number of samples collected. + public function getSampleCount () : Number; + + /// Returns the number of times a method was executed. + public function getInvocationCount (obj:Object, qname:QName) : Number; + + /// Returns the number of times a set function was executed. + public function getSetterInvocationCount (obj:Object, qname:QName) : Number; + + /// Returns the number of times a get function was executed. + public function getGetterInvocationCount (obj:Object, qname:QName) : Number; + + /// Checks to see if a property is defined by a get/set function. + public function isGetterSetter (obj:Object, qname:QName) : Boolean; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/ApplicationDomain.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/ApplicationDomain.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,31 @@ +package flash.system +{ + import flash.system.ApplicationDomain; + import flash.utils.ByteArray; + + /// The ApplicationDomain class is a container for discrete groups of class definitions. + public class ApplicationDomain extends Object + { + /// Gets the current application domain in which your code is executing. + public static function get currentDomain () : ApplicationDomain; + + /// Gets and sets the object on which domain-global memory operations will operate within this ApplicationDomain. + public function get domainMemory () : ByteArray; + public function set domainMemory (mem:ByteArray) : void; + + /// Gets the minimum memory object length required to be used as ApplicationDomain.domainMemory. + public static function get MIN_DOMAIN_MEMORY_LENGTH () : uint; + + /// Gets the parent domain of this application domain. + public function get parentDomain () : ApplicationDomain; + + /// Creates a new application domain. + public function ApplicationDomain (parentDomain:ApplicationDomain = null); + + /// Gets a public definition from the specified application domain. + public function getDefinition (name:String) : Object; + + /// Checks to see if a public definition exists within the specified application domain. + public function hasDefinition (name:String) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Capabilities.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Capabilities.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,94 @@ +package flash.system +{ + /// The Capabilities class provides properties that describe the system and player that are hosting a SWF file. + public class Capabilities extends Object + { + public static function get _internal () : uint; + + /// Specifies whether access to the user's camera and microphone has been administratively prohibited (true) or allowed (false). + public static function get avHardwareDisable () : Boolean; + + /// Specifies whether the system supports (true) or does not support (false) communication with accessibility aids. + public static function get hasAccessibility () : Boolean; + + /// Specifies whether the system has audio capabilities. + public static function get hasAudio () : Boolean; + + /// Specifies whether the system can (true) or cannot (false) encode an audio stream, such as that coming from a microphone. + public static function get hasAudioEncoder () : Boolean; + + /// Specifies whether the system supports (true) or does not support (false) embedded video. + public static function get hasEmbeddedVideo () : Boolean; + + /// Specifies whether the system does (true) or does not (false) have an input method editor (IME) installed. + public static function get hasIME () : Boolean; + + /// Specifies whether the system does (true) or does not (false) have an MP3 decoder. + public static function get hasMP3 () : Boolean; + + /// Specifies whether the system does (true) or does not (false) support printing. + public static function get hasPrinting () : Boolean; + + /// Specifies whether the system does (true) or does not (false) support the development of screen broadcast applications to be run through Flash Media Server. + public static function get hasScreenBroadcast () : Boolean; + + /// Specifies whether the system does (true) or does not (false) support the playback of screen broadcast applications that are being run through Flash Media Server. + public static function get hasScreenPlayback () : Boolean; + + /// Specifies whether the system can (true) or cannot (false) play streaming audio. + public static function get hasStreamingAudio () : Boolean; + + /// Specifies whether the system can (true) or cannot (false) play streaming video. + public static function get hasStreamingVideo () : Boolean; + + /// Specifies whether the system supports native SSL sockets through NetConnection (true) or does not (false). + public static function get hasTLS () : Boolean; + + /// Specifies whether the system can (true) or cannot (false) encode a video stream, such as that coming from a web camera. + public static function get hasVideoEncoder () : Boolean; + + /// Specifies whether the system is using special debugging software (true) or an officially released version (false). + public static function get isDebugger () : Boolean; + + /// Specifies the language code of the system on which the content is running. + public static function get language () : String; + + /// Specifies whether read access to the user's hard disk has been administratively prohibited (true) or allowed (false). + public static function get localFileReadDisable () : Boolean; + + /// Specifies the manufacturer of the running version of Flash Player or the AIR runtime, in the format "Adobe OSName". + public static function get manufacturer () : String; + + /// Retrieves the highest H.264 Level IDC that the client hardware supports. + public static function get maxLevelIDC () : String; + + /// Specifies the current operating system. + public static function get os () : String; + + /// Specifies the pixel aspect ratio of the screen. + public static function get pixelAspectRatio () : Number; + + /// Specifies the type of runtime environment. + public static function get playerType () : String; + + /// Specifies the screen color. + public static function get screenColor () : String; + + /// Specifies the dots-per-inch (dpi) resolution of the screen, in pixels. + public static function get screenDPI () : Number; + + /// Specifies the maximum horizontal resolution of the screen. + public static function get screenResolutionX () : Number; + + /// Specifies the maximum vertical resolution of the screen. + public static function get screenResolutionY () : Number; + + /// A URL-encoded string that specifies values for each Capabilities property. + public static function get serverString () : String; + + /// Specifies the Flash Player or Adobe AIR platform and version information. + public static function get version () : String; + + public function Capabilities (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/FSCommand.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/FSCommand.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.system +{ + public class FSCommand extends Object + { + public static function _fscommand (command:String, args:String) : void; + + public function FSCommand (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IME.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IME.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,32 @@ +package flash.system +{ + import flash.events.EventDispatcher; + + /** + * Dispatched when a user has completed an input method editor (IME) composition and the reading string is available. + * @eventType flash.events.IMEEvent.IME_COMPOSITION + */ + [Event(name="imeComposition", type="flash.events.IMEEvent")] + + /// The IME class lets you directly manipulate the operating system's input method editor (IME) in the Flash Player application that is running on a client computer. + public class IME extends EventDispatcher + { + public static function set constructOK (construct:Boolean) : void; + + /// The conversion mode of the current IME. + public static function get conversionMode () : String; + public static function set conversionMode (mode:String) : void; + + /// Indicates whether the system IME is enabled (true) or disabled (false). + public static function get enabled () : Boolean; + public static function set enabled (enabled:Boolean) : void; + + /// Instructs the IME to select the first candidate for the current composition string. + public static function doConversion () : void; + + public function IME (); + + /// Sets the IME composition string. + public static function setCompositionString (composition:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IMEConversionMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/IMEConversionMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,25 @@ +package flash.system +{ + /// This class contains constants for use with the IME.conversionMode property. + public class IMEConversionMode extends Object + { + /// The string "ALPHANUMERIC_FULL", for use with the IME.conversionMode property. + public static const ALPHANUMERIC_FULL : String; + /// The string "ALPHANUMERIC_HALF", for use with the IME.conversionMode property. + public static const ALPHANUMERIC_HALF : String; + /// The string "CHINESE", for use with the IME.conversionMode property. + public static const CHINESE : String; + /// The string "JAPANESE_HIRAGANA", for use with the IME.conversionMode property. + public static const JAPANESE_HIRAGANA : String; + /// The string "JAPANESE_KATAKANA_FULL", for use with the IME.conversionMode property. + public static const JAPANESE_KATAKANA_FULL : String; + /// The string "JAPANESE_KATAKANA_HALF", for use with the IME.conversionMode property. + public static const JAPANESE_KATAKANA_HALF : String; + /// The string "KOREAN", for use with the IME.conversionMode property. + public static const KOREAN : String; + /// The string "UNKNOWN", which can be returned by a call to the IME.conversionMode property. + public static const UNKNOWN : String; + + public function IMEConversionMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/JPEGLoaderContext.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/JPEGLoaderContext.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,12 @@ +package flash.system +{ + import flash.system.ApplicationDomain; + import flash.system.SecurityDomain; + + public class JPEGLoaderContext extends LoaderContext + { + public var deblockingFilter : Number; + + public function JPEGLoaderContext (deblockingFilter:Number = 0, checkPolicyFile:Boolean = false, applicationDomain:ApplicationDomain = null, securityDomain:SecurityDomain = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/LoaderContext.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/LoaderContext.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.system +{ + import flash.system.ApplicationDomain; + import flash.system.SecurityDomain; + + /// The LoaderContext class provides options for loading SWF files and other media by using the Loader class. + public class LoaderContext extends Object + { + /// Specifies the application domain to use for the Loader.load() or Loader.loadBytes() method. + public var applicationDomain : ApplicationDomain; + /// Specifies whether Flash Player should attempt to download a URL policy file from the loaded object's server before beginning to load the object itself. + public var checkPolicyFile : Boolean; + /// Specifies the security domain to use for a Loader.load() operation. + public var securityDomain : SecurityDomain; + + /// Creates a new LoaderContext object, with the specified settings. + public function LoaderContext (checkPolicyFile:Boolean = false, applicationDomain:ApplicationDomain = null, securityDomain:SecurityDomain = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Security.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/Security.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +package flash.system +{ + /// The Security class lets you specify how content in different domains can communicate with each other. + public class Security extends Object + { + /// The file is a local file and has been trusted by the user, using either the Flash Player Settings Manager or a FlashPlayerTrust configuration file. + public static const LOCAL_TRUSTED : String; + /// The file is a local file, has not been trusted by the user, and it is not a SWF file that was published with a networking designation. + public static const LOCAL_WITH_FILE : String; + /// The file is a local file, has not been trusted by the user, and it is a SWF file that was published with a networking designation. + public static const LOCAL_WITH_NETWORK : String; + /// The file is from an Internet URL and operates under domain-based sandbox rules. + public static const REMOTE : String; + + public static function get disableAVM1Loading () : Boolean; + public static function set disableAVM1Loading (value:Boolean) : void; + + /// Determines how Flash Player or AIR chooses the domain to use for certain content settings, including settings for camera and microphone permissions, storage quotas, and storage of persistent shared objects. + public static function get exactSettings () : Boolean; + public static function set exactSettings (value:Boolean) : void; + + /// Indicates the type of security sandbox in which the calling file is operating. + public static function get sandboxType () : String; + + /// Lets SWF files and HTML files access objects and variables in the calling SWF file. + public static function allowDomain (...rest) : void; + + /// Lets SWF and HTML files hosted using the HTTPS protocol, access objects and variables in the calling SWF file. + public static function allowInsecureDomain (...rest) : void; + + /// Loads a URL policy file from a location specified by the url parameter. + public static function loadPolicyFile (url:String) : void; + + public function Security (); + + /// Displays the Security Settings panel in Flash Player. + public static function showSettings (panel:String = "default") : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityDomain.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityDomain.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.system +{ + import flash.system.SecurityDomain; + + /// The SecurityDomain class represents the current security "sandbox," also known as a security domain. + public class SecurityDomain extends Object + { + /// Gets the current security domain. + public static function get currentDomain () : SecurityDomain; + + public function SecurityDomain (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityPanel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/SecurityPanel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.system +{ + /// The SecurityPanel class provides values for specifying which Security Settings panel you want to display. + public class SecurityPanel extends Object + { + /// When passed to Security.showSettings(), displays the Camera panel in Flash Player Settings. + public static const CAMERA : String; + /// When passed to Security.showSettings(), displays the panel that was open the last time the user closed the Flash Player Settings. + public static const DEFAULT : String; + /// When passed to Security.showSettings(), displays the Display panel in Flash Player Settings. + public static const DISPLAY : String; + /// When passed to Security.showSettings(), displays the Local Storage Settings panel in Flash Player Settings. + public static const LOCAL_STORAGE : String; + /// When passed to Security.showSettings(), displays the Microphone panel in Flash Player Settings. + public static const MICROPHONE : String; + /// When passed to Security.showSettings(), displays the Privacy Settings panel in Flash Player Settings. + public static const PRIVACY : String; + /// When passed to Security.showSettings(), displays the Settings Manager (in a separate browser window). + public static const SETTINGS_MANAGER : String; + + public function SecurityPanel (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/System.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/System.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,37 @@ +package flash.system +{ + import flash.system.IME; + + /// The System class contains properties related to certain operations that take place on the user's computer, such as operations with shared objects, local settings for cameras and microphones, and use of the Clipboard. + public class System extends Object + { + /// The currently installed system IME. + public static function get ime () : IME; + + /// The amount of memory (in bytes) currently in use by Adobe Flash Player. + public static function get totalMemory () : uint; + + /// A Boolean value that tells Flash Player which code page to use to interpret external text files. + public static function get useCodePage () : Boolean; + public static function set useCodePage (value:Boolean) : void; + + public static function get vmVersion () : String; + + /// Closes the Flash Player. + public static function exit (code:uint) : void; + + /// Forces the garbage collection process. + public static function gc () : void; + + /// Pauses the Flash Player. + public static function pause () : void; + + /// Resumes the Flash Player after using System.pause(). + public static function resume () : void; + + /// Replaces the contents of the Clipboard with a specified text string. + public static function setClipboard (string:String) : void; + + public function System (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/system/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +package flash.system +{ + /// Lets the SWF file communicate with either Flash Player or the program hosting Flash Player, such as a web browser. + public function fscommand (command:String, args:String) : void; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/AntiAliasType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/AntiAliasType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text +{ + /// The AntiAliasType class provides values for anti-aliasing in the flash.text.TextField class. + public class AntiAliasType extends Object + { + /// Sets anti-aliasing to advanced anti-aliasing. + public static const ADVANCED : String; + /// Sets anti-aliasing to the anti-aliasing that is used in Flash Player 7 and earlier. + public static const NORMAL : String; + + public function AntiAliasType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/CSMSettings.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/CSMSettings.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,16 @@ +package flash.text +{ + /// The CSMSettings class contains properties for use with the TextRenderer.setAdvancedAntiAliasingTable() method to provide continuous stroke modulation (CSM). + public class CSMSettings extends Object + { + /// The size, in pixels, for which the settings apply. + public var fontSize : Number; + /// The inside cutoff value, above which densities are set to a maximum density value (such as 255). + public var insideCutoff : Number; + /// The outside cutoff value, below which densities are set to zero. + public var outsideCutoff : Number; + + /// Creates a new CSMSettings object which stores stroke values for custom anti-aliasing settings. + public function CSMSettings (fontSize:Number, insideCutoff:Number, outsideCutoff:Number); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/Font.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/Font.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,26 @@ +package flash.text +{ + /// The Font class is used to manage embedded fonts in SWF files. + public class Font extends Object + { + /// The name of an embedded font. + public function get fontName () : String; + + /// The style of the font. + public function get fontStyle () : String; + + /// The type of the font. + public function get fontType () : String; + + /// Specifies whether to provide a list of the currently available embedded fonts. + public static function enumerateFonts (enumerateDeviceFonts:Boolean = false) : Array; + + public function Font (); + + /// Specifies whether a provided string can be displayed using the currently assigned font. + public function hasGlyphs (str:String) : Boolean; + + /// Registers a font class in the global font list. + public static function registerFont (font:Class) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontStyle.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontStyle.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text +{ + /// The FontStyle class provides values for the TextRenderer class. + public class FontStyle extends Object + { + /// Defines the bold style of a font for the fontStyle parameter in the setAdvancedAntiAliasingTable() method. + public static const BOLD : String; + /// Defines the combined bold and italic style of a font for the fontStyle parameter in the setAdvancedAntiAliasingTable() method. + public static const BOLD_ITALIC : String; + /// Defines the italic style of a font for the fontStyle parameter in the setAdvancedAntiAliasingTable() method. + public static const ITALIC : String; + /// Defines the plain style of a font for the fontStyle parameter in the setAdvancedAntiAliasingTable() method. + public static const REGULAR : String; + + public function FontStyle (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/FontType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text +{ + /// The FontType class contains the enumerated constants "embedded" and "device" for the fontType property of the Font class. + public class FontType extends Object + { + /// Indicates that this is a device font. + public static const DEVICE : String; + /// Indicates that this is an embedded font. + public static const EMBEDDED : String; + /// Indicates that this is an embedded CFF font. + public static const EMBEDDED_CFF : String; + + public function FontType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/GridFitType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/GridFitType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text +{ + /// The GridFitType class defines values for grid fitting in the TextField class. + public class GridFitType extends Object + { + /// Doesn't set grid fitting. + public static const NONE : String; + /// Fits strong horizontal and vertical lines to the pixel grid. + public static const PIXEL : String; + /// Fits strong horizontal and vertical lines to the sub-pixel grid on LCD monitors. + public static const SUBPIXEL : String; + + public function GridFitType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StaticText.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StaticText.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text +{ + import flash.display.DisplayObject; + + /// This class represents StaticText objects on the display list. + public class StaticText extends DisplayObject + { + /// Returns the current text of the static text field. + public function get text () : String; + + public function StaticText (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StyleSheet.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/StyleSheet.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package flash.text +{ + import flash.events.EventDispatcher; + import flash.text.TextFormat; + + /// The StyleSheet class lets you create a StyleSheet object that contains text formatting rules for font size, color, and other styles. + public class StyleSheet extends EventDispatcher + { + /// An array that contains the names (as strings) of all of the styles registered in this style sheet. + public function get styleNames () : Array; + + /// Removes all styles from the style sheet object. + public function clear () : void; + + /// Returns a copy of the style object associated with the style named styleName. + public function getStyle (styleName:String) : Object; + + /// Parses the CSS in cssText and loads the StyleSheet with it. + public function parseCSS (CSSText:String) : void; + + /// Adds a new style with the specified name to the style sheet object. + public function setStyle (styleName:String, styleObject:Object) : void; + + /// Creates a new StyleSheet object. + public function StyleSheet (); + + /// Extends the CSS parsing capability. + public function transform (formatObject:Object) : TextFormat; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextColorType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextColorType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text +{ + /// The TextColorType class provides color values for the flash.text.TextRenderer class. + public class TextColorType extends Object + { + /// Used in the colorType parameter in the setAdvancedAntiAliasingTable() method. + public static const DARK_COLOR : String; + /// Used in the colorType parameter in the setAdvancedAntiAliasingTable() method. + public static const LIGHT_COLOR : String; + + public function TextColorType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextDisplayMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextDisplayMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text +{ + /// The TextDisplayMode class contains values that control the subpixel anti-aliasing of the advanced anti-aliasing system. + public class TextDisplayMode extends Object + { + /// Forces Flash Player to display grayscale anti-aliasing. + public static const CRT : String; + /// Allows Flash Player to choose LCD or CRT mode. + public static const DEFAULT : String; + /// Forces Flash Player to use LCD subpixel anti-aliasing. + public static const LCD : String; + + public function TextDisplayMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextExtent.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextExtent.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +package flash.text +{ + public class TextExtent extends Object + { + public var ascent : Number; + public var descent : Number; + public var height : Number; + public var textFieldHeight : Number; + public var textFieldWidth : Number; + public var width : Number; + + public function TextExtent (width:Number, height:Number, textFieldWidth:Number, textFieldHeight:Number, ascent:Number, descent:Number); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextField.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextField.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,246 @@ +package flash.text +{ + import flash.display.InteractiveObject; + import flash.text.TextFormat; + import flash.display.DisplayObject; + import flash.geom.Rectangle; + import flash.text.StyleSheet; + import flash.text.TextLineMetrics; + + /** + * Flash Player dispatches the textInput event when a user enters one or more characters of text. + * @eventType flash.events.TextEvent.TEXT_INPUT + */ + [Event(name="textInput", type="flash.events.TextEvent")] + + /** + * Dispatched by a TextField object after the user scrolls. + * @eventType flash.events.Event.SCROLL + */ + [Event(name="scroll", type="flash.events.Event")] + + /** + * Dispatched when a user clicks a hyperlink in an HTML-enabled text field, where the URL begins with "event:". + * @eventType flash.events.TextEvent.LINK + */ + [Event(name="link", type="flash.events.TextEvent")] + + /** + * Dispatched after a control value is modified, unlike the textInput event, which is dispatched before the value is modified. + * @eventType flash.events.Event.CHANGE + */ + [Event(name="change", type="flash.events.Event")] + + /// The TextField class is used to create display objects for text display and input. + public class TextField extends InteractiveObject + { + /// When set to true and the text field is not in focus, Flash Player highlights the selection in the text field in gray. + public function get alwaysShowSelection () : Boolean; + public function set alwaysShowSelection (value:Boolean) : void; + + /// The type of anti-aliasing used for this text field. + public function get antiAliasType () : String; + public function set antiAliasType (antiAliasType:String) : void; + + /// Controls automatic sizing and alignment of text fields. + public function get autoSize () : String; + public function set autoSize (value:String) : void; + + /// Specifies whether the text field has a background fill. + public function get background () : Boolean; + public function set background (value:Boolean) : void; + + /// The color of the text field background. + public function get backgroundColor () : uint; + public function set backgroundColor (value:uint) : void; + + /// Specifies whether the text field has a border. + public function get border () : Boolean; + public function set border (value:Boolean) : void; + + /// The color of the text field border. + public function get borderColor () : uint; + public function set borderColor (value:uint) : void; + + /// An integer (1-based index) that indicates the bottommost line that is currently visible in the specified text field. + public function get bottomScrollV () : int; + + /// The index of the insertion point (caret) position. + public function get caretIndex () : int; + + /// A Boolean value that specifies whether extra white space (spaces, line breaks, and so on) in a text field with HTML text is removed. + public function get condenseWhite () : Boolean; + public function set condenseWhite (value:Boolean) : void; + + /// Specifies the format applied to newly inserted text, such as text inserted with the replaceSelectedText() method or text entered by a user. + public function get defaultTextFormat () : TextFormat; + public function set defaultTextFormat (format:TextFormat) : void; + + /// Specifies whether the text field is a password text field. + public function get displayAsPassword () : Boolean; + public function set displayAsPassword (value:Boolean) : void; + + /// Specifies whether to render by using embedded font outlines. + public function get embedFonts () : Boolean; + public function set embedFonts (value:Boolean) : void; + + /// The type of grid fitting used for this text field. + public function get gridFitType () : String; + public function set gridFitType (gridFitType:String) : void; + + /// Contains the HTML representation of the text field contents. + public function get htmlText () : String; + public function set htmlText (value:String) : void; + + /// The number of characters in a text field. + public function get length () : int; + + /// The maximum number of characters that the text field can contain, as entered by a user. + public function get maxChars () : int; + public function set maxChars (value:int) : void; + + /// The maximum value of scrollH. + public function get maxScrollH () : int; + + /// The maximum value of scrollV. + public function get maxScrollV () : int; + + /// A Boolean value that indicates whether Flash Player automatically scrolls multiline text fields when the user clicks a text field and rolls the mouse wheel. + public function get mouseWheelEnabled () : Boolean; + public function set mouseWheelEnabled (value:Boolean) : void; + + /// Indicates whether field is a multiline text field. + public function get multiline () : Boolean; + public function set multiline (value:Boolean) : void; + + /// Defines the number of text lines in a multiline text field. + public function get numLines () : int; + + /// Indicates the set of characters that a user can enter into the text field. + public function get restrict () : String; + public function set restrict (value:String) : void; + + /// The current horizontal scrolling position. + public function get scrollH () : int; + public function set scrollH (value:int) : void; + + /// The vertical position of text in a text field. + public function get scrollV () : int; + public function set scrollV (value:int) : void; + + /// A Boolean value that indicates whether the text field is selectable. + public function get selectable () : Boolean; + public function set selectable (value:Boolean) : void; + + public function get selectedText () : String; + + /// The zero-based character index value of the first character in the current selection. + public function get selectionBeginIndex () : int; + + /// The zero-based character index value of the last character in the current selection. + public function get selectionEndIndex () : int; + + /// The sharpness of the glyph edges in this text field. + public function get sharpness () : Number; + public function set sharpness (value:Number) : void; + + /// Attaches a style sheet to the text field. + public function get styleSheet () : StyleSheet; + public function set styleSheet (value:StyleSheet) : void; + + /// A string that is the current text in the text field. + public function get text () : String; + public function set text (value:String) : void; + + /// The color of the text in a text field, in hexadecimal format. + public function get textColor () : uint; + public function set textColor (value:uint) : void; + + /// The height of the text in pixels. + public function get textHeight () : Number; + + /// The width of the text in pixels. + public function get textWidth () : Number; + + /// The thickness of the glyph edges in this text field. + public function get thickness () : Number; + public function set thickness (value:Number) : void; + + /// The type of the text field. + public function get type () : String; + public function set type (value:String) : void; + + /// Specifies whether to copy and paste the text formatting along with the text. + public function get useRichTextClipboard () : Boolean; + public function set useRichTextClipboard (value:Boolean) : void; + + /// A Boolean value that indicates whether the text field has word wrap. + public function get wordWrap () : Boolean; + public function set wordWrap (value:Boolean) : void; + + /// Appends text to the end of the existing text of the TextField. + public function appendText (newText:String) : void; + + /// Returns a rectangle that is the bounding box of the character. + public function getCharBoundaries (charIndex:int) : Rectangle; + + /// Returns the zero-based index value of the character. + public function getCharIndexAtPoint (x:Number, y:Number) : int; + + /// The zero-based index value of the character. + public function getFirstCharInParagraph (charIndex:int) : int; + + /// Returns a DisplayObject reference for the given id, for an image or SWF file that has been added to an HTML-formatted text field by using an tag. + public function getImageReference (id:String) : DisplayObject; + + /// The zero-based index value of the line at a specified point. + public function getLineIndexAtPoint (x:Number, y:Number) : int; + + /// The zero-based index value of the line containing the character that the the charIndex parameter specifies. + public function getLineIndexOfChar (charIndex:int) : int; + + /// Returns the number of characters in a specific text line. + public function getLineLength (lineIndex:int) : int; + + /// Returns metrics information about a given text line. + public function getLineMetrics (lineIndex:int) : TextLineMetrics; + + /// The zero-based index value of the first character in the line. + public function getLineOffset (lineIndex:int) : int; + + /// The text string contained in the specified line. + public function getLineText (lineIndex:int) : String; + + /// The zero-based index value of the character. + public function getParagraphLength (charIndex:int) : int; + + public function getRawText () : String; + + /// Returns a TextFormat object. + public function getTextFormat (beginIndex:int = -1, endIndex:int = -1) : TextFormat; + + public function getTextRuns (beginIndex:int = 0, endIndex:int = 2147483647) : Array; + + public function getXMLText (beginIndex:int = 0, endIndex:int = 2147483647) : String; + + public function insertXMLText (beginIndex:int, endIndex:int, richText:String, pasting:Boolean = false) : void; + + /// Returns true if an embedded font is available with the specified fontName and fontStyle where Font.fontType is flash.text.FontType.EMBEDDED. + public static function isFontCompatible (fontName:String, fontStyle:String) : Boolean; + + /// Replaces the current selection with the contents of the value parameter. + public function replaceSelectedText (value:String) : void; + + /// Replaces a range of characters. + public function replaceText (beginIndex:int, endIndex:int, newText:String) : void; + + /// Sets a new text selection. + public function setSelection (beginIndex:int, endIndex:int) : void; + + /// Applies text formatting. + public function setTextFormat (format:TextFormat, beginIndex:int = -1, endIndex:int = -1) : void; + + /// Creates a new TextField instance. + public function TextField (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldAutoSize.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldAutoSize.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text +{ + /// The TextFieldAutoSize class is an enumeration of constant values used in setting the autoSize property of the TextField class. + public class TextFieldAutoSize extends Object + { + /// Specifies that the text is to be treated as center-justified text. + public static const CENTER : String; + /// Specifies that the text is to be treated as left-justified text, meaning that the left side of the text field remains fixed and any resizing of a single line is on the right side. + public static const LEFT : String; + /// Specifies that no resizing is to occur. + public static const NONE : String; + /// Specifies that the text is to be treated as right-justified text, meaning that the right side of the text field remains fixed and any resizing of a single line is on the left side. + public static const RIGHT : String; + + public function TextFieldAutoSize (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFieldType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text +{ + /// The TextFieldType class is an enumeration of constant values used in setting the type property of the TextField class. + public class TextFieldType extends Object + { + /// Used to specify a dynamic TextField. + public static const DYNAMIC : String; + /// Used to specify an input TextField. + public static const INPUT : String; + + public function TextFieldType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormat.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormat.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,84 @@ +package flash.text +{ + /// The TextFormat class represents character formatting information. + public class TextFormat extends Object + { + /// Indicates the alignment of the paragraph. + public function get align () : String; + public function set align (value:String) : void; + + /// Indicates the block indentation in pixels. + public function get blockIndent () : Object; + public function set blockIndent (value:Object) : void; + + /// Specifies whether the text is boldface. + public function get bold () : Object; + public function set bold (value:Object) : void; + + /// Indicates that the text is part of a bulleted list. + public function get bullet () : Object; + public function set bullet (value:Object) : void; + + /// Indicates the color of the text. + public function get color () : Object; + public function set color (value:Object) : void; + + public function get display () : String; + public function set display (value:String) : void; + + /// The name of the font for text in this text format, as a string. + public function get font () : String; + public function set font (value:String) : void; + + /// Indicates the indentation from the left margin to the first character in the paragraph. + public function get indent () : Object; + public function set indent (value:Object) : void; + + /// Indicates whether text in this text format is italicized. + public function get italic () : Object; + public function set italic (value:Object) : void; + + /// A Boolean value that indicates whether kerning is enabled (true) or disabled (false). + public function get kerning () : Object; + public function set kerning (value:Object) : void; + + /// An integer representing the amount of vertical space (called leading) between lines. + public function get leading () : Object; + public function set leading (value:Object) : void; + + /// The left margin of the paragraph, in pixels. + public function get leftMargin () : Object; + public function set leftMargin (value:Object) : void; + + /// A number representing the amount of space that is uniformly distributed between all characters. + public function get letterSpacing () : Object; + public function set letterSpacing (value:Object) : void; + + /// The right margin of the paragraph, in pixels. + public function get rightMargin () : Object; + public function set rightMargin (value:Object) : void; + + /// The point size of text in this text format. + public function get size () : Object; + public function set size (value:Object) : void; + + /// Specifies custom tab stops as an array of non-negative integers. + public function get tabStops () : Array; + public function set tabStops (value:Array) : void; + + /// Indicates the target window where the hyperlink is displayed. + public function get target () : String; + public function set target (value:String) : void; + + /// Indicates whether the text that uses this text format is underlined (true) or not (false). + public function get underline () : Object; + public function set underline (value:Object) : void; + + /// Indicates the target URL for the text in this text format. + public function get url () : String; + public function set url (value:String) : void; + + /// Creates a TextFormat object with the specified properties. + public function TextFormat (font:String = null, size:Object = null, color:Object = null, bold:Object = null, italic:Object = null, underline:Object = null, url:String = null, target:String = null, align:String = null, leftMargin:Object = null, rightMargin:Object = null, indent:Object = null, leading:Object = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatAlign.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatAlign.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text +{ + /// The TextFormatAlign class provides values for text alignment in the TextFormat class. + public class TextFormatAlign extends Object + { + /// Constant; centers the text in the text field. + public static const CENTER : String; + /// Constant; justifies text within the text field. + public static const JUSTIFY : String; + /// Constant; aligns text to the left within the text field. + public static const LEFT : String; + /// Constant; aligns text to the right within the text field. + public static const RIGHT : String; + + public function TextFormatAlign (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatDisplay.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextFormatDisplay.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,10 @@ +package flash.text +{ + public class TextFormatDisplay extends Object + { + public static const BLOCK : String; + public static const INLINE : String; + + public function TextFormatDisplay (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextLineMetrics.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextLineMetrics.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +package flash.text +{ + /// The TextLineMetrics class contains information about the text position and measurements of a line of text within a text field. + public class TextLineMetrics extends Object + { + /// The ascent value of the text is the length from the baseline to the top of the line height in pixels. + public var ascent : Number; + /// The descent value of the text is the length from the baseline to the bottom depth of the line in pixels. + public var descent : Number; + /// The height value of the text of the selected lines (not necessarily the complete text) in pixels. + public var height : Number; + /// The leading value is the measurement of the vertical distance between the lines of text. + public var leading : Number; + /// The width value is the width of the text of the selected lines (not necessarily the complete text) in pixels. + public var width : Number; + /// The x value is the left position of the first character in pixels. + public var x : Number; + + /// Contains information about the text position and measurements of a line of text in a text field. + public function TextLineMetrics (x:Number, width:Number, height:Number, ascent:Number, descent:Number, leading:Number); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRenderer.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRenderer.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +package flash.text +{ + /// The TextRenderer class provides functionality for the advanced anti-aliasing capability of embedded fonts. + public class TextRenderer extends Object + { + public static function get antiAliasType () : String; + public static function set antiAliasType (value:String) : void; + + /// Controls the rendering of advanced anti-aliased text. + public static function get displayMode () : String; + public static function set displayMode (value:String) : void; + + /// The adaptively sampled distance fields (ADFs) quality level for advanced anti-aliasing. + public static function get maxLevel () : int; + public static function set maxLevel (value:int) : void; + + /// Sets a custom continuous stroke modulation (CSM) lookup table for a font. + public static function setAdvancedAntiAliasingTable (fontName:String, fontStyle:String, colorType:String, advancedAntiAliasingTable:Array) : void; + + public function TextRenderer (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRun.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextRun.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text +{ + import flash.text.TextFormat; + + public class TextRun extends Object + { + public var beginIndex : int; + public var endIndex : int; + public var textFormat : TextFormat; + + public function TextRun (beginIndex:int, endIndex:int, textFormat:TextFormat); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextSnapshot.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/TextSnapshot.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,35 @@ +package flash.text +{ + /// TextSnapshot objects let you work with static text in a movie clip. + public class TextSnapshot extends Object + { + /// The number of characters in a TextSnapshot object. + public function get charCount () : int; + + /// Searches the specified TextSnapshot object and returns the position of the first occurrence of textToFind found at or after beginIndex. + public function findText (beginIndex:int, textToFind:String, caseSensitive:Boolean) : int; + + /// Returns a Boolean value that specifies whether a TextSnapshot object contains selected text in the specified range. + public function getSelected (beginIndex:int, endIndex:int) : Boolean; + + /// Returns a string that contains all the characters specified by the corresponding setSelected() method. + public function getSelectedText (includeLineEndings:Boolean = false) : String; + + /// Returns a string that contains all the characters specified by the beginIndex and endIndex parameters. + public function getText (beginIndex:int, endIndex:int, includeLineEndings:Boolean = false) : String; + + /// Returns an array of objects that contains information about a run of text. + public function getTextRunInfo (beginIndex:int, endIndex:int) : Array; + + /// Lets you determine which character within a TextSnapshot object is on or near the specified x, y coordinates of the movie clip containing the text in the TextSnapshot object. + public function hitTestTextNearPos (x:Number, y:Number, maxDistance:Number = 0) : Number; + + /// Specifies the color to use when highlighting characters that have been selected with the setSelected() method. + public function setSelectColor (hexColor:uint = 16776960) : void; + + /// Specifies a range of characters in a TextSnapshot object to be selected or deselected. + public function setSelected (beginIndex:int, endIndex:int, select:Boolean) : void; + + public function TextSnapshot (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/BreakOpportunity.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/BreakOpportunity.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text.engine +{ + /// The BreakOpportunity class is an enumeration of constant values that you can use to set the breakOpportunity property of the ElementFormat class. + public class BreakOpportunity extends Object + { + /// Treats all characters in the ContentElement object as mandatory line break opportunities. + public static const ALL : String; + /// Treats any character in the ContentElement object as a line break opportunity. + public static const ANY : String; + /// Bases line break opportunities on Unicode character properties. + public static const AUTO : String; + /// Treats no characters in the ContentElement object as line break opportunities. + public static const NONE : String; + + public function BreakOpportunity (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/CFFHinting.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/CFFHinting.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text.engine +{ + /// The CFFHinting class defines values for cff hinting in the FontDescription class. + public class CFFHinting extends Object + { + /// Fits strong horizontal stems to the pixel grid for improved readability. + public static const HORIZONTAL_STEM : String; + /// No hinting is applied. + public static const NONE : String; + + public function CFFHinting (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ContentElement.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ContentElement.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,46 @@ +package flash.text.engine +{ + import flash.text.engine.TextBlock; + import flash.text.engine.ElementFormat; + import flash.events.EventDispatcher; + import flash.text.engine.GroupElement; + + /// The ContentElement class serves as a base class for the element types that can appear in a GroupElement, namely a GraphicElement, another GroupElement, or a TextElement. + public class ContentElement extends Object + { + /// Indicates the presence a graphic element in the text. + public static const GRAPHIC_ELEMENT : uint; + /// Provides a way for the author to associate arbitrary data with the element. + public var userData : *; + + /// The ElementFormat object used for the element. + public function get elementFormat () : ElementFormat; + public function set elementFormat (value:ElementFormat) : void; + + /// The EventDispatcher object that receives copies of every event dispatched to valid text lines based on this content element. + public function get eventMirror () : EventDispatcher; + public function set eventMirror (value:EventDispatcher) : void; + + /// The GroupElement object that contains this element, or null if it is not in a group. + public function get groupElement () : GroupElement; + + /// A copy of the text in the element, including the U+FDEF characters. + public function get rawText () : String; + + /// A copy of the text in the element, not including the U+FDEF characters, which represent graphic elements in the String. + public function get text () : String; + + /// The TextBlock to which this element belongs. + public function get textBlock () : TextBlock; + + /// The index in the text block of the first character of this element. + public function get textBlockBeginIndex () : int; + + /// The rotation to apply to the element as a unit. + public function get textRotation () : String; + public function set textRotation (value:String) : void; + + /// Calling the new ContentElement() constructor throws an ArgumentError exception. + public function ContentElement (elementFormat:ElementFormat = null, eventMirror:EventDispatcher = null, textRotation:String = "rotate0"); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitCase.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitCase.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text.engine +{ + /// The DigitCase class is an enumeration of constant values used in setting the digitCase property of the ElementFormat class. + public class DigitCase extends Object + { + /// Used to specify default digit case. + public static const DEFAULT : String; + /// Used to specify lining digit case. + public static const LINING : String; + /// Used to specify old style digit case. + public static const OLD_STYLE : String; + + public function DigitCase (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitWidth.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/DigitWidth.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text.engine +{ + /// The DigitWidth class is an enumeration of constant values used in setting the digitWidth property of the ElementFormat class. + public class DigitWidth extends Object + { + /// Used to specify default digit width. + public static const DEFAULT : String; + /// Used to specify proportional digit width. + public static const PROPORTIONAL : String; + /// Used to specify tabular digit width. + public static const TABULAR : String; + + public function DigitWidth (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/EastAsianJustifier.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/EastAsianJustifier.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.text.engine +{ + import flash.text.engine.TextJustifier; + + /// The EastAsianJustifier class has properties to control the justification options for text lines whose content is primarily East Asian text. + public class EastAsianJustifier extends TextJustifier + { + /// Specifies the justification style for the text in a text block. + public function get justificationStyle () : String; + public function set justificationStyle (value:String) : void; + + /// Constructs a cloned copy of the EastAsianJustifier. + public function clone () : TextJustifier; + + /// Creates a EastAsianJustifier object. + public function EastAsianJustifier (locale:String = "ja", lineJustification:String = "allButLast", justificationStyle:String = "pushInKinsoku"); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ElementFormat.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/ElementFormat.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,91 @@ +package flash.text.engine +{ + import flash.text.engine.FontDescription; + import flash.text.engine.ElementFormat; + import flash.text.engine.FontMetrics; + + /// The ElementFormat class represents formatting information which can be applied to a ContentElement. + public class ElementFormat extends Object + { + /// Specifies which of the baselines of the line containing the element the dominantBaseline snaps to, thus determining the vertical position of the element in the line. + public function get alignmentBaseline () : String; + public function set alignmentBaseline (alignmentBaseline:String) : void; + + /// Provides a way for the author to automatically set the alpha property of all line atoms based on the element format to the specified Number. + public function get alpha () : Number; + public function set alpha (value:Number) : void; + + /// Indicates the baseline shift for the element in pixels. + public function get baselineShift () : Number; + public function set baselineShift (value:Number) : void; + + /// The line break opportunity applied to this text. + public function get breakOpportunity () : String; + public function set breakOpportunity (opportunityType:String) : void; + + /// Indicates the color of the text. + public function get color () : uint; + public function set color (value:uint) : void; + + /// The digit case used for this text. + public function get digitCase () : String; + public function set digitCase (digitCaseType:String) : void; + + /// The digit width used for this text. + public function get digitWidth () : String; + public function set digitWidth (digitWidthType:String) : void; + + /// Specifies which of the baselines of the element snaps to the alignmentBaseline to determine the vertical position of the element on the line. + public function get dominantBaseline () : String; + public function set dominantBaseline (dominantBaseline:String) : void; + + /// An object which encapsulates properties necessary to describe a font. + public function get fontDescription () : FontDescription; + public function set fontDescription (value:FontDescription) : void; + + /// The point size of text. + public function get fontSize () : Number; + public function set fontSize (value:Number) : void; + + /// The kerning used for this text. + public function get kerning () : String; + public function set kerning (value:String) : void; + + /// The ligature level used for this text. + public function get ligatureLevel () : String; + public function set ligatureLevel (ligatureLevelType:String) : void; + + /// The locale of the text. + public function get locale () : String; + public function set locale (value:String) : void; + + /// Indicates whether or not the ElementFormat is locked. + public function get locked () : Boolean; + public function set locked (value:Boolean) : void; + + /// Sets the rotation applied to individual glyphs. + public function get textRotation () : String; + public function set textRotation (value:String) : void; + + /// The tracking or manual kerning applied to the left of each glyph in pixels. + public function get trackingLeft () : Number; + public function set trackingLeft (value:Number) : void; + + /// The tracking or manual kerning applied to the right of each glyph in pixels. + public function get trackingRight () : Number; + public function set trackingRight (value:Number) : void; + + /// The typographic case used for this text. + public function get typographicCase () : String; + public function set typographicCase (typographicCaseType:String) : void; + + /// Constructs an unlocked, cloned copy of the ElementFormat. + public function clone () : ElementFormat; + + /// Creates an ElementFormat object. + public function ElementFormat (fontDescription:FontDescription = null, fontSize:Number = 12, color:uint = 0, alpha:Number = 1, textRotation:String = "auto", dominantBaseline:String = "roman", alignmentBaseline:String = "useDominantBaseline", baselineShift:Number = 0, kerning:String = "on", trackingRight:Number = 0, trackingLeft:Number = 0, locale:String = "en", breakOpportunity:String = "auto", digitCase:String = "default", digitWidth:String = "default", ligatureLevel:String = "common", typographicCase:String = "default"); + + /// Returns a FontMetrics object with properties which describe the emBox, strikethrough position, strikethrough thickness, underline position, and underline thickness for the font specified by fontDescription and fontSize. + public function getFontMetrics () : FontMetrics; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontDescription.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontDescription.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,45 @@ +package flash.text.engine +{ + import flash.text.engine.FontDescription; + + /// The FontDescription class represents properties necessary to describe a font. + public class FontDescription extends Object + { + /// The type of CFF hinting used for this text. + public function get cffHinting () : String; + public function set cffHinting (value:String) : void; + + /// Specifies how the font should be looked up. + public function get fontLookup () : String; + public function set fontLookup (value:String) : void; + + /// The name of the font to use, or a comma-separated list of font names. + public function get fontName () : String; + public function set fontName (value:String) : void; + + /// Specifies the font posture. + public function get fontPosture () : String; + public function set fontPosture (value:String) : void; + + /// Specifies the font weight. + public function get fontWeight () : String; + public function set fontWeight (value:String) : void; + + /// Indicates whether or not the FontDescription is locked. + public function get locked () : Boolean; + public function set locked (value:Boolean) : void; + + /// The rendering mode used for this text. + public function get renderingMode () : String; + public function set renderingMode (value:String) : void; + + /// Constructs an unlocked, cloned copy of the FontDescription. + public function clone () : FontDescription; + + /// Creates a FontDescription object. + public function FontDescription (fontName:String = "_serif", fontWeight:String = "normal", fontPosture:String = "normal", fontLookup:String = "device", renderingMode:String = "cff", cffHinting:String = "horizontalStem"); + + /// Returns true if an embedded font is available with the specified fontName, fontWeight, and fontPosture where Font.fontType is flash.text.FontType.EMBEDDED_CFF. + public static function isFontCompatible (fontName:String, fontWeight:String, fontPosture:String) : Boolean; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontLookup.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontLookup.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text.engine +{ + /// The FontLookup class is an enumeration of constant values used with FontDescription.fontLookup. + public class FontLookup extends Object + { + /// Used to indicate device font lookup. + public static const DEVICE : String; + /// Used to indicate embedded CFF font lookup. + public static const EMBEDDED_CFF : String; + + public function FontLookup (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontMetrics.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontMetrics.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package flash.text.engine +{ + import flash.geom.Rectangle; + + /// The FontMetrics class contains measurement and offset information about a font. + public class FontMetrics extends Object + { + /// The emBox value represents the design space of the font and is used to place Chinese, Korean, or Japanese glyphs relative to the Roman baseline. + public var emBox : Rectangle; + /// The strikethroughOffset value is the suggested vertical offset from the Roman baseline for a strikethrough. + public var strikethroughOffset : Number; + /// The strikethroughThickness value is the suggested thickness for a strikethrough. + public var strikethroughThickness : Number; + /// The subscriptOffset value is the suggested vertical offset from the Roman baseline for a subscript. + public var subscriptOffset : Number; + /// The subscriptScale value is the suggested scale factor to apply to the point size for a subscript. + public var subscriptScale : Number; + /// The superscriptOffset value is the suggested vertical offset from the Roman baseline for a superscript. + public var superscriptOffset : Number; + /// The superscriptScale value is the suggested scale factor to apply to the point size for a superscript. + public var superscriptScale : Number; + /// The underlineOffset value is the suggested vertical offset from the Roman baseline for an underline. + public var underlineOffset : Number; + /// The underlineThickness value is the suggested thickness for an underline. + public var underlineThickness : Number; + + /// Creates a FontMetrics object. + public function FontMetrics (emBox:Rectangle, strikethroughOffset:Number, strikethroughThickness:Number, underlineOffset:Number, underlineThickness:Number, subscriptOffset:Number, subscriptScale:Number, superscriptOffset:Number, superscriptScale:Number); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontPosture.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontPosture.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text.engine +{ + /// The FontPosture class is an enumeration of constant values used with FontDescription.fontPosture. + public class FontPosture extends Object + { + /// Used to indicate italic font posture. + public static const ITALIC : String; + /// Used to indicate normal font posture. + public static const NORMAL : String; + + public function FontPosture (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontWeight.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/FontWeight.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text.engine +{ + /// The FontWeight class is an enumeration of constant values used with FontDescription.fontWeight. + public class FontWeight extends Object + { + /// Used to indicate bold font weight. + public static const BOLD : String; + /// Used to indicate normal font weight. + public static const NORMAL : String; + + public function FontWeight (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GraphicElement.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GraphicElement.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,25 @@ +package flash.text.engine +{ + import flash.display.DisplayObject; + import flash.text.engine.ElementFormat; + import flash.events.EventDispatcher; + + /// The GraphicElement class represents a graphic element in a TextBlock or GroupElement object. + public class GraphicElement extends ContentElement + { + /// The height in pixels to reserve for the graphic in the line. + public function get elementHeight () : Number; + public function set elementHeight (value:Number) : void; + + /// The width in pixels to reserve for the graphic in the line. + public function get elementWidth () : Number; + public function set elementWidth (value:Number) : void; + + /// The DisplayObject to be used as a graphic for the GraphicElement. + public function get graphic () : DisplayObject; + public function set graphic (value:DisplayObject) : void; + + /// Creates a new GraphicElement instance. + public function GraphicElement (graphic:DisplayObject = null, elementWidth:Number = 15, elementHeight:Number = 15, elementFormat:ElementFormat = null, eventMirror:EventDispatcher = null, textRotation:String = "rotate0"); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GroupElement.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/GroupElement.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,45 @@ +package flash.text.engine +{ + import flash.text.engine.ContentElement; + import flash.text.engine.TextElement; + import flash.text.engine.GroupElement; + import flash.text.engine.ElementFormat; + import flash.events.EventDispatcher; + + /// A GroupElement object groups a collection of TextElement, GraphicElement, or other GroupElement objects that you can assign as a whole to the content property of a TextBlock object. + public class GroupElement extends ContentElement + { + /// The number of elements in the group. + public function get elementCount () : int; + + /// Retrieves an element from within the group. + public function getElementAt (index:int) : ContentElement; + + /// Returns the element containing the character specified by the charIndex parameter. + public function getElementAtCharIndex (charIndex:int) : ContentElement; + + /// Returns the index of the element specified by the element parameter. + public function getElementIndex (element:ContentElement) : int; + + /// Creates a new GroupElement instance. + public function GroupElement (elements:Vector. = null, elementFormat:ElementFormat = null, eventMirror:EventDispatcher = null, textRotation:String = "rotate0"); + + /// Replaces the range of elements that the beginIndex and endIndex parameters specify with a new GroupElement containing those elements. + public function groupElements (beginIndex:int, endIndex:int) : GroupElement; + + /// Merges the text from the range of elements that the beginIndex and endIndex parameters specify into the element specified by beginIndex without affecting the format of that element. + public function mergeTextElements (beginIndex:int, endIndex:int) : TextElement; + + /// Replaces the range of elements that the beginIndex and endIndex parameters specify with the contents of the newElements parameter. + public function replaceElements (beginIndex:int, endIndex:int, newElements:Vector.) : Vector.; + + /// Sets the elements in the group to the contents of the Vector. + public function setElements (value:Vector.) : void; + + /// Splits a portion of a TextElement in the group into a new TextElement which is inserted into the group following the specified TextElement. + public function splitTextElement (elementIndex:int, splitIndex:int) : TextElement; + + /// Ungroups the elements in a nested GroupElement that groupIndex specifies within an outer GroupElement object. + public function ungroupElements (groupIndex:int) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/JustificationStyle.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/JustificationStyle.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text.engine +{ + /// The JustificationStyle class is an enumeration of constant values for setting the justificationStyle property of the EastAsianJustifier class. + public class JustificationStyle extends Object + { + /// Bases justification on either expanding or compressing the line, whichever gives a result closest to the desired width. + public static const PRIORITIZE_LEAST_ADJUSTMENT : String; + /// Bases justification on compressing kinsoku at the end of the line, or expanding it if no kinsoku occurs or if that space is insufficient. + public static const PUSH_IN_KINSOKU : String; + /// Bases justification on expanding the line. + public static const PUSH_OUT_ONLY : String; + + public function JustificationStyle (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/Kerning.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/Kerning.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text.engine +{ + /// The Kerning class is an enumeration of constant values used with ElementFormat.kerning. + public class Kerning extends Object + { + /// Used to indicate kerning is enabled only for characters appropriate in Asian typography. + public static const AUTO : String; + /// Used to indicate kerning is disabled. + public static const OFF : String; + /// Used to indicate kerning is enabled. + public static const ON : String; + + public function Kerning (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LigatureLevel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LigatureLevel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.text.engine +{ + /// The LigatureLevel class is an enumeration of constant values used in setting the ligatureLevel property of the ElementFormat class. + public class LigatureLevel extends Object + { + /// Used to specify common ligatures. + public static const COMMON : String; + /// Used to specify exotic ligatures. + public static const EXOTIC : String; + /// Used to specify minimum ligatures. + public static const MINIMUM : String; + /// Used to specify no ligatures. + public static const NONE : String; + /// Used to specify uncommon ligatures. + public static const UNCOMMON : String; + + public function LigatureLevel (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LineJustification.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/LineJustification.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.text.engine +{ + /// The LineJustification class is an enumeration of constant values used in setting the lineJustfication property of the TextJustifier subclasses. + public class LineJustification extends Object + { + /// Directs the text engine to justify all but the last line. + public static const ALL_BUT_LAST : String; + /// Directs the text engine to justify all lines. + public static const ALL_INCLUDING_LAST : String; + /// Directs the text engine to generate unjustified lines. + public static const UNJUSTIFIED : String; + + public function LineJustification (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/RenderingMode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/RenderingMode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.text.engine +{ + /// The RenderingMode class provides values for rendering mode in the flash.text.engine.FontDescription class. + public class RenderingMode extends Object + { + /// Sets rendering mode to CFF. + public static const CFF : String; + /// Sets rendering mode to the rendering mode that is used in Flash Player 7 and earlier. + public static const NORMAL : String; + + public function RenderingMode (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/SpaceJustifier.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/SpaceJustifier.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.text.engine +{ + import flash.text.engine.TextJustifier; + + /// The SpaceJustifier class represents properties that control the justification options for text lines in a text block. + public class SpaceJustifier extends TextJustifier + { + /// Specifies whether to use letter spacing during justification. + public function get letterSpacing () : Boolean; + public function set letterSpacing (value:Boolean) : void; + + /// Constructs a cloned copy of the SpaceJustifier. + public function clone () : TextJustifier; + + /// Creates a SpaceJustifier object. + public function SpaceJustifier (locale:String = "en", lineJustification:String = "unjustified", letterSpacing:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabAlignment.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabAlignment.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text.engine +{ + /// The TabAlignment class is an enumeration of constant values that you can use to set the tabAlignment property of the TabStop class. + public class TabAlignment extends Object + { + /// Positions the center of the tabbed text at the tab stop. + public static const CENTER : String; + /// Positions the alignment token of the tabbed text at the tab stop. + public static const DECIMAL : String; + /// Positions the end of the tabbed text at the tab stop. + public static const END : String; + /// Positions the start of the tabbed text at the tab stop. + public static const START : String; + + public function TabAlignment (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabStop.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TabStop.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,21 @@ +package flash.text.engine +{ + /// The TabStop class represents the properties of a tab stop in a text block. + public class TabStop extends Object + { + /// Specifies the tab alignment for this tab stop. + public function get alignment () : String; + public function set alignment (value:String) : void; + + /// Specifies the alignment token to use when you set the alignment property to TabAlignment.DECIMAL. + public function get decimalAlignmentToken () : String; + public function set decimalAlignmentToken (value:String) : void; + + /// The position of the tab stop, in pixels, relative to the start of the text line. + public function get position () : Number; + public function set position (value:Number) : void; + + /// Creates a new TabStop. + public function TabStop (alignment:String = "start", position:Number = 0, decimalAlignmentToken:String = ""); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBaseline.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBaseline.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.text.engine +{ + /// The TextBaseline class is an enumeration of constant values to use in setting the dominantBaseline andalignmentBaseline properties of the ElementFormat class. + public class TextBaseline extends Object + { + /// Specifies an ascent baseline. + public static const ASCENT : String; + /// Specifies a descent baseline. + public static const DESCENT : String; + /// Specifies an ideographic bottom baseline. + public static const IDEOGRAPHIC_BOTTOM : String; + /// Specifies an ideographic center baseline. + public static const IDEOGRAPHIC_CENTER : String; + /// Specifies an ideographic top baseline. + public static const IDEOGRAPHIC_TOP : String; + /// Specifies a roman baseline. + public static const ROMAN : String; + /// Specifies that the alignmentBaseline is the same as the dominantBaseline. + public static const USE_DOMINANT_BASELINE : String; + + public function TextBaseline (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBlock.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextBlock.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,90 @@ +package flash.text.engine +{ + import flash.text.engine.TextJustifier; + import flash.text.engine.TextLine; + import flash.text.engine.ContentElement; + import flash.text.engine.FontDescription; + import flash.text.engine.TabStop; + + /// The TextBlock class is a factory for the creation of TextLine objects, which you can render by placing them on the display list. + public class TextBlock extends Object + { + /// Provides a way for the author to associate arbitrary data with the text block. + public var userData : *; + + /// Specifies that you want to enhance screen appearance at the expense of what-you-see-is-what-you-get (WYSIWYG) print fidelity. + public function get applyNonLinearFontScaling () : Boolean; + public function set applyNonLinearFontScaling (value:Boolean) : void; + + /// The font used to determine the baselines for all the lines created from the block, independent of their content. + public function get baselineFontDescription () : FontDescription; + public function set baselineFontDescription (value:FontDescription) : void; + + /// The font size used to calculate the baselines for the lines created from the block. + public function get baselineFontSize () : Number; + public function set baselineFontSize (value:Number) : void; + + /// Specifies which baseline is at y=0 for lines created from this block. + public function get baselineZero () : String; + public function set baselineZero (value:String) : void; + + /// Specifies the default bidirectional embedding level of the text in the text block. + public function get bidiLevel () : int; + public function set bidiLevel (value:int) : void; + + /// Holds the contents of the text block. + public function get content () : ContentElement; + public function set content (value:ContentElement) : void; + + /// Identifies the first line in the text block in which TextLine.validity is not equal to TextLineValidity.VALID. + public function get firstInvalidLine () : TextLine; + + /// The first TextLine in the TextBlock, if any. + public function get firstLine () : TextLine; + + /// The last TextLine in the TextBlock, if any. + public function get lastLine () : TextLine; + + /// Rotates the text lines in the text block as a unit. + public function get lineRotation () : String; + public function set lineRotation (value:String) : void; + + /// Specifies the tab stops for the text in the text block, in the form of a Vector of TabStop objects. + public function get tabStops () : Vector.; + public function set tabStops (value:Vector.) : void; + + /// Specifies the TextJustifier to use during line creation. + public function get textJustifier () : TextJustifier; + public function set textJustifier (value:TextJustifier) : void; + + /// Indicates the result of a createTextLine() operation. + public function get textLineCreationResult () : String; + + /// Instructs the text block to create a line of text from its content, beginning at the point specified by the previousLine parameter and breaking at the point specified by the width parameter. + public function createTextLine (previousLine:TextLine = null, width:Number = 1000000, lineOffset:Number = 0, fitSomething:Boolean = false) : TextLine; + + /// Dumps the underlying contents of the TextBlock as an XML string. + public function dump () : String; + + /// Finds the index of the next Atom boundary from the specified character index, not including the character at the specified index. + public function findNextAtomBoundary (afterCharIndex:int) : int; + + /// Finds the index of the next word boundary from the specified character index, not including the character at the specified index. + public function findNextWordBoundary (afterCharIndex:int) : int; + + /// Finds the index of the previous atom boundary to the specified character index, not including the character at the specified index. + public function findPreviousAtomBoundary (beforeCharIndex:int) : int; + + /// Finds the index of the previous word boundary to the specified character index, not including the character at the specified index. + public function findPreviousWordBoundary (beforeCharIndex:int) : int; + + /// Returns the TextLine containing the character specified by the charIndex parameter. + public function getTextLineAtCharIndex (charIndex:int) : TextLine; + + /// Removes a range of text lines from the list of lines maintained by the TextBlock. + public function releaseLines (firstLine:TextLine, lastLine:TextLine) : void; + + /// Creates a TextBlock object + public function TextBlock (content:ContentElement = null, tabStops:Vector. = null, textJustifier:TextJustifier = null, lineRotation:String = "rotate0", baselineZero:String = "roman", bidiLevel:int = 0, applyNonLinearFontScaling:Boolean = true, baselineFontDescription:FontDescription = null, baselineFontSize:Number = 12); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextElement.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextElement.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.text.engine +{ + import flash.text.engine.ElementFormat; + import flash.events.EventDispatcher; + + /// The TextElement class represents a string of formatted text. + public class TextElement extends ContentElement + { + /// Receives the text that is the content of the element. + public function set text (value:String) : void; + + /// Replaces the range of characters that the beginIndex and endIndex parameters specify with the contents of the newText parameter. + public function replaceText (beginIndex:int, endIndex:int, newText:String) : void; + + /// Creates a new TextElement instance. + public function TextElement (text:String = null, elementFormat:ElementFormat = null, eventMirror:EventDispatcher = null, textRotation:String = "rotate0"); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextJustifier.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextJustifier.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package flash.text.engine +{ + import flash.text.engine.TextJustifier; + + /// The TextJustifier class is an abstract base class for the justifier types that you can apply to a TextBlock, specifically the EastAsianJustifier and SpaceJustifier classes. + public class TextJustifier extends Object + { + /// Specifies the line justification for the text in a text block. + public function get lineJustification () : String; + public function set lineJustification (value:String) : void; + + /// Specifies the locale to determine the justification rules for the text in a text block. + public function get locale () : String; + + /// Constructs a cloned copy of the TextJustifier. + public function clone () : TextJustifier; + + /// Constructs a default TextJustifier subclass appropriate to the specified locale. + public static function getJustifierForLocale (locale:String) : TextJustifier; + + /// Calling the new TextJustifier() constructor throws an ArgumentError exception. + public function TextJustifier (locale:String, lineJustification:String); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLine.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLine.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,120 @@ +package flash.text.engine +{ + import flash.display.DisplayObjectContainer; + import flash.text.engine.TextLineMirrorRegion; + import flash.display.DisplayObject; + import flash.text.engine.TextLine; + import flash.events.EventDispatcher; + import flash.ui.ContextMenu; + import flash.text.engine.TextBlock; + import flash.geom.Rectangle; + + /// The TextLine class is used to display text on the display list. + public class TextLine extends DisplayObjectContainer + { + /// The maximum requested width of a text line, in pixels. + public static const MAX_LINE_WIDTH : int; + /// Provides a way for the author to associate arbitrary data with the text line. + public var userData : *; + + /// Specifies the number of pixels from the baseline to the top of the tallest characters in the line. + public function get ascent () : Number; + + /// The number of atoms in the line, which is the number of indivisible elements, including spaces and graphic elements. + public function get atomCount () : int; + + public function set contextMenu (cm:ContextMenu) : void; + + /// Specifies the number of pixels from the baseline to the bottom of the lowest-descending characters in the line. + public function get descent () : Number; + + public function set focusRect (focusRect:Object) : void; + + /// Indicates whether the text line contains any graphic elements. + public function get hasGraphicElement () : Boolean; + + /// A Vector containing the TextLineMirrorRegion objects associated with the line, or null if none exist. + public function get mirrorRegions () : Vector.; + + /// The next TextLine in the TextBlock, or null if the current line is the last line in the block or the validity of the line is TextLineValidity.STATIC. + public function get nextLine () : TextLine; + + /// The previous TextLine in the TextBlock, or null if the line is the first line in the block or the validity of the line is TextLineValidity.STATIC. + public function get previousLine () : TextLine; + + /// The length of the raw text in the text block that became the line, including the U+FDEF characters representing graphic elements and any trailing spaces, which are part of the line but not are displayed. + public function get rawTextLength () : int; + + /// The width that was specified to the TextBlock.createTextLine() method when it created the line. + public function get specifiedWidth () : Number; + + public function set tabChildren (enable:Boolean) : void; + + public function set tabEnabled (enabled:Boolean) : void; + + public function set tabIndex (index:int) : void; + + /// The TextBlock containing this text line, or null if the validity of the line is TextLineValidity.STATIC. + public function get textBlock () : TextBlock; + + /// The index of the first character of the line in the raw text of the text block. + public function get textBlockBeginIndex () : int; + + /// The logical height of the text line, which is equal to ascent + descent. + public function get textHeight () : Number; + + /// The logical width of the text line, which is the width that the text engine uses to lay out the line. + public function get textWidth () : Number; + + /// The width of the line if it was not justified. + public function get unjustifiedTextWidth () : Number; + + /// Specifies the current validity of the text line. + public function get validity () : String; + public function set validity (value:String) : void; + + /// Dumps the underlying contents of the TextLine as an XML string. + public function dump () : String; + + /// Releases the atom data of the line for garbage collection. + public function flushAtomData () : void; + + /// Gets the bidirectional level of the atom at the specified index. + public function getAtomBidiLevel (atomIndex:int) : int; + + /// Gets the bounds of the atom at the specified index relative to the text line. + public function getAtomBounds (atomIndex:int) : Rectangle; + + /// Gets the center of the atom as measured along the baseline at the specified index. + public function getAtomCenter (atomIndex:int) : Number; + + /// Gets the graphic of the atom at the specified index, or null if the atom is a character. + public function getAtomGraphic (atomIndex:int) : DisplayObject; + + /// Returns the index of the atom containing the character specified by the charIndex parameter, or -1 if the character does not contribute to any atom in the line. + public function getAtomIndexAtCharIndex (charIndex:int) : int; + + /// Returns the index of the atom at the point specified by the x and y parameters, or -1 if no atom exists at that point. + public function getAtomIndexAtPoint (stageX:Number, stageY:Number) : int; + + /// Gets the text block begin index of the atom at the specified index. + public function getAtomTextBlockBeginIndex (atomIndex:int) : int; + + /// Gets the text block end index of the atom at the specified index. + public function getAtomTextBlockEndIndex (atomIndex:int) : int; + + /// Gets the rotation of the atom at the specified index. + public function getAtomTextRotation (atomIndex:int) : String; + + /// Indicates whether a word boundary occurs to the left of the atom at the specified index. + public function getAtomWordBoundaryOnLeft (atomIndex:int) : Boolean; + + /// Gets the position of the specified baseline, relative to TextBlock.baselineZero. + public function getBaselinePosition (baseline:String) : Number; + + /// Returns the first TextLineMirrorRegion on the line whose mirror property matches that specified by the mirror parameter, or null if no match exists. + public function getMirrorRegion (mirror:EventDispatcher) : TextLineMirrorRegion; + + public function TextLine (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineCreationResult.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineCreationResult.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text.engine +{ + /// The TextLineCreationResult class is an enumeration of constant values used with TextBlock.textLineCreationResult. + public class TextLineCreationResult extends Object + { + /// Indicates no line was created because all text in the block had already been broken. + public static const COMPLETE : String; + /// Indicates the line was created with an emergency break because no break opportunity was available in the specified width. + public static const EMERGENCY : String; + /// Indicates no line was created because no text could fit in the specified width and fitSomething was not specified in the call to createTextLine(). + public static const INSUFFICIENT_WIDTH : String; + /// Indicates the line was successfully broken. + public static const SUCCESS : String; + + public function TextLineCreationResult (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineMirrorRegion.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineMirrorRegion.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,32 @@ +package flash.text.engine +{ + import flash.text.engine.TextLine; + import flash.geom.Rectangle; + import flash.text.engine.TextLineMirrorRegion; + import flash.text.engine.ContentElement; + import flash.events.EventDispatcher; + + /// The TextLineMirrorRegion class represents a portion of a text line wherein events are mirrored to another event dispatcher. + public class TextLineMirrorRegion extends Object + { + /// The bounds of the mirror region, relative to the text line. + public function get bounds () : Rectangle; + + /// The ContentElement object from which the mirror region was derived. + public function get element () : ContentElement; + + /// The EventDispatcher object to which events affecting the mirror region are mirrored. + public function get mirror () : EventDispatcher; + + /// The next TextLineMirrorRegion in the set derived from the text element, or null if the current region is the last mirror region in the set. + public function get nextRegion () : TextLineMirrorRegion; + + /// The previous TextLineMirrorRegion in the set derived from the text element, or null if the current region is the first mirror region in the set. + public function get previousRegion () : TextLineMirrorRegion; + + /// The TextLine containing this mirror region. + public function get textLine () : TextLine; + + public function TextLineMirrorRegion (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineValidity.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextLineValidity.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.text.engine +{ + /// The TextLineValidity class is an enumeration of constant values for setting the validity property of the TextLine class. + public class TextLineValidity extends Object + { + /// Specifies that the line is invalid. + public static const INVALID : String; + /// Specifies that the text line is possibly invalid. + public static const POSSIBLY_INVALID : String; + /// Specifies that the line is static, and that the connection between the line and the text block has been severed, and the atom data that is unnecessary for rendering has been discarded. + public static const STATIC : String; + /// Specifies that the text line is valid. + public static const VALID : String; + + public function TextLineValidity (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextRotation.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TextRotation.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.text.engine +{ + /// The TextRotation class is an enumeration of constant values used with the following properties:ElementFormat.textRotation, ContentElement.textRotation,TextBlock.lineRotation, and TextLine.getAtomTextRotation(). + public class TextRotation extends Object + { + /// Specifies a 90 degree counter clockwise rotation for full width and wide glyphs only, as determined by the Unicode properties of the glyph. + public static const AUTO : String; + /// Specifies no rotation. + public static const ROTATE_0 : String; + /// Specifies a 180 degree rotation. + public static const ROTATE_180 : String; + /// Specifies a 270 degree clockwise rotation. + public static const ROTATE_270 : String; + /// Specifies a 90 degree clockwise rotation. + public static const ROTATE_90 : String; + + public function TextRotation (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TypographicCase.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/text/engine/TypographicCase.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.text.engine +{ + /// The TypographicCase class is an enumeration of constant values for setting the typographicCase property of the ElementFormat class. + public class TypographicCase extends Object + { + /// Specifies that spacing is adjusted for uppercase characters on output. + public static const CAPS : String; + /// Specifies that all lowercase characters use small-caps glyphs on output. + public static const CAPS_AND_SMALL_CAPS : String; + /// Specifies default typographic case. + public static const DEFAULT : String; + /// Specifies that all characters use lowercase glyphs on output. + public static const LOWERCASE : String; + /// Specifies that uppercase characters use small-caps glyphs on output. + public static const SMALL_CAPS : String; + /// Specifies that uppercase characters use title glyphs on output. + public static const TITLE : String; + /// Specifies that all characters use uppercase glyphs on output. + public static const UPPERCASE : String; + + public function TypographicCase (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/trace/Trace.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/trace/Trace.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +package flash.trace +{ + public class Trace extends Object + { + public static const FILE : *; + public static const LISTENER : *; + public static const METHODS : int; + public static const METHODS_AND_LINES : int; + public static const METHODS_AND_LINES_WITH_ARGS : int; + public static const METHODS_WITH_ARGS : int; + public static const OFF : int; + + public static function getLevel (target:int = 2) : int; + + public static function getListener () : Function; + + public static function setLevel (l:int, target:int = 2) : *; + + public static function setListener (f:Function) : *; + + public function Trace (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenu.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenu.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,47 @@ +package flash.ui +{ + import flash.events.EventDispatcher; + import flash.ui.ContextMenuBuiltInItems; + import flash.ui.ContextMenu; + import flash.ui.ContextMenuClipboardItems; + import flash.net.URLRequest; + + /** + * Dispatched when a user first generates a context menu but before the contents of the context menu are displayed. + * @eventType flash.events.ContextMenuEvent.MENU_SELECT + */ + [Event(name="menuSelect", type="flash.events.ContextMenuEvent")] + + /// The ContextMenu class provides control over the items in the Flash Player context menu. + public class ContextMenu extends EventDispatcher + { + /// An object that has the following properties of the ContextMenuBuiltInItems class: forwardAndBack, loop, play, print, quality, rewind, save, and zoom. + public function get builtInItems () : ContextMenuBuiltInItems; + public function set builtInItems (value:ContextMenuBuiltInItems) : void; + + /// An object that has the following properties of the ContextMenuClipboardItems class: cut, copy, paste, delete, selectAll. + public function get clipboardItems () : ContextMenuClipboardItems; + public function set clipboardItems (value:ContextMenuClipboardItems) : void; + + /// Specifies whether or not the clipboard menu should be used. + public function get clipboardMenu () : Boolean; + public function set clipboardMenu (value:Boolean) : void; + + /// An array of ContextMenuItem objects. + public function get customItems () : Array; + public function set customItems (value:Array) : void; + + /// The URLRequest of the link. + public function get link () : URLRequest; + public function set link (value:URLRequest) : void; + + /// Creates a copy of the specified ContextMenu object. + public function clone () : ContextMenu; + + /// Creates a ContextMenu object. + public function ContextMenu (); + + /// Hides all built-in menu items (except Settings) in the specified ContextMenu object. + public function hideBuiltInItems () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuBuiltInItems.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuBuiltInItems.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package flash.ui +{ + import flash.ui.ContextMenuBuiltInItems; + + /// The ContextMenuBuiltInItems class describes the items that are built in to a context menu. + public class ContextMenuBuiltInItems extends Object + { + /// Lets the user move forward or backward one frame in a SWF file at run time (does not appear for a single-frame SWF file). + public var forwardAndBack : Boolean; + /// Lets the user set a SWF file to start over automatically when it reaches the final frame (does not appear for a single-frame SWF file). + public var loop : Boolean; + /// Lets the user start a paused SWF file (does not appear for a single-frame SWF file). + public var play : Boolean; + /// Lets the user send the displayed frame image to a printer. + public var print : Boolean; + /// Lets the user set the resolution of the SWF file at run time. + public var quality : Boolean; + /// Lets the user set a SWF file to play from the first frame when selected, at any time (does not appear for a single-frame SWF file). + public var rewind : Boolean; + /// Lets the user with Shockmachine installed save a SWF file. + public var save : Boolean; + /// Lets the user zoom in and out on a SWF file at run time. + public var zoom : Boolean; + + public function clone () : ContextMenuBuiltInItems; + + /// Creates a new ContextMenuBuiltInItems object so that you can set the properties for Flash Player to display or hide each menu item. + public function ContextMenuBuiltInItems (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuClipboardItems.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuClipboardItems.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +package flash.ui +{ + import flash.ui.ContextMenuClipboardItems; + + /// The ContextMenuClipboardItems class determines which items are enabled or disabled on the clipboard context menu. + public class ContextMenuClipboardItems extends Object + { + /// Enables or disables the 'Delete' (Windows) / 'Clear' (Mac) item on the clipboard menu. + public var clear : Boolean; + /// Enables or disables the 'Copy' item on the clipboard menu. + public var copy : Boolean; + /// Enables or disables the 'Cut' item on the clipboard menu. + public var cut : Boolean; + /// Enables or disables the 'Paste' item on the clipboard menu. + public var paste : Boolean; + /// Enables or disables the 'Select All' item on the clipboard menu. + public var selectAll : Boolean; + + public function clone () : ContextMenuClipboardItems; + + /// Creates a new ContextMenuClipboardItems object so that you can set the properties for Flash Player to enable or disable each menu item. + public function ContextMenuClipboardItems (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuItem.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/ContextMenuItem.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,37 @@ +package flash.ui +{ + import flash.events.EventDispatcher; + import flash.ui.ContextMenuItem; + + /** + * Dispatched when a user selects an item from a context menu. + * @eventType flash.events.ContextMenuEvent.MENU_ITEM_SELECT + */ + [Event(name="menuItemSelect", type="flash.events.ContextMenuEvent")] + + /// Use the ContextMenuItem class to create custom menu items to display in the Flash Player context menu. + public class ContextMenuItem extends EventDispatcher + { + /// Specifies the menu item caption (text) displayed in the context menu. + public function get caption () : String; + public function set caption (value:String) : void; + + /// Indicates whether the specified menu item is enabled or disabled. + public function get enabled () : Boolean; + public function set enabled (value:Boolean) : void; + + /// Indicates whether a separator bar should appear above the specified menu item. + public function get separatorBefore () : Boolean; + public function set separatorBefore (value:Boolean) : void; + + /// Indicates whether the specified menu item is visible when the Flash Player context menu is displayed. + public function get visible () : Boolean; + public function set visible (value:Boolean) : void; + + /// Creates and returns a copy of the specified ContextMenuItem object. + public function clone () : ContextMenuItem; + + /// Creates a new ContextMenuItem object that can be added to the ContextMenu.customItems array. + public function ContextMenuItem (caption:String, separatorBefore:Boolean = false, enabled:Boolean = true, visible:Boolean = true); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/KeyLocation.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/KeyLocation.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ +package flash.ui +{ + /// The KeyLocation class contains constants that indicate the location of a key pressed on the keyboard. + public class KeyLocation extends Object + { + /// Indicates the key activated is in the left key location (there is more than one possible location for this key). + public static const LEFT : uint; + /// Indicates the key activation originated on the numeric keypad or with a virtual key corresponding to the numeric keypad. + public static const NUM_PAD : uint; + /// Indicates the key activated is in the right key location (there is more than one possible location for this key). + public static const RIGHT : uint; + /// Indicates the key activation is not distinguished as the left or right version of the key, and did not originate on the numeric keypad (or did not originate with a virtual key corresponding to the numeric keypad). + public static const STANDARD : uint; + + public function KeyLocation (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Keyboard.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Keyboard.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,116 @@ +package flash.ui +{ + /// The Keyboard class is used to build an interface that can be controlled by a user with a standard keyboard. + public class Keyboard extends Object + { + /// Constant associated with the key code value for the Backspace key (8). + public static const BACKSPACE : uint; + /// Constant associated with the key code value for the Caps Lock key (20). + public static const CAPS_LOCK : uint; + /// Constant associated with the key code value for the Control key (17). + public static const CONTROL : uint; + /// Constant associated with the key code value for the Delete key (46). + public static const DELETE : uint; + /// Constant associated with the key code value for the Down Arrow key (40). + public static const DOWN : uint; + /// Constant associated with the key code value for the End key (35). + public static const END : uint; + /// Constant associated with the key code value for the Enter key (13). + public static const ENTER : uint; + /// Constant associated with the key code value for the Escape key (27). + public static const ESCAPE : uint; + /// Constant associated with the key code value for the F1 key (112). + public static const F1 : uint; + /// Constant associated with the key code value for the F10 key (121). + public static const F10 : uint; + /// Constant associated with the key code value for the F11 key (122). + public static const F11 : uint; + /// Constant associated with the key code value for the F12 key (123). + public static const F12 : uint; + /// Constant associated with the key code value for the F13 key (124). + public static const F13 : uint; + /// Constant associated with the key code value for the F14 key (125). + public static const F14 : uint; + /// Constant associated with the key code value for the F15 key (126). + public static const F15 : uint; + /// Constant associated with the key code value for the F2 key (113). + public static const F2 : uint; + /// Constant associated with the key code value for the F3 key (114). + public static const F3 : uint; + /// Constant associated with the key code value for the F4 key (115). + public static const F4 : uint; + /// Constant associated with the key code value for the F5 key (116). + public static const F5 : uint; + /// Constant associated with the key code value for the F6 key (117). + public static const F6 : uint; + /// Constant associated with the key code value for the F7 key (118). + public static const F7 : uint; + /// Constant associated with the key code value for the F8 key (119). + public static const F8 : uint; + /// Constant associated with the key code value for the F9 key (120). + public static const F9 : uint; + /// Constant associated with the key code value for the Home key (36). + public static const HOME : uint; + /// Constant associated with the key code value for the Insert key (45). + public static const INSERT : uint; + /// Constant associated with the key code value for the Left Arrow key (37). + public static const LEFT : uint; + /// Constant associated with the key code value for the number 0 key on the number pad (96). + public static const NUMPAD_0 : uint; + /// Constant associated with the key code value for the number 1 key on the number pad (97). + public static const NUMPAD_1 : uint; + /// Constant associated with the key code value for the number 2 key on the number pad (98). + public static const NUMPAD_2 : uint; + /// Constant associated with the key code value for the number 3 key on the number pad (99). + public static const NUMPAD_3 : uint; + /// Constant associated with the key code value for the number 4 key on the number pad (100). + public static const NUMPAD_4 : uint; + /// Constant associated with the key code value for the number 5 key on the number pad (101). + public static const NUMPAD_5 : uint; + /// Constant associated with the key code value for the number 6 key on the number pad (102). + public static const NUMPAD_6 : uint; + /// Constant associated with the key code value for the number 7 key on the number pad (103). + public static const NUMPAD_7 : uint; + /// Constant associated with the key code value for the number 8 key on the number pad (104). + public static const NUMPAD_8 : uint; + /// Constant associated with the key code value for the number 9 key on the number pad (105). + public static const NUMPAD_9 : uint; + /// Constant associated with the key code value for the addition key on the number pad (107). + public static const NUMPAD_ADD : uint; + /// Constant associated with the key code value for the decimal key on the number pad (110). + public static const NUMPAD_DECIMAL : uint; + /// Constant associated with the key code value for the division key on the number pad (111). + public static const NUMPAD_DIVIDE : uint; + /// Constant associated with the key code value for the Enter key on the number pad (108). + public static const NUMPAD_ENTER : uint; + /// Constant associated with the key code value for the multiplication key on the number pad (106). + public static const NUMPAD_MULTIPLY : uint; + /// Constant associated with the key code value for the subtraction key on the number pad (109). + public static const NUMPAD_SUBTRACT : uint; + /// Constant associated with the key code value for the Page Down key (34). + public static const PAGE_DOWN : uint; + /// Constant associated with the key code value for the Page Up key (33). + public static const PAGE_UP : uint; + /// Constant associated with the key code value for the Right Arrow key (39). + public static const RIGHT : uint; + /// Constant associated with the key code value for the Shift key (16). + public static const SHIFT : uint; + /// Constant associated with the key code value for the Spacebar (32). + public static const SPACE : uint; + /// Constant associated with the key code value for the Tab key (9). + public static const TAB : uint; + /// Constant associated with the key code value for the Up Arrow key (38). + public static const UP : uint; + + /// Specifies whether the Caps Lock key is activated (true) or not (false). + public static function get capsLock () : Boolean; + + /// Specifies whether the Num Lock key is activated (true) or not (false). + public static function get numLock () : Boolean; + + /// Specifies whether the last key pressed is accessible by other SWF files. + public static function isAccessible () : Boolean; + + public function Keyboard (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Mouse.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/Mouse.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.ui +{ + /// The methods of the Mouse class are used to hide and show the mouse pointer, or to set the pointer to a specific style. + public class Mouse extends Object + { + /// Sets the mouse cursor. + public static function get cursor () : String; + public static function set cursor (value:String) : void; + + /// Hides the pointer. + public static function hide () : void; + + public function Mouse (); + + /// Displays the pointer. + public static function show () : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/MouseCursor.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/ui/MouseCursor.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.ui +{ + /// The MouseCursor class is an enumeration of constant values used in setting the cursor property of the Mouse class. + public class MouseCursor extends Object + { + /// Used to specify that the arrow cursor should be used. + public static const ARROW : String; + /// Used to specify that the cursor should be selected automatically based on the object under the mouse. + public static const AUTO : String; + /// Used to specify that the button pressing hand cursor should be used. + public static const BUTTON : String; + /// Used to specify that the dragging hand cursor should be used. + public static const HAND : String; + /// Used to specify that the I-beam cursor should be used. + public static const IBEAM : String; + + public function MouseCursor (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ByteArray.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ByteArray.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,130 @@ +package flash.utils +{ + import flash.utils.ByteArray; + + /// The ByteArray class provides methods and properties to optimize reading, writing, and working with binary data. + public class ByteArray extends Object implements IDataInput, IDataOutput + { + /// The number of bytes of data available for reading from the current position in the byte array to the end of the array. + public function get bytesAvailable () : uint; + + /// Denotes the default object encoding for the ByteArray class to use for a new ByteArray instance. + public static function get defaultObjectEncoding () : uint; + public static function set defaultObjectEncoding (version:uint) : void; + + /// Changes or reads the byte order for the data; either Endian.BIG_ENDIAN or Endian.LITTLE_ENDIAN. + public function get endian () : String; + public function set endian (type:String) : void; + + /// The length of the ByteArray object, in bytes. + public function get length () : uint; + public function set length (value:uint) : void; + + /// Used to determine whether the ActionScript 3.0, ActionScript 2.0, or ActionScript 1.0 format should be used when writing to, or reading from, a ByteArray instance. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// Moves, or returns the current position, in bytes, of the file pointer into the ByteArray object. + public function get position () : uint; + public function set position (offset:uint) : void; + + /// Creates a ByteArray instance representing a packed array of bytes, so that you can use the methods and properties in this class to optimize your data storage and stream. + public function ByteArray (); + + /// Clears the contents of the byte array and resets the length and position properties to 0. + public function clear () : void; + + /// Compresses the byte array. + public function compress () : void; + + /// Compresses the byte array using the DEFLATE compression algorithm. + public function deflate () : void; + + /// Decompresses the byte array. + public function inflate () : void; + + /// Reads a Boolean value from the byte stream. + public function readBoolean () : Boolean; + + /// Reads a signed byte from the byte stream. + public function readByte () : int; + + /// Reads the number of data bytes, specified by the length parameter, from the byte stream. + public function readBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Reads an IEEE 754 double-precision (64-bit) floating-point number from the byte stream. + public function readDouble () : Number; + + /// Reads an IEEE 754 single-precision (32-bit) floating-point number from the byte stream. + public function readFloat () : Number; + + /// Reads a signed 32-bit integer from the byte stream. + public function readInt () : int; + + /// Reads a multibyte string of specified length from the byte stream using the specified character set. + public function readMultiByte (length:uint, charSet:String) : String; + + /// Reads an object from the byte array, encoded in AMF serialized format. + public function readObject () : *; + + /// Reads a signed 16-bit integer from the byte stream. + public function readShort () : int; + + /// Reads an unsigned byte from the byte stream. + public function readUnsignedByte () : uint; + + /// Reads an unsigned 32-bit integer from the byte stream. + public function readUnsignedInt () : uint; + + /// Reads an unsigned 16-bit integer from the byte stream. + public function readUnsignedShort () : uint; + + /// Reads a UTF-8 string from the byte stream. + public function readUTF () : String; + + /// Reads a sequence of UTF-8 bytes specified by the length parameter from the byte stream and returns a string. + public function readUTFBytes (length:uint) : String; + + /// Converts the byte array to a string. + public function toString () : String; + + /// Decompresses the byte array. + public function uncompress () : void; + + /// Writes a Boolean value. + public function writeBoolean (value:Boolean) : void; + + /// Writes a byte to the byte stream. + public function writeByte (value:int) : void; + + /// Writes a sequence of length bytes from the specified byte array, bytes, starting offset (zero-based index) bytes into the byte stream. + public function writeBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Writes an IEEE 754 double-precision (64-bit) floating-point number to the byte stream. + public function writeDouble (value:Number) : void; + + /// Writes an IEEE 754 single-precision (32-bit) floating-point number to the byte stream. + public function writeFloat (value:Number) : void; + + /// Writes a 32-bit signed integer to the byte stream. + public function writeInt (value:int) : void; + + /// Writes a multibyte string to the byte stream using the specified character set. + public function writeMultiByte (value:String, charSet:String) : void; + + /// Writes an object into the byte array in AMF serialized format. + public function writeObject (object:*) : void; + + /// Writes a 16-bit integer to the byte stream. + public function writeShort (value:int) : void; + + /// Writes a 32-bit unsigned integer to the byte stream. + public function writeUnsignedInt (value:uint) : void; + + /// Writes a UTF-8 string to the byte stream. + public function writeUTF (value:String) : void; + + /// Writes a UTF-8 string to the byte stream. + public function writeUTFBytes (value:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Dictionary.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Dictionary.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.utils +{ + /// The Dictionary class lets you create a dynamic collection of properties, which uses strict equality (===) for key comparison. + public class Dictionary extends Object + { + /// Creates a new Dictionary object. + public function Dictionary (weakKeys:Boolean = false); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Endian.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Endian.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.utils +{ + /// The Endian class contains values that denote the byte order used to represent multibyte numbers. + public class Endian extends Object + { + /// Indicates the most significant byte of the multibyte number appears first in the sequence of bytes. + public static const BIG_ENDIAN : String; + /// Indicates the least significant byte of the multibyte number appears first in the sequence of bytes. + public static const LITTLE_ENDIAN : String; + + public function Endian (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataInput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataInput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,61 @@ +package flash.utils +{ + import flash.utils.ByteArray; + + /// The IDataInput interface provides a set of methods for reading binary data. + public interface IDataInput + { + /// Returns the number of bytes of data available for reading in the input buffer. + public function get bytesAvailable () : uint; + + /// The byte order for the data, either the "bigEndian" or "littleEndian" constant from the Endian class. + public function get endian () : String; + public function set endian (type:String) : void; + + /// Used to determine whether the ActionScript 3.0, ActionScript 2.0, or ActionScript 1.0 format should be used when writing to, or reading binary data. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// Reads a Boolean value from the byte stream or byte array. + public function readBoolean () : Boolean; + + /// Reads a signed byte from the byte stream or byte array. + public function readByte () : int; + + /// Reads the number of data bytes, specified by the length parameter, from the byte stream or byte array. + public function readBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Reads an IEEE 754 double-precision floating point number from the byte stream or byte array. + public function readDouble () : Number; + + /// Reads an IEEE 754 single-precision floating point number from the byte stream or byte array. + public function readFloat () : Number; + + /// Reads a signed 32-bit integer from the byte stream or byte array. + public function readInt () : int; + + /// Reads a multibyte string of specified length from the byte stream using the specified character set. + public function readMultiByte (length:uint, charSet:String) : String; + + /// Reads an object from the byte stream or byte array, encoded in AMF serialized format. + public function readObject () : *; + + /// Reads a signed 16-bit integer from the byte stream or byte array. + public function readShort () : int; + + /// Reads an unsigned byte from the byte stream or byte array. + public function readUnsignedByte () : uint; + + /// Reads an unsigned 32-bit integer from the byte stream or byte array. + public function readUnsignedInt () : uint; + + /// Reads an unsigned 16-bit integer from the byte stream or byte array. + public function readUnsignedShort () : uint; + + /// Reads a UTF-8 string from the byte stream or byte array. + public function readUTF () : String; + + /// Reads a sequence of length UTF-8 bytes from the byte stream or byte array and returns a string. + public function readUTFBytes (length:uint) : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataOutput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IDataOutput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,52 @@ +package flash.utils +{ + import flash.utils.ByteArray; + + /// The IDataOutput interface provides a set of methods for writing binary data. + public interface IDataOutput + { + /// The byte order for the data, either the "bigEndian" or "littleEndian" constant from the Endian class. + public function get endian () : String; + public function set endian (type:String) : void; + + /// Used to determine whether the ActionScript 3.0, ActionScript 2.0, or ActionScript 1.0 format should be used when writing to, or reading binary data. + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + /// Writes a Boolean value. + public function writeBoolean (value:Boolean) : void; + + /// Writes a byte. + public function writeByte (value:int) : void; + + /// Writes a sequence of length bytes from the specified byte array, bytes, starting offset(zero-based index) bytes into the byte stream. + public function writeBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + /// Writes an IEEE 754 double-precision (64-bit) floating point number. + public function writeDouble (value:Number) : void; + + /// Writes an IEEE 754 single-precision (32-bit) floating point number. + public function writeFloat (value:Number) : void; + + /// Writes a 32-bit signed integer. + public function writeInt (value:int) : void; + + /// Writes a multibyte string to the byte stream using the specified character set. + public function writeMultiByte (value:String, charSet:String) : void; + + /// Writes an object to the byte stream or byte array in AMF serialized format. + public function writeObject (object:*) : void; + + /// Writes a 16-bit integer. + public function writeShort (value:int) : void; + + /// Writes a 32-bit unsigned integer. + public function writeUnsignedInt (value:uint) : void; + + /// Writes a UTF-8 string to the byte stream. + public function writeUTF (value:String) : void; + + /// Writes a UTF-8 string. + public function writeUTFBytes (value:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IExternalizable.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/IExternalizable.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,15 @@ +package flash.utils +{ + import flash.utils.IDataInput; + import flash.utils.IDataOutput; + + /// The IExternalizable interface provides control over serialization of a class as it is encoded into a data stream. + public interface IExternalizable + { + /// A class implements this method to decode itself from a data stream by calling the methods of the IDataInput interface. + public function readExternal (input:IDataInput) : void; + + /// A class implements this method to encode itself for a data stream by calling the methods of the IDataOutput interface. + public function writeExternal (output:IDataOutput) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectInput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectInput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,45 @@ +package flash.utils +{ + import flash.utils.ByteArray; + + public class ObjectInput extends Object implements IDataInput + { + public function get bytesAvailable () : uint; + + public function get endian () : String; + public function set endian (type:String) : void; + + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + public function ObjectInput (); + + public function readBoolean () : Boolean; + + public function readByte () : int; + + public function readBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + public function readDouble () : Number; + + public function readFloat () : Number; + + public function readInt () : int; + + public function readMultiByte (length:uint, charSet:String) : String; + + public function readObject () : *; + + public function readShort () : int; + + public function readUnsignedByte () : uint; + + public function readUnsignedInt () : uint; + + public function readUnsignedShort () : uint; + + public function readUTF () : String; + + public function readUTFBytes (length:uint) : String; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectOutput.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/ObjectOutput.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +package flash.utils +{ + import flash.utils.ByteArray; + + public class ObjectOutput extends Object implements IDataOutput + { + public function get endian () : String; + public function set endian (type:String) : void; + + public function get objectEncoding () : uint; + public function set objectEncoding (version:uint) : void; + + public function ObjectOutput (); + + public function writeBoolean (value:Boolean) : void; + + public function writeByte (value:int) : void; + + public function writeBytes (bytes:ByteArray, offset:uint = 0, length:uint = 0) : void; + + public function writeDouble (value:Number) : void; + + public function writeFloat (value:Number) : void; + + public function writeInt (value:int) : void; + + public function writeMultiByte (value:String, charSet:String) : void; + + public function writeObject (object:*) : void; + + public function writeShort (value:int) : void; + + public function writeUnsignedInt (value:uint) : void; + + public function writeUTF (value:String) : void; + + public function writeUTFBytes (value:String) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Proxy.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Proxy.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,38 @@ +package flash.utils +{ + /// The Proxy class lets you override the default behavior of ActionScript operations (such as retrieving and modifying properties) on an object. + public class Proxy extends Object + { + /// Overrides the behavior of an object property that can be called as a function. + flash_proxy function callProperty (name:*, ...rest) : *; + + /// Overrides the request to delete a property. + flash_proxy function deleteProperty (name:*) : Boolean; + + /// Overrides the use of the descendant operator. + flash_proxy function getDescendants (name:*) : *; + + /// Overrides any request for a property's value. + flash_proxy function getProperty (name:*) : *; + + /// Overrides a request to check whether an object has a particular property by name. + flash_proxy function hasProperty (name:*) : Boolean; + + /// Checks whether a supplied QName is also marked as an attribute. + flash_proxy function isAttribute (name:*) : Boolean; + + /// Allows enumeration of the proxied object's properties by index number to retrieve property names. + flash_proxy function nextName (index:int) : String; + + /// Allows enumeration of the proxied object's properties by index number. + flash_proxy function nextNameIndex (index:int) : int; + + /// Allows enumeration of the proxied object's properties by index number to retrieve property values. + flash_proxy function nextValue (index:int) : *; + + flash_proxy function Proxy (); + + /// Overrides a call to change a property's value. + flash_proxy function setProperty (name:*, value:*) : void; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/SetIntervalTimer.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/SetIntervalTimer.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,9 @@ +package flash.utils +{ + import flash.events.Event; + + public class SetIntervalTimer extends Timer + { + public function SetIntervalTimer (closure:Function, delay:Number, repeats:Boolean, rest:Array); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Timer.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/Timer.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,46 @@ +package flash.utils +{ + import flash.events.EventDispatcher; + + /** + * Dispatched whenever it has completed the number of requests set by Timer.repeatCount. + * @eventType flash.events.TimerEvent.TIMER_COMPLETE + */ + [Event(name="timerComplete", type="flash.events.TimerEvent")] + + /** + * Dispatched whenever a Timer object reaches an interval specified according to the Timer.delay property. + * @eventType flash.events.TimerEvent.TIMER + */ + [Event(name="timer", type="flash.events.TimerEvent")] + + /// The Timer class is the interface to Flash Player timers. + public class Timer extends EventDispatcher + { + /// The total number of times the timer has fired since it started at zero. + public function get currentCount () : int; + + /// The delay, in milliseconds, between timer events. + public function get delay () : Number; + public function set delay (value:Number) : void; + + /// The total number of times the timer is set to run. + public function get repeatCount () : int; + public function set repeatCount (value:int) : void; + + /// The timer's current state; true if the timer is running, otherwise false. + public function get running () : Boolean; + + /// Stops the timer, if it is running, and sets the currentCount property back to 0, like the reset button of a stopwatch. + public function reset () : void; + + /// Starts the timer, if it is not already running. + public function start () : void; + + /// Stops the timer. + public function stop () : void; + + /// Constructs a new Timer object with the specified delay and repeatCount states. + public function Timer (delay:Number, repeatCount:int = 0); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/package.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/utils/package.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,40 @@ +package flash.utils +{ + /// Proxy methods namespace + public namespace flash_proxy; + + /// Runs a function at a specified interval (in milliseconds). + public function setInterval (closure:Function, delay:Number, ...arguments) : uint; + + /// Runs a specified function after a specified delay (in milliseconds). + public function setTimeout (closure:Function, delay:Number, ...arguments) : uint; + + /// Cancels a specified setInterval() call. + public function clearInterval (id:uint) : void; + + /// Cancels a specified setTimeout() call. + public function clearTimeout (id:uint) : void; + + /// Produces an XML object that describes the ActionScript object named as the parameter of the method. + public function describeType (value:*) : XML; + + /// Returns the fully qualified class name of an object. + public function getQualifiedClassName (value:*) : String; + + /// Returns a reference to the class object of the class specified by the name parameter. + public function getDefinitionByName (name:String) : Object; + + /// Returns the fully qualified class name of the base class of the object specified by the value parameter. + public function getQualifiedSuperclassName (value:*) : String; + + /// Returns the number of milliseconds that have elapsed since Flash Player was initialized, and is used to compute relative time. + public function getTimer () : int; + + /// Returns an escaped copy of the input string encoded as either UTF-8 or system code page, depending on the value of System.useCodePage. + public function escapeMultiByte (value:String) : String; + + /// Returns an unescaped copy of the input string, which is decoded from either system code page page or UTF-8 depending on the value of System.useCodePage. + public function unescapeMultiByte (value:String) : String; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLDocument.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLDocument.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,32 @@ +package flash.xml +{ + import flash.xml.XMLNode; + + /// The XMLDocument class represents the legacy XML object that was present in ActionScript 2.0. + public class XMLDocument extends XMLNode + { + /// Specifies information about the XML document's DOCTYPE declaration. + public var docTypeDecl : Object; + /// An Object containing the nodes of the XML that have an id attribute assigned. + public var idMap : Object; + /// When set to true, text nodes that contain only white space are discarded during the parsing process. + public var ignoreWhite : Boolean; + /// A string that specifies information about a document's XML declaration. + public var xmlDecl : Object; + + /// Creates a new XMLNode object with the name specified in the parameter. + public function createElement (name:String) : XMLNode; + + /// Creates a new XML text node with the specified text. + public function createTextNode (text:String) : XMLNode; + + /// Parses the XML text specified in the value parameter and populates the specified XMLDocument object with the resulting XML tree. + public function parseXML (source:String) : void; + + /// Returns a string representation of the XML object. + public function toString () : String; + + /// Creates a new XMLDocument object. + public function XMLDocument (source:String = null); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNode.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNode.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,68 @@ +package flash.xml +{ + import flash.xml.XMLNode; + + /// The XMLNode class represents the legacy XML object that was present in ActionScript 2.0 and that was renamed in ActionScript 3.0. + public class XMLNode extends Object + { + /// Evaluates the specified XMLDocument object and references the first child in the parent node's child list. + public var firstChild : XMLNode; + /// An XMLNode value that references the last child in the node's child list. + public var lastChild : XMLNode; + /// An XMLNode value that references the next sibling in the parent node's child list. + public var nextSibling : XMLNode; + /// A string representing the node name of the XMLNode object. + public var nodeName : String; + /// A nodeType constant value, either XMLNodeType.ELEMENT_NODE for an XML element or XMLNodeType.TEXT_NODE for a text node. + public var nodeType : uint; + /// The node value of the XMLDocument object. + public var nodeValue : String; + /// An XMLNode value that references the parent node of the specified XML object, or returns null if the node has no parent. + public var parentNode : XMLNode; + /// An XMLNode value that references the previous sibling in the parent node's child list. + public var previousSibling : XMLNode; + + /// An object containing all of the attributes of the specified XMLNode instance. + public function get attributes () : Object; + public function set attributes (value:Object) : void; + + /// An array of the specified XMLNode object's children. + public function get childNodes () : Array; + + /// The local name portion of the XML node's name. + public function get localName () : String; + + /// If the XML node has a prefix, namespaceURI is the value of the xmlns declaration for that prefix (the URI), which is typically called the namespace URI. + public function get namespaceURI () : String; + + /// The prefix portion of the XML node name. + public function get prefix () : String; + + /// Appends the specified node to the XML object's child list. + public function appendChild (node:XMLNode) : void; + + /// Constructs and returns a new XML node of the same type, name, value, and attributes as the specified XML object. + public function cloneNode (deep:Boolean) : XMLNode; + + /// Returns the namespace URI that is associated with the specified prefix for the node. + public function getNamespaceForPrefix (prefix:String) : String; + + /// Returns the prefix that is associated with the specified namespace URI for the node. + public function getPrefixForNamespace (ns:String) : String; + + /// Indicates whether the specified XMLNode object has child nodes. + public function hasChildNodes () : Boolean; + + /// Inserts a new child node into the XML object's child list, before the beforeNode node. + public function insertBefore (node:XMLNode, before:XMLNode) : void; + + /// Removes the specified XML object from its parent. + public function removeNode () : void; + + /// Evaluates the specified XMLNode object, constructs a textual representation of the XML structure, including the node, children, and attributes, and returns the result as a string. + public function toString () : String; + + /// Creates a new XMLNode object. + public function XMLNode (type:uint, value:String); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNodeType.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLNodeType.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +package flash.xml +{ + /// The XMLNodeType class contains constants used with XMLNode.nodeType. + public class XMLNodeType extends Object + { + public static const CDATA_NODE : uint; + public static const COMMENT_NODE : uint; + public static const DOCUMENT_TYPE_NODE : uint; + /// Specifies that the node is an element. + public static const ELEMENT_NODE : uint; + public static const PROCESSING_INSTRUCTION_NODE : uint; + /// Specifies that the node is a text node. + public static const TEXT_NODE : uint; + public static const XML_DECLARATION : uint; + + public function XMLNodeType (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLParser.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLParser.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +package flash.xml +{ + import flash.xml.XMLTag; + + public class XMLParser extends Object + { + public function getNext (tag:XMLTag) : int; + + public function startParse (source:String, ignoreWhite:Boolean) : void; + + public function XMLParser (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLTag.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/flash/xml/XMLTag.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +package flash.xml +{ + public class XMLTag extends Object + { + public function get attrs () : Object; + public function set attrs (value:Object) : void; + + public function get empty () : Boolean; + public function set empty (value:Boolean) : void; + + public function get type () : uint; + public function set type (value:uint) : void; + + public function get value () : String; + public function set value (v:String) : void; + + public function XMLTag (); + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/int.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/int.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package +{ + /// The int class lets you work with the data type representing a 32-bit signed integer. + public class int extends Object + { + public static const length : int; + /// The largest representable 32-bit signed integer, which is 2,147,483,647. + public static const MAX_VALUE : int; + /// The smallest representable 32-bit signed integer, which is -2,147,483,648. + public static const MIN_VALUE : int; + + /// Constructor; creates a new int object. + public function int (value:* = 0); + + /// Returns a string representation of the number in exponential notation. + public function toExponential (p:* = 0) : String; + + /// Returns a string representation of the number in fixed-point notation. + public function toFixed (p:* = 0) : String; + + /// Returns a string representation of the number either in exponential notation or in fixed-point notation. + public function toPrecision (p:* = 0) : String; + + /// Returns the string representation of an int object. + public function toString (radix:* = 10) : String; + + /// Returns the primitive value of the specified int object. + public function valueOf () : int; + } +} Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/toplevel.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/toplevel.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,49 @@ +package +{ + /// A special value representing positive Infinity. + public var Infinity:Number; + + /// A special member of the Number data type that represents a value that is "not a number" (NaN). + public var NaN:Number; + + /// A special value that applies to untyped variables that have not been initialized or dynamic object properties that are not initialized. + public var undefined:*; + + /// Displays expressions, or writes to log files, while debugging. + public function trace (...arguments) : void; + + /// Decodes an encoded URI into a string. + public function decodeURI (uri:String) : String; + + /// Decodes an encoded URI component into a string. + public function decodeURIComponent (uri:String) : String; + + /// Encodes a string into a valid URI (Uniform Resource Identifier). + public function encodeURI (uri:String) : String; + + /// Encodes a string into a valid URI component. + public function encodeURIComponent (uri:String) : String; + + /// Converts the parameter to a string and encodes it in a URL-encoded format, where most nonalphanumeric characters are replaced with % hexadecimal sequences. + public function escape (str:String) : String; + + /// Returns true if the value is a finite number, or false if the value is Infinity or -Infinity. + public function isFinite (num:Number) : Boolean; + + /// Returns true if the value is NaN(not a number). + public function isNaN (num:Number) : Boolean; + + /// Determines whether the specified string is a valid name for an XML element or attribute. + public function isXMLName (str:String) : Boolean; + + /// Converts a string to an integer. + public function parseInt (str:String, radix:uint = 0) : Number; + + /// Converts a string to a floating-point number. + public function parseFloat (str:String) : Number; + + /// Evaluates the parameter str as a string, decodes the string from URL-encoded format (converting all hexadecimal sequences to ASCII characters), and returns the string. + public function unescape (str:String) : String; + +} + Added: pypy/branch/avm/pypy/translator/avm2/intrinsic/src/uint.as ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/intrinsic/src/uint.as Wed Nov 4 00:51:35 2009 @@ -0,0 +1,30 @@ +package +{ + /// The uint class provides methods for working with a data type representing a 32-bit unsigned integer. + public class uint extends Object + { + public static const length : int; + /// The largest representable 32-bit unsigned integer, which is 4,294,967,295. + public static const MAX_VALUE : uint; + /// The smallest representable unsigned integer, which is 0. + public static const MIN_VALUE : uint; + + /// Returns a string representation of the number in exponential notation. + public function toExponential (p:* = 0) : String; + + /// Returns a string representation of the number in fixed-point notation. + public function toFixed (p:* = 0) : String; + + /// Returns a string representation of the number either in exponential notation or in fixed-point notation. + public function toPrecision (p:* = 0) : String; + + /// Returns the string representation of a uint object. + public function toString (radix:* = 10) : String; + + /// Creates a new uint object. + public function uint (value:* = 0); + + /// Returns the primitive uint type value of the specified uint object. + public function valueOf () : uint; + } +} Added: pypy/branch/avm/pypy/translator/avm2/metavm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/metavm.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,297 @@ +from pypy.translator.cli import oopspec +from pypy.rpython.ootypesystem import ootype +from pypy.translator.oosupport.metavm import InstructionList, MicroInstruction, \ + PushAllArgs, StoreResult, GetField, SetField, DownCast +from pypy.translator.oosupport.metavm import _Call as _OOCall +# from pypy.translator.cli.comparer import EqualityComparer +from pypy.translator.avm2.runtime import _static_meth, NativeInstance + +STRING_HELPER_CLASS = '[pypylib]pypy.runtime.String' + +def functype_to_cts(cts, FUNC): + ret_type = cts.lltype_to_cts(FUNC.RESULT) + arg_types = [cts.lltype_to_cts(arg).typename() + for arg in FUNC.ARGS + if arg is not ootype.Void] + return ret_type, arg_types + +class _Call(_OOCall): + + def render(self, generator, op): + callee = op.args[0].value + if isinstance(callee, _static_meth): + self._render_native_function(generator, callee, op.args) + else: + _OOCall.render(self, generator, op) + + def _render_native_function(self, generator, funcdesc, args): + for func_arg in args[1:]: # push parameters + self._load_arg_or_null(generator, func_arg) + cts = generator.cts + #ret_type, arg_types = functype_to_cts(cts, funcdesc._TYPE) + #arg_list = ', '.join(arg_types) + #signature = '%s %s::%s(%s)' % (ret_type, funcdesc._cls._name, funcdesc._name, arg_list) + generator.call_signature(signature) + + def _load_arg_or_null(self, generator, arg): + if arg.concretetype is ootype.Void: + if arg.value is None: + generator.ilasm.opcode('ldnull') # special-case: use None as a null value + else: + assert False, "Don't know how to load this arg" + else: + generator.load(arg) + + +class _CallMethod(_Call): + def render(self, generator, op): + method = op.args[0] + self._render_method(generator, method.value, op.args[1:]) + + def _render_method(self, generator, method_name, args): + this = args[0] + native = isinstance(this.concretetype, NativeInstance) + for arg in args: # push parametes + if native: + self._load_arg_or_null(generator, arg) + else: + generator.load(arg) + + # XXX: very hackish, need refactoring + if this.concretetype in (ootype.String, ootype.Unicode): + # special case for string: don't use methods, but plain functions + METH = this.concretetype._METHODS[method_name] + cts = generator.cts + ret_type, arg_types = functype_to_cts(cts, METH) + arg_types.insert(0, cts.lltype_to_cts(ootype.String).typename()) + generator.call_method(signature) + elif isinstance(this.concretetype, ootype.Array) and this.concretetype.ITEM is not ootype.Void: + v_array = args[0] + ARRAY = v_array.concretetype + if method_name == 'll_setitem_fast': + generator.array_setitem(ARRAY) + elif method_name == 'll_getitem_fast': + generator.array_getitem(ARRAY) + elif method_name == 'll_length': + generator.array_length(ARRAY) + else: + assert False + else: + generator.call_method(this.concretetype, method_name) + + # special case: DictItemsIterator(XXX, + # Void).ll_current_value needs to return an int32 because + # we can't use 'void' as a parameter of a Generic. This + # means that after the call to ll_current_value there will + # be a value on the stack, and we need to explicitly pop + # it. + if isinstance(this.concretetype, ootype.DictItemsIterator) and \ + ((this.concretetype._VALUETYPE is ootype.Void and \ + method_name == 'll_current_value') or \ + (this.concretetype._KEYTYPE is ootype.Void and \ + method_name == 'll_current_key')): + generator.ilasm.pop() + + +class _IndirectCall(_CallMethod): + def render(self, generator, op): + # discard the last argument because it's used only for analysis + self._render_method(generator, 'Invoke', op.args[:-1]) + +class _RuntimeNew(MicroInstruction): + def render(self, generator, op): + generator.load(op.args[0]) + generator.call_signature('object [pypylib]pypy.runtime.Utils::RuntimeNew(class [mscorlib]System.Type)') + generator.cast_to(op.result.concretetype) + +class CallConstantMethod(MicroInstruction): + pass + +# class _NewCustomDict(MicroInstruction): +# def render(self, generator, op): +# DICT = op.args[0].value +# comparer = EqualityComparer(generator.db, DICT._KEYTYPE, +# (op.args[1], op.args[2], op.args[3]), +# (op.args[4], op.args[5], op.args[6])) +# generator.db.pending_node(comparer) +# dict_type = generator.cts.lltype_to_cts(DICT) + +# generator.ilasm.new(comparer.get_ctor()) +# generator.ilasm.new('instance void %s::.ctor(class' +# '[mscorlib]System.Collections.Generic.IEqualityComparer`1)' +# % dict_type) + +#XXX adapt to new way of things +#class _CastWeakAdrToPtr(MicroInstruction): +# def render(self, generator, op): +# RESULTTYPE = op.result.concretetype +# resulttype = generator.cts.lltype_to_cts(RESULTTYPE) +# generator.load(op.args[0]) +# generator.ilasm.call_method('object class %s::get_Target()' % WEAKREF, True) +# generator.ilasm.opcode('castclass', resulttype) + +# class MapException(MicroInstruction): +# COUNT = 0 + +# def __init__(self, instr, mapping): +# if isinstance(instr, str): +# self.instr = InstructionList([PushAllArgs, instr, StoreResult]) +# else: +# self.instr = InstructionList(instr) +# self.mapping = mapping + +# def render(self, generator, op): +# ilasm = generator.ilasm +# label = '__check_block_%d' % MapException.COUNT +# MapException.COUNT += 1 +# ilasm.begin_try() +# self.instr.render(generator, op) +# ilasm.leave(label) +# ilasm.end_try() +# for cli_exc, py_exc in self.mapping: +# ilasm.begin_catch(cli_exc) +# ilasm.new('instance void class %s::.ctor()' % py_exc) +# ilasm.opcode('throw') +# ilasm.end_catch() +# ilasm.label(label) +# ilasm.opcode('nop') + +# class _Box(MicroInstruction): +# def render(self, generator, op): +# generator.load(op.args[0]) +# TYPE = op.args[0].concretetype +# boxtype = generator.cts.lltype_to_cts(TYPE) +# generator.ilasm.opcode('box', boxtype) + +# class _Unbox(MicroInstruction): +# def render(self, generator, op): +# v_obj, v_type = op.args +# assert v_type.concretetype is ootype.Void +# TYPE = v_type.value +# boxtype = generator.cts.lltype_to_cts(TYPE) +# generator.load(v_obj) +# generator.ilasm.opcode('unbox.any', boxtype) + +class _NewArray(MicroInstruction): + def render(self, generator, op): + v_type, v_length = op.args + assert v_type.concretetype is ootype.Void + TYPE = v_type.value._INSTANCE + typetok = generator.cts.lltype_to_cts(TYPE) + generator.load(v_length) + generator.ilasm.opcode('newarr', typetok) + +class _GetArrayElem(MicroInstruction): + def render(self, generator, op): + generator.load(op.args[0]) + generator.load(op.args[1]) + rettype = generator.cts.lltype_to_cts(op.result.concretetype) + generator.ilasm.opcode('ldelem', rettype) + +class _SetArrayElem(MicroInstruction): + def render(self, generator, op): + v_array, v_index, v_elem = op.args + generator.load(v_array) + generator.load(v_index) + if v_elem.concretetype is ootype.Void and v_elem.value is None: + generator.ilasm.opcode('ldnull') + else: + generator.load(v_elem) + elemtype = generator.cts.lltype_to_cts(v_array.concretetype) + generator.ilasm.opcode('stelem', elemtype) + +class _TypeOf(MicroInstruction): + def render(self, generator, op): + c_type, = op.args + assert c_type.concretetype is ootype.Void + if isinstance(c_type.value, ootype.StaticMethod): + FUNC = c_type.value + fullname = generator.cts.lltype_to_cts(FUNC) + else: + cliClass = c_type.value + fullname = cliClass._INSTANCE._name + generator.ilasm.opcode('ldtoken', fullname) + generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + +class _EventHandler(MicroInstruction): + def render(self, generator, op): + cts = generator.cts + v_obj, c_methname = op.args + assert c_methname.concretetype is ootype.Void + TYPE = v_obj.concretetype + classname = TYPE._name + methname = 'o' + c_methname.value # XXX: do proper mangling + _, meth = TYPE._lookup(methname) + METH = ootype.typeOf(meth) + ret_type, arg_types = functype_to_cts(cts, METH) + arg_list = ', '.join(arg_types) + generator.load(v_obj) + desc = '%s class %s::%s(%s)' % (ret_type, classname, methname, arg_list) + generator.ilasm.opcode('ldftn instance', desc) + generator.ilasm.opcode('newobj', 'instance void class [mscorlib]System.EventHandler::.ctor(object, native int)') + +class _GetStaticField(MicroInstruction): + def render(self, generator, op): + cli_class = op.args[0].value + fldname = op.args[1].value + TYPE = op.result.concretetype + cts_type = generator.cts.lltype_to_cts(TYPE) + desc = '%s::%s' % (cli_class._name, fldname) + generator.ilasm.load_static_field(cts_type, desc) + +class _SetStaticField(MicroInstruction): + def render(self, generator, op): + cli_class = op.args[0].value + fldname = op.args[1].value + TYPE = op.result.concretetype + cts_type = generator.cts.lltype_to_cts(TYPE) + desc = '%s::%s' % (cli_class._name, fldname) + generator.load(op.args[2]) + generator.ilasm.store_static_field(cts_type, desc) + +class _FieldInfoForConst(MicroInstruction): + def render(self, generator, op): + from pypy.translator.cli.constant import CONST_CLASS + llvalue = op.args[0].value + constgen = generator.db.constant_generator + const = constgen.record_const(llvalue) + generator.ilasm.opcode('ldtoken', CONST_CLASS) + generator.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') + generator.ilasm.opcode('ldstr', '"%s"' % const.name) + generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) + + +OOTYPE_TO_MNEMONIC = { + ootype.Bool: 'i1', + ootype.Char: 'i2', + ootype.UniChar: 'i2', + ootype.Signed: 'i4', + ootype.SignedLongLong: 'i8', + ootype.Unsigned: 'u4', + ootype.UnsignedLongLong: 'u8', + ootype.Float: 'r8', + } + +class _CastPrimitive(MicroInstruction): + def render(self, generator, op): + TO = op.result.concretetype + mnemonic = OOTYPE_TO_MNEMONIC[TO] + generator.ilasm.opcode('conv.%s' % mnemonic) + +Call = _Call() +CallMethod = _CallMethod() +IndirectCall = _IndirectCall() +RuntimeNew = _RuntimeNew() +NewCustomDict = _NewCustomDict() +#CastWeakAdrToPtr = _CastWeakAdrToPtr() +Box = _Box() +Unbox = _Unbox() +NewArray = _NewArray() +GetArrayElem = _GetArrayElem() +SetArrayElem = _SetArrayElem() +TypeOf = _TypeOf() +EventHandler = _EventHandler() +GetStaticField = _GetStaticField() +SetStaticField = _SetStaticField() +FieldInfoForConst = _FieldInfoForConst() +CastPrimitive = _CastPrimitive() Added: pypy/branch/avm/pypy/translator/avm2/opcodes.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/opcodes.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,141 @@ + +from pypy.translator.oosupport import metavm as om +from pypy.translator.avm1 import metavm as am, avm2 as a + +DoNothing = [om.PushAllArgs] + +misc_ops = { + 'new': [om.New], + 'runtimenew': [om.RuntimeNew], + 'oosetfield': [am.SetField], + 'oogetfield': [am.GetField], + 'oonewarray': [om.OONewArray], + 'oosend': [am.CallMethod], +} + +unary_ops = { + 'same_as': DoNothing, + 'bool_not': 'not', + 'int_neg': 'negate', + 'int_neg_ovf': 'negate', + 'int_abs': [am.PushArgsForFunctionCall, am.CallConstantMethod("Math", "abs")], + + 'cast_int_to_char': [am.PushArgsForFunctionCall, am.CallConstantMethod("String", "fromCharCode")], + 'cast_int_to_unichar': [am.PushArgsForFunctionCall, am.CallConstantMethod("String", "fromCharCode")], + 'cast_int_to_float': DoNothing, + 'cast_int_to_longlong': DoNothing, + 'cast_int_to_uint': DoNothing, + 'cast_int_to_long': DoNothing, + 'cast_uint_to_float': DoNothing, + 'cast_uint_to_longlong': DoNothing, + 'cast_uint_to_int' : DoNothing, + 'cast_uint_to_long': DoNothing, + + 'cast_bool_to_int': 'convert_to_number', + 'cast_bool_to_uint': 'convert_to_number', + 'cast_bool_to_float': 'convert_to_number', + + +} + +binary_ops = { + 'int_add': 'typed_add', + 'int_sub': 'subtract', + 'int_mul': 'multiply', + 'int_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'int_mod': 'modulo', + 'int_lt': 'typed_less', + 'int_le': [om.PushAllArgs, 'greater', 'not'], + 'int_eq': 'typed_equals', + 'int_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'int_gt': 'greater', + 'int_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'int_and': 'bit_and', + 'int_or': 'bit_or', + 'int_lshift': 'shift_left', + 'int_rshift': 'shift_right', + 'int_xor': 'bit_xor', + + 'uint_add': 'typed_add', + 'uint_sub': 'subtract', + 'uint_mul': 'multiply', + 'uint_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'uint_mod': 'modulo', + 'uint_lt': 'typed_less', + 'uint_le': [om.PushAllArgs, 'greater', 'not'], + 'uint_eq': 'typed_equals', + 'uint_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'uint_gt': 'greater', + 'uint_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'uint_and': 'bit_and', + 'uint_or': 'bit_or', + 'uint_lshift': 'shift_left', + 'uint_rshift': 'shift_right', + 'uint_xor': 'bit_xor', + + 'float_add': 'typed_add', + 'float_sub': 'subtract', + 'float_mul': 'multiply', + 'float_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'float_mod': 'modulo', + 'float_lt': 'typed_less', + 'float_le': [om.PushAllArgs, 'greater', 'not'], + 'float_eq': 'typed_equals', + 'float_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'float_gt': 'greater', + 'float_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'float_and': 'bit_and', + 'float_or': 'bit_or', + 'float_lshift': 'shift_left', + 'float_rshift': 'shift_right', + 'float_xor': 'bit_xor', + + 'llong_add': 'typed_add', + 'llong_sub': 'subtract', + 'llong_mul': 'multiply', + 'llong_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'llong_mod': 'modulo', + 'llong_lt': 'typed_less', + 'llong_le': [om.PushAllArgs, 'greater', 'not'], + 'llong_eq': 'typed_equals', + 'llong_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'llong_gt': 'greater', + 'llong_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'llong_and': 'bit_and', + 'llong_or': 'bit_or', + 'llong_lshift': 'shift_left', + 'llong_rshift': 'shift_right', + 'llong_xor': 'bit_xor', + + 'ullong_add': 'typed_add', + 'ullong_sub': 'subtract', + 'ullong_mul': 'multiply', + 'ullong_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'ullong_mod': 'modulo', + 'ullong_lt': 'typed_less', + 'ullong_le': [om.PushAllArgs, 'greater', 'not'], + 'ullong_eq': 'typed_equals', + 'ullong_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'ullong_gt': 'greater', + 'ullong_ge': [om.PushAllArgs, 'typed_less', 'not'], + 'ullong_lshift': 'shift_left', + 'ullong_rshift': 'shift_right', +} + +opcodes = misc_ops.copy() +opcodes.update(unary_ops) +opcodes.update(binary_ops) + +for key, value in opcodes.iteritems(): + if isinstance(value, str): + value = [am.StoreResultStart, om.PushAllArgs, value, am.StoreResultEnd] + if am.StoreResultStart not in value: + value.insert(0, am.StoreResultStart) + if am.StoreResultEnd not in value: + value.append(am.StoreResultEnd) + value = om.InstructionList(value) + opcodes[key] = value Added: pypy/branch/avm/pypy/translator/avm2/test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/__init__.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/avm/pypy/translator/avm2/test/autopath.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/autopath.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,135 @@ +""" +self cloning, automatic path configuration + +copy this into any subdirectory of pypy from which scripts need +to be run, typically all of the test subdirs. +The idea is that any such script simply issues + + import autopath + +and this will make sure that the parent directory containing "pypy" +is in sys.path. + +If you modify the master "autopath.py" version (in pypy/tool/autopath.py) +you can directly run it which will copy itself on all autopath.py files +it finds under the pypy root directory. + +This module always provides these attributes: + + pypydir pypy root directory path + this_dir directory where this autopath.py resides + +""" + +def __dirinfo(part): + """ return (partdir, this_dir) and insert parent of partdir + into sys.path. If the parent directories don't have the part + an EnvironmentError is raised.""" + + import sys, os + try: + head = this_dir = os.path.realpath(os.path.dirname(__file__)) + except NameError: + head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + + error = None + while head: + partdir = head + head, tail = os.path.split(head) + if tail == part: + # check if "../py/__init__.py" exists + checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) + break + else: + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) + + pypy_root = os.path.join(head, '') + try: + sys.path.remove(head) + except ValueError: + pass + sys.path.insert(0, head) + + munged = {} + for name, mod in sys.modules.items(): + if '.' in name: + continue + fn = getattr(mod, '__file__', None) + if not isinstance(fn, str): + continue + newname = os.path.splitext(os.path.basename(fn))[0] + if not newname.startswith(part + '.'): + continue + path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') + if path.startswith(pypy_root) and newname != part: + modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) + if newname != '__init__': + modpaths.append(newname) + modpath = '.'.join(modpaths) + if modpath not in sys.modules: + munged[modpath] = mod + + for name, mod in munged.iteritems(): + if name not in sys.modules: + sys.modules[name] = mod + if '.' in name: + prename = name[:name.rfind('.')] + postname = name[len(prename)+1:] + if prename not in sys.modules: + __import__(prename) + if not hasattr(sys.modules[prename], postname): + setattr(sys.modules[prename], postname, mod) + + return partdir, this_dir + +def __clone(): + """ clone master version of autopath.py into all subdirs """ + from os.path import join, walk + if not this_dir.endswith(join('pypy','tool')): + raise EnvironmentError("can only clone master version " + "'%s'" % join(pypydir, 'tool',_myname)) + + + def sync_walker(arg, dirname, fnames): + if _myname in fnames: + fn = join(dirname, _myname) + f = open(fn, 'rwb+') + try: + if f.read() == arg: + print "checkok", fn + else: + print "syncing", fn + f = open(fn, 'w') + f.write(arg) + finally: + f.close() + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) + +_myname = 'autopath.py' + +# set guaranteed attributes + +pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) + +if __name__ == '__main__': + __clone() Added: pypy/branch/avm/pypy/translator/avm2/test/bootstrap.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/bootstrap.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +import autopath + +from pypy.translator.avm import swf as s, avm1 as a, tags as t, records as r + +if __name__ == "__main__": + with open("test.swf", "w") as f: + data = s.SwfData() + data.add_tag(t.SetBackgroundColor(0x333333)) + data.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "txt", "Testing!", color=r.RGBA(0xFFFFFF))) + data.add_tag(t.PlaceObject(1, 2)) + actions = t.DoAction() + actions.add_action(a.ActionPush("txt", "txt")) + actions.add_action(a.ActionGetVariable()) + actions.add_action(a.ActionPush("\nTesting 2!")) + actions.add_action(a.ActionStringAdd()) + actions.add_action(a.ActionSetVariable()) + actions.add_action(a.ActionPush("/test.result", "")) + actions.add_action(a.ActionGetURL2("POST")) + data.add_tag(actions) + data.add_tag(t.ShowFrame()) + data.add_tag(t.End()) + f.write(data.serialize()) Added: pypy/branch/avm/pypy/translator/avm2/test/browsertest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/browsertest.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,99 @@ +from BaseHTTPServer import HTTPServer as BaseHTTPServer, BaseHTTPRequestHandler +from cgi import parse_qs + +import time +import webbrowser + +class HTTPServer(BaseHTTPServer): + allow_reuse_address = True + +class config(object): + http_port = 10001 + + html_page = """ + +PyPy AVM Backend Test Case: %s + + + + + + + +""" + + crossdomain_xml=""" + +""" + +def parse_result(string): + if string == "true": + return True + elif string == "false": + return False + elif string == "undefined" or string == "null": + return None + elif all(c in "123456789-" for c in string): + return int(string) + elif "," in string: + return [parse_result(s) for s in string.split(",")] + return string + +class TestCase(object): + def __init__(self, name, swfdata): + self.name = name + self.swfdata = swfdata + self.result = None + +class TestHandler(BaseHTTPRequestHandler): + """The HTTP handler class that provides the tests and handles results""" + + def do_GET(self): + testcase = self.server.testcase + if self.path == "/test.html": + data = config.html_page % (testcase.name, testcase.swfdata.width, testcase.swfdata.height) + mime = 'text/html' + elif self.path == "/test.swf": + data = testcase.swfdata.serialize() + mime = 'application/x-shockwave-flash' + elif self.path == "/crossdomain.xml": + data = config.crossdomain_xml + mime = 'text/xml' + self.serve_data(mime, data) + + def do_POST(self): + if self.path == "/test.result": + form = parse_qs(self.rfile.read(int(self.headers['content-length']))) + self.server.testcase.result = form['result'][0] + + def serve_data(self, content_type, data): + self.send_response(200) + self.send_header("Content-type", content_type) + self.send_header("Content-length", len(data)) + self.end_headers() + self.wfile.write(data) + +class BrowserTest(object): + """The browser driver""" + + def start_server(self, port, testcase): + server_address = ('', port) + self.httpd = HTTPServer(server_address, TestHandler) + self.httpd.testcase = testcase + + def get_result(self): + testcase = self.httpd.testcase + start_time = time.time() + while testcase.result is None: + if time.time() - start_time > 10: + assert False + self.httpd.handle_request() + return testcase.result + +def browsertest(name, swfdata): + testcase = TestCase(str(name), swfdata) + driver = BrowserTest() + driver.start_server(config.http_port, testcase) + webbrowser.open('http://localhost:%d/test.html' % config.http_port) + + return parse_result(driver.get_result()) Added: pypy/branch/avm/pypy/translator/avm2/test/harness.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/harness.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,68 @@ + +from pypy.translator.avm1 import swf as s, tags as t, records as r +from pypy.translator.avm2.test import browsertest as b +from pypy.translator.avm2 import avm2gen as m + +class TestHarness(object): + def __init__(self, name): + self.testname = name + self.swf = s.SwfData() + self.swf.add_tag(t.SetBackgroundColor(0x333333)) + self.swf.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "txt", + "Testing %s." % (name,), color=r.RGBA(0xFFFFFF))) + self.swf.add_tag(t.PlaceObject2(1, 2)) + self.actions = g.AVM1Gen(t.DoAction()) + self.swf.add_tag(self.actions.block) + self.swf.add_tag(t.ShowFrame()) + self.swf.add_tag(t.End()) + self.start_test() + + def print_text(self, text): + self.actions.push_const("txt", "\n" + text) + self.actions.push_var("txt") + self.actions.swap() + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_var(self, prefix, varname): + self.actions.push_const("txt") + self.actions.push_var("txt") + self.actions.push_const("\n" + prefix) + self.actions.push_var(varname) + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_stack(self, prefix): + self.actions.push_const("txt") + self.actions.swap() + self.actions.push_var("txt") + self.actions.swap() + self.actions.push_const("\n" + prefix) + self.actions.swap() + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def start_test(self): + self.print_text("Running test %s." % self.testname) + self.actions.push_const("result") + + def finish_test(self): + # result value is on the stack, + # followed by the string "result" + self.actions.set_variable() + self.print_var("Got: ", "result") + self.actions.push_const("/test.result", "") # URL, target + self.actions.action(a.ActionGetURL2("POST", True, True)) + self.actions.push_const("javascript:window.close()", "") # Close the window. + self.actions.action(a.ActionGetURL2("")) + + def do_test(self, debug=False): + self.finish_test() + self.actions.scope.block.seal() + if debug: + f = open("test.swf", "w") + f.write(self.swf.serialize()) + f.close() + return b.browsertest(self.testname, self.swf) Added: pypy/branch/avm/pypy/translator/avm2/test/mylib.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/mylib.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.carbonpython import export + + at export(int, int) +def sum(a, b): + return a+b Added: pypy/branch/avm/pypy/translator/avm2/test/runtest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/runtest.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,138 @@ +import platform + +import py +from pypy.translator.translator import TranslationContext +from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin +from pypy.rpython.lltypesystem.lltype import typeOf +from pypy.rpython.ootypesystem import ootype +from pypy.annotation.model import lltype_to_annotation +from pypy.translator.backendopt.all import backend_optimizations +from pypy.translator.backendopt.checkvirtual import check_virtual_methods +from pypy.translator.oosupport.support import patch_os, unpatch_os +from pypy.translator.avm2.test.harness import TestHarness +from pypy.translator.avm2.genavm import GenAVM1 + +# def translate_space_op(gen, op): +# if op.opname == "cast_int_to_char": +# gen.push_arg(op.args[0]) +# gen.push_const(1) +# gen.push_var("String") +# gen.call_method("fromCharCode") + +def compile_function(func, name, annotation=[], graph=None, backendopt=True, + auto_raise_exc=False, exctrans=False, + annotatorpolicy=None, nowrap=False): + olddefs = patch_os() + gen = _build_gen(func, annotation, name, graph, backendopt, + exctrans, annotatorpolicy, nowrap) + harness = TestHarness(name) + gen.ilasm = harness.actions + gen.generate_source() + unpatch_os(olddefs) # restore original values + return gen, harness + +def _build_gen(func, annotation, name, graph=None, backendopt=True, exctrans=False, + annotatorpolicy=None, nowrap=False): + try: + func = func.im_func + except AttributeError: + pass + t = TranslationContext() + if graph is not None: + graph.func = func + ann = t.buildannotator(policy=annotatorpolicy) + inputcells = [ann.typeannotation(an) for an in annotation] + ann.build_graph_types(graph, inputcells) + t.graphs.insert(0, graph) + else: + ann = t.buildannotator(policy=annotatorpolicy) + ann.build_types(func, annotation) + + t.buildrtyper(type_system="ootype").specialize() + if backendopt: + check_virtual_methods(ootype.ROOT) + backend_optimizations(t) + + main_graph = t.graphs[0] + tmpdir = py.path.local('.') + + return GenAVM1(tmpdir, t, None, exctrans) + +class AVM1Test(BaseRtypingTest, OORtypeMixin): + def __init__(self): + self._func = None + self._ann = None + self._genoo = None + self._harness = None + + def _compile(self, fn, args, ann=None, backendopt=True, auto_raise_exc=False, exctrans=False): + if ann is None: + ann = [lltype_to_annotation(typeOf(x)) for x in args] + self._genoo, self._harness = compile_function(fn, + "%s.%s" % (self.__class__.__name__, fn.func_name), + ann, + backendopt=backendopt, + auto_raise_exc=auto_raise_exc, + exctrans=exctrans) + self._func = fn + self._ann = ann + return self._harness + + def _skip_win(self, reason): + if platform.system() == 'Windows': + py.test.skip('Windows --> %s' % reason) + + def _skip_powerpc(self, reason): + if platform.processor() == 'powerpc': + py.test.skip('PowerPC --> %s' % reason) + + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + pass + + def _get_backendopt(self, backendopt): + if backendopt is None: + backendopt = getattr(self, 'backendopt', True) # enable it by default + return backendopt + + def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False): + backendopt = self._get_backendopt(backendopt) + harness = self._compile(fn, args, annotation, backendopt=backendopt, exctrans=exctrans) + harness.actions.call_function_constargs(fn.func_name, *args) + result = harness.do_test(True) + return result + + def interpret_raises(self, exception, fn, args, backendopt=None, exctrans=False): + import exceptions # needed by eval + backendopt = self._get_backendopt(backendopt) + try: + self.interpret(fn, args, backendopt=backendopt, exctrans=exctrans) + except ExceptionWrapper, ex: + assert issubclass(eval(ex.class_name), exception) + else: + assert False, 'function did raise no exception at all' + + float_eq = BaseRtypingTest.float_eq_approx + + def is_of_type(self, x, type_): + return True # we can't really test the type + + def ll_to_string(self, s): + return s + + def ll_to_unicode(self, s): + return s + + def ll_to_list(self, l): + return l + + def ll_to_tuple(self, t): + return t + + def class_name(self, value): + return value.class_name.split(".")[-1] + + def is_of_instance_type(self, val): + return isinstance(val, InstanceWrapper) + + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on gencli tests') Added: pypy/branch/avm/pypy/translator/avm2/test/runtest2.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/runtest2.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,213 @@ + +import py, re, subprocess +from pypy.translator.translator import TranslationContext +from pypy.translator.avm.test.browsertest import browsertest +# from pypy.translator.backendopt.all import backend_optimizations +# from pypy.translator.js.js import JS +# from pypy.translator.js.test.browsertest import jstest +# from pypy.translator.js import conftest +#from pypy.translator.js.log import log +from pypy.conftest import option +from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin +# from pypy.rlib.nonconst import NonConstant +# from pypy.rpython.ootypesystem import ootype + +from pypy.rpython.llinterp import LLException + +log = log.runtest + +port = 8080 + +class AVMException(LLException): + pass + +#def _CLI_is_on_path(): +# if py.path.local.sysfind('js') is None: #we recommend Spidermonkey +# return False +# return True + +class compile_function(object): + def __init__(self, function, annotations, stackless=False, view=False, html=None, is_interactive=False, root = None, run_browser = True, policy = None): + + self.html = html + self.is_interactive = is_interactive + t = TranslationContext() + + if policy is None: + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() + policy.allow_someobjects = False + + ann = t.buildannotator(policy=policy) + ann.build_types(function, annotations) + if view or option.view: + t.view() + t.buildrtyper(type_system="ootype").specialize() + + if view or option.view: + t.view() + #self.js = JS(t, [function, callback_function], stackless) + self.js = JS(t, function, stackless) + self.js.write_source() + if root is None and use_tg: + from pypy.translator.js.demo.jsdemo.controllers import Root + self.root = Root + else: + self.root = root + self.run_browser = run_browser + self.function_calls = [] + + def source(self): + return self.js.tmpfile.open().read() + + def _conv(self, v): + if isinstance(v, str): + return repr(v) + return str(v).lower() + + def __call__(self, *kwds): + return self.call(None, kwds) + + def call(self, entry_function, kwds): + args = ', '.join([self._conv(kw) for kw in kwds]) #lowerstr for (py)False->(js)false, etc. + + if entry_function is None: + entry_function = self.js.translator.graphs[0].name + else: + entry_function = self.js.translator.annotator.bookkeeper.getdesc(entry_function).cached_graph(None) + function_call = "%s(%s)" % (entry_function, args) + self.function_calls.append(function_call) + #if self.js.stackless: + # function_call = "slp_entry_point('%s')" % function_call + + if use_browsertest: + if not use_tg: + log("Used html: %r" % self.html) + output = browsertest(self.js.filename, function_call) + else: + global port + from pypy.translator.js.test.tgtest import run_tgtest + out = run_tgtest(self, tg_root = self.root, port=port, run_browser=self.run_browser).results + assert out[1] == 'undefined' or out[1] == "" + output = out[0] + port += 1 + return self.reinterpret(output) + else: +# cmd = 'echo "load(\'%s\'); print(%s)" | js 2>&1' % (self.js.filename, function_call) +# log(cmd) +# output = os.popen(cmd).read().strip() + js = subprocess.Popen(["js"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + input = "load(%r);\n" % self.js.filename.strpath + for call in self.function_calls[:-1]: + input += "%s;\n" % call + input += "print(\"'\" + %s + \"'\");\n" % self.function_calls[-1] + js.stdin.write(input) + stdout, stderr = js.communicate() + output = (stderr + stdout).strip() + for s in output.split('\n'): + log(s) + + m = re.match("'(.*)'", output, re.DOTALL) + if not m: + log("Error: %s" % output) + raise JSException(output) + return self.reinterpret(m.group(1)) + + def reinterpret(cls, s): + #while s.startswith(" "): + # s = s[1:] # :-) quite inneficient, but who cares + if s == 'false': + res = False + elif s == 'true': + res = True + elif s == 'undefined': + res = None + elif s == 'inf': + res = 1e300 * 1e300 + elif s == 'NaN': + res = (1e300 * 1e300) / (1e300 * 1e300) + elif s.startswith('[') or s.startswith('('): + l = s[1:-1].split(',') + res = [cls.reinterpret(i) for i in l] + else: + try: + res = float(s) + if float(int(res)) == res: + return int(res) + except ValueError: + res = str(s) + return res + reinterpret = classmethod(reinterpret) + +class JsTest(BaseRtypingTest, OORtypeMixin): + def _compile(self, _fn, args, policy=None): + argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] + func_name = _fn.func_name + if func_name == '': + func_name = 'func' + source = py.code.Source(""" + def %s(): + from pypy.rlib.nonconst import NonConstant + res = _fn(%s) + if isinstance(res, type(None)): + return None + else: + return str(res)""" + % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for + name, i in zip(argnames, args)]))) + exec source.compile() in locals() + return compile_function(locals()[func_name], [], policy=policy) + + def string_to_ll(self, s): + return s + + def interpret(self, fn, args, policy=None): + f = self._compile(fn, args, policy) + res = f(*args) + return res + + def interpret_raises(self, exception, fn, args): + #import exceptions # needed by eval + #try: + #import pdb; pdb.set_trace() + try: + res = self.interpret(fn, args) + except JSException, e: + s = e.args[0] + assert s.startswith('uncaught exception:') + assert re.search(exception.__name__, s) + else: + raise AssertionError("Did not raise, returned %s" % res) + #except ExceptionWrapper, ex: + # assert issubclass(eval(ex.class_name), exception) + #else: + # assert False, 'function did raise no exception at all' + + def ll_to_string(self, s): + return str(s) + + def ll_to_list(self, l): + return l + + def ll_unpack_tuple(self, t, length): + assert len(t) == length + return tuple(t) + + def class_name(self, value): + return value[:-8].split('.')[-1] + + def is_of_instance_type(self, val): + m = re.match("^<.* object>$", val) + return bool(m) + + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on genjs tests') + +def check_source_contains(compiled_function, pattern): + import re + + source = compiled_function.js.tmpfile.open().read() + return re.search(pattern, source) Added: pypy/branch/avm/pypy/translator/avm2/test/simpletest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/simpletest.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,17 @@ + +from pypy.translator.avm.swf import SwfData +from pypy.translator.avm.tags import SetBackgroundColor, DefineShape, End +from pypy.translator.avm.records import ShapeWithStyle, LineStyle, StraightEdgeRecord, StyleChangeRecord + +linestyle = LineStyle(3, 0x000000) + +shape = ShapeWithStyle() +shape.add_line_style(linestyle) +shape.add_shape_record(StyleChangeRecord(20, 20, linestyle)) +shape.add_shape_record(StraightEdgeRecord(100, 100)) + +swf = SwfData(400, 300) +swf.add_tag(SetBackgroundColor(0x333333)) +swf.add_tag(DefineShape(shape)) +swf.add_tag(End()) +print swf.serialize() Added: pypy/branch/avm/pypy/translator/avm2/test/test_backendopt.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_backendopt.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import compile_function +from pypy.translator.oosupport.test_template.backendopt import BaseTestOptimizedSwitch + +class TestOptimizedSwitch(BaseTestOptimizedSwitch): + def getcompiled(self, fn, annotation): + return compile_function(fn, annotation, backendopt=True) Added: pypy/branch/avm/pypy/translator/avm2/test/test_bool.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_bool.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rbool import BaseTestRbool + +class TestCliBool(CliTest, BaseTestRbool): + pass + Added: pypy/branch/avm/pypy/translator/avm2/test/test_builtin.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_builtin.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,29 @@ +import platform +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.builtin import BaseTestBuiltin, BaseTestTime + + +def skip_os(self): + py.test.skip("CLI doesn't support the os module, yet") + +def skip_win(): + if platform.system() == 'Windows': + py.test.skip("Doesn't work on Windows, yet") + +class TestCliBuiltin(CliTest, BaseTestBuiltin): + test_os_path_exists = skip_os + test_os_isdir = skip_os + test_os_dup_oo = skip_os + test_os_access = skip_os + + def test_builtin_math_frexp(self): + self._skip_powerpc("Mono math floating point problem") + BaseTestBuiltin.test_builtin_math_frexp(self) + + def test_debug_llinterpcall(self): + py.test.skip("so far, debug_llinterpcall is only used on lltypesystem") + + +class TestCliTime(CliTest, BaseTestTime): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_carbonpython.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_carbonpython.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,175 @@ +import py +py.test.skip("it passes usually, but fails on buildbot, no clue why") + +import os +import os.path +from pypy.tool import udir +from pypy.translator.cli.rte import Target +from pypy.translator.cli.carbonpython import DllDef, export, collect_entrypoints,\ + collect_class_entrypoints, compile_dll +from pypy.translator.cli.test.runtest import CliFunctionWrapper, CliTest + +TEMPLATE = """ +using System; +using System.Collections; +class CarbonPytonTest { + public static void Main() { + %s + } +} +""" + +class TestCarbonPython(CliTest): + + def _csharp(self, source, references=[], netmodules=[]): + tmpfile = udir.udir.join('tmp.cs') + tmpfile.write(TEMPLATE % source) + flags = ['/r:%s' % ref for ref in references] + flags += ['/addmodule:%s' % mod for mod in netmodules] + + class MyTarget(Target): + SOURCES = [str(tmpfile)] + FLAGS = flags + OUTPUT = 'tmp.exe' + SRC_DIR = str(udir.udir) + + func = CliFunctionWrapper(MyTarget.get()) + return func() + + def test_compilation(self): + res = self._csharp('Console.WriteLine(42);') + assert res == 42 + + def test_func_namespace(self): + def foo(x): + return x+1 + def bar(x): + return foo(x) + foo._namespace_ = 'MyNamespace.MyClass' + bar._namespace_ = 'MyClass' + res = self.interpret(bar, [41], backendopt=False) + assert res == 42 + + def test_simple_functions(self): + def foo(x): + return x+1 + def bar(x): + return x*2 + dll = DllDef('test', 'Test', [(foo, [int]), + (bar, [int])]) + dll.compile() + res = self._csharp('Console.WriteLine("{0}, {1}", Test.foo(42), Test.bar(42));', ['test']) + assert res == (43, 84) + + def test_export(self): + @export(int, float) + def foo(x, y): + pass + @export(int, float, namespace='test') + def bar(x, y): + pass + @export + def baz(): + pass + + assert foo._inputtypes_ == (int, float) + assert not hasattr(foo, '_namespace_') + assert bar._inputtypes_ == (int, float) + assert bar._namespace_ == 'test' + assert baz._inputtypes_ == () + + def test_collect_entrypoints(self): + @export(int, float) + def foo(x, y): + pass + def bar(x, y): + pass + mydict = dict(foo=foo, bar=bar, x=42) + entrypoints = collect_entrypoints(mydict) + assert entrypoints == [(foo, (int, float))] + + def test_collect_class_entrypoints(self): + class NotExported: + def __init__(self): + pass + + class MyClass: + @export + def __init__(self): + pass + @export(int) + def foo(self, x): + return x + + assert collect_class_entrypoints(NotExported) == [] + entrypoints = collect_class_entrypoints(MyClass) + assert len(entrypoints) == 2 + assert entrypoints[0][1] == () # __init__ inputtypes + assert entrypoints[1][1] == (MyClass, int) # foo inputtypes + + def test_compile_class(self): + py.test.skip('This test fails every other day. No clue why :-(') + class MyClass: + @export(int) + def __init__(self, x): + self.x = x + @export(int, int) + def add(self, y, z): + return self.x + y + z + MyClass.__module__ = 'Test' # put the class in the Test namespace + + entrypoints = collect_entrypoints({'MyClass': MyClass}) + dll = DllDef('test', 'Test', entrypoints) + dll.compile() + res = self._csharp(""" + Test.MyClass obj = new Test.MyClass(); + obj.__init__(39); + Console.WriteLine(obj.add(1, 2)); + """, ['test']) + assert res == 42 + + def test_export_cliclass(self): + py.test.skip('it fails every other day on builbot, no clue why') + from pypy.translator.cli.dotnet import CLR + + @export(CLR.System.Collections.ArrayList, int) + def getitem(obj, i): + return obj.get_Item(i) + + entrypoints = collect_entrypoints({'getitem': getitem}) + dll = DllDef('test', 'Test', entrypoints) + dll.compile() + res = self._csharp(""" + ArrayList obj = new ArrayList(); + obj.Add(42); + Console.WriteLine(Test.getitem(obj, 0)); + """, ['test']) + assert res == 42 + + def test_compile_dll(self): + py.test.skip('This test fails every other day. No clue why :-(') + cwd, _ = os.path.split(__file__) + mylib_py = os.path.join(cwd, 'mylib.py') + compile_dll(mylib_py, copy_dll=False) + res = self._csharp(""" + Console.WriteLine(mylib.sum(20, 22)); + """, ['mylib']) + assert res == 42 + + def test_compile_dll_alternative_name(self): + cwd, _ = os.path.split(__file__) + mylib_py = os.path.join(cwd, 'mylib.py') + compile_dll(mylib_py, 'mylibxxx.dll', copy_dll=False) + res = self._csharp(""" + Console.WriteLine(mylibxxx.sum(20, 22)); + """, ['mylibxxx']) + assert res == 42 + + def test_compile_netmodule(self): + def foo(x): + return x+1 + dll = DllDef('mymodule', 'Test', [(foo, [int])], isnetmodule=True) + dll.compile() + res = self._csharp('Console.WriteLine("{0}", Test.foo(41));', + netmodules = ['mymodule']) + Added: pypy/branch/avm/pypy/translator/avm2/test/test_cast.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_cast.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.cast import BaseTestCast + +class TestCast(BaseTestCast, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_class.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_class.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,11 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.class_ import BaseTestClass, BaseTestSpecialcase + +# ====> ../../oosupport/test_template/class_.py + +class TestCliClass(CliTest, BaseTestClass): + pass + +class TestCliSpecialCase(CliTest, BaseTestSpecialcase): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_constant.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_constant.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,5 @@ +from pypy.translator.oosupport.test_template.constant import BaseTestConstant +from pypy.translator.cli.test.runtest import CliTest + +class TestConstant(BaseTestConstant, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_cts.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_cts.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +from pypy.translator.cli import cts + +def test_primitive(): + void = cts.CliPrimitiveType('void') + assert str(void) == void.typename() == 'void' + assert void == cts.CliPrimitiveType('void') + +def test_class(): + Math = cts.CliClassType('mscorlib', 'System.Math') + assert str(Math) == Math.typename() == 'class [mscorlib]System.Math' + assert Math.classname() == '[mscorlib]System.Math' + assert Math == cts.CliClassType('mscorlib', 'System.Math') + +def test_generic(): + Dict = cts.CliGenericType('mscorlib', 'System.Dict', 2) + assert str(Dict) == Dict.typename() == 'class [mscorlib]System.Dict`2' + + int32 = cts.CliPrimitiveType('int32') + Math = cts.CliClassType('mscorlib', 'System.Math') + MyDict = Dict.specialize(int32, Math) + assert isinstance(MyDict, cts.CliSpecializedType) + classname = '[mscorlib]System.Dict`2' + assert str(MyDict) == MyDict.typename() == 'class ' + classname + assert MyDict.classname() == classname Added: pypy/branch/avm/pypy/translator/avm2/test/test_dict.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_dict.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,24 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +import pypy.translator.oosupport.test_template.dict as oodict + +class TestCliDict(CliTest, oodict.BaseTestDict): + def test_dict_of_dict(self): + py.test.skip("CLI doesn't support recursive dicts") + + def test_recursive(self): + py.test.skip("CLI doesn't support recursive dicts") + + def test_dict_of_void_special_case(self): + def fn(n): + d = {} + for i in xrange(n): + d[i] = None + return d[0] + assert self.interpret(fn, [2]) is None + +class TestCliEmptyDict(CliTest, oodict.BaseTestEmptyDict): + pass + +class TestCliConstantDict(CliTest, oodict.BaseTestConstantDict): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_dotnet.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_dotnet.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,711 @@ +import py +from pypy.annotation.annrpython import RPythonAnnotator +from pypy.annotation import model as annmodel +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.ootypesystem.ootype import meth, Meth, Char, Signed, Float, String,\ + ROOT, overload, Instance, new +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ + NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ + native_exc, new_array, init_array, typeof, eventhandler, clidowncast,\ + fieldinfo_for_const, classof, cast_record_to_object, cast_object_to_record + +System = CLR.System +ArrayList = CLR.System.Collections.ArrayList +OpCodes = System.Reflection.Emit.OpCodes +DynamicMethod = System.Reflection.Emit.DynamicMethod +Utils = CLR.pypy.runtime.Utils +FUNCTYPE = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) + +class TestDotnetAnnotation(object): + + def test_overloaded_meth_string(self): + C = Instance("test", ROOT, {}, + {'foo': overload(meth(Meth([Char], Signed)), + meth(Meth([String], Float)), + resolver=OverloadingResolver), + 'bar': overload(meth(Meth([Signed], Char)), + meth(Meth([Float], String)), + resolver=OverloadingResolver)}) + def fn1(): + return new(C).foo('a') + def fn2(): + return new(C).foo('aa') + def fn3(x): + return new(C).bar(x) + a = RPythonAnnotator() + assert isinstance(a.build_types(fn1, []), annmodel.SomeInteger) + assert isinstance(a.build_types(fn2, []), annmodel.SomeFloat) + assert isinstance(a.build_types(fn3, [int]), annmodel.SomeChar) + assert isinstance(a.build_types(fn3, [float]), annmodel.SomeString) + + def test_class(self): + def fn(): + return System.Math + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliClass) + assert s.const is System.Math + + def test_fullname(self): + def fn(): + return CLR.System.Math + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliClass) + assert s.const is System.Math + + def test_staticmeth(self): + def fn(): + return System.Math.Abs + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliStaticMethod) + assert s.cli_class is System.Math + assert s.meth_name == 'Abs' + + def test_staticmeth_call(self): + def fn1(): + return System.Math.Abs(42) + def fn2(): + return System.Math.Abs(42.5) + a = RPythonAnnotator() + assert type(a.build_types(fn1, [])) is annmodel.SomeInteger + assert type(a.build_types(fn2, [])) is annmodel.SomeFloat + + def test_new_instance(self): + def fn(): + return ArrayList() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert isinstance(s.ootype, NativeInstance) + assert s.ootype._name == '[mscorlib]System.Collections.ArrayList' + + def test_box(self): + def fn(): + return box(42) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + assert not s.can_be_None + + def test_box_can_be_None(self): + def fn(flag): + if flag: + return box(42) + else: + return box(None) + a = RPythonAnnotator() + s = a.build_types(fn, [bool]) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + assert s.can_be_None + + def test_unbox(self): + def fn(): + x = box(42) + return unbox(x, ootype.Signed) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_unbox_can_be_None(self): + class Foo: + pass + def fn(): + x = box(42) + return unbox(x, Foo) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInstance) + assert s.can_be_None + + def test_array(self): + def fn(): + x = ArrayList() + return x.ToArray() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._isArray + assert s.ootype._ELEMENT._name == '[mscorlib]System.Object' + + def test_array_getitem(self): + def fn(): + x = ArrayList().ToArray() + return x[0] + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + + def test_mix_None_and_instance(self): + def fn(x): + if x: + return None + else: + return box(42) + a = RPythonAnnotator() + s = a.build_types(fn, [bool]) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.can_be_None == True + + def test_box_instance(self): + class Foo: + pass + def fn(): + return box(Foo()) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + + def test_unbox_instance(self): + class Foo: + pass + def fn(): + boxed = box(Foo()) + return unbox(boxed, Foo) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInstance) + assert s.classdef.name.endswith('Foo') + + def test_can_be_None(self): + def fn(): + ttype = System.Type.GetType('foo') + return ttype.get_Namespace() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeString) + assert s.can_be_None + + +class TestDotnetRtyping(CliTest): + def _skip_pythonnet(self, msg): + pass + + def test_staticmeth_call(self): + def fn(x): + return System.Math.Abs(x) + assert self.interpret(fn, [-42]) == 42 + + def test_staticmeth_overload(self): + self._skip_pythonnet('Pythonnet bug!') + def fn(x, y): + return System.Math.Abs(x), System.Math.Abs(y) + res = self.interpret(fn, [-42, -42.5]) + item0, item1 = self.ll_to_tuple(res) + assert item0 == 42 + assert item1 == 42.5 + + def test_tostring(self): + StringBuilder = CLR.System.Text.StringBuilder + def fn(): + x = StringBuilder() + x.Append(box("foo")).Append(box("bar")) + return x.ToString() + res = self.ll_to_string(self.interpret(fn, [])) + assert res == 'foobar' + + def test_box(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + x.Add(box('Foo')) + return x.get_Count() + assert self.interpret(fn, []) == 2 + + def test_whitout_box(self): + def fn(): + x = ArrayList() + x.Add(42) # note we have forgot box() + py.test.raises(TypeError, self.interpret, fn, []) + + def test_unbox(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + return unbox(x.get_Item(0), ootype.Signed) + assert self.interpret(fn, []) == 42 + + def test_unbox_string(self): + def fn(): + x = ArrayList() + x.Add(box('foo')) + return unbox(x.get_Item(0), ootype.String) + assert self.interpret(fn, []) == 'foo' + + def test_box_method(self): + def fn(): + x = box(42) + t = x.GetType() + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'Int32' + + def test_box_object(self): + def fn(): + return box(System.Object()).ToString() + res = self.interpret(fn, []) + assert res == 'System.Object' + + def test_array(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + array = x.ToArray() + return unbox(array[0], ootype.Signed) + assert self.interpret(fn, []) == 42 + + def test_new_array(self): + def fn(): + x = new_array(System.Object, 2) + x[0] = box(42) + x[1] = box(43) + return unbox(x[0], ootype.Signed) + unbox(x[1], ootype.Signed) + assert self.interpret(fn, []) == 42+43 + + def test_init_array(self): + def fn(): + x = init_array(System.Object, box(42), box(43)) + return unbox(x[0], ootype.Signed) + unbox(x[1], ootype.Signed) + assert self.interpret(fn, []) == 42+43 + + def test_array_setitem_None(self): + py.test.skip('Mono bug :-(') + def fn(): + x = init_array(System.Object, box(42), box(43)) + x[0] = None + return x[0] + assert self.interpret(fn, []) is None + + def test_array_length(self): + def fn(): + x = init_array(System.Object, box(42), box(43)) + return len(x) + assert self.interpret(fn, []) == 2 + + def test_null(self): + def fn(): + return System.Object.Equals(None, None) + assert self.interpret(fn, []) == True + + def test_null_bound_method(self): + def fn(): + x = ArrayList() + x.Add(None) + return x.get_Item(0) + assert self.interpret(fn, []) is None + + def test_native_exception_precise(self): + ArgumentOutOfRangeException = NativeException(CLR.System.ArgumentOutOfRangeException) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return False + except ArgumentOutOfRangeException: + return True + assert self.interpret(fn, []) == True + + def test_native_exception_superclass(self): + SystemException = NativeException(CLR.System.Exception) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return False + except SystemException: + return True + assert self.interpret(fn, []) == True + + def test_native_exception_object(self): + SystemException = NativeException(CLR.System.Exception) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return "Impossible!" + except SystemException, e: + ex = native_exc(e) + return ex.get_Message() + res = self.ll_to_string(self.interpret(fn, [])) + assert res.startswith("Index is less than 0") + + def test_native_exception_invoke(self): + TargetInvocationException = NativeException(CLR.System.Reflection.TargetInvocationException) + def fn(): + x = ArrayList() + t = x.GetType() + meth = t.GetMethod('get_Item') + args = init_array(System.Object, box(0)) + try: + meth.Invoke(x, args) + return "Impossible!" + except TargetInvocationException, e: + inner = native_exc(e).get_InnerException() + message = str(inner.get_Message()) + return message + res = self.ll_to_string(self.interpret(fn, [])) + assert res.startswith("Index is less than 0") + + def test_typeof(self): + def fn(): + x = box(42) + return x.GetType() == typeof(System.Int32) + res = self.interpret(fn, []) + assert res is True + + def test_typeof_pypylib(self): + DelegateType = CLR.pypy.test.DelegateType_int__int_2 + def fn(): + return typeof(DelegateType) is not None + res = self.interpret(fn, []) + assert res is True + + def test_typeof_functype(self): + # this test is overridden in TestPythonnet + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + + def test_clidowncast(self): + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(c, ArrayList) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + + def test_clidowncast_lltype(self): + ARRAY_LIST = ArrayList._INSTANCE + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(c, ARRAY_LIST) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + + def test_mix_None_and_instance(self): + def g(x): + return x + def fn(flag): + if flag: + x = None + else: + x = box(42) + return g(x) + res = self.interpret(fn, [1]) + assert res is None + + def test_box_unbox_instance(self): + class Foo: + pass + def fn(): + obj = Foo() + b_obj = box(obj) + obj2 = unbox(b_obj, Foo) + return obj is obj2 + res = self.interpret(fn, []) + assert res is True + + def test_unbox_instance_fail(self): + class Foo: + pass + def fn(): + b_obj = box(42) + return unbox(b_obj, Foo) + res = self.interpret(fn, []) + assert res is None + + def test_box_unbox_ooinstance(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + + def test_box_unbox_ooinstance_fail(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + b_obj = System.Object() + a2 = unbox(b_obj, A) + return a2 + res = self.interpret(fn, [True]) + assert res is None + + def test_box_unbox_oorecord(self): + A = ootype.Record({'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + + def test_instance_wrapping(self): + class Foo: + pass + def fn(): + obj = Foo() + x = ArrayList() + x.Add(box(obj)) + obj2 = unbox(x.get_Item(0), Foo) + return obj is obj2 + res = self.interpret(fn, []) + assert res is True + + def test_compare_string_None(self): + from pypy.rlib.nonconst import NonConstant + def null(): + if NonConstant(True): + return None + else: + return "" + + def fn(): + ttype = System.Type.GetType('Consts, mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089') + namespace = ttype.get_Namespace() + if namespace is not None: + return False + else: + return True + res = self.interpret(fn, [], backendopt=False) + assert res is True + + def test_delegate(self): + class Foo: + def __init__(self): + self.x = 0 + def m(self, sender, args): + self.x = 42 + + def fn(flag): + f = Foo() + if flag: + f.m(None, None) + delegate = eventhandler(f.m) + delegate.Invoke(None, None) + return f.x + res = self.interpret(fn, [False]) + assert res == 42 + + def test_static_fields(self): + DummyClass = CLR.pypy.test.DummyClass + def fn(): + obj = System.Object() + DummyClass.myfield = obj + return DummyClass.myfield is obj + res = self.interpret(fn, []) + assert res + + def test_pypylib(self): + def fn(): + return CLR.pypy.runtime.Utils.OOString(42, -1) + res = self.interpret(fn, []) + assert self.ll_to_string(res) == '42' + + def test_call_delegate(self): + def build_fn(): + tInt = typeof(System.Int32) + args = init_array(System.Type, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_0) + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE)) + return myfunc + + def fn(x, y): + myfunc = unbox(build_fn(), FUNCTYPE) + a = myfunc(x, y) + mytuple = (x, y) + b = myfunc(*mytuple) + return a+b + res = self.interpret(fn, [10, 11]) + assert res == 42 + + def test_bound_delegate(self): + def build_fn(): + tObjArray = System.Type.GetType("System.Object[]") + tInt = typeof(System.Int32) + args = init_array(System.Type, tObjArray, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Ldarg_2) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + array = new_array(System.Object, 0) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE), array) + return myfunc + + def fn(): + myfunc = unbox(build_fn(), FUNCTYPE) + return myfunc(30, 12) + res = self.interpret(fn, []) + assert res == 42 + + def test_valuetype_field(self): + class Foo: + def __init__(self, x): + self.x = x + + def fn(): + f = Foo(OpCodes.Add) + return f + self.interpret(fn, []) + + def test_fieldinfo_for_const(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + const.xx = 42 + def fn(): + fieldinfo = fieldinfo_for_const(const) + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 + + def test_fieldinfo_for_const_pbc(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + fieldinfo = fieldinfo_for_const(const) + def fn(): + const.xx = 42 + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 + + def test_classof(self): + int32_class = classof(System.Int32) + def fn(): + int32_obj = box(int32_class) + int32_type = clidowncast(int32_obj, System.Type) + return int32_type.get_Name() + assert self.interpret(fn, []) == 'Int32' + + def test_classof_compare(self): + int32_a = classof(System.Int32) + int32_b = classof(System.Int32) + def fn(): + return int32_a is int32_b + assert self.interpret(fn, []) + + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + + def test_mix_classof(self): + a = classof(System.Int32) + b = classof(FUNCTYPE) + def fn(flag): + if flag: + x = a + else: + x = b + return clidowncast(box(x), System.Type).get_Name() + res = self.interpret(fn, [True]) + assert res == 'Int32' + + def test_cast_record(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + def fn(flag): + if flag: + obj = cast_record_to_object(record) + else: + obj = System.Object() + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, [True]) + assert res + + def test_cast_record_pbc(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + record.x = 42 + obj = cast_record_to_object(record) + def fn(): + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, []) + assert res + + def test_cast_record_mix_object(self): + T = ootype.Record({'x': ootype.Signed}) + NULL = ootype.null(System.Object._INSTANCE) + record = cast_record_to_object(ootype.new(T)) + assert record != NULL + assert NULL != record + + +class TestPythonnet(TestDotnetRtyping): + # don't interpreter functions but execute them directly through pythonnet + def interpret(self, f, args, backendopt='ignored'): + return f(*args) + + def _skip_pythonnet(self, msg): + py.test.skip(msg) + + def test_whitout_box(self): + pass # it makes sense only during translation + + def test_typeof_functype(self): + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' + + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' + + def test_fieldinfo_for_const(self): + pass # it makes sense only during translation + + def test_fieldinfo_for_const_pbc(self): + pass # it makes sense only during translation Added: pypy/branch/avm/pypy/translator/avm2/test/test_exception.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_exception.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,22 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.exception import BaseTestException + +class TestCliException(CliTest, BaseTestException): + use_exception_transformer = False + backendopt = False + + def interpret(self, *args, **kwds): + kwds['exctrans'] = self.use_exception_transformer + return CliTest.interpret(self, *args, **kwds) + + def test_raise_and_catch_other(self): + pass + + def test_raise_prebuilt_and_catch_other(self): + pass + + +class TestCliExceptionTransformer(TestCliException): + use_exception_transformer = True + backendopt = False Added: pypy/branch/avm/pypy/translator/avm2/test/test_float.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_float.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,26 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rfloat import BaseTestRfloat + +class TestCliFloat(CliTest, BaseTestRfloat): + + inf = 'Infinity' + minus_inf = '-Infinity' + nan = 'NaN' + + def test_parse_float(self): + ex = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] + def fn(i): + s = ex[i] + try: + return float(s) + except ValueError: + return -999.0 + + for i in range(len(ex)): + expected = fn(i) + res = self.interpret(fn, [i]) + assert res == expected + + def test_r_singlefloat(self): + py.test.skip("not implemented: single-precision floats") Added: pypy/branch/avm/pypy/translator/avm2/test/test_harness.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_harness.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,14 @@ +import autopath + +from pypy.translator.avm.test import browsertest, harness as h +from pypy.translator.avm import avm1 as a + +def test_harness(): + harness = h.TestHarness("harness") + harness.start_test("harness") + harness.actions.add_action(a.ActionPush(True)) + harness.finish_test(True) + harness.do_test() + +if __name__ == "__main__": + test_harness() Added: pypy/branch/avm/pypy/translator/avm2/test/test_int.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_int.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,50 @@ +import autopath +import py +from pypy.translator.avm1.test.runtest import AVM1Test +from pypy.rpython.test.test_rint import BaseTestRint +from pypy.rlib.rarithmetic import r_longlong + +class TestAVM1Int(AVM1Test, BaseTestRint): + def test_char_constant(self): + def dummyfn(i): + return chr(i) + _ = self.interpret(dummyfn, [ord(' ')]) + assert _ == ' ' + _ = self.interpret(dummyfn, [ord('a')]) + assert _ == 'a' + + def test_rarithmetic(self): + pass # it doesn't make sense here + + div_mod_iteration_count = 20 + + def test_div_mod(self): + import random + + for inttype in (int, r_longlong): + + # def d(x, y): + # return x/y + + # for i in range(self.div_mod_iteration_count): + # x = inttype(random.randint(-100000, 100000)) + # y = inttype(random.randint(-100000, 100000)) + # if not y: continue + # res = self.interpret(d, [x, y]) + # print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", d(x, y) + # assert res == d(x, y) + + def m(x, y): + return x%y + + for i in range(self.div_mod_iteration_count): + x = inttype(random.randint(-100000, 100000)) + y = inttype(random.randint(-100000, 100000)) + if not y: continue + res = self.interpret(m, [x, y]) + print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", m(x, y) + assert res == m(x, y) + + +if __name__=="__main__": + TestAVM1Int().test_div_mod() Added: pypy/branch/avm/pypy/translator/avm2/test/test_list.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_list.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,27 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rlist import BaseTestRlist +from pypy.rlib.rarithmetic import r_uint + +class TestCliList(CliTest, BaseTestRlist): + def test_recursive(self): + py.test.skip("CLI doesn't support recursive lists") + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_list_unsigned(self): + def fn(x): + lst = [r_uint(0), r_uint(1)] + lst[0] = r_uint(x) + return lst[0] + res = self.interpret(fn, [42]) + assert res == 42 + + def test_list_bool(self): + def fn(x): + lst = [True, False] + lst[0] = x + return lst[0] + res = self.interpret(fn, [False]) + assert res == False Added: pypy/branch/avm/pypy/translator/avm2/test/test_objectmodel.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_objectmodel.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,10 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.objectmodel import \ + BaseTestObjectModel + +def skip_r_dict(self): + py.test.skip('r_dict support is still incomplete') + +class TestCliObjectModel(CliTest, BaseTestObjectModel): + test_rtype_r_dict_bm = skip_r_dict Added: pypy/branch/avm/pypy/translator/avm2/test/test_oo.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_oo.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,125 @@ +from pypy.translator.cli.test.runtest import CliTest + +class MyClass: + INCREMENT = 1 + + def __init__(self, x, y): + self.x = x + self.y = y + + def compute(self): + return self.x + self.y + + def compute_and_multiply(self, factor): + return self.compute() * factor + + def static_meth(x, y): + return x*y + static_meth = staticmethod(static_meth) + + def class_attribute(self): + return self.x + self.INCREMENT + +class MyDerivedClass(MyClass): + INCREMENT = 2 + + def __init__(self, x, y): + MyClass.__init__(self, x+12, y+34) + + def compute(self): + return self.x - self.y + + +# helper functions +def call_method(obj): + return obj.compute() + +def init_and_compute(cls, x, y): + return cls(x, y).compute() + +def nonnull_helper(lst): + if lst is None: + return 1 + else: + return 2 + + +class TestOO(CliTest): + def test_indirect_call(self): + def f(): + return 1 + def g(): + return 2 + def fn(flag): + if flag: + x = f + else: + x = g + return x() + assert self.interpret(fn, [True]) == 1 + assert self.interpret(fn, [False]) == 2 + + def test_indirect_call_arguments(self): + def f(x): + return x+1 + def g(x): + return x+2 + def fn(flag, n): + if flag: + x = f + else: + x = g + return x(n) + assert self.interpret(fn, [True, 42]) == 43 + + + def test_compute(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_compute_multiply(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_inheritance(self): + def fn(x, y): + obj = MyDerivedClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_liskov(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return call_method(base) + call_method(derived) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_static_method(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.static_meth(x,y) + derived.static_meth(x, y)\ + + MyClass.static_meth(x, y) + MyDerivedClass.static_meth(x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_class_attribute(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.class_attribute() + derived.class_attribute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_runtimenew(self): + def fn(x, y): + return init_and_compute(MyClass, x, y) + init_and_compute(MyDerivedClass, x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_nonnull(self): + def fn(x, y): + return nonnull_helper([]) + nonnull_helper(None) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + Added: pypy/branch/avm/pypy/translator/avm2/test/test_op.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_op.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,19 @@ +import sys +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.operations import BaseTestOperations +from pypy.rlib.rarithmetic import ovfcheck + +# ====> ../../oosupport/test_template/operations.py + +class TestOperations(CliTest, BaseTestOperations): + def test_int_div_overflow(self): + import py + py.test.skip('fixme!') + def fn(x, y): + try: + return x//y + except OverflowError: + return 42 + res = self.interpret(fn, [-sys.maxint-1, -1]) + assert res == 42 + Added: pypy/branch/avm/pypy/translator/avm2/test/test_overflow.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_overflow.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.overflow import BaseTestOverflow + +class TestOverflow(BaseTestOverflow, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_pbc.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_pbc.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rpbc import BaseTestRPBC + +class TestCliPBC(CliTest, BaseTestRPBC): + pass Added: pypy/branch/avm/pypy/translator/avm2/test/test_primitive.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_primitive.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,13 @@ +import os +import time + +from pypy.translator.cli.test.runtest import CliTest + +class TestPrimitive(CliTest): + + def test_time_time(self): + def fn(): + return time.time() + t1 = self.interpret(fn, []) + t2 = self.interpret(fn, []) + assert t1 <= t2 Added: pypy/branch/avm/pypy/translator/avm2/test/test_query.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_query.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,39 @@ +import py +from pypy.translator.cli import query +from pypy.translator.cli.dotnet import CLR, CliNamespace + +def setup_module(module): + from pypy.translator.cli.query import load_assembly, mscorlib + load_assembly(mscorlib) + +def test_load_assembly(): + query.load_assembly(query.mscorlib) + assert 'System.Math' in query.Types + assert 'System.Collections.ArrayList' in query.Types + +def test_namespaces(): + assert CLR.System._name == 'System' + assert CLR.System.Collections._name == 'System.Collections' + py.test.raises(AttributeError, getattr, CLR, 'Foo') + py.test.raises(AttributeError, getattr, CLR.System, 'Foo') + +def test_CLR_getattr(): + System = CLR.System + assert isinstance(System, CliNamespace) + assert System._name == 'System' + assert hasattr(CLR, 'System') + +def test_static_fields(): + desc = query.get_class_desc('System.Reflection.Emit.OpCodes') + assert ('Add', 'System.Reflection.Emit.OpCode') in desc.StaticFields + +def test_System_Object(): + Object = CLR.System.Object + assert Object._name == '[mscorlib]System.Object' + assert 'Equals' in Object._static_methods + assert 'ToString' in Object._INSTANCE._methods + +def test_array(): + cls = query.get_cli_class('System.Object[]') + assert cls._INSTANCE._isArray + assert cls._INSTANCE._ELEMENT is CLR.System.Object._INSTANCE Added: pypy/branch/avm/pypy/translator/avm2/test/test_range.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_range.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rrange import BaseTestRrange + +class TestCliRange(CliTest, BaseTestRrange): + def test_rlist_range(self): + pass # it doesn't make sense here Added: pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,18 @@ +import autopath +import py +from pypy.translator.oosupport.test_template.runtest import BaseTestRunTest +from pypy.translator.avm1.test.runtest import AVM1Test + +class TestRunTest(BaseTestRunTest, AVM1Test): + + def test_auto_raise_exc(self): + def fn(): + raise ValueError + f = self._compile(fn, [], auto_raise_exc=True) + py.test.raises(ValueError, f) + + def test_big_arglist(self): + def fn(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9): + return a0 + res = self.interpret(fn, [42]*10) + assert res == 42 Added: pypy/branch/avm/pypy/translator/avm2/test/test_snippet.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_snippet.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,47 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.snippets import BaseTestSnippets + +class TestSnippets(BaseTestSnippets, CliTest): + def test_llshl(self): + py.test.skip('llshl currently broken on CLI') + + def test_link_SSA(self): + def fn(): + lst = [42, 43, 44] + for i in range(len(lst)): + item = lst[i] + if i < 10: + lst[i] = item+10 + return lst + res = self.ll_to_list(self.interpret(fn, [])) + assert res == [52, 53, 54] + + def test_mangle(self): + class Foo: + def le(self): + return 42 + + def fn(): + f = Foo() + return f.le() + res = self.interpret(fn, [], backendopt=False) + + def test_link_vars_overlapping(self): + from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift + def fn(maxofs): + lastofs = 0 + ofs = 1 + while ofs < maxofs: + lastofs = ofs + try: + ofs = ovfcheck_lshift(ofs, 1) + except OverflowError: + ofs = maxofs + else: + ofs = ofs + 1 + return lastofs + res = self.interpret(fn, [64]) + expected = fn(64) + assert res == expected + Added: pypy/branch/avm/pypy/translator/avm2/test/test_streamio.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_streamio.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,28 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rlib.test.test_streamio import BaseTestBufferingInputStreamTests,\ + BaseTestBufferingOutputStream, BaseTestLineBufferingOutputStream,\ + BaseTestCRLFFilter, BaseTestBufferingInputOutputStreamTests,\ + BaseTestTextInputFilter, BaseTestTextOutputFilter + +class TestBufferingInputStreamTests(CliTest, BaseTestBufferingInputStreamTests): + pass + +class TestBufferingOutputStream(CliTest, BaseTestBufferingOutputStream): + pass + +class TestLineBufferingOutputStream(CliTest, BaseTestLineBufferingOutputStream): + pass + +class TestCRLFFilter(CliTest, BaseTestCRLFFilter): + pass + +class TestBufferingInputOutputStreamTests(CliTest, BaseTestBufferingInputOutputStreamTests): + pass + +class TestTextInputFilter(CliTest, BaseTestTextInputFilter): + pass + +class TestTextOutputFilter(CliTest, BaseTestTextOutputFilter): + pass + Added: pypy/branch/avm/pypy/translator/avm2/test/test_string.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_string.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,40 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +import pypy.translator.oosupport.test_template.string as oostring + +class TestCliString(CliTest, oostring.BaseTestString): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("CLI interpret doesn't support unicode for input arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_upper(self): + py.test.skip("CLI doens't support backquotes inside string literals") + test_lower = test_upper + + def test_hlstr(self): + py.test.skip("CLI tests can't have string as input arguments") + + test_inplace_add = test_hlstr + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_compare(self): + strings = ['aa', 'ZZ'] + def fn(i, j): + return strings[i] < strings[j] + assert self.interpret(fn, [0, 1], backendopt=False) == fn(0, 1) + + def test_literal_length(self): + strings = ['aa', 'a\x01', 'a\x00'] + def fn(): + for s in strings: + assert len(s) == 2 + self.interpret(fn, [], backendopt=False) Added: pypy/branch/avm/pypy/translator/avm2/test/test_tuple.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_tuple.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,12 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rtuple import BaseTestRtuple + +class TestCliTuple(CliTest, BaseTestRtuple): + def test_builtin_records(self): + def fn(x, y): + return x, y + res = self.interpret(fn, [1.0, 1]) + assert res.item0 == 1.0 and res.item1 == 1 + res = self.interpret(fn, [1.0, 1.0]) + assert res.item0 == 1.0 and res.item1 == 1.0 Added: pypy/branch/avm/pypy/translator/avm2/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_unicode.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,23 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_runicode import BaseTestRUnicode + +# ====> ../../../rpython/test/test_runicode.py + +class TestCliUnicode(CliTest, BaseTestRUnicode): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("CLI interpret doesn't support unicode for input arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_inplace_add(self): + py.test.skip("CLI tests can't have string as input arguments") Added: pypy/branch/avm/pypy/translator/avm2/test/test_weakref.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/test/test_weakref.py Wed Nov 4 00:51:35 2009 @@ -0,0 +1,6 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rweakref import BaseTestRweakref + +class TestCliWeakRef(CliTest, BaseTestRweakref): + pass From fijal at codespeak.net Wed Nov 4 10:34:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:34:33 +0100 (CET) Subject: [pypy-svn] r68953 - pypy/trunk/pypy/doc Message-ID: <20091104093433.02B9F168437@codespeak.net> Author: fijal Date: Wed Nov 4 10:34:32 2009 New Revision: 68953 Modified: pypy/trunk/pypy/doc/faq.txt Log: (MostAwesomeDude) A faq entry about SELinux & building pypy Modified: pypy/trunk/pypy/doc/faq.txt ============================================================================== --- pypy/trunk/pypy/doc/faq.txt (original) +++ pypy/trunk/pypy/doc/faq.txt Wed Nov 4 10:34:32 2009 @@ -205,6 +205,20 @@ .. _`py.cleanup`: http://codespeak.net/py/current/doc/bin.html +------------------------------------------------------------- +OSError: ... cannot restore segment prot after reloc... Help? +------------------------------------------------------------- + +On Linux, if SELinux is enabled, you may get errors along the lines of +"OSError: externmod.so: cannot restore segment prot after reloc: Permission +denied." This is caused by a slight abuse of the C compiler during +configuration, and can be disabled by running the following command with root +privileges:: + + # setenforce 0 + +This will disable SELinux's protection and allow PyPy to configure correctly. +Be sure to enable it again if you need it! PyPy translation tool chain ======================================================================== From fijal at codespeak.net Wed Nov 4 10:50:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:50:56 +0100 (CET) Subject: [pypy-svn] r68954 - pypy/branch/c-backend-fun Message-ID: <20091104095056.553B7168437@codespeak.net> Author: fijal Date: Wed Nov 4 10:50:56 2009 New Revision: 68954 Removed: pypy/branch/c-backend-fun/ Log: Kill dead branch From fijal at codespeak.net Wed Nov 4 10:51:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:51:42 +0100 (CET) Subject: [pypy-svn] r68955 - pypy/branch/cbuild-llvm Message-ID: <20091104095142.EF307168438@codespeak.net> Author: fijal Date: Wed Nov 4 10:51:42 2009 New Revision: 68955 Removed: pypy/branch/cbuild-llvm/ Log: this branch is at least seriously outdated, kill From fijal at codespeak.net Wed Nov 4 10:52:48 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:52:48 +0100 (CET) Subject: [pypy-svn] r68956 - pypy/branch/cross-compilation Message-ID: <20091104095248.4466A168438@codespeak.net> Author: fijal Date: Wed Nov 4 10:52:47 2009 New Revision: 68956 Removed: pypy/branch/cross-compilation/ Log: This branch was merged From fijal at codespeak.net Wed Nov 4 10:53:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:53:07 +0100 (CET) Subject: [pypy-svn] r68957 - pypy/branch/ctypes-stable Message-ID: <20091104095307.2851B168438@codespeak.net> Author: fijal Date: Wed Nov 4 10:53:06 2009 New Revision: 68957 Removed: pypy/branch/ctypes-stable/ Log: We don't need any more a stable version of ctypes From fijal at codespeak.net Wed Nov 4 10:53:40 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:53:40 +0100 (CET) Subject: [pypy-svn] r68958 - pypy/branch/faster-ctypes Message-ID: <20091104095340.0EB97168439@codespeak.net> Author: fijal Date: Wed Nov 4 10:53:39 2009 New Revision: 68958 Removed: pypy/branch/faster-ctypes/ Log: this attempt should be started from scratch From fijal at codespeak.net Wed Nov 4 10:54:02 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:54:02 +0100 (CET) Subject: [pypy-svn] r68959 - pypy/branch/no-trace-if-frame-escapes Message-ID: <20091104095402.067BF168009@codespeak.net> Author: fijal Date: Wed Nov 4 10:54:02 2009 New Revision: 68959 Removed: pypy/branch/no-trace-if-frame-escapes/ Log: Kill this branch (will branch again) From arigo at codespeak.net Wed Nov 4 10:55:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 10:55:33 +0100 (CET) Subject: [pypy-svn] r68960 - in pypy/branch/gc-jit-hack/pypy/rpython/memory/gc: . test Message-ID: <20091104095533.8F8D5168438@codespeak.net> Author: arigo Date: Wed Nov 4 10:55:33 2009 New Revision: 68960 Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py Log: Implement resizing the nursery at runtime. Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Wed Nov 4 10:55:33 2009 @@ -102,6 +102,16 @@ self.nursery_free = NULL def set_nursery_size(self, newsize): + # Invariants: nursery <= nursery_free <= nursery_top + # nursery_size == nursery_top - nursery + # + # In the usual case: nursery_size == allocated_nursery_size + # == next_nursery_size + # + # In the presence of resize_nursery(), however, the nursery currently + # in use may be only a fraction of the larger allocated nursery. + # next_nursery_size <= nursery_size <= allocated_nursery_size + # debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size @@ -111,6 +121,9 @@ # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust self.nursery_size = newsize + self.allocated_nursery_size = newsize + self.next_nursery_size = newsize + self.minimal_setting_for_nursery_size = newsize self.largest_young_fixedsize = self.get_young_fixedsize(newsize) self.largest_young_var_basesize = self.get_young_var_basesize(newsize) scale = 0 @@ -321,42 +334,57 @@ # Implementation of nursery-only collections def collect_nursery(self): - if self.nursery_size > self.top_of_space - self.free: + while self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) - ll_assert(self.nursery_size <= self.top_of_space - self.free, - "obtain_free_space failed to do its job") + # in rare cases it is possible that self.nursery_size changed + # when we ran the finalizers... that's why we loop. if self.nursery: debug_start("gc-minor") - debug_print("--- minor collect ---") - debug_print("nursery:", self.nursery, "to", self.nursery_top) - # a nursery-only collection - scan = beginning = self.free - self.collect_oldrefs_to_nursery() - self.collect_roots_in_nursery() - scan = self.scan_objects_just_copied_out_of_nursery(scan) - # at this point, all static and old objects have got their - # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery - if self.young_objects_with_weakrefs.non_empty(): - self.invalidate_young_weakrefs() - if self.young_objects_with_id.length() > 0: - self.update_young_objects_with_id() + self.minor_collection() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - debug_print("survived (fraction of the size):", - float(scan - beginning) / self.nursery_size) debug_stop("gc-minor") #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either # just above or by some previous non-nursery-based allocation. # Grab a piece of the current space for the nursery. + self.allocated_nursery_size = self.next_nursery_size self.nursery = self.free - self.nursery_top = self.nursery + self.nursery_size - self.free = self.nursery_top + self.free += self.allocated_nursery_size + self.nursery_size = self.next_nursery_size + self.nursery_top = self.nursery + self.nursery_size self.nursery_free = self.nursery return self.nursery_free + def drop_nursery(self): + while self.nursery: + if self.nursery_size <= self.top_of_space - self.free: + self.minor_collection() + # mark as free, but don't bother filling it with zeroes + llarena.arena_reset(self.nursery, self.nursery_size, 0) + self.reset_nursery() + return + self.obtain_free_space(self.nursery_size) + + def minor_collection(self): + debug_print("--- minor collect ---") + debug_print("nursery:", self.nursery, "to", self.nursery_top) + # a nursery-only collection + scan = beginning = self.free + self.collect_oldrefs_to_nursery() + self.collect_roots_in_nursery() + scan = self.scan_objects_just_copied_out_of_nursery(scan) + # at this point, all static and old objects have got their + # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery + if self.young_objects_with_weakrefs.non_empty(): + self.invalidate_young_weakrefs() + if self.young_objects_with_id.length() > 0: + self.update_young_objects_with_id() + debug_print("survived (fraction of the size):", + float(scan - beginning) / self.nursery_size) + # NB. we can use self.copy() to move objects out of the nursery, # but only if the object was really in the nursery. @@ -564,6 +592,48 @@ else: SemiSpaceGC.debug_check_can_copy(self, obj) + # ---------- resizing the nursery at runtime ---------- + + def resize_nursery(self, newsize): + debug_start("gc-resize-nursery") + if newsize < self.minimal_setting_for_nursery_size: + newsize = self.minimal_setting_for_nursery_size + if not self.nursery or newsize > self.allocated_nursery_size: + self.drop_nursery() + self.nursery_size = newsize + self.allocated_nursery_size = newsize + self.next_nursery_size = newsize + else: + # situation of the nursery: [###########.......|...........] + # \-- nursery_size --/ + # \--- allocated_nursery_size ---/ + # + # we have to change in-place the nursery_size and nursery_top. + # + self.next_nursery_size = newsize + if self.nursery_size < self.next_nursery_size: + # restore this invariant, i.e. grow nursery_top + self.nursery_size = self.next_nursery_size + self.nursery_top = self.nursery + self.nursery_size + else: + # this is the case where nursery_size is larger than + # next_nursery_size. If it is *much* larger, then it might + # still have room for more than newsize bytes (in addition + # to the already-allocated objects). In that case limit + # that extra space to newsize bytes. + currently_free = self.nursery_top - self.nursery_free + if currently_free > newsize: + self.nursery_top = self.nursery_free + newsize + self.nursery_size = self.nursery_top - self.nursery + ll_assert(self.next_nursery_size <= self.nursery_size, + "bogus invariant in resize_nursery (1)") + ll_assert(self.nursery + self.nursery_size == self.nursery_top, + "bogus invariant in resize_nursery (2)") + ll_assert(self.nursery_free <= self.nursery_top, + "bogus invariant in resize_nursery (3)") + # + debug_stop("gc-resize-nursery") + # ____________________________________________________________ import os Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py Wed Nov 4 10:55:33 2009 @@ -271,6 +271,8 @@ class TestGenerationGC(TestSemiSpaceGC): from pypy.rpython.memory.gc.generation import GenerationGC as GCClass + GC_PARAMS = {'nursery_size': 128} + def test_collect_gen(self): gc = self.gc old_semispace_collect = gc.semispace_collect @@ -313,6 +315,107 @@ assert s0.next.x == 1 + def test_resize_nursery(self): + if self.GC_PARAMS['nursery_size'] != 128: + py.test.skip("test only for nursery_size == 128") + + gc = self.gc + old_minor_collection = gc.minor_collection + counters = [0, 0] + + def minor_collection(): + counters[1] += 1 + return old_minor_collection() + gc.minor_collection = minor_collection + + def alloc(): + counters[0] += 1 + self.stackroots.append(self.malloc(S)) + + # for a default nursery size of 128, and a space size of 4096 + while counters[1] == 0: + alloc() + assert counters[1] == 1 + assert counters[0] in (5, 9) + # nursery contains 1 object (out of a maximum of 4/8) + + # almost fill the nursery again + for i in range(counters[0]-3): + alloc() + assert counters[1] == 1 + assert counters[0] in (7, 15) + # nursery contains 3/7 objects + + # now make the nursery grow to 1024 + gc.resize_nursery(1024) + assert counters[1] == 2 + assert counters[0] in (7, 15) + counters[0] = 0 # empty + + # check its size + while counters[1] == 2: + alloc() + assert counters[1] == 3 + assert counters[0] in (33, 65) + + # almost fill it again + for i in range(counters[0]-3): + alloc() + assert counters[1] == 3 + assert counters[0] in (63, 127) + + # now mark it as "must shrink back" + gc.resize_nursery(0) + assert counters[1] == 3 + assert counters[0] in (63, 127) + + # check that it will shrink back after two allocations + alloc() + assert counters[1] == 3 + alloc() + assert counters[1] == 4 + counters[0] = 1 # only the most recently allocated object + + # check its size + while counters[1] == 4: + alloc() + assert counters[1] == 5 + assert counters[0] in (5, 9) + + # flush the nursery + gc.collect() + assert counters[1] == 5 + assert counters[0] in (5, 9) + counters[0] = 0 # nursery not allocated at all + + # resize from a fresh nursery: should not do another collect + gc.resize_nursery(512) + assert counters[1] == 5 + + # fill it partially only + for i in range(20): + alloc() + assert counters[1] == 5 + assert counters[0] == 20 + + # now mark it as "must shrink back", + # which should still leave room for 4/8 objects + gc.resize_nursery(0) + assert counters[1] == 5 + assert counters[0] == 20 + + # check its size + while counters[1] == 5: + alloc() + assert counters[1] == 6 + assert counters[0] in (25, 29) + + # it should now have shrunk to the standard size of 4/8 objects + while counters[1] == 6: + alloc() + assert counters[1] == 7 + assert counters[0] in (29, 37) + class TestHybridGC(TestGenerationGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass From fijal at codespeak.net Wed Nov 4 10:56:00 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 10:56:00 +0100 (CET) Subject: [pypy-svn] r68961 - pypy/branch/no-trace-if-frame-escapes Message-ID: <20091104095600.679B3168438@codespeak.net> Author: fijal Date: Wed Nov 4 10:56:00 2009 New Revision: 68961 Added: pypy/branch/no-trace-if-frame-escapes/ - copied from r68959, pypy/trunk/ Log: A branch to experiment with abandoning trace in case virtualizable escapes From arigo at codespeak.net Wed Nov 4 11:27:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 11:27:53 +0100 (CET) Subject: [pypy-svn] r68965 - in pypy/branch/gc-jit-hack/pypy: rpython/lltypesystem rpython/memory/gctransform translator/c/test Message-ID: <20091104102753.3BB1D168437@codespeak.net> Author: arigo Date: Wed Nov 4 11:27:52 2009 New Revision: 68965 Modified: pypy/branch/gc-jit-hack/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Log: Interface resize_nursery() in framework.py, and test it in test_newgc. Modified: pypy/branch/gc-jit-hack/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/lltypesystem/lloperation.py Wed Nov 4 11:27:52 2009 @@ -467,7 +467,9 @@ # ^^^ returns an address of nursery free pointer, for later modifications 'gc_adr_of_nursery_top' : LLOp(), # ^^^ returns an address of pointer, since it can change at runtime - + 'gc_resize_nursery': LLOp(), + + # ------- experimental ------- # experimental operations in support of thread cloning, only # implemented by the Mark&Sweep GC 'gc_x_swap_pool': LLOp(canraise=(MemoryError,), canunwindgc=True), Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py Wed Nov 4 11:27:52 2009 @@ -348,6 +348,10 @@ self.obtainfreespace_ptr = getfn(GCClass.obtain_free_space.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeAddress()) + if getattr(GCClass, 'resize_nursery', False): + self.resizenursery_ptr = getfn(GCClass.resize_nursery.im_func, + [s_gc, annmodel.SomeInteger()], + annmodel.s_None) if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, @@ -800,6 +804,16 @@ resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) + def gct_gc_resize_nursery(self, hop): + if not hasattr(self, 'resizenursery_ptr'): + raise NotImplementedError("gc_resize_nursery: " + "only for generational gcs") + livevars = self.push_roots(hop) + [v_number] = hop.spaceop.args + hop.genop("direct_call", + [self.resizenursery_ptr, self.c_const_gc, v_number]) + self.pop_roots(hop, livevars) + def gct_gc_set_max_heap_size(self, hop): [v_size] = hop.spaceop.args hop.genop("direct_call", [self.set_max_heap_size_ptr, Modified: pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Wed Nov 4 11:27:52 2009 @@ -932,10 +932,54 @@ res = self.run('string_builder_over_allocation') assert res[1000] == 'y' + class TestGenerationalGC(TestSemiSpaceGC): gcpolicy = "generation" should_be_moving = True + def define_resize_nursery(self): + # this is only a "does not crash" kind of test + from pypy.rlib.objectmodel import compute_identity_hash + import random + seed = random.randrange(0, 10000) + print 'resize_nursery: random seed =', seed + r = random.Random(seed) + events = [] + maxcount = 10000000 + for i in range(50): + events.append((r.randrange(0, maxcount), + r.randrange(0, 9000)*1024)) + events.sort() + events.append((maxcount, 0)) # sentinel + # + class A: + pass + def fn(): + prev = None + i = 0 + j = 0 + while i < maxcount: + if i == events[j][0]: + llop.gc_resize_nursery(lltype.Void, events[j][1]) + j += 1 + a = A() + a.thehash = compute_identity_hash(a) + a.prev = prev + prev = a + i += 1 + while i > 0: + i -= 1 + assert a.thehash == compute_identity_hash(a) + a = a.prev + assert a is None + return 0 + return fn + + def test_resize_nursery(self): + res = self.run('resize_nursery') + assert res == 0 + + class TestHybridGC(TestGenerationalGC): gcpolicy = "hybrid" should_be_moving = True From arigo at codespeak.net Wed Nov 4 11:34:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 11:34:16 +0100 (CET) Subject: [pypy-svn] r68966 - pypy/trunk/pypy/objspace/flow Message-ID: <20091104103416.9F81B168437@codespeak.net> Author: arigo Date: Wed Nov 4 11:34:15 2009 New Revision: 68966 Modified: pypy/trunk/pypy/objspace/flow/objspace.py Log: Typo. Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Wed Nov 4 11:34:15 2009 @@ -441,7 +441,7 @@ # XXX XXX Ha Ha # the reason to do this is: if you interrupt the flowing of a function # with the bytecode interpreter will raise an applevel - # KeyboardInterrup and you will get an AttributeError: space does not + # KeyboardInterrupt and you will get an AttributeError: space does not # have w_KeyboardInterrupt, which is not very helpful raise KeyboardInterrupt w_KeyboardInterrupt = property(w_KeyboardInterrupt) From arigo at codespeak.net Wed Nov 4 11:38:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 11:38:13 +0100 (CET) Subject: [pypy-svn] r68967 - pypy/branch/gc-jit-hack/pypy/rpython/memory/gc Message-ID: <20091104103813.C1BBE168437@codespeak.net> Author: arigo Date: Wed Nov 4 11:38:13 2009 New Revision: 68967 Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/semispace.py Log: In record_red_zone() we know that the nursery is going to be allocated very soon, and it was already allocated previously. So it should really be counted as part of the allocated objects. The result is that we resize the semispaces a bit more eagerly. Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Wed Nov 4 11:38:13 2009 @@ -265,6 +265,11 @@ self.reset_nursery() SemiSpaceGC.semispace_collect(self, size_changing) + def record_red_zone(self, overhead=0): + if not self.nursery: + overhead += self.next_nursery_size + SemiSpaceGC.record_red_zone(self, overhead) + def make_a_copy(self, obj, objsize): tid = self.header(obj).tid # During a full collect, all copied objects might implicitly come Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/semispace.py Wed Nov 4 11:38:13 2009 @@ -283,14 +283,14 @@ def finished_full_collect(self): pass # hook for the HybridGC - def record_red_zone(self): + def record_red_zone(self, overhead=0): # red zone: if the space is more than 80% full, the next collection # should double its size. If it is more than 66% full twice in a row, # then it should double its size too. (XXX adjust) # The goal is to avoid many repeated collection that don't free a lot # of memory each, if the size of the live object set is just below the # size of the space. - free_after_collection = self.top_of_space - self.free + free_after_collection = self.top_of_space - self.free - overhead if free_after_collection > self.space_size // 3: self.red_zone = 0 else: From arigo at codespeak.net Wed Nov 4 11:42:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 11:42:21 +0100 (CET) Subject: [pypy-svn] r68968 - pypy/branch/gc-jit-hack/pypy/rpython/memory/gc Message-ID: <20091104104221.B5B64168437@codespeak.net> Author: arigo Date: Wed Nov 4 11:42:21 2009 New Revision: 68968 Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Log: Comment out the lines that bound the nursery size to half the initial space size, with explanation. Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Wed Nov 4 11:42:21 2009 @@ -115,8 +115,11 @@ debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size - if newsize > self.space_size // 2: - newsize = self.space_size // 2 + # --- the following lines are commented out: if we want a nursery + # --- that is too large for the current space, well, then the + # --- space will be resized in collect_nursery(). + #if newsize > self.space_size // 2: + # newsize = self.space_size // 2 # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust From arigo at codespeak.net Wed Nov 4 12:02:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 12:02:29 +0100 (CET) Subject: [pypy-svn] r68969 - in pypy/branch/gc-jit-hack/pypy: jit/metainterp rpython/memory/gctransform Message-ID: <20091104110229.66B4C168436@codespeak.net> Author: arigo Date: Wed Nov 4 12:02:29 2009 New Revision: 68969 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py pypy/branch/gc-jit-hack/pypy/jit/metainterp/warmspot.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py Log: Use resize_nursery() in the JIT. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Wed Nov 4 12:02:29 2009 @@ -1079,6 +1079,20 @@ def log(self, msg): debug_print(msg) + # ---- tweaking the size of the GC nursery -------- + + def enlarge_nursery(self, enlarge): + warmrunnerdesc = self.warmrunnerdesc + if warmrunnerdesc is not None and warmrunnerdesc.can_resize_nursery: + from pypy.rpython.lltypesystem import lltype, lloperation + if enlarge: + newsize = 10*1024*1024 # XXX a rather custom number + if sys.maxint > 2147483647: + newsize *= 2 + else: + newsize = 0 + lloperation.llop.gc_resize_nursery(lltype.Void, newsize) + # ____________________________________________________________ class MetaInterpGlobalData(object): @@ -1367,10 +1381,12 @@ def compile_and_run_once(self, *args): debug_start('jit-tracing') self.staticdata._setup_once() + self.staticdata.enlarge_nursery(True) self.create_empty_history() try: return self._compile_and_run_once(*args) finally: + self.staticdata.enlarge_nursery(False) if self.history is None: debug_stop('jit-blackhole') else: @@ -1397,12 +1413,15 @@ must_compile = warmrunnerstate.must_compile_from_failure(key) if must_compile: debug_start('jit-tracing') + self.staticdata.enlarge_nursery(True) else: debug_start('jit-blackhole') self.initialize_state_from_guard_failure(key, must_compile) try: return self._handle_guard_failure(key) finally: + if must_compile: + self.staticdata.enlarge_nursery(False) if self.history is None: debug_stop('jit-blackhole') else: Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/warmspot.py Wed Nov 4 12:02:29 2009 @@ -192,6 +192,8 @@ self.translator = translator self.rtyper = translator.rtyper self.gcdescr = gc.get_description(translator.config) + self.can_resize_nursery = ( + translator.config.translation.gctransformer == "framework") def find_portal(self): graphs = self.translator.graphs Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py Wed Nov 4 12:02:29 2009 @@ -806,8 +806,7 @@ def gct_gc_resize_nursery(self, hop): if not hasattr(self, 'resizenursery_ptr'): - raise NotImplementedError("gc_resize_nursery: " - "only for generational gcs") + return # only for generational gcs livevars = self.push_roots(hop) [v_number] = hop.spaceop.args hop.genop("direct_call", From cfbolz at codespeak.net Wed Nov 4 12:04:41 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 4 Nov 2009 12:04:41 +0100 (CET) Subject: [pypy-svn] r68970 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20091104110441.17C0C168436@codespeak.net> Author: cfbolz Date: Wed Nov 4 12:04:40 2009 New Revision: 68970 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_repr.py Log: Forgot to merge r68825 from the shrink-multidict branch Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_repr.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_repr.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_repr.py Wed Nov 4 12:04:40 2009 @@ -148,8 +148,7 @@ # Functions eq(repr(hash), '') # Methods - self.failUnless(repr(''.split).find( - "bound method str.split of '' at 0x") > -1) + self.assertEquals(repr(''.split), "") def test_xrange(self): import warnings From fijal at codespeak.net Wed Nov 4 12:40:52 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 12:40:52 +0100 (CET) Subject: [pypy-svn] r68971 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091104114052.C368A16843E@codespeak.net> Author: fijal Date: Wed Nov 4 12:40:52 2009 New Revision: 68971 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py Log: actually print time spent blackholing and running Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Wed Nov 4 12:40:52 2009 @@ -153,8 +153,8 @@ calls = self.calls self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) - self._print_intline("Running asm", cnt[RUNNING]) - self._print_intline("Blackhole", cnt[BLACKHOLE]) + self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) + self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) os.write(2, line) self._print_intline("ops", cnt[OPS]) From fijal at codespeak.net Wed Nov 4 12:47:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 12:47:28 +0100 (CET) Subject: [pypy-svn] r68972 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091104114728.44F0E16843E@codespeak.net> Author: fijal Date: Wed Nov 4 12:47:27 2009 New Revision: 68972 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py Log: Revert 68791 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Wed Nov 4 12:47:27 2009 @@ -153,8 +153,8 @@ calls = self.calls self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) - self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) - self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) + self._print_intline("Running asm", cnt[RUNNING]) + self._print_intline("Blackhole", cnt[BLACKHOLE]) line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) os.write(2, line) self._print_intline("ops", cnt[OPS]) From cfbolz at codespeak.net Wed Nov 4 13:08:23 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 4 Nov 2009 13:08:23 +0100 (CET) Subject: [pypy-svn] r68973 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091104120823.B9C68168436@codespeak.net> Author: cfbolz Date: Wed Nov 4 13:08:23 2009 New Revision: 68973 Modified: pypy/trunk/pypy/objspace/std/sharingdict.py pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Log: Bah. I managed to write wrong code and a test that completely by chance manages to work anyway. Fix. Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Wed Nov 4 13:08:23 2009 @@ -133,8 +133,8 @@ num_back = struct_len - pos - 1 if num_back > 0: - for i in range(pos, struct_len): - self.entries[pos] = self.entries[pos + 1] + for i in range(pos, struct_len - 1): + self.entries[i] = self.entries[i + 1] # don't make the entries list shorter, new keys might be added soon self.entries[struct_len - 1] = None structure = self.structure Modified: pypy/trunk/pypy/objspace/std/test/test_sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_sharingdict.py Wed Nov 4 13:08:23 2009 @@ -52,3 +52,17 @@ d.delitem("a") assert d.entries == [None, None, None] assert d.structure.keys == {} + + d = SharedDictImplementation(space) + d.setitem_str("a", 1) + d.setitem_str("b", 2) + d.setitem_str("c", 3) + d.setitem_str("d", 4) + d.setitem_str("e", 5) + d.setitem_str("f", 6) + d.setitem_str("g", 7) + d.setitem_str("h", 8) + d.setitem_str("i", 9) + d.delitem("d") + assert d.entries == [1, 2, 3, 5, 6, 7, 8, 9, None] + assert d.structure.keys == {"a": 0, "b": 1, "c": 2, "e": 3, "f": 4, "g": 5, "h": 6, "i": 7} From arigo at codespeak.net Wed Nov 4 16:08:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 16:08:15 +0100 (CET) Subject: [pypy-svn] r68974 - pypy/branch/gc-jit-hack/pypy/jit/metainterp Message-ID: <20091104150815.972F5168439@codespeak.net> Author: arigo Date: Wed Nov 4 16:08:14 2009 New Revision: 68974 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Log: "import sys" missing. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Wed Nov 4 16:08:14 2009 @@ -1,4 +1,4 @@ -import py, os +import py, os, sys from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated @@ -419,7 +419,6 @@ @arguments("orgpc", "box", "box") def opimpl_check_div_overflow(self, pc, box1, box2): # detect the combination "box1 = -sys.maxint-1, box2 = -1". - import sys tmp1 = self.metainterp.execute_and_record( # combination to detect: rop.INT_ADD, None, box1, ConstInt(sys.maxint)) # tmp1=-1, box2=-1 tmp2 = self.metainterp.execute_and_record( @@ -1373,7 +1372,6 @@ try: self._interpret() except: - import sys if sys.exc_info()[0] is not None: codewriter.log.info(sys.exc_info()[0].__name__) raise From fijal at codespeak.net Wed Nov 4 16:15:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 16:15:21 +0100 (CET) Subject: [pypy-svn] r68975 - pypy/trunk/pypy/rpython/tool Message-ID: <20091104151521.38C3C168439@codespeak.net> Author: fijal Date: Wed Nov 4 16:15:15 2009 New Revision: 68975 Modified: pypy/trunk/pypy/rpython/tool/rffi_platform.py Log: A fix good enough to try compiling pypy on top of pypy, without crashing too early Modified: pypy/trunk/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/trunk/pypy/rpython/tool/rffi_platform.py Wed Nov 4 16:15:15 2009 @@ -566,7 +566,7 @@ def get_python_include_dir(): from distutils import sysconfig gcv = sysconfig.get_config_vars() - return gcv['INCLUDEPY'] + return gcv.get('INCLUDEPY', '.') # this is for running on top of pypy-c def configure_external_library(name, eci, configurations, symbol=None, _cache={}): From arigo at codespeak.net Wed Nov 4 17:21:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 17:21:55 +0100 (CET) Subject: [pypy-svn] r68976 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091104162155.2C674168433@codespeak.net> Author: arigo Date: Wed Nov 4 17:21:54 2009 New Revision: 68976 Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py Log: Add a comment to help migration away from jitprof.py. Note that this might be a temporary way. Please fix the comment if you add another solution. Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Wed Nov 4 17:21:54 2009 @@ -130,6 +130,10 @@ def start_backend(self): self._start(BACKEND) def end_backend(self): self._end (BACKEND) + # Don't record times for 'running' and 'blackhole' because there are + # too many of them: calling time.time() is a major blocker. + # If you are interested in these numbers, use 'PYPYLOG=file' and + # look at the resulting file with pypy/tool/logparser.py. def start_running(self): self.count(RUNNING) def end_running(self): pass From arigo at codespeak.net Wed Nov 4 17:42:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 17:42:54 +0100 (CET) Subject: [pypy-svn] r68977 - pypy/trunk/pypy/tool Message-ID: <20091104164254.10993168436@codespeak.net> Author: arigo Date: Wed Nov 4 17:42:54 2009 New Revision: 68977 Modified: pypy/trunk/pypy/tool/logparser.py (props changed) Log: Mark this as executable. From fijal at codespeak.net Wed Nov 4 19:13:50 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 19:13:50 +0100 (CET) Subject: [pypy-svn] r68978 - pypy/trunk/pypy/lang/prolog/interpreter Message-ID: <20091104181350.4E8FE16843E@codespeak.net> Author: fijal Date: Wed Nov 4 19:13:49 2009 New Revision: 68978 Modified: pypy/trunk/pypy/lang/prolog/interpreter/arithmetic.py Log: kill unnecessary import Modified: pypy/trunk/pypy/lang/prolog/interpreter/arithmetic.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/arithmetic.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/arithmetic.py Wed Nov 4 19:13:49 2009 @@ -4,7 +4,6 @@ from pypy.lang.prolog.interpreter import engine, helper, term, error from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound from pypy.rlib.rarithmetic import intmask -from pypy.rlib.jit import we_are_jitted, hint from pypy.rlib.unroll import unrolling_iterable arithmetic_functions = {} From fijal at codespeak.net Wed Nov 4 19:17:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 19:17:31 +0100 (CET) Subject: [pypy-svn] r68979 - pypy/trunk/pypy/objspace/std Message-ID: <20091104181731.A59B0168438@codespeak.net> Author: fijal Date: Wed Nov 4 19:17:31 2009 New Revision: 68979 Modified: pypy/trunk/pypy/objspace/std/objspace.py Log: unused import Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Wed Nov 4 19:17:31 2009 @@ -18,7 +18,7 @@ from pypy.objspace.std import stdtypedef from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.jit import hint, we_are_jitted +from pypy.rlib.jit import hint from pypy.rlib.unroll import unrolling_iterable import sys import os From fijal at codespeak.net Wed Nov 4 20:25:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 4 Nov 2009 20:25:01 +0100 (CET) Subject: [pypy-svn] r68980 - pypy/trunk/pypy/interpreter Message-ID: <20091104192501.16BCF168438@codespeak.net> Author: fijal Date: Wed Nov 4 20:25:01 2009 New Revision: 68980 Modified: pypy/trunk/pypy/interpreter/nestedscope.py Log: mark MAKE_CLOSURE as unroll_safe Modified: pypy/trunk/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/trunk/pypy/interpreter/nestedscope.py (original) +++ pypy/trunk/pypy/interpreter/nestedscope.py Wed Nov 4 20:25:01 2009 @@ -201,6 +201,7 @@ cell = f.cells[varindex] cell.set(w_newvalue) + @jit.unroll_safe def MAKE_CLOSURE(f, numdefaults, *ignored): w_codeobj = f.popvalue() codeobj = f.space.interp_w(pycode.PyCode, w_codeobj) From arigo at codespeak.net Wed Nov 4 23:36:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 4 Nov 2009 23:36:35 +0100 (CET) Subject: [pypy-svn] r68981 - in pypy/branch/gc-jit-hack/pypy: jit/metainterp rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform translator/c/test Message-ID: <20091104223635.B8E33168439@codespeak.net> Author: arigo Date: Wed Nov 4 23:36:34 2009 New Revision: 68981 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Log: Force a minor collection at the start of jit-tracing. Experimental. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Wed Nov 4 23:36:34 2009 @@ -1090,7 +1090,7 @@ newsize *= 2 else: newsize = 0 - lloperation.llop.gc_resize_nursery(lltype.Void, newsize) + lloperation.llop.gc_resize_nursery(lltype.Void, newsize, enlarge) # ____________________________________________________________ Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Wed Nov 4 23:36:34 2009 @@ -602,11 +602,11 @@ # ---------- resizing the nursery at runtime ---------- - def resize_nursery(self, newsize): + def resize_nursery(self, newsize, forced): debug_start("gc-resize-nursery") if newsize < self.minimal_setting_for_nursery_size: newsize = self.minimal_setting_for_nursery_size - if not self.nursery or newsize > self.allocated_nursery_size: + if forced or not self.nursery or newsize > self.allocated_nursery_size: self.drop_nursery() self.nursery_size = newsize self.allocated_nursery_size = newsize Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py Wed Nov 4 23:36:34 2009 @@ -347,7 +347,7 @@ # nursery contains 3/7 objects # now make the nursery grow to 1024 - gc.resize_nursery(1024) + gc.resize_nursery(1024, False) assert counters[1] == 2 assert counters[0] in (7, 15) counters[0] = 0 # empty @@ -365,7 +365,7 @@ assert counters[0] in (63, 127) # now mark it as "must shrink back" - gc.resize_nursery(0) + gc.resize_nursery(0, False) assert counters[1] == 3 assert counters[0] in (63, 127) @@ -389,7 +389,7 @@ counters[0] = 0 # nursery not allocated at all # resize from a fresh nursery: should not do another collect - gc.resize_nursery(512) + gc.resize_nursery(512, False) assert counters[1] == 5 # fill it partially only @@ -400,7 +400,7 @@ # now mark it as "must shrink back", # which should still leave room for 4/8 objects - gc.resize_nursery(0) + gc.resize_nursery(0, False) assert counters[1] == 5 assert counters[0] == 20 Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py Wed Nov 4 23:36:34 2009 @@ -350,7 +350,8 @@ annmodel.SomeAddress()) if getattr(GCClass, 'resize_nursery', False): self.resizenursery_ptr = getfn(GCClass.resize_nursery.im_func, - [s_gc, annmodel.SomeInteger()], + [s_gc, annmodel.SomeInteger(), + annmodel.s_Bool], annmodel.s_None) if GCClass.moving_gc: @@ -808,9 +809,10 @@ if not hasattr(self, 'resizenursery_ptr'): return # only for generational gcs livevars = self.push_roots(hop) - [v_number] = hop.spaceop.args + [v_number, v_forced] = hop.spaceop.args hop.genop("direct_call", - [self.resizenursery_ptr, self.c_const_gc, v_number]) + [self.resizenursery_ptr, self.c_const_gc, + v_number, v_forced]) self.pop_roots(hop, livevars) def gct_gc_set_max_heap_size(self, hop): Modified: pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Wed Nov 4 23:36:34 2009 @@ -960,7 +960,7 @@ j = 0 while i < maxcount: if i == events[j][0]: - llop.gc_resize_nursery(lltype.Void, events[j][1]) + llop.gc_resize_nursery(lltype.Void, events[j][1], False) j += 1 a = A() a.thehash = compute_identity_hash(a) From arigo at codespeak.net Thu Nov 5 10:05:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 10:05:17 +0100 (CET) Subject: [pypy-svn] r68983 - in pypy/branch/gc-jit-hack/pypy: jit/metainterp rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform translator/c/test Message-ID: <20091105090517.5125116843C@codespeak.net> Author: arigo Date: Thu Nov 5 10:05:14 2009 New Revision: 68983 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Log: Revert r68981. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Thu Nov 5 10:05:14 2009 @@ -1090,7 +1090,7 @@ newsize *= 2 else: newsize = 0 - lloperation.llop.gc_resize_nursery(lltype.Void, newsize, enlarge) + lloperation.llop.gc_resize_nursery(lltype.Void, newsize) # ____________________________________________________________ Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/generation.py Thu Nov 5 10:05:14 2009 @@ -602,11 +602,11 @@ # ---------- resizing the nursery at runtime ---------- - def resize_nursery(self, newsize, forced): + def resize_nursery(self, newsize): debug_start("gc-resize-nursery") if newsize < self.minimal_setting_for_nursery_size: newsize = self.minimal_setting_for_nursery_size - if forced or not self.nursery or newsize > self.allocated_nursery_size: + if not self.nursery or newsize > self.allocated_nursery_size: self.drop_nursery() self.nursery_size = newsize self.allocated_nursery_size = newsize Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gc/test/test_direct.py Thu Nov 5 10:05:14 2009 @@ -347,7 +347,7 @@ # nursery contains 3/7 objects # now make the nursery grow to 1024 - gc.resize_nursery(1024, False) + gc.resize_nursery(1024) assert counters[1] == 2 assert counters[0] in (7, 15) counters[0] = 0 # empty @@ -365,7 +365,7 @@ assert counters[0] in (63, 127) # now mark it as "must shrink back" - gc.resize_nursery(0, False) + gc.resize_nursery(0) assert counters[1] == 3 assert counters[0] in (63, 127) @@ -389,7 +389,7 @@ counters[0] = 0 # nursery not allocated at all # resize from a fresh nursery: should not do another collect - gc.resize_nursery(512, False) + gc.resize_nursery(512) assert counters[1] == 5 # fill it partially only @@ -400,7 +400,7 @@ # now mark it as "must shrink back", # which should still leave room for 4/8 objects - gc.resize_nursery(0, False) + gc.resize_nursery(0) assert counters[1] == 5 assert counters[0] == 20 Modified: pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-jit-hack/pypy/rpython/memory/gctransform/framework.py Thu Nov 5 10:05:14 2009 @@ -350,8 +350,7 @@ annmodel.SomeAddress()) if getattr(GCClass, 'resize_nursery', False): self.resizenursery_ptr = getfn(GCClass.resize_nursery.im_func, - [s_gc, annmodel.SomeInteger(), - annmodel.s_Bool], + [s_gc, annmodel.SomeInteger()], annmodel.s_None) if GCClass.moving_gc: @@ -809,10 +808,9 @@ if not hasattr(self, 'resizenursery_ptr'): return # only for generational gcs livevars = self.push_roots(hop) - [v_number, v_forced] = hop.spaceop.args + [v_number] = hop.spaceop.args hop.genop("direct_call", - [self.resizenursery_ptr, self.c_const_gc, - v_number, v_forced]) + [self.resizenursery_ptr, self.c_const_gc, v_number]) self.pop_roots(hop, livevars) def gct_gc_set_max_heap_size(self, hop): Modified: pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-jit-hack/pypy/translator/c/test/test_newgc.py Thu Nov 5 10:05:14 2009 @@ -960,7 +960,7 @@ j = 0 while i < maxcount: if i == events[j][0]: - llop.gc_resize_nursery(lltype.Void, events[j][1], False) + llop.gc_resize_nursery(lltype.Void, events[j][1]) j += 1 a = A() a.thehash = compute_identity_hash(a) From arigo at codespeak.net Thu Nov 5 11:15:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 11:15:46 +0100 (CET) Subject: [pypy-svn] r68984 - pypy/branch/gc-jit-hack/pypy/jit/metainterp Message-ID: <20091105101546.18B71168430@codespeak.net> Author: arigo Date: Thu Nov 5 11:15:46 2009 New Revision: 68984 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Log: The current solution I think best. It still gives mixed results. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Thu Nov 5 11:15:46 2009 @@ -1084,6 +1084,7 @@ warmrunnerdesc = self.warmrunnerdesc if warmrunnerdesc is not None and warmrunnerdesc.can_resize_nursery: from pypy.rpython.lltypesystem import lltype, lloperation + from pypy.rlib import rgc if enlarge: newsize = 10*1024*1024 # XXX a rather custom number if sys.maxint > 2147483647: @@ -1091,6 +1092,8 @@ else: newsize = 0 lloperation.llop.gc_resize_nursery(lltype.Void, newsize) + if enlarge: + rgc.collect(0) # ____________________________________________________________ From fijal at codespeak.net Thu Nov 5 11:47:40 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 5 Nov 2009 11:47:40 +0100 (CET) Subject: [pypy-svn] r68985 - pypy/build/bot2/pypybuildbot Message-ID: <20091105104740.AD996168435@codespeak.net> Author: fijal Date: Thu Nov 5 11:47:40 2009 New Revision: 68985 Modified: pypy/build/bot2/pypybuildbot/master.py Log: make own category, benchmark Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Nov 5 11:47:40 2009 @@ -151,7 +151,7 @@ "slavenames": ["bigdogvm2"], "builddir": JITBENCH, "factory": pypyJITBenchmarkFactory, - "category": 'other', + "category": 'benchmark', }, ], From arigo at codespeak.net Thu Nov 5 12:00:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 12:00:18 +0100 (CET) Subject: [pypy-svn] r68986 - pypy/trunk/pypy/doc Message-ID: <20091105110018.C0455168436@codespeak.net> Author: arigo Date: Thu Nov 5 12:00:18 2009 New Revision: 68986 Modified: pypy/trunk/pypy/doc/getting-started-python.txt Log: Small updates to getting-started-python. Modified: pypy/trunk/pypy/doc/getting-started-python.txt ============================================================================== --- pypy/trunk/pypy/doc/getting-started-python.txt (original) +++ pypy/trunk/pypy/doc/getting-started-python.txt Thu Nov 5 12:00:18 2009 @@ -55,17 +55,26 @@ `optimization level`_ `1` in the next step. A level of `2` or `3` gives much better results, though. + Let me stress this another time: at ``--opt=1`` you get the Boehm + GC, which is here mostly for historical and for testing reasons. + You really do not want to pick it. The resulting ``pypy-c`` is + slow. + + Alternatively, if you want to enable the experimental JIT, choose + the optimization level ``jit``. + 3. Run:: cd pypy/translator/goal python translate.py --opt=3 targetpypystandalone.py - possibly replacing ``--opt=3`` with ``--opt=1`` or another + possibly replacing ``--opt=3`` with ``--opt=jit`` or another `optimization level`_ of your choice. - On Linux 32-bit Intel machines, if you don't need threads, you - can get some extra speed (and extra translation time) by adding - ``--gcrootfinder=asmgcc`` just after the ``--opt`` option. + On Linux 32-bit Intel machines, you can get some extra speed + (and extra translation time) by adding ``--gcrootfinder=asmgcc`` + just after the ``--opt`` option. (This option is implied by + ``--opt=jit``.) .. _`optimization level`: config/opt.html From arigo at codespeak.net Thu Nov 5 12:46:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 12:46:10 +0100 (CET) Subject: [pypy-svn] r68987 - pypy/branch/gc-jit-hack/pypy/jit/metainterp Message-ID: <20091105114610.9207D318137@codespeak.net> Author: arigo Date: Thu Nov 5 12:46:09 2009 New Revision: 68987 Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Log: This is the best result so far. At least in this case I understand the results: they seem to clearly show that a lot of the objects allocated during jit-tracing actually survive. The results are not good enough -- they say that jit-tracing takes only half the time, but the truth is that it's always soon followed by a lengthy minor collection, so that the total time (as well as the total GC time) end up not really improved. Closing this branch for now. Modified: pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-jit-hack/pypy/jit/metainterp/pyjitpl.py Thu Nov 5 12:46:09 2009 @@ -1086,14 +1086,13 @@ from pypy.rpython.lltypesystem import lltype, lloperation from pypy.rlib import rgc if enlarge: + rgc.collect(0) newsize = 10*1024*1024 # XXX a rather custom number if sys.maxint > 2147483647: newsize *= 2 else: newsize = 0 lloperation.llop.gc_resize_nursery(lltype.Void, newsize) - if enlarge: - rgc.collect(0) # ____________________________________________________________ From afa at codespeak.net Thu Nov 5 15:01:37 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 15:01:37 +0100 (CET) Subject: [pypy-svn] r68989 - pypy/branch/msvc-asmgcroot/pypy/translator/platform Message-ID: <20091105140137.95D82168436@codespeak.net> Author: afa Date: Thu Nov 5 15:01:37 2009 New Revision: 68989 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Log: Avoid the __thread attribute on Windows: when intermediate assembler is used, the thread-local slot does not seem to be correctly allocated. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Thu Nov 5 15:01:37 2009 @@ -142,6 +142,12 @@ def _args_for_shared(self, args): return ['/dll'] + args + def check___thread(self): + # __declspec(thread) does not seem to work when using assembler. + # Returning False will cause the program to use TlsAlloc functions. + # see src/thread_nt.h + return False + def _link_args_from_eci(self, eci, standalone): # Windows needs to resolve all symbols even for DLLs args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) From afa at codespeak.net Thu Nov 5 16:34:52 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 16:34:52 +0100 (CET) Subject: [pypy-svn] r68990 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test Message-ID: <20091105153452.EA71E168433@codespeak.net> Author: afa Date: Thu Nov 5 16:34:52 2009 New Revision: 68990 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py Log: Merge r68621 from trunk Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Nov 5 16:34:52 2009 @@ -52,7 +52,11 @@ def run(arg0, arg1): lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 - g = os.popen('"%s" %d %d' % (exe_name, arg0, arg1), 'r') + if sys.platform == 'win32': + redirect = ' 2> NUL' + else: + redirect = '' + g = os.popen('"%s" %d %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) From afa at codespeak.net Thu Nov 5 17:26:21 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 17:26:21 +0100 (CET) Subject: [pypy-svn] r68991 - pypy/branch/msvc-asmgcroot/pypy/translator/platform Message-ID: <20091105162621.D1A20168433@codespeak.net> Author: afa Date: Thu Nov 5 17:26:21 2009 New Revision: 68991 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Log: Fix makefile generic rule for windows Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Thu Nov 5 17:26:21 2009 @@ -247,7 +247,7 @@ rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('.c.o', '', '$(CC) $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), + ('.c.obj', '', '$(CC) $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: From arigo at codespeak.net Thu Nov 5 18:04:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 18:04:44 +0100 (CET) Subject: [pypy-svn] r68992 - pypy/trunk/pypy/doc Message-ID: <20091105170444.5B157168433@codespeak.net> Author: arigo Date: Thu Nov 5 18:04:42 2009 New Revision: 68992 Modified: pypy/trunk/pypy/doc/cpython_differences.txt Log: Write a note about when __del__s are called in PyPy. Also mention the issue with weak proxies. Modified: pypy/trunk/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/trunk/pypy/doc/cpython_differences.txt (original) +++ pypy/trunk/pypy/doc/cpython_differences.txt Thu Nov 5 18:04:42 2009 @@ -120,11 +120,22 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html +Note that the time at which __del__ is called is not well-defined in any +implementation apart from CPython. A __del__ method may appear to be +called sometimes later in PyPy; for example, a file may stay open for a +bit longer, which can have visible effects (e.g. a file opened for +writing might still contain data not flushed yet). This also makes +"weak proxies" less useful (see ``weakref.proxy()``). They will appear +to stay alive a bit longer in PyPy, and suddenly they will really be +dead, raising a ``ReferenceError`` on the next access. Any code that +uses weak proxies must carefully catch such ``ReferenceError`` at any +place that uses them. + The built-in function ``id()`` returns numbers that are not addresses for most of PyPy's garbage collectors. This is most visible in the default repr: a typical PyPy object can pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because the physical address can change for some GCs). Calling +its real address (because an object can move around in some GCs). Calling ``id`` a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to From afa at codespeak.net Thu Nov 5 18:06:16 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 18:06:16 +0100 (CET) Subject: [pypy-svn] r68993 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . gcc gcc/test gcc/test/msvc src Message-ID: <20091105170616.3609C168433@codespeak.net> Author: afa Date: Thu Nov 5 18:06:15 2009 New Revision: 68993 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Log: Shave the tragcroot yack until it produces a working pypy-c-jit with the Microsoft compiler. It now works with optimizations enabled, which is good. I changed the gcroot marker: there seems to be no way to prevent the optimizer from reusing the old value, unless we use some memory input outside the current stack, e.g a global variable. Unfortunately this produces two operations, that we have to match to remove the global variable access. Also take care of some of the many inventive constructs generated by the optimizer. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Thu Nov 5 18:06:15 2009 @@ -117,7 +117,8 @@ ; 1487 : l_v406 = pypy_asm_gcroot(l_v405); - test DWORD PTR _l_v405$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v405$[esp+24] ; 1488 : l_items_2 = l_v406; /* for moving GCs */ ; 1489 : l_v408 = RPyField((&pypy_g_ExcData), ed_exc_type); Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s Thu Nov 5 18:06:15 2009 @@ -212,7 +212,8 @@ ; 1096 : l_v354 = pypy_asm_gcroot(l_v353); - test DWORD PTR _l_v353$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v353$[esp+24] ; 1097 : l_a2_1 = l_v354; /* for moving GCs */ ; 1098 : l_v356 = (void*)l_a5_1; @@ -222,7 +223,8 @@ ; 1099 : l_v357 = pypy_asm_gcroot(l_v356); - test DWORD PTR _l_v356$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v356$[esp+24] ; 1100 : l_a5_1 = l_v357; /* for moving GCs */ ; 1101 : l_v359 = (void*)l_a6_1; @@ -232,7 +234,8 @@ ; 1102 : l_v360 = pypy_asm_gcroot(l_v359); - test DWORD PTR _l_v359$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v359$[esp+24] ; 1103 : l_a6_1 = l_v360; /* for moving GCs */ ; 1104 : l_v362 = (void*)l_a1_1; @@ -241,7 +244,8 @@ ; 1105 : l_v363 = pypy_asm_gcroot(l_v362); - test DWORD PTR _l_v362$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v362$[esp+24] ; 1106 : l_a1_1 = l_v363; /* for moving GCs */ ; 1107 : l_v365 = (void*)l_a3_1; @@ -250,7 +254,8 @@ ; 1108 : l_v366 = pypy_asm_gcroot(l_v365); - test DWORD PTR _l_v365$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v365$[esp+24] ; 1109 : l_a3_1 = l_v366; /* for moving GCs */ ; 1110 : l_v368 = (void*)l_a4_1; @@ -260,7 +265,8 @@ ; 1111 : l_v369 = pypy_asm_gcroot(l_v368); - test DWORD PTR _l_v368$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v368$[esp+24] ; 1112 : l_a4_1 = l_v369; /* for moving GCs */ ; 1113 : l_v335 = (struct pypy_src8_A0 *)l_v428; @@ -302,7 +308,8 @@ ; 1126 : l_v377 = pypy_asm_gcroot(l_v376); - test DWORD PTR _l_v376$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v376$[esp+24] ; 1127 : l_a2_1 = l_v377; /* for moving GCs */ ; 1128 : l_v379 = (void*)l_a6_1; @@ -312,7 +319,8 @@ ; 1129 : l_v380 = pypy_asm_gcroot(l_v379); - test DWORD PTR _l_v379$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v379$[esp+24] ; 1130 : l_a6_1 = l_v380; /* for moving GCs */ ; 1131 : l_v382 = (void*)l_a1_1; @@ -321,7 +329,8 @@ ; 1132 : l_v383 = pypy_asm_gcroot(l_v382); - test DWORD PTR _l_v382$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v382$[esp+24] ; 1133 : l_a1_1 = l_v383; /* for moving GCs */ ; 1134 : l_v385 = (void*)l_v335; @@ -330,7 +339,8 @@ ; 1135 : l_v386 = pypy_asm_gcroot(l_v385); - test DWORD PTR _l_v385$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v385$[esp+24] ; 1136 : l_v335 = l_v386; /* for moving GCs */ ; 1137 : l_v388 = (void*)l_a3_1; @@ -339,7 +349,8 @@ ; 1138 : l_v389 = pypy_asm_gcroot(l_v388); - test DWORD PTR _l_v388$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v388$[esp+24] ; 1139 : l_a3_1 = l_v389; /* for moving GCs */ ; 1140 : l_v391 = (void*)l_a5_1; @@ -349,7 +360,8 @@ ; 1141 : l_v392 = pypy_asm_gcroot(l_v391); - test DWORD PTR _l_v391$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v391$[esp+24] ; 1142 : l_a5_1 = l_v392; /* for moving GCs */ ; 1143 : l_v394 = (void*)l_a4_1; @@ -359,7 +371,8 @@ ; 1144 : l_v395 = pypy_asm_gcroot(l_v394); - test DWORD PTR _l_v394$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v394$[esp+24] ; 1145 : l_a4_1 = l_v395; /* for moving GCs */ ; 1146 : l_v397 = RPyField((&pypy_g_ExcData), ed_exc_type); @@ -398,7 +411,8 @@ ; 1157 : l_v402 = pypy_asm_gcroot(l_v401); - test DWORD PTR _l_v401$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v401$[esp+24] ; 1158 : l_a2_1 = l_v402; /* for moving GCs */ ; 1159 : l_v404 = (void*)l_a1_1; @@ -407,7 +421,8 @@ ; 1160 : l_v405 = pypy_asm_gcroot(l_v404); - test DWORD PTR _l_v404$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v404$[esp+24] ; 1161 : l_a1_1 = l_v405; /* for moving GCs */ ; 1162 : l_v407 = (void*)l_a3_1; @@ -416,7 +431,8 @@ ; 1163 : l_v408 = pypy_asm_gcroot(l_v407); - test DWORD PTR _l_v407$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v407$[esp+24] ; 1164 : l_a3_1 = l_v408; /* for moving GCs */ ; 1165 : l_v410 = (void*)l_a6_1; @@ -426,7 +442,8 @@ ; 1166 : l_v411 = pypy_asm_gcroot(l_v410); - test DWORD PTR _l_v410$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v410$[esp+24] ; 1167 : l_a6_1 = l_v411; /* for moving GCs */ ; 1168 : l_v413 = (void*)l_a5_1; @@ -436,7 +453,8 @@ ; 1169 : l_v414 = pypy_asm_gcroot(l_v413); - test DWORD PTR _l_v413$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v413$[esp+24] ; 1170 : l_a5_1 = l_v414; /* for moving GCs */ ; 1171 : l_v416 = (void*)l_a4_1; @@ -446,7 +464,8 @@ ; 1172 : l_v417 = pypy_asm_gcroot(l_v416); - test DWORD PTR _l_v416$[esp+24], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v416$[esp+24] ; 1173 : l_a4_1 = l_v417; /* for moving GCs */ ; 1174 : l_v419 = RPyField((&pypy_g_ExcData), ed_exc_type); Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s Thu Nov 5 18:06:15 2009 @@ -131,7 +131,8 @@ ; 15663: l_v271559 = pypy_asm_gcroot(l_v271558); - test DWORD PTR _l_v271558$[ebp], 0 + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v271558$[ebp] mov edx, DWORD PTR _l_v271558$[ebp] mov DWORD PTR _l_v271559$[ebp], edx Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Nov 5 18:06:15 2009 @@ -154,13 +154,6 @@ res = self.run('callback_simple') assert res == 4900 - if sys.platform == 'win32': - def test_callback_with_collect(self): - py.test.skip("No libffi yet with mingw32") - - def define_callback_with_collect(cls): - return lambda: 0 - class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): # for the individual tests see @@ -181,3 +174,10 @@ config = TestAsmGCRootWithSemiSpaceGC.make_config() config.translation.cc = 'mingw32' return config + + + def test_callback_with_collect(self): + py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Thu Nov 5 18:06:15 2009 @@ -122,7 +122,8 @@ yield check_computegcmaptable, format, path -r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") +r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") +r_gcroot_constant = re.compile(r";\tmov\t.+, .+_constant_always_one_") def check_computegcmaptable(format, path): if format == 'msvc': @@ -159,6 +160,11 @@ else: expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + if format == 'msvc' and r_gcroot_constant.match(line): + expectedlines[i] = ';' + expectedlines[i] + expectedlines[i+1] = (expectedlines[i+1] + .replace('\timul\t', '\tmov\t') + + '\t; GCROOT\n') prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Nov 5 18:06:15 2009 @@ -1,47 +1,36 @@ #! /usr/bin/env python import autopath - import re, sys, os, random -from pypy.translator.c.gcc.instruction import * - -LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' - -# darwin -r_textstart = re.compile(r"\t.text\s*$") -# see -# http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html -OTHERSECTIONS = ['section', 'zerofill', - 'const', 'static_const', 'cstring', - 'literal4', 'literal8', 'literal16', - 'constructor', 'desctructor', - 'symbol_stub', - 'data', 'static_data', - 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', - 'dyld', 'mod_init_func', 'mod_term_func', - 'const_data' - ] -r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") +from pypy.translator.c.gcc.instruction import Insn, Label, InsnCall, InsnRet +from pypy.translator.c.gcc.instruction import InsnFunctionStart, InsnStop +from pypy.translator.c.gcc.instruction import InsnSetLocal, InsnCopyLocal +from pypy.translator.c.gcc.instruction import InsnPrologue, InsnEpilogue +from pypy.translator.c.gcc.instruction import InsnGCROOT +from pypy.translator.c.gcc.instruction import InsnStackAdjust +from pypy.translator.c.gcc.instruction import InsnCannotFollowEsp +from pypy.translator.c.gcc.instruction import LocalVar, somenewvalue, frameloc +from pypy.translator.c.gcc.instruction import LOC_REG, LOC_NOWHERE, LOC_MASK +from pypy.translator.c.gcc.instruction import LOC_EBP_BASED, LOC_ESP_BASED OFFSET_LABELS = 2**30 -# inside functions -r_label = re.compile(LABEL+"[:]\s*$") -r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") -r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) -r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") - class FunctionGcRootTracker(object): skip = 0 @classmethod def init_regexp(cls): + cls.r_label = re.compile(cls.LABEL+"[:]\s*$") + cls.r_globl = re.compile(r"\t[.]globl\t"+cls.LABEL+"\s*$") + cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) + + cls.r_insn = re.compile(r"\t([a-z]\w*)\s") cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") + cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") def __init__(self, funcname, lines, filetag=0): self.funcname = funcname @@ -116,7 +105,7 @@ def findlabels(self): self.labels = {} # {name: Label()} for lineno, line in enumerate(self.lines): - match = r_label.match(line) + match = self.r_label.match(line) if match: label = match.group(1) assert label not in self.labels, "duplicate label" @@ -142,7 +131,7 @@ continue self.currentlineno = lineno insn = [] - match = r_insn.match(line) + match = self.r_insn.match(line) if self.r_bottom_marker.match(line): self.is_stack_bottom = True @@ -163,7 +152,7 @@ elif line == '\t/* end_ignore_in_trackgcroot */\n': ignore_insns = False else: - match = r_label.match(line) + match = self.r_label.match(line) if match: insn = self.labels[match.group(1)] @@ -207,13 +196,15 @@ for insn in self.insns: if isinstance(insn, (InsnRet, InsnEpilogue, InsnGCROOT)): deltas = {} + if isinstance(insn, InsnRet): + deltas[insn] = 0 self.walk_instructions_backwards(walker, insn, 0) size_at_insn = [] for insn1, delta1 in deltas.items(): if hasattr(insn1, 'framesize'): size_at_insn.append(insn1.framesize + delta1) - assert len(size_at_insn) > 0, ( - "cannot reach the start of the function??") + if not size_at_insn: + continue size_at_insn = size_at_insn[0] for insn1, delta1 in deltas.items(): size_at_insn1 = size_at_insn - delta1 @@ -308,10 +299,10 @@ # script); otherwise invent a name and add the label to tracker.lines. label = None # this checks for a ".globl NAME" followed by "NAME:" - match = r_globl.match(self.lines[call.lineno+1]) + match = self.r_globl.match(self.lines[call.lineno+1]) if match: label1 = match.group(1) - match = r_globllabel.match(self.lines[call.lineno+2]) + match = self.r_globllabel.match(self.lines[call.lineno+2]) if match: label2 = match.group(1) if label1 == label2: @@ -525,7 +516,7 @@ def visit_jmp(self, line): tablelabels = [] - match = r_jmp_switch.match(line) + match = self.r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like @@ -543,13 +534,13 @@ for s in insn.all_sources_of(loc): # if the source looks like 8(%eax,%edx,4) # %eax is the real source, %edx is an offset. - match = r_jmp_source.match(s) + match = self.r_jmp_source.match(s) if match and not self.r_localvar_esp.match(s): sources.append(match.group(1)) else: sources.append(s) for source in sources: - label_match = re.compile(LABEL).match(source) + label_match = re.compile(self.LABEL).match(source) if label_match: tablelabels.append(label_match.group(0)) return @@ -589,7 +580,8 @@ return InsnStop() def register_jump_to(self, label): - self.labels[label].previous_insns.append(self.insns[-1]) + if not isinstance(self.insns[-1], InsnStop): + self.labels[label].previous_insns.append(self.insns[-1]) def conditional_jump(self, line): match = self.r_jump.match(line) @@ -652,14 +644,41 @@ insns = [InsnCall(self.currentlineno), InsnSetLocal(self.EAX)] # the result is there if sys.platform == 'win32': + # On win32, the address of a foreign function must be + # computed, the optimizer may store it in a register. We + # could ignore this, except when the function has a + # __stdcall calling convention... + def find_register(target): + reg = [] + def walker(insn, locs): + sources = [] + for loc in locs: + for s in insn.all_sources_of(loc): + sources.append(s) + for source in sources: + if re.match("DWORD PTR .+@", source): + reg.append(source) + if reg: + return + yield tuple(sources) + insn = InsnStop() + insn.previous_insns = [self.insns[-1]] + self.walk_instructions_backwards(walker, insn, (target,)) + return reg + + if match and self.r_localvarfp.match(target): + sources = find_register(target) + if sources: + target, = sources + # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size - if match and '@' in target: + if match and '@' in target and not target.startswith('@'): insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) - # XXX some functions seem use the "fastcall" calling convention - # without their declaration, how can we guess the stack effect? - if match and target in ['__alldiv', '__allrem', '__allmul']: + # Some (intrinsic?) functions use the "fastcall" calling convention + # XXX without any declaration, how can we guess the stack effect? + if match and target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: insns.append(InsnStackAdjust(16)) return insns @@ -734,12 +753,12 @@ TOP_OF_STACK = 'DWORD PTR [esp]' OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' - LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' r_functionstart = re.compile(r"; Function compile flags: ") r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") - r_symboldefine = re.compile(r"([_a-z0-9]+\$) = ([-0-9]+)\s*;.+\n") + r_symboldefine = re.compile(r"([_A-Za-z0-9$]+) = ([-0-9]+)\s*;.+\n") LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]?\d*\]" LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" @@ -748,7 +767,8 @@ r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") - r_gcroot_marker = re.compile(r";.+ = pypy_asm_gcroot\(") + r_gcroot_marker = re.compile(r"$1") # never matches + r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot") r_bottom_marker = re.compile(r"\tcall\t_pypy_asm_stack_bottom\s*") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") @@ -759,7 +779,7 @@ def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+r"),\s*(?P"+cls.OPERAND+r")\s*(?:;.+)?$") - cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+LABEL+"\s*$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+cls.LABEL+"\s*$") def __init__(self, lines, filetag=0): self.defines = {} @@ -792,6 +812,7 @@ locals()['visit_' + name] = getattr(FunctionGcRootTracker, 'visit_' + name + 'l') + visit_int = FunctionGcRootTracker.visit_nop visit_npad = FunctionGcRootTracker.visit_nop # probably not GC pointers visit_cdq = FunctionGcRootTracker.visit_nop @@ -802,12 +823,75 @@ except ValueError: return None - def _visit_gcroot_marker(self, line): - assert self.lines[self.currentlineno + 1] == "\n" - assert self.lines[self.currentlineno + 2].startswith("\ttest\tDWORD PTR") - match = self.r_binaryinsn.match(self.lines[self.currentlineno + 2]) - loc = match.group("target") - return InsnGCROOT(self.replace_symbols(loc)) + def _visit_gcroot_marker(self, line=None): + # two possible patterns: + # 1. mov reg, DWORD PTR _always_one_ + # imul target, reg + # 2. mov reg, DWORD PTR _always_one_ + # imul reg, target + assert self.lines[self.currentlineno].startswith("\tmov\t") + mov = self.r_binaryinsn.match(self.lines[self.currentlineno]) + assert re.match("DWORD PTR .+_always_one_", mov.group("source")) + reg = mov.group("target") + + self.lines[self.currentlineno] = ";" + self.lines[self.currentlineno] + + # the 'imul' must appear in the same block; the 'reg' must not + # appear in the instructions between + imul = None + lineno = self.currentlineno + 1 + stop = False + while not stop: + line = self.lines[lineno] + if line == '\n': + stop = True + elif line.startswith("\tjmp\t"): + stop = True + elif self.r_gcroot_marker_var.search(line): + stop = True + elif line.startswith("\tmov\t%s," % (reg,)): + # mov reg, + stop = True + elif line.startswith("\txor\t%s, %s" % (reg, reg)): + # xor reg, reg + stop = True + elif line.startswith("\timul\t"): + imul = self.r_binaryinsn.match(line) + imul_arg1 = imul.group("target") + imul_arg2 = imul.group("source") + break + # the register may not appear in other instructions + elif reg in line: + assert False, (line, lineno) + + lineno += 1 + else: + # No imul, the returned value is not used in this function + return [] + + if reg == imul_arg2: + self.lines[lineno] = ";" + self.lines[lineno] + return InsnGCROOT(self.replace_symbols(imul_arg1)) + else: + assert reg == imul_arg1 + self.lines[lineno] = "\tmov\t%s, %s\n" % (imul_arg1, imul_arg2) + if imul_arg2.startswith('OFFSET '): + # ignore static global variables + pass + else: + self.lines[lineno] += "\t; GCROOT\n" + + return [] + + def insns_for_copy(self, source, target): + if self.r_gcroot_marker_var.match(source): + return self._visit_gcroot_marker() + if self.lines[self.currentlineno].endswith("\t; GCROOT\n"): + insns = [InsnGCROOT(self.replace_symbols(source))] + else: + insns = [] + return insns + super(MsvcFunctionGcRootTracker, self).insns_for_copy(source, target) + MsvcFunctionGcRootTracker.init_regexp() @@ -881,16 +965,32 @@ format = "darwin" FunctionGcRootTracker = DarwinFunctionGcRootTracker + r_textstart = re.compile(r"\t.text\s*$") + + # see + # http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html + OTHERSECTIONS = ['section', 'zerofill', + 'const', 'static_const', 'cstring', + 'literal4', 'literal8', 'literal16', + 'constructor', 'desctructor', + 'symbol_stub', + 'data', 'static_data', + 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', + 'dyld', 'mod_init_func', 'mod_term_func', + 'const_data' + ] + r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + @classmethod def find_functions(cls, iterlines): functionlines = [] in_text = False in_function = False for n, line in enumerate(iterlines): - if r_textstart.match(line): + if cls.r_textstart.match(line): assert not in_text, "unexpected repeated .text start: %d" % n in_text = True - elif r_sectionstart.match(line): + elif cls.r_sectionstart.match(line): if in_function: yield in_function, functionlines functionlines = [] @@ -920,9 +1020,9 @@ in_text = False in_function = False for n, line in enumerate(iterlines): - if r_textstart.match(line): + if cls.r_textstart.match(line): in_text = True - elif r_sectionstart.match(line): + elif cls.r_sectionstart.match(line): in_text = False elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines @@ -966,18 +1066,27 @@ def write_newfile(self, newfile, lines, grist): newlines = [] for line in lines: + # truncate long comments + if line.startswith(";"): + line = line[:-1][:500] + '\n' + # Workaround a bug in the .s files generated by msvc - # compiler: every float constant is exported with a name - # correcponding to its value, and will conflict with other - # modules. - line = line.replace('__real@', - '__%s__real@' % grist) + # compiler: every string or float constant is exported + # with a name built after its value, and will conflict + # with other modules. + if line.startswith("PUBLIC\t__real@"): + line = '; ' + line + elif line.startswith("PUBLIC\t??_C@"): + line = '; ' + line + # Because we insert labels in the code, some "SHORT" jumps # are now longer than 127 bytes. We turn them all into # "NEAR" jumps. Note that the assembler allocates space # for a near jump, but still generates a short jump when # it can. line = line.replace('\tjmp\tSHORT ', '\tjmp\t') + line = line.replace('\tjne\tSHORT ', '\tjne\t') + line = line.replace('\tje\tSHORT ', '\tje\t') newlines.append(line) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Thu Nov 5 18:06:15 2009 @@ -486,9 +486,13 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - sfiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] - lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] - gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] + trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] + if self.translator.platform.name == 'msvc': + trackgcfiles = [f for f in trackgcfiles + if f.startswith(('implement', 'testing'))] + sfiles = ['%s.s' % (c,) for c in trackgcfiles] + lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] + gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] mk.definition('ASMFILES', sfiles) mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) @@ -500,11 +504,20 @@ python = '' if self.translator.platform.name == 'msvc': - lblofiles = ['%s.lbl.obj' % (cfile[:-2],) for cfile in mk.cfiles] + lblofiles = [] + for cfile in mk.cfiles: + f = cfile[:-2] + if f in trackgcfiles: + ofile = '%s.lbl.obj' % (f,) + else: + ofile = '%s.obj' % (f,) + + lblofiles.append(ofile) mk.definition('ASMLBLOBJFILES', lblofiles) mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') - # almost all optimizations implied by /O2, except /Og - mk.definition('ASM_CFLAGS', '$(CFLAGS) /Od /Oi /Ot /Oy /Ob2 /GF /Gy') + # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory + # even in debug builds + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Od /Oi /Ob1') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/mem.h Thu Nov 5 18:06:15 2009 @@ -17,16 +17,18 @@ could make two copies of the local variable (e.g. one in the stack and one in a register), pass one to GCROOT, and later use the other one. In practice the pypy_asm_gcroot() is often a no-op in the final - machine code and doesn't prevent most optimizations. Getting the - asm() right was tricky, though. The asm() is not volatile so that - gcc is free to delete it if the output variable is not used at all. - We need to prevent gcc from moving the asm() *before* the call that - could cause a collection; this is the purpose of the (unused) - __gcnoreorderhack input argument. Any memory input argument would - have this effect: as far as gcc knows the call instruction can modify - arbitrary memory, thus creating the order dependency that we want. */ + machine code and doesn't prevent most optimizations. */ #ifndef _MSC_VER +/* With gcc, getting the asm() right was tricky, though. The asm() is + not volatile so that gcc is free to delete it if the output variable + is not used at all. We need to prevent gcc from moving the asm() + *before* the call that could cause a collection; this is the purpose + of the (unused) __gcnoreorderhack input argument. Any memory input + argument would have this effect: as far as gcc knows the call + instruction can modify arbitrary memory, thus creating the order + dependency that we want. */ + #define pypy_asm_gcroot(p) ({void*_r; \ asm ("/* GCROOT %0 */" : "=g" (_r) : \ "0" (p), "m" (__gcnoreorderhack)); \ @@ -39,13 +41,23 @@ #define pypy_asm_stack_bottom() asm volatile ("/* GC_STACK_BOTTOM */" : : ) #else -/* Microsoft Compiler */ -static __forceinline -void* pypy_asm_gcroot(void* _r1) + +/* With the msvc Microsoft Compiler, the optimizer seems free to move + any code (even asm) that involves local memory (registers and stack). + The _ReadWriteBarrier function has an effect only where the content + of a global variable is *really* used. trackgcroot.py will remove + the extra instructions: the access to _constant_always_one_ is + removed, and the multiplication is replaced with a simple move. */ + +static __forceinline void* +pypy_asm_gcroot(void* _r1) { - __asm test _r1, 0 + static volatile int _constant_always_one_ = 1; + (long)_r1 *= _constant_always_one_; + _ReadWriteBarrier(); return _r1; } + #define pypy_asm_keepalive(v) __asm { } static __declspec(noinline) void pypy_asm_stack_bottom() { } From arigo at codespeak.net Thu Nov 5 18:20:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 18:20:50 +0100 (CET) Subject: [pypy-svn] r68994 - in pypy/trunk/pypy: jit/metainterp translator/c/src translator/c/test Message-ID: <20091105172050.5A1B3168433@codespeak.net> Author: arigo Date: Thu Nov 5 18:20:48 2009 New Revision: 68994 Modified: pypy/trunk/pypy/jit/metainterp/logger.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/translator/c/src/debug.h pypy/trunk/pypy/translator/c/test/test_standalone.py Log: Fix the debugging logic to include nested subsections even if the parent section was disabled. Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Thu Nov 5 18:20:48 2009 @@ -12,8 +12,6 @@ self.guard_number=guard_number def log_loop(self, inputargs, operations, number=0, type=None): - if not have_debug_prints(): - return if type is None: debug_start("jit-log-noopt-loop") self._log_operations(inputargs, operations) @@ -26,8 +24,6 @@ debug_stop("jit-log-opt-loop") def log_bridge(self, inputargs, operations, number=-1): - if not have_debug_prints(): - return if number == -1: debug_start("jit-log-noopt-bridge") self._log_operations(inputargs, operations) Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Thu Nov 5 18:20:48 2009 @@ -257,8 +257,7 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if have_debug_prints(): - dump_storage(storage, liveboxes) + dump_storage(storage, liveboxes) return liveboxes[:] def _number_virtuals(self, liveboxes): @@ -431,27 +430,28 @@ "For profiling only." from pypy.rlib.objectmodel import compute_unique_id debug_start("jit-resume") - debug_print('Log storage', compute_unique_id(storage)) - frameinfo = storage.rd_frame_info_list - while frameinfo is not None: - try: - jitcodename = frameinfo.jitcode.name - except AttributeError: - jitcodename = str(compute_unique_id(frameinfo.jitcode)) - debug_print('\tjitcode/pc', jitcodename, - frameinfo.pc, frameinfo.exception_target, - 'at', compute_unique_id(frameinfo)) - frameinfo = frameinfo.prev - numb = storage.rd_numb - while numb is not None: - debug_print('\tnumb', str([untag(i) for i in numb.nums]), - 'at', compute_unique_id(numb)) - numb = numb.prev - for const in storage.rd_consts: - debug_print('\tconst', const.repr_rpython()) - for box in liveboxes: - debug_print('\tbox', box.repr_rpython()) - if storage.rd_virtuals is not None: - for virtual in storage.rd_virtuals: - virtual.debug_prints() + if have_debug_prints(): + debug_print('Log storage', compute_unique_id(storage)) + frameinfo = storage.rd_frame_info_list + while frameinfo is not None: + try: + jitcodename = frameinfo.jitcode.name + except AttributeError: + jitcodename = str(compute_unique_id(frameinfo.jitcode)) + debug_print('\tjitcode/pc', jitcodename, + frameinfo.pc, frameinfo.exception_target, + 'at', compute_unique_id(frameinfo)) + frameinfo = frameinfo.prev + numb = storage.rd_numb + while numb is not None: + debug_print('\tnumb', str([untag(i) for i in numb.nums]), + 'at', compute_unique_id(numb)) + numb = numb.prev + for const in storage.rd_consts: + debug_print('\tconst', const.repr_rpython()) + for box in liveboxes: + debug_print('\tbox', box.repr_rpython()) + if storage.rd_virtuals is not None: + for virtual in storage.rd_virtuals: + virtual.debug_prints() debug_stop("jit-resume") Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Thu Nov 5 18:20:48 2009 @@ -21,12 +21,12 @@ /* macros used by the generated code */ -#define PYPY_HAVE_DEBUG_PRINTS (pypy_ignoring_nested_prints ? 0 : \ - (pypy_debug_ensure_opened(), 1)) +#define PYPY_HAVE_DEBUG_PRINTS (pypy_have_debug_prints & 1 ? \ + (pypy_debug_ensure_opened(), 1) : 0) #define PYPY_DEBUG_FILE pypy_debug_file #define PYPY_DEBUG_START(cat) pypy_debug_start(cat) #define PYPY_DEBUG_STOP(cat) pypy_debug_stop(cat) -#define OP_HAVE_DEBUG_PRINTS(r) r = !pypy_ignoring_nested_prints +#define OP_HAVE_DEBUG_PRINTS(r) r = (pypy_have_debug_prints & 1) /************************************************************/ @@ -36,7 +36,7 @@ void pypy_debug_start(const char *category); void pypy_debug_stop(const char *category); -extern int pypy_ignoring_nested_prints; +extern long pypy_have_debug_prints; extern FILE *pypy_debug_file; @@ -45,7 +45,7 @@ #ifndef PYPY_NOT_MAIN_FILE #include -int pypy_ignoring_nested_prints = 0; +long pypy_have_debug_prints = -1; FILE *pypy_debug_file = NULL; static bool_t debug_ready = 0; static bool_t debug_profile = 0; @@ -139,39 +139,30 @@ void pypy_debug_start(const char *category) { pypy_debug_ensure_opened(); - if (debug_profile) - { - /* profiling version */ - pypy_ignoring_nested_prints++; /* disable nested debug_print */ - } - else + /* Enter a nesting level. Nested debug_prints are disabled by default + because the following left shift introduces a 0 in the last bit. + Note that this logic assumes that we are never going to nest + debug_starts more than 31 levels (63 on 64-bits). */ + pypy_have_debug_prints <<= 1; + if (!debug_profile) { /* non-profiling version */ - if (pypy_ignoring_nested_prints > 0) - { - /* already ignoring the parent section */ - pypy_ignoring_nested_prints++; - return; - } if (!debug_prefix || !startswith(category, debug_prefix)) { /* wrong section name, or no PYPYLOG at all, skip it */ - pypy_ignoring_nested_prints = 1; return; } + /* else make this subsection active */ + pypy_have_debug_prints |= 1; } display_startstop("{", "", category, debug_start_colors_1); } void pypy_debug_stop(const char *category) { - if (pypy_ignoring_nested_prints > 0) - { - pypy_ignoring_nested_prints--; - if (!debug_profile) - return; - } - display_startstop("", "}", category, debug_start_colors_2); + if (debug_profile | (pypy_have_debug_prints & 1)) + display_startstop("", "}", category, debug_start_colors_2); + pypy_have_debug_prints >>= 1; } #endif /* PYPY_NOT_MAIN_FILE */ Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_standalone.py (original) +++ pypy/trunk/pypy/translator/c/test/test_standalone.py Thu Nov 5 18:20:48 2009 @@ -353,15 +353,15 @@ path = udir.join('test_debug_xxx_cat.log') out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': 'cat:%s' % path}) - assert out.strip() == 'got:a.' + assert out.strip() == 'got:ca.' assert not err assert path.check(file=1) data = path.read() assert 'toplevel' in path.read() assert 'mycat' not in path.read() assert 'foo 2 bar 3' not in path.read() - assert 'cat2' not in data # because it is nested - assert 'baz' not in data + assert 'cat2' in data + assert 'baz' in data assert 'bok' not in data # # finally, check compiling with logging disabled From arigo at codespeak.net Thu Nov 5 18:29:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 18:29:13 +0100 (CET) Subject: [pypy-svn] r68995 - pypy/trunk/pypy/doc Message-ID: <20091105172913.43343168436@codespeak.net> Author: arigo Date: Thu Nov 5 18:29:12 2009 New Revision: 68995 Modified: pypy/trunk/pypy/doc/cpython_differences.txt Log: Kill the new paragraph; rephrase and move its content closer to the start, which was already explaining part of it. Modified: pypy/trunk/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/trunk/pypy/doc/cpython_differences.txt (original) +++ pypy/trunk/pypy/doc/cpython_differences.txt Thu Nov 5 18:29:12 2009 @@ -107,6 +107,15 @@ adopted by Jython or IronPython (or any other port of Python to Java or .NET, like PyPy itself). +This affects the precise time at which __del__ methods are called, which +is not reliable in PyPy (nor Jython nor IronPython). It also means that +weak references may stay alive for a bit longer than expected. This +makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less +useful: they will appear to stay alive for a bit longer in PyPy, and +suddenly they will really be dead, raising a ``ReferenceError`` on the +next access. Any code that uses weak proxies must carefully catch such +``ReferenceError`` at any place that uses them. + There are a few extra implications for the difference in the GC. Most notably, if an object has a __del__, the __del__ is never called more than once in PyPy; but CPython will call the same __del__ several times @@ -120,17 +129,6 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html -Note that the time at which __del__ is called is not well-defined in any -implementation apart from CPython. A __del__ method may appear to be -called sometimes later in PyPy; for example, a file may stay open for a -bit longer, which can have visible effects (e.g. a file opened for -writing might still contain data not flushed yet). This also makes -"weak proxies" less useful (see ``weakref.proxy()``). They will appear -to stay alive a bit longer in PyPy, and suddenly they will really be -dead, raising a ``ReferenceError`` on the next access. Any code that -uses weak proxies must carefully catch such ``ReferenceError`` at any -place that uses them. - The built-in function ``id()`` returns numbers that are not addresses for most of PyPy's garbage collectors. This is most visible in the default repr: a typical PyPy object can From arigo at codespeak.net Thu Nov 5 18:42:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 18:42:01 +0100 (CET) Subject: [pypy-svn] r68997 - pypy/trunk/pypy/doc Message-ID: <20091105174201.51264168439@codespeak.net> Author: arigo Date: Thu Nov 5 18:42:00 2009 New Revision: 68997 Modified: pypy/trunk/pypy/doc/docindex.txt Log: Link to cpython_differences from here. Modified: pypy/trunk/pypy/doc/docindex.txt ============================================================================== --- pypy/trunk/pypy/doc/docindex.txt (original) +++ pypy/trunk/pypy/doc/docindex.txt Thu Nov 5 18:42:00 2009 @@ -21,6 +21,7 @@ New features of PyPy's Python Interpreter and Translation Framework: + * `Differences between PyPy and CPython`_ * `What PyPy can do for your objects`_ * `Stackless and coroutines`_ * `JIT Generation in PyPy`_ @@ -296,6 +297,7 @@ .. _`taint object space`: objspace-proxies.html#taint .. _`thunk object space`: objspace-proxies.html#thunk .. _`transparent proxies`: objspace-proxies.html#tproxy +.. _`Differences between PyPy and CPython`: cpython_differences.html .. _`What PyPy can do for your objects`: objspace-proxies.html .. _`Stackless and coroutines`: stackless.html .. _StdObjSpace: objspace.html#the-standard-object-space From arigo at codespeak.net Thu Nov 5 18:45:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 18:45:28 +0100 (CET) Subject: [pypy-svn] r68998 - pypy/trunk/pypy/jit/metainterp Message-ID: <20091105174528.B3CB6168439@codespeak.net> Author: arigo Date: Thu Nov 5 18:45:28 2009 New Revision: 68998 Modified: pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/resoperation.py Log: Kill the "XXX temp", now that it's clear that the corresponding attributes are not used any more. Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Thu Nov 5 18:45:28 2009 @@ -685,7 +685,6 @@ inputargs = None operations = None token = None - specnodes = property(lambda x: crash, lambda x, y: crash) # XXX temp def __init__(self, name): self.name = name Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Thu Nov 5 18:45:28 2009 @@ -4,12 +4,7 @@ class ResOperation(object): """The central ResOperation class, representing one operation.""" - # for 'jump': points to the target loop; - jump_target = property(lambda x: crash, lambda x, y: crash) # XXX temp - # for 'guard_*' - suboperations = property(lambda x: crash, lambda x, y: crash) # XXX temp - optimized = property(lambda x: crash, lambda x, y: crash) # XXX temp fail_args = None # debug From arigo at codespeak.net Thu Nov 5 19:01:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 19:01:12 +0100 (CET) Subject: [pypy-svn] r68999 - pypy/branch/gc-dump-malloc Message-ID: <20091105180112.558B8168433@codespeak.net> Author: arigo Date: Thu Nov 5 19:01:11 2009 New Revision: 68999 Added: pypy/branch/gc-dump-malloc/ - copied from r68998, pypy/trunk/ Log: A branch to try to add a few counters. For now they will just stay on that branch. From arigo at codespeak.net Thu Nov 5 19:18:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 19:18:21 +0100 (CET) Subject: [pypy-svn] r69000 - in pypy/branch/gc-dump-malloc/pypy/rpython/memory: . gc Message-ID: <20091105181821.DCD53168436@codespeak.net> Author: arigo Date: Thu Nov 5 19:18:21 2009 New Revision: 69000 Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/base.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Log: Add a counter on all allocations, by typeid. What is left: find a way to display that information when the process exits... Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/base.py Thu Nov 5 19:18:21 2009 @@ -48,7 +48,8 @@ varsize_offset_to_variable_part, varsize_offset_to_length, varsize_offsets_to_gcpointers_in_var_part, - weakpointer_offset): + weakpointer_offset, + count_allocation): self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize @@ -60,6 +61,7 @@ self.varsize_offset_to_length = varsize_offset_to_length self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part self.weakpointer_offset = weakpointer_offset + self.count_allocation = count_allocation def set_root_walker(self, root_walker): self.root_walker = root_walker Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py Thu Nov 5 19:18:21 2009 @@ -428,6 +428,7 @@ def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.tid = self.combine(typeid16, flags) + self.count_allocation(typeid16) def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Thu Nov 5 19:18:21 2009 @@ -24,6 +24,7 @@ ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), + ("counter", lltype.Signed), hints={'immutable': True}, ) VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", @@ -93,6 +94,9 @@ return weakptr_offset return -1 + def q_count_allocation(self, typeid): + self.get(typeid).counter += 1 + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -105,7 +109,8 @@ self.q_varsize_offset_to_variable_part, self.q_varsize_offset_to_length, self.q_varsize_offsets_to_gcpointers_in_var_part, - self.q_weakpointer_offset) + self.q_weakpointer_offset, + self.q_count_allocation) T_IS_VARSIZE = 0x01 @@ -160,6 +165,7 @@ if TYPE == WEAKREF: infobits |= T_IS_WEAKREF info.infobits = infobits + info.counter = 0 # ____________________________________________________________ From arigo at codespeak.net Thu Nov 5 20:05:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 20:05:00 +0100 (CET) Subject: [pypy-svn] r69001 - in pypy/branch/gc-dump-malloc/pypy: rpython/lltypesystem rpython/memory translator/c translator/c/src Message-ID: <20091105190500.43497168430@codespeak.net> Author: arigo Date: Thu Nov 5 20:04:59 2009 New Revision: 69001 Modified: pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/llgroup.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-dump-malloc/pypy/translator/c/node.py pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h Log: Quick hacks to print the result at the end of the C program. Modified: pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/llgroup.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/llgroup.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/llgroup.py Thu Nov 5 20:04:59 2009 @@ -21,8 +21,9 @@ def __init__(self, name): self.name = name self.members = [] + self.extrainfos = [] - def add_member(self, structptr): + def add_member(self, structptr, extrainfo=None): TYPE = lltype.typeOf(structptr) assert isinstance(TYPE.TO, lltype.Struct) assert TYPE.TO._gckind == 'raw' @@ -34,6 +35,7 @@ assert struct._parentstructure() is None index = len(self.members) self.members.append(struct) + self.extrainfos.append(extrainfo) _membership[struct] = self return GroupMemberOffset(self, index) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Thu Nov 5 20:04:59 2009 @@ -20,11 +20,11 @@ # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", + ("counter", lltype.Signed), ("infobits", lltype.Signed), # combination of the T_xxx consts ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - ("counter", lltype.Signed), hints={'immutable': True}, ) VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", @@ -223,7 +223,7 @@ else: self._pending_type_shapes.append((info, TYPE)) # store it - type_id = self.type_info_group.add_member(fullinfo) + type_id = self.type_info_group.add_member(fullinfo, str(TYPE)) self.id_of_type[TYPE] = type_id self.add_vtable_after_typeinfo(TYPE) return type_id Modified: pypy/branch/gc-dump-malloc/pypy/translator/c/node.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/translator/c/node.py (original) +++ pypy/branch/gc-dump-malloc/pypy/translator/c/node.py Thu Nov 5 20:04:59 2009 @@ -992,6 +992,16 @@ structnode = self.db.getcontainernode(member) yield '#define %s %s.member%d' % (structnode.name, self.name, i) + # + from pypy.translator.c.support import c_string_constant + yield '#define DUMP_GROUP_INFO \\' + for i, extrainfo in enumerate(self.obj.extrainfos): + if extrainfo: + text = '\tdump_group_info(%s, (long*)(&%s.member%d)); \\' % ( + c_string_constant(extrainfo), self.name, i) + yield text.replace('\n', '\\\n') + yield '\t;' + # yield '' def initializationexpr(self): Modified: pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h (original) +++ pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h Thu Nov 5 20:04:59 2009 @@ -15,6 +15,12 @@ #ifndef PYPY_NOT_MAIN_FILE +void dump_group_info(const char *typename, long *counter) +{ + if (*counter) + fprintf(stderr, "|%12lu %s\n", (unsigned long)*counter, typename); +} + int main(int argc, char *argv[]) { char *errmsg; @@ -35,6 +41,7 @@ } exitcode = STANDALONE_ENTRY_POINT(list); + DUMP_GROUP_INFO if (RPyExceptionOccurred()) { /* fish for the exception type, at least */ #ifndef AVR From magcius at codespeak.net Thu Nov 5 20:27:33 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 5 Nov 2009 20:27:33 +0100 (CET) Subject: [pypy-svn] r69002 - in pypy/branch/avm/pypy: . interpreter interpreter/test module objspace/std objspace/std/test Message-ID: <20091105192733.32E61318137@codespeak.net> Author: magcius Date: Thu Nov 5 20:27:32 2009 New Revision: 69002 Added: pypy/branch/avm/pypy/TODO - copied unchanged from r4823, pypy/trunk/src/pypy/TODO pypy/branch/avm/pypy/interpreter/autopath.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/autopath.py pypy/branch/avm/pypy/interpreter/extmodule.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/extmodule.py pypy/branch/avm/pypy/interpreter/py.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/py.py pypy/branch/avm/pypy/interpreter/test/autopath.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/test/autopath.py pypy/branch/avm/pypy/interpreter/test/foomodule.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/test/foomodule.py pypy/branch/avm/pypy/interpreter/test/test_extmodule.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/test/test_extmodule.py pypy/branch/avm/pypy/interpreter/unittest_w.py - copied unchanged from r4823, pypy/trunk/src/pypy/interpreter/unittest_w.py pypy/branch/avm/pypy/module/__builtin__interp.py - copied unchanged from r4823, pypy/trunk/src/pypy/module/__builtin__interp.py pypy/branch/avm/pypy/module/__builtin__module.py - copied unchanged from r4823, pypy/trunk/src/pypy/module/__builtin__module.py pypy/branch/avm/pypy/module/sysinterp.py - copied unchanged from r4823, pypy/trunk/src/pypy/module/sysinterp.py pypy/branch/avm/pypy/module/sysmodule.py - copied unchanged from r4823, pypy/trunk/src/pypy/module/sysmodule.py pypy/branch/avm/pypy/objspace/std/test/broken_test_floatobject.py - copied unchanged from r4823, pypy/trunk/src/pypy/objspace/std/test/broken_test_floatobject.py pypy/branch/avm/pypy/objspace/std/userobject.py - copied unchanged from r4823, pypy/trunk/src/pypy/objspace/std/userobject.py Modified: pypy/branch/avm/pypy/ (props changed) pypy/branch/avm/pypy/interpreter/ (props changed) pypy/branch/avm/pypy/interpreter/baseobjspace.py pypy/branch/avm/pypy/interpreter/executioncontext.py pypy/branch/avm/pypy/interpreter/interactive.py pypy/branch/avm/pypy/interpreter/pyframe.py pypy/branch/avm/pypy/interpreter/test/test_interpreter.py pypy/branch/avm/pypy/objspace/std/intobject.py pypy/branch/avm/pypy/objspace/std/listobject.py pypy/branch/avm/pypy/objspace/std/noneobject.py pypy/branch/avm/pypy/objspace/std/sliceobject.py pypy/branch/avm/pypy/objspace/std/stringobject.py pypy/branch/avm/pypy/objspace/std/test/test_multimethod.py Log: Finally. More work on AVM2. Modified: pypy/branch/avm/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/avm/pypy/interpreter/baseobjspace.py Thu Nov 5 20:27:32 2009 @@ -1,594 +1,130 @@ -from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag -from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction +from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.error import OperationError -from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack -from pypy.interpreter.pycompiler import CPythonCompiler, PythonAstCompiler -from pypy.interpreter.miscutils import ThreadLocals -from pypy.tool.cache import Cache -from pypy.tool.uid import HUGEVAL_BYTES -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.timer import DummyTimer, Timer -import os, sys - -__all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] - - -class W_Root(object): - """This is the abstract root class of all wrapped objects that live - in a 'normal' object space like StdObjSpace.""" - __slots__ = () - _settled_ = True - - def getdict(self): - return None - - def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) - - def getdictvalue(self, space, w_attr): - w_dict = self.getdict() - if w_dict is not None: - return space.finditem(w_dict, w_attr) - return None - - def getdictvalue_attr_is_in_class(self, space, w_attr): - return self.getdictvalue(space, w_attr) - - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): - w_dict = self.getdict() - if w_dict is not None: - space.set_str_keyed_item(w_dict, w_attr, w_value, shadows_type) - return True - return False +from pypy.interpreter.miscutils import Stack, getthreadlocals +import pypy.module - def deldictvalue(self, space, w_name): - w_dict = self.getdict() - if w_dict is not None: - try: - space.delitem(w_dict, w_name) - return True - except OperationError, ex: - if not ex.match(space, space.w_KeyError): - raise - return False +__all__ = ['ObjSpace', 'OperationError', 'NoValue'] - def setdict(self, space, w_dict): - typename = space.type(self).getname(space, '?') - raise OperationError(space.w_TypeError, - space.wrap("attribute '__dict__' of %s objects " - "is not writable" % typename)) - - # to be used directly only by space.type implementations - def getclass(self, space): - return space.gettypeobject(self.typedef) - - def setclass(self, space, w_subtype): - raise OperationError(space.w_TypeError, - space.wrap("__class__ assignment: only for heap types")) - def user_setup(self, space, w_subtype): - assert False, "only for interp-level user subclasses from typedef.py" +class Wrappable(object): + """A subclass of Wrappable is an internal, interpreter-level class + that can nevertheless be exposed at application-level by space.wrap().""" - def getname(self, space, default): + def get_wdict(self): + space = self.space try: - return space.str_w(space.getattr(self, space.wrap('__name__'))) - except OperationError, e: - if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError): - return default - raise - - def getaddrstring(self, space): - # XXX slowish - w_id = space.id(self) - w_4 = space.wrap(4) - w_0x0F = space.wrap(0x0F) - i = 2 * HUGEVAL_BYTES - addrstring = [' '] * i - while True: - n = space.int_w(space.and_(w_id, w_0x0F)) - n += ord('0') - if n > ord('9'): - n += (ord('a') - ord('9') - 1) - i -= 1 - addrstring[i] = chr(n) - if i == 0: - break - w_id = space.rshift(w_id, w_4) - return ''.join(addrstring) - - def getrepr(self, space, info, moreinfo=''): - addrstring = self.getaddrstring(space) - return space.wrap("<%s at 0x%s%s>" % (info, addrstring, - moreinfo)) - - def getslotvalue(self, index): - raise NotImplementedError + return self.w_dict + except AttributeError: + w_dict = self.w_dict = space.newdict([]) + for name,w_value in self.app_visible(): + space.setitem(w_dict,space.wrap(name),w_value) + return w_dict - def setslotvalue(self, index, w_val): + def app_visible(self): + """ returns [(name,w_value)...] for application-level visible attributes """ raise NotImplementedError - def descr_call_mismatch(self, space, opname, RequiredClass, args): - if RequiredClass is None: - classname = '?' - else: - classname = wrappable_class_name(RequiredClass) - msg = "'%s' object expected, got '%s' instead" % ( - classname, self.getclass(space).getname(space, '?')) - raise OperationError(space.w_TypeError, space.wrap(msg)) - - # used by _weakref implemenation - - def getweakref(self): - return None - - def setweakref(self, space, weakreflifeline): - typename = space.type(self).getname(space, '?') - raise OperationError(space.w_TypeError, space.wrap( - "cannot create weak reference to '%s' object" % typename)) - - def clear_all_weakrefs(self): - """Call this at the beginning of interp-level __del__() methods - in subclasses. It ensures that weakrefs (if any) are cleared - before the object is further destroyed. - """ - lifeline = self.getweakref() - if lifeline is not None: - # Clear all weakrefs to this object before we proceed with - # the destruction of the object. We detach the lifeline - # first: if the code following before_del() calls the - # app-level, e.g. a user-defined __del__(), and this code - # tries to use weakrefs again, it won't reuse the broken - # (already-cleared) weakrefs from this lifeline. - self.setweakref(lifeline.space, None) - lifeline.clear_all_weakrefs() - - __already_enqueued_for_destruction = False - - def _enqueue_for_destruction(self, space): - """Put the object in the destructor queue of the space. - At a later, safe point in time, UserDelAction will use - space.userdel() to call the object's app-level __del__ method. - """ - # this function always resurect the object, so when - # running on top of CPython we must manually ensure that - # we enqueue it only once - if not we_are_translated(): - if self.__already_enqueued_for_destruction: - return - self.__already_enqueued_for_destruction = True - self.clear_all_weakrefs() - space.user_del_action.register_dying_object(self) - - def _call_builtin_destructor(self): - pass # method overridden in typedef.py - + def pypy_getattr(self, w_name): + space = self.space + w_dict = self.get_wdict() + try: + return space.getitem(w_dict,w_name) + except OperationError,e: + if not e.match(space,space.w_KeyError): + raise + raise OperationError(space.w_AttributeError,w_name) -class Wrappable(W_Root): - """A subclass of Wrappable is an internal, interpreter-level class - that can nevertheless be exposed at application-level by space.wrap().""" - __slots__ = () - _settled_ = True - def __spacebind__(self, space): - return self +class NoValue(Exception): + """Raised to signal absence of value, e.g. in the iterator accessing + method 'op.next()' of object spaces.""" -class InternalSpaceCache(Cache): - """A generic cache for an object space. Arbitrary information can - be attached to the space by defining a function or class 'f' which - can be called as 'f(space)'. Its result is stored in this - ObjSpaceCache. - """ - def __init__(self, space): - Cache.__init__(self) - self.space = space - def _build(self, callable): - return callable(self.space) - -class SpaceCache(Cache): - """A base class for all our concrete caches.""" - def __init__(self, space): - Cache.__init__(self) - self.space = space - def _build(self, key): - val = self.space.enter_cache_building_mode() - try: - return self.build(key) - finally: - self.space.leave_cache_building_mode(val) - def _ready(self, result): - val = self.space.enter_cache_building_mode() - try: - return self.ready(result) - finally: - self.space.leave_cache_building_mode(val) - def ready(self, result): - pass - -class UnpackValueError(ValueError): - def __init__(self, msg): - self.msg = msg - def __str__(self): - return self.msg - -class DescrMismatch(Exception): - pass - -def wrappable_class_name(Class): - try: - return Class.typedef.name - except AttributeError: - return 'internal subclass of %s' % (Class.__name__,) -wrappable_class_name._annspecialcase_ = 'specialize:memo' -# ____________________________________________________________ - -class ObjSpace(object): +class ObjSpace: """Base class for the interpreter-level implementations of object spaces. - http://codespeak.net/pypy/dist/pypy/doc/objspace.html""" - + http://codespeak.net/moin/pypy/moin.cgi/ObjectSpace""" + full_exceptions = True # full support for exceptions (normalization & more) - def __init__(self, config=None): - "NOT_RPYTHON: Basic initialization of objects." - self.fromcache = InternalSpaceCache(self).getorbuild - self.threadlocals = ThreadLocals() - # set recursion limit - # sets all the internal descriptors - if config is None: - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=False) - self.config = config - - # import extra modules for side-effects - import pypy.interpreter.nestedscope # register *_DEREF bytecodes - - self.interned_strings = {} - self.actionflag = ActionFlag() # changed by the signal module - self.user_del_action = UserDelAction(self) - self.frame_trace_action = FrameTraceAction(self) - self.actionflag.register_action(self.user_del_action) - self.actionflag.register_action(self.frame_trace_action) - - from pypy.interpreter.pyframe import PyFrame - self.FrameClass = PyFrame # can be overridden to a subclass - - if self.config.objspace.logbytecodes: - self.bytecodecounts = [0] * 256 - self.bytecodetransitioncount = {} - - if self.config.objspace.timing: - self.timer = Timer() - else: - self.timer = DummyTimer() - + def __init__(self): + "Basic initialization of objects." self.initialize() - def startup(self): - # To be called before using the space - - # Initialize all builtin modules - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): - import time - self.timer.start("startup " + modname) - mod.startup(self) - self.timer.stop("startup " + modname) - - def finish(self): - w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') - if w_exitfunc is not None: - self.call_function(w_exitfunc) - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable( - self.sys.get('builtin_module_names')): - modname = self.str_w(w_modname) - mod = self.interpclass_w(self.getbuiltinmodule(modname)) - if isinstance(mod, Module): - mod.shutdown(self) - if self.config.objspace.std.withdictmeasurement: - from pypy.objspace.std.dictmultiobject import report - report() - if self.config.objspace.logbytecodes: - self.reportbytecodecounts() - if self.config.objspace.std.logspaceoptypes: - for s in self.FrameClass._space_op_types: - print s - - def reportbytecodecounts(self): - os.write(2, "Starting bytecode report.\n") - fd = os.open('bytecode.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) - os.write(fd, "bytecodecounts = {\n") - for opcode in range(len(self.bytecodecounts)): - count = self.bytecodecounts[opcode] - if not count: - continue - os.write(fd, " %s: %s,\n" % (opcode, count)) - os.write(fd, "}\n") - os.write(fd, "bytecodetransitioncount = {\n") - for opcode, probs in self.bytecodetransitioncount.iteritems(): - os.write(fd, " %s: {\n" % (opcode, )) - for nextcode, count in probs.iteritems(): - os.write(fd, " %s: %s,\n" % (nextcode, count)) - os.write(fd, " },\n") - os.write(fd, "}\n") - os.close(fd) - os.write(2, "Reporting done.\n") - - def __repr__(self): - try: - return self._this_space_repr_ - except AttributeError: - return self.__class__.__name__ - - def setbuiltinmodule(self, importname): - """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules""" - import sys - - fullname = "pypy.module.%s" % importname - - Module = __import__(fullname, - None, None, ["Module"]).Module - if Module.applevel_name is not None: - name = Module.applevel_name - else: - name = importname - - w_name = self.wrap(name) - w_mod = self.wrap(Module(self, w_name)) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, w_mod) - return name - - def getbuiltinmodule(self, name): - w_name = self.wrap(name) - w_modules = self.sys.get('modules') - return self.getitem(w_modules, w_name) - - def get_builtinmodule_to_install(self): - """NOT_RPYTHON""" - try: - return self._builtinmodule_list - except AttributeError: - pass - - modules = [] - - # You can enable more modules by specifying --usemodules=xxx,yyy - for name, value in self.config.objspace.usemodules: - if value and name not in modules: - modules.append(name) - - # a bit of custom logic: time2 or rctime take precedence over time - # XXX this could probably be done as a "requires" in the config - if ('time2' in modules or 'rctime' in modules) and 'time' in modules: - modules.remove('time') - - import pypy - if not self.config.objspace.nofaking: - for modname in self.ALL_BUILTIN_MODULES: - if not (os.path.exists( - os.path.join(os.path.dirname(pypy.__file__), - 'lib', modname+'.py'))): - modules.append('faked+'+modname) - - self._builtinmodule_list = modules - return self._builtinmodule_list - - ALL_BUILTIN_MODULES = [ - 'posix', 'nt', 'os2', 'mac', 'ce', 'riscos', - 'math', 'array', 'select', - '_random', '_sre', 'time', '_socket', 'errno', - 'unicodedata', - 'parser', 'fcntl', '_codecs', 'binascii' - ] - def make_builtins(self): - "NOT_RPYTHON: only for initializing the space." - - from pypy.module.sys import Module - w_name = self.wrap('sys') - self.sys = Module(self, w_name) - w_modules = self.sys.get('modules') - self.setitem(w_modules, w_name, self.wrap(self.sys)) - - from pypy.module.__builtin__ import Module - w_name = self.wrap('__builtin__') - self.builtin = Module(self, w_name) - w_builtin = self.wrap(self.builtin) - self.setitem(w_modules, w_name, w_builtin) - self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) - - bootstrap_modules = ['sys', '__builtin__', 'exceptions'] - installed_builtin_modules = bootstrap_modules[:] + # initializing builtins may require creating a frame which in + # turn already accesses space.w_builtins, provide a dummy one ... + self.w_builtins = self.newdict([]) + + assert not hasattr(self, 'builtin') + if not hasattr(self, 'sys'): + self.make_sys() + + from pypy.interpreter.extmodule import BuiltinModule + + # the builtins are iteratively initialized + self.builtin = BuiltinModule(self, '__builtin__', self.w_builtins) + self.w_builtin = self.wrap(self.builtin) # initialize with "bootstrap types" from objspace (e.g. w_None) for name, value in self.__dict__.items(): - if name.startswith('w_') and not name.endswith('Type'): + if name.startswith('w_'): name = name[2:] + if name.startswith('builtin') or name.startswith('sys'): + continue #print "setitem: space instance %-20s into builtins" % name - self.setitem(self.builtin.w_dict, self.wrap(name), value) + self.setitem(self.w_builtins, self.wrap(name), value) - # install mixed and faked modules and set builtin_module_names on sys - for mixedname in self.get_builtinmodule_to_install(): - if (mixedname not in bootstrap_modules - and not mixedname.startswith('faked+')): - self.install_mixedmodule(mixedname, installed_builtin_modules) - for mixedname in self.get_builtinmodule_to_install(): - if mixedname.startswith('faked+'): - modname = mixedname[6:] - self.install_faked_module(modname, installed_builtin_modules) - - installed_builtin_modules.sort() - w_builtin_module_names = self.newtuple( - [self.wrap(fn) for fn in installed_builtin_modules]) - - # force this value into the dict without unlazyfying everything - self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'), - w_builtin_module_names) - - def install_mixedmodule(self, mixedname, installed_builtin_modules): - """NOT_RPYTHON""" - modname = self.setbuiltinmodule(mixedname) - if modname: - assert modname not in installed_builtin_modules, ( - "duplicate interp-level module enabled for the " - "app-level module %r" % (modname,)) - installed_builtin_modules.append(modname) - - def load_cpython_module(self, modname): - "NOT_RPYTHON. Steal a module from CPython." - cpy_module = __import__(modname, {}, {}, ['*']) - return cpy_module - - def install_faked_module(self, modname, installed_builtin_modules): - """NOT_RPYTHON""" - if modname in installed_builtin_modules: - return - try: - module = self.load_cpython_module(modname) - except ImportError: - return - else: - w_modules = self.sys.get('modules') - self.setitem(w_modules, self.wrap(modname), self.wrap(module)) - installed_builtin_modules.append(modname) - - def setup_builtin_modules(self): - "NOT_RPYTHON: only for initializing the space." - from pypy.interpreter.module import Module - for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): - modname = self.unwrap(w_modname) - mod = self.getbuiltinmodule(modname) - if isinstance(mod, Module): - mod.setup_after_space_initialization() + self.sys.setbuiltinmodule(self.w_builtin, '__builtin__') - def initialize(self): - """NOT_RPYTHON: Abstract method that should put some minimal - content into the w_builtins.""" + def make_sys(self): + from pypy.interpreter.extmodule import BuiltinModule + assert not hasattr(self, 'sys') + self.sys = BuiltinModule(self, 'sys') + self.w_sys = self.wrap(self.sys) + self.sys.setbuiltinmodule(self.w_sys, 'sys') + + def get_builtin_module(self, name): + if name not in self.sys.builtin_modules: + return None + module = self.sys.builtin_modules[name] + if module is None: + from pypy.interpreter.extmodule import BuiltinModule + module = BuiltinModule(self, name) + self.sys.builtin_modules[name] = module + w_module = self.wrap(module) + self.sys.setbuiltinmodule(w_module, name) + return w_module - def enter_cache_building_mode(self): - "hook for the flow object space" - def leave_cache_building_mode(self, val): - "hook for the flow object space" + def initialize(self): + """Abstract method that should put some minimal content into the + w_builtins.""" def getexecutioncontext(self): "Return what we consider to be the active execution context." - # Important: the annotator must not see a prebuilt ExecutionContext - # for reasons related to the specialization of the framestack attribute - # so we make sure that the threadlocals never *have* an - # ExecutionContext during translation. - if self.config.translating and not we_are_translated(): - assert self.threadlocals.getvalue() is None, ( - "threadlocals got an ExecutionContext during translation!") - try: - return self._ec_during_translation - except AttributeError: - ec = self.createexecutioncontext() - self._ec_during_translation = ec - return ec - # normal case follows. The 'thread' module installs a real - # thread-local object in self.threadlocals, so this builds - # and caches a new ec in each thread. - ec = self.threadlocals.getvalue() + ec = getthreadlocals().executioncontext #it's allways None (dec. 2003) if ec is None: ec = self.createexecutioncontext() - self.threadlocals.setvalue(ec) return ec - def _freeze_(self): - return True - def createexecutioncontext(self): "Factory function for execution contexts." return ExecutionContext(self) - def createcompiler(self): - "Factory function creating a compiler object." - # XXX simple selection logic for now - try: - return self.default_compiler - except AttributeError: - if self.config.objspace.compiler == 'cpython': - compiler = CPythonCompiler(self) - elif self.config.objspace.compiler == 'ast': - compiler = PythonAstCompiler(self) - else: - raise ValueError('unknown --compiler option value: %r' % ( - self.config.objspace.compiler,)) - self.default_compiler = compiler - return compiler - - def createframe(self, code, w_globals, closure=None): - "Create an empty PyFrame suitable for this code object." - return self.FrameClass(self, code, w_globals, closure) - - def allocate_lock(self): - """Return an interp-level Lock object if threads are enabled, - and a dummy object if they are not.""" - if self.config.objspace.usemodules.thread: - # we use a sub-function to avoid putting the 'import' statement - # here, where the flow space would see it even if thread=False - return self.__allocate_lock() - else: - return dummy_lock - - def __allocate_lock(self): - from pypy.module.thread.ll_thread import allocate_lock, error - try: - return allocate_lock() - except error: - raise OperationError(self.w_RuntimeError, - self.wrap("out of resources")) - # Following is a friendly interface to common object space operations # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. - #def is_(self, w_x, w_y): -- not really useful. Must be subclassed - # "'x is y'." - # w_id_x = self.id(w_x) - # w_id_y = self.id(w_y) - # return self.eq(w_id_x, w_id_y) - - def not_(self, w_obj): - return self.wrap(not self.is_true(w_obj)) - - def eq_w(self, w_obj1, w_obj2): - """shortcut for space.is_true(space.eq(w_obj1, w_obj2))""" - return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2)) - - def is_w(self, w_obj1, w_obj2): - """shortcut for space.is_true(space.is_(w_obj1, w_obj2))""" - return self.is_true(self.is_(w_obj1, w_obj2)) - - def hash_w(self, w_obj): - """shortcut for space.int_w(space.hash(w_obj))""" - return self.int_w(self.hash(w_obj)) - - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): - return self.setitem(w_obj, w_key, w_value) - - def finditem(self, w_obj, w_key): - try: - return self.getitem(w_obj, w_key) - except OperationError, e: - if e.match(self, self.w_KeyError): - return None - raise - - def findattr(self, w_object, w_name): - try: - return self.getattr(w_object, w_name) - except OperationError, e: - # a PyPy extension: let SystemExit and KeyboardInterrupt go through - if e.async(self): - raise - return None + def is_(self, w_x, w_y): + "'x is y'." + w_id_x = self.id(w_x) + w_id_y = self.id(w_y) + return self.eq(w_id_x, w_id_y) + + def unwrapdefault(self, w_value, default): + if w_value is None or w_value == self.w_None: + return default + else: + return self.unwrap(w_value) def newbool(self, b): if b: @@ -596,462 +132,78 @@ else: return self.w_False - def new_interned_w_str(self, w_s): - s = self.str_w(w_s) - try: - return self.interned_strings[s] - except KeyError: - pass - self.interned_strings[s] = w_s - return w_s - - def new_interned_str(self, s): - try: - return self.interned_strings[s] - except KeyError: - pass - w_s = self.interned_strings[s] = self.wrap(s) - return w_s - - def interpclass_w(space, w_obj): - """ - If w_obj is a wrapped internal interpreter class instance unwrap to it, - otherwise return None. (Can be overridden in specific spaces; you - should generally use the helper space.interp_w() instead.) - """ - if isinstance(w_obj, Wrappable): - return w_obj - return None - - def descr_self_interp_w(self, RequiredClass, w_obj): - obj = self.interpclass_w(w_obj) - if not isinstance(obj, RequiredClass): - raise DescrMismatch() - return obj - descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)' - - def interp_w(self, RequiredClass, w_obj, can_be_None=False): - """ - Unwrap w_obj, checking that it is an instance of the required internal - interpreter class (a subclass of Wrappable). - """ - assert RequiredClass is not None - if can_be_None and self.is_w(w_obj, self.w_None): - return None - obj = self.interpclass_w(w_obj) - if not isinstance(obj, RequiredClass): # or obj is None - msg = "'%s' object expected, got '%s' instead" % ( - wrappable_class_name(RequiredClass), - w_obj.getclass(self).getname(self, '?')) - raise OperationError(self.w_TypeError, self.wrap(msg)) - return obj - interp_w._annspecialcase_ = 'specialize:arg(1)' - - def unpackiterable(self, w_iterable, expected_length=-1): + def unpackiterable(self, w_iterable, expected_length=None): """Unpack an iterable object into a real (interpreter-level) list. - Raise a real (subclass of) ValueError if the length is wrong.""" + Raise a real ValueError if the length is wrong.""" w_iterator = self.iter(w_iterable) items = [] while True: try: w_item = self.next(w_iterator) - except OperationError, e: - if not e.match(self, self.w_StopIteration): - raise + except NoValue: break # done - if expected_length != -1 and len(items) == expected_length: - raise UnpackValueError("too many values to unpack") + if expected_length is not None and len(items) == expected_length: + raise ValueError, "too many values to unpack" items.append(w_item) - if expected_length != -1 and len(items) < expected_length: + if expected_length is not None and len(items) < expected_length: i = len(items) if i == 1: plural = "" else: plural = "s" - raise UnpackValueError("need more than %d value%s to unpack" % - (i, plural)) + raise ValueError, "need more than %d value%s to unpack" % (i, plural) return items - def viewiterable(self, w_iterable, expected_length=-1): - """ More or less the same as unpackiterable, but does not return - a copy. Please don't modify the result - """ - return make_sure_not_resized(self.unpackiterable(w_iterable, - expected_length)[:]) + def unpacktuple(self, w_tuple, expected_length=None): + """Same as unpackiterable(), but only for tuples. + Only use for bootstrapping or performance reasons.""" + tuple_length = self.unwrap(self.len(w_tuple)) + if expected_length is not None and tuple_length != expected_length: + raise ValueError, "got a tuple of length %d instead of %d" % ( + tuple_length, expected_length) + items = [ + self.getitem(w_tuple, self.wrap(i)) for i in range(tuple_length)] + return items def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" - if self.is_w(w_exc_type, w_check_class): - return True # fast path (also here to handle string exceptions) - try: - return self.abstract_issubclass_w(w_exc_type, w_check_class) - except OperationError, e: - if e.match(self, self.w_TypeError): # string exceptions maybe - return False - raise - - def call_obj_args(self, w_callable, w_obj, args): - if not self.config.objspace.disable_call_speedhacks: - # XXX start of hack for performance - from pypy.interpreter.function import Function - if isinstance(w_callable, Function): - return w_callable.call_obj_args(w_obj, args) - # XXX end of hack for performance - return self.call_args(w_callable, args.prepend(w_obj)) - - def call(self, w_callable, w_args, w_kwds=None): - args = Arguments.frompacked(self, w_args, w_kwds) - return self.call_args(w_callable, args) - - def call_function(self, w_func, *args_w): - nargs = len(args_w) # used for pruning funccall versions - if not self.config.objspace.disable_call_speedhacks and nargs < 5: - # XXX start of hack for performance - from pypy.interpreter.function import Function, Method - if isinstance(w_func, Method): - w_inst = w_func.w_instance - if w_inst is not None: - if nargs < 4: - func = w_func.w_function - if isinstance(func, Function): - return func.funccall(w_inst, *args_w) - elif args_w and ( - self.abstract_isinstance_w(args_w[0], w_func.w_class)): - w_func = w_func.w_function - - if isinstance(w_func, Function): - return w_func.funccall(*args_w) - # XXX end of hack for performance - - args = Arguments(self, list(args_w)) - return self.call_args(w_func, args) - - def call_valuestack(self, w_func, nargs, frame): - from pypy.interpreter.function import Function, Method, is_builtin_code - if frame.is_being_profiled and is_builtin_code(w_func): - # XXX: this code is copied&pasted :-( from the slow path below - # call_valuestack(). - args = frame.make_arguments(nargs) + check_list = [w_check_class] + while check_list: + w_item = check_list.pop() + # Match identical items. + if self.is_true(self.is_(w_exc_type, w_item)): + return True try: - return self.call_args_and_c_profile(frame, w_func, args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None - - if not self.config.objspace.disable_call_speedhacks: - # XXX start of hack for performance - if isinstance(w_func, Method): - w_inst = w_func.w_instance - if w_inst is not None: - w_func = w_func.w_function - # reuse callable stack place for w_inst - frame.settopvalue(w_inst, nargs) - nargs += 1 - elif nargs > 0 and ( - self.abstract_isinstance_w(frame.peekvalue(nargs-1), # :-( - w_func.w_class)): - w_func = w_func.w_function - - if isinstance(w_func, Function): - return w_func.funccall_valuestack(nargs, frame) - # XXX end of hack for performance + # Match subclasses. + if self.is_true(self.issubtype(w_exc_type, w_item)): + return True + except OperationError: + # Assume that this is a TypeError: w_item not a type, + # and assume that w_item is then actually a tuple. + exclst = self.unpackiterable(w_item) + check_list.extend(exclst) + return False - args = frame.make_arguments(nargs) - try: - return self.call_args(w_func, args) - finally: - if isinstance(args, ArgumentsFromValuestack): - args.frame = None - - def call_args_and_c_profile(self, frame, w_func, args): - ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) - try: - w_res = self.call_args(w_func, args) - except OperationError, e: - ec.c_exception_trace(frame, e.w_value) - raise - ec.c_return_trace(frame, w_func) - return w_res + def call_function(self, w_func, *args_w, **kw_w): + w_kw = self.newdict([(self.wrap(k), w_v) for k, w_v in kw_w.iteritems()]) + return self.call(w_func, self.newtuple(list(args_w)), w_kw) - def call_method(self, w_obj, methname, *arg_w): + def call_method(self, w_obj, methname, *arg_w, **kw_w): w_meth = self.getattr(w_obj, self.wrap(methname)) - return self.call_function(w_meth, *arg_w) - - def lookup(self, w_obj, name): - w_type = self.type(w_obj) - w_mro = self.getattr(w_type, self.wrap("__mro__")) - for w_supertype in self.unpackiterable(w_mro): - w_value = w_supertype.getdictvalue_w(self, name) - if w_value is not None: - return w_value - return None - - def is_oldstyle_instance(self, w_obj): - # xxx hack hack hack - from pypy.module.__builtin__.interp_classobj import W_InstanceObject - obj = self.interpclass_w(w_obj) - return obj is not None and isinstance(obj, W_InstanceObject) - - def callable(self, w_obj): - if self.lookup(w_obj, "__call__") is not None: - if self.is_oldstyle_instance(w_obj): - # ugly old style class special treatment, but well ... - try: - self.getattr(w_obj, self.wrap("__call__")) - return self.w_True - except OperationError, e: - if not e.match(self, self.w_AttributeError): - raise - return self.w_False - else: - return self.w_True - return self.w_False + return self.call_function(w_meth, *arg_w, **kw_w) def isinstance(self, w_obj, w_type): w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) - def abstract_issubclass_w(self, w_cls1, w_cls2): - # Equivalent to 'issubclass(cls1, cls2)'. The code below only works - # for the simple case (new-style class, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. - return self.is_true(self.issubtype(w_cls1, w_cls2)) - - def abstract_isinstance_w(self, w_obj, w_cls): - # Equivalent to 'isinstance(obj, cls)'. The code below only works - # for the simple case (new-style instance, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. - return self.is_true(self.isinstance(w_obj, w_cls)) - - def abstract_isclass_w(self, w_obj): - # Equivalent to 'isinstance(obj, type)'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. - return self.is_true(self.isinstance(w_obj, self.w_type)) - - def abstract_getclass(self, w_obj): - # Equivalent to 'obj.__class__'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. - return self.type(w_obj) - - def eval(self, expression, w_globals, w_locals, hidden_applevel=False): - "NOT_RPYTHON: For internal debugging." - import types - from pypy.interpreter.pycode import PyCode - if isinstance(expression, str): - expression = compile(expression, '?', 'eval') - if isinstance(expression, types.CodeType): - expression = PyCode._from_code(self, expression, - hidden_applevel=hidden_applevel) - if not isinstance(expression, PyCode): - raise TypeError, 'space.eval(): expected a string, code or PyCode object' - return expression.exec_code(self, w_globals, w_locals) - - def exec_(self, statement, w_globals, w_locals, hidden_applevel=False): - "NOT_RPYTHON: For internal debugging." - import types - from pypy.interpreter.pycode import PyCode - if isinstance(statement, str): - statement = compile(statement, '?', 'exec') - if isinstance(statement, types.CodeType): - statement = PyCode._from_code(self, statement, - hidden_applevel=hidden_applevel) - if not isinstance(statement, PyCode): - raise TypeError, 'space.exec_(): expected a string, code or PyCode object' - w_key = self.wrap('__builtins__') - if not self.is_true(self.contains(w_globals, w_key)): - self.setitem(w_globals, w_key, self.wrap(self.builtin)) - return statement.exec_code(self, w_globals, w_locals) - - def appexec(self, posargs_w, source): - """ return value from executing given source at applevel. - EXPERIMENTAL. The source must look like - '''(x, y): - do_stuff... - return result - ''' - """ - w_func = self.fromcache(AppExecCache).getorbuild(source) - args = Arguments(self, list(posargs_w)) - return self.call_args(w_func, args) - appexec._annspecialcase_ = 'specialize:arg(2)' - - def decode_index(self, w_index_or_slice, seqlength): - """Helper for custom sequence implementations - -> (index, 0, 0) or - (start, stop, step) - """ - if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): - w_indices = self.call_method(w_index_or_slice, "indices", - self.wrap(seqlength)) - w_start, w_stop, w_step = self.unpackiterable(w_indices, 3) - start = self.int_w(w_start) - stop = self.int_w(w_stop) - step = self.int_w(w_step) - if step == 0: - raise OperationError(self.w_ValueError, - self.wrap("slice step cannot be zero")) - if start < 0: - start = 0 - if stop < start: - stop = start - assert stop <= seqlength - else: - start = self.int_w(w_index_or_slice) - if start < 0: - start += seqlength - if not (0 <= start < seqlength): - raise OperationError(self.w_IndexError, - self.wrap("index out of range")) - stop = 0 - step = 0 - return start, stop, step - - def getindex_w(self, w_obj, w_exception, objdescr=None): - """Return w_obj.__index__() as an RPython int. - If w_exception is None, silently clamp in case of overflow; - else raise w_exception. - """ - try: - w_index = self.index(w_obj) - except OperationError, err: - if objdescr is None or not err.match(self, self.w_TypeError): - raise - msg = "%s must be an integer, not %s" % ( - objdescr, self.type(w_obj).getname(self, '?')) - raise OperationError(self.w_TypeError, self.wrap(msg)) - try: - index = self.int_w(w_index) - except OperationError, err: - if not err.match(self, self.w_OverflowError): - raise - if not w_exception: - # w_index should be a long object, but can't be sure of that - if self.is_true(self.lt(w_index, self.wrap(0))): - return -sys.maxint-1 - else: - return sys.maxint - else: - raise OperationError( - w_exception, self.wrap( - "cannot fit '%s' into an index-sized " - "integer" % self.type(w_obj).getname(self, '?'))) - else: - return index - - def r_longlong_w(self, w_obj): - bigint = self.bigint_w(w_obj) - try: - return bigint.tolonglong() - except OverflowError: - raise OperationError(self.w_OverflowError, - self.wrap('integer too large')) - - def r_ulonglong_w(self, w_obj): - bigint = self.bigint_w(w_obj) - try: - return bigint.toulonglong() - except OverflowError: - raise OperationError(self.w_OverflowError, - self.wrap('integer too large')) - except ValueError: - raise OperationError(self.w_ValueError, - self.wrap('cannot convert negative integer ' - 'to unsigned int')) - - def buffer_w(self, w_obj): - # returns a Buffer instance - from pypy.interpreter.buffer import Buffer - w_buffer = self.buffer(w_obj) - return self.interp_w(Buffer, w_buffer) - - def rwbuffer_w(self, w_obj): - # returns a RWBuffer instance - from pypy.interpreter.buffer import RWBuffer - buffer = self.buffer_w(w_obj) - if not isinstance(buffer, RWBuffer): - raise OperationError(self.w_TypeError, - self.wrap('read-write buffer expected')) - return buffer - - def bufferstr_w(self, w_obj): - # Directly returns an interp-level str. Note that if w_obj is a - # unicode string, this is different from str_w(buffer(w_obj)): - # indeed, the latter returns a string with the raw bytes from - # the underlying unicode buffer, but bufferstr_w() just converts - # the unicode to an ascii string. This inconsistency is kind of - # needed because CPython has the same issue. (Well, it's - # unclear if there is any use at all for getting the bytes in - # the unicode buffer.) - try: - return self.str_w(w_obj) - except OperationError, e: - if not e.match(self, self.w_TypeError): - raise - buffer = self.buffer_w(w_obj) - return buffer.as_str() - - def bool_w(self, w_obj): - # Unwraps a bool, also accepting an int for compatibility. - # This is here mostly just for gateway.int_unwrapping_space_method(). - return bool(self.int_w(w_obj)) - - def nonnegint_w(self, w_obj): - # Like space.int_w(), but raises an app-level ValueError if - # the integer is negative. Mostly here for gateway.py. - value = self.int_w(w_obj) - if value < 0: - raise OperationError(self.w_ValueError, - self.wrap("expected a non-negative integer")) - return value - - def warn(self, msg, w_warningcls): - self.appexec([self.wrap(msg), w_warningcls], """(msg, warningcls): - import warnings - warnings.warn(msg, warningcls, stacklevel=2) - """) - - def resolve_target(self, w_obj): - """ A space method that can be used by special object spaces (like - thunk) to replace an object by another. """ - return w_obj - - -class AppExecCache(SpaceCache): - def build(cache, source): - """ NOT_RPYTHON """ - space = cache.space - # XXX will change once we have our own compiler - import py - source = source.lstrip() - assert source.startswith('('), "incorrect header in:\n%s" % (source,) - source = py.code.Source("def anonymous%s\n" % source) - w_glob = space.newdict() - space.exec_(source.compile(), w_glob, w_glob) - return space.getitem(w_glob, space.wrap('anonymous')) - -class DummyLock(object): - def acquire(self, flag): - return True - def release(self): - pass - def _freeze_(self): - return True -dummy_lock = DummyLock() ## Table describing the regular part of the interface of object spaces, ## namely all methods which only take w_ arguments and return a w_ result -## (if any). Note: keep in sync with pypy.objspace.flow.operation.Table. +## (if any). XXX Maybe we should say that these methods must be accessed +## as 'space.op.xxx()' instead of directly 'space.xxx()'. ObjSpace.MethodTable = [ # method name # symbol # number of arguments # special method name(s) - ('is_', 'is', 2, []), ('id', 'id', 1, []), ('type', 'type', 1, []), ('issubtype', 'issubtype', 2, []), # not for old-style classes @@ -1065,15 +217,13 @@ ('getitem', 'getitem', 2, ['__getitem__']), ('setitem', 'setitem', 3, ['__setitem__']), ('delitem', 'delitem', 2, ['__delitem__']), - ('getslice', 'getslice', 3, ['__getslice__']), - ('setslice', 'setslice', 4, ['__setslice__']), - ('delslice', 'delslice', 3, ['__delslice__']), ('pos', 'pos', 1, ['__pos__']), ('neg', 'neg', 1, ['__neg__']), - ('nonzero', 'truth', 1, ['__nonzero__']), + ('not_', 'not', 1, []), ('abs' , 'abs', 1, ['__abs__']), ('hex', 'hex', 1, ['__hex__']), ('oct', 'oct', 1, ['__oct__']), + ('round', 'round', 2, []), ('ord', 'ord', 1, []), ('invert', '~', 1, ['__invert__']), ('add', '+', 2, ['__add__', '__radd__']), @@ -1091,9 +241,7 @@ ('or_', '|', 2, ['__or__', '__ror__']), ('xor', '^', 2, ['__xor__', '__rxor__']), ('int', 'int', 1, ['__int__']), - ('index', 'index', 1, ['__index__']), ('float', 'float', 1, ['__float__']), - ('long', 'long', 1, ['__long__']), ('inplace_add', '+=', 2, ['__iadd__']), ('inplace_sub', '-=', 2, ['__isub__']), ('inplace_mul', '*=', 2, ['__imul__']), @@ -1113,17 +261,12 @@ ('ne', '!=', 2, ['__ne__', '__ne__']), ('gt', '>', 2, ['__gt__', '__lt__']), ('ge', '>=', 2, ['__ge__', '__le__']), - ('cmp', 'cmp', 2, ['__cmp__']), # rich cmps preferred - ('coerce', 'coerce', 2, ['__coerce__', '__coerce__']), ('contains', 'contains', 2, ['__contains__']), ('iter', 'iter', 1, ['__iter__']), - ('next', 'next', 1, ['next']), -# ('call', 'call', 3, ['__call__']), + ('call', 'call', 3, ['__call__']), ('get', 'get', 3, ['__get__']), ('set', 'set', 3, ['__set__']), ('delete', 'delete', 2, ['__delete__']), - ('userdel', 'del', 1, ['__del__']), - ('buffer', 'buffer', 1, ['__buffer__']), # see buffer.py ] ObjSpace.BuiltinModuleTable = [ @@ -1176,38 +319,13 @@ ## Irregular part of the interface: # -# wrap(x) -> w_x -# str_w(w_str) -> str -# int_w(w_ival or w_long_ival) -> ival -# float_w(w_floatval) -> floatval -# uint_w(w_ival or w_long_ival) -> r_uint_val (unsigned int value) -# bigint_w(w_ival or w_long_ival) -> rbigint -#interpclass_w(w_interpclass_inst or w_obj) -> interpclass_inst|w_obj -# unwrap(w_x) -> x -# is_true(w_x) -> True or False -# newtuple([w_1, w_2,...]) -> w_tuple -# newlist([w_1, w_2,...]) -> w_list -# newdict() -> empty w_dict -# newslice(w_start,w_stop,w_step) -> w_slice -# call_args(w_obj,Arguments()) -> w_result - -ObjSpace.IrregularOpTable = [ - 'wrap', - 'str_w', - 'int_w', - 'float_w', - 'uint_w', - 'bigint_w', - 'unicode_w', - 'interpclass_w', - 'unwrap', - 'is_true', - 'is_w', - 'newtuple', - 'newlist', - 'newdict', - 'newslice', - 'call_args', - 'marshal_w', - ] - +# wrap(x) -> w_x +# unwrap(w_x) -> x +# is_true(w_x) -> True or False +# newtuple([w_1, w_2,...]) -> w_tuple +# newlist([w_1, w_2,...]) -> w_list +# newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) +# newdict([(w_key,w_value),...]) -> w_dict +#newslice(w_start,w_stop,w_step) -> w_slice (any argument may be a real None) +# next(w_iter) -> w_value or raise NoValue +# Modified: pypy/branch/avm/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/avm/pypy/interpreter/executioncontext.py Thu Nov 5 20:27:32 2009 @@ -1,173 +1,48 @@ -import sys -from pypy.interpreter.miscutils import Stack -from pypy.interpreter.error import OperationError -from pypy.rlib.rarithmetic import LONG_BIT -from pypy.rlib.unroll import unrolling_iterable - -def new_framestack(): - return Stack() - -def app_profile_call(space, w_callable, frame, event, w_arg): - space.call_function(w_callable, - space.wrap(frame), - space.wrap(event), w_arg) +from pypy.interpreter.miscutils import getthreadlocals, Stack class ExecutionContext: """An ExecutionContext holds the state of an execution thread in the Python interpreter.""" - + def __init__(self, space): + # Note that self.framestack only contains PyFrames self.space = space - self.framestack = new_framestack() - # tracing: space.frame_trace_action.fire() must be called to ensure - # that tracing occurs whenever self.w_tracefunc or self.is_tracing - # is modified. - self.w_tracefunc = None - self.is_tracing = 0 - self.compiler = space.createcompiler() - self.profilefunc = None - self.w_profilefuncarg = None + self.framestack = Stack() def enter(self, frame): - if self.framestack.depth() > self.space.sys.recursionlimit: - raise OperationError(self.space.w_RuntimeError, - self.space.wrap("maximum recursion depth exceeded")) - try: - frame.f_back = self.framestack.top() - except IndexError: - frame.f_back = None - - if not frame.hide(): - self.framestack.push(frame) - - def leave(self, frame): - if self.profilefunc: - self._trace(frame, 'leaveframe', self.space.w_None) - - if not frame.hide(): - self.framestack.pop() - if self.w_tracefunc is not None: - self.space.frame_trace_action.fire() - - - class Subcontext(object): - # coroutine: subcontext support - - def __init__(self): - self.framestack = new_framestack() - self.w_tracefunc = None - self.profilefunc = None - self.w_profilefuncarg = None - self.is_tracing = 0 - - def enter(self, ec): - ec.framestack = self.framestack - ec.w_tracefunc = self.w_tracefunc - ec.profilefunc = self.profilefunc - ec.w_profilefuncarg = self.w_profilefuncarg - ec.is_tracing = self.is_tracing - ec.space.frame_trace_action.fire() - - def leave(self, ec): - self.framestack = ec.framestack - self.w_tracefunc = ec.w_tracefunc - self.profilefunc = ec.profilefunc - self.w_profilefuncarg = ec.w_profilefuncarg - self.is_tracing = ec.is_tracing - - # the following interface is for pickling and unpickling - def getstate(self, space): - # we just save the framestack - items = [space.wrap(item) for item in self.framestack.items] - return space.newtuple(items) - - def setstate(self, space, w_state): - from pypy.interpreter.pyframe import PyFrame - items = [space.interp_w(PyFrame, w_item) - for w_item in space.unpackiterable(w_state)] - self.framestack.items = items - # coroutine: I think this is all, folks! - - - def get_builtin(self): - try: - return self.framestack.top().builtin - except IndexError: - return self.space.builtin + locals = getthreadlocals() + self.framestack.push(frame) + previous_ec = locals.executioncontext + locals.executioncontext = self + return previous_ec + + def leave(self, previous_ec): + locals = getthreadlocals() + locals.executioncontext = previous_ec + self.framestack.pop() + + def get_w_builtins(self): + if self.framestack.empty(): + return self.space.w_builtins + else: + return self.framestack.top().w_builtins - # XXX this one should probably be dropped in favor of a module def make_standard_w_globals(self): "Create a new empty 'globals' dictionary." w_key = self.space.wrap("__builtins__") - w_value = self.space.wrap(self.get_builtin()) - w_globals = self.space.newdict() - space.setitem(w_globals, w_key, w_value) + w_value = self.get_w_builtins() + w_globals = self.space.newdict([(w_key, w_value)]) return w_globals - def c_call_trace(self, frame, w_func): - "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) - - def c_return_trace(self, frame, w_retval): - "Profile the return from a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_return', w_retval) - - def c_exception_trace(self, frame, w_exc): - "Profile function called upon OperationError." - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_exception', w_exc) - - def _llprofile(self, event, w_arg): - fr = self.framestack.items - space = self.space - w_callback = self.profilefunc - if w_callback is not None: - frame = None - if fr: - frame = fr[0] - self.profilefunc(space, self.w_profilefuncarg, frame, event, w_arg) - - def call_trace(self, frame): - "Trace the call of a function" - if self.w_tracefunc is not None or self.profilefunc is not None: - self._trace(frame, 'call', self.space.w_None) - if self.profilefunc: - frame.is_being_profiled = True - - def return_trace(self, frame, w_retval): - "Trace the return from a function" - if self.w_tracefunc is not None: - self._trace(frame, 'return', w_retval) - def bytecode_trace(self, frame): "Trace function called before each bytecode." - # this is split into a fast path and a slower path that is - # not invoked every time bytecode_trace() is. - actionflag = self.space.actionflag - ticker = actionflag.get() - if actionflag.has_bytecode_counter: # this "if" is constant-folded - ticker += 1 - actionflag.set(ticker) - if ticker & actionflag.interesting_bits: # fast check - actionflag.action_dispatcher(self, frame) # slow path - bytecode_trace._always_inline_ = True - def exception_trace(self, frame, operationerr): + def exception_trace(self, operationerr): "Trace function called upon OperationError." operationerr.record_interpreter_traceback() - if self.w_tracefunc is not None: - self._trace(frame, 'exception', None, operationerr) #operationerr.print_detailed_traceback(self.space) - def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!! + def sys_exc_info(self): """Implements sys.exc_info(). Return an OperationError instance or None.""" for i in range(self.framestack.depth()): @@ -175,339 +50,3 @@ if frame.last_exception is not None: return frame.last_exception return None - - def settrace(self, w_func): - """Set the global trace function.""" - if self.space.is_w(w_func, self.space.w_None): - self.w_tracefunc = None - else: - self.w_tracefunc = w_func - self.space.frame_trace_action.fire() - - def setprofile(self, w_func): - """Set the global trace function.""" - if self.space.is_w(w_func, self.space.w_None): - self.profilefunc = None - self.w_profilefuncarg = None - else: - self.setllprofile(app_profile_call, w_func) - - def setllprofile(self, func, w_arg): - self.profilefunc = func - if func is not None: - if w_arg is None: - raise ValueError("Cannot call setllprofile with real None") - for frame in self.framestack.items: - frame.is_being_profiled = True - self.w_profilefuncarg = w_arg - - def call_tracing(self, w_func, w_args): - is_tracing = self.is_tracing - self.is_tracing = 0 - try: - self.space.frame_trace_action.fire() - return self.space.call(w_func, w_args) - finally: - self.is_tracing = is_tracing - - def _trace(self, frame, event, w_arg, operr=None): - if self.is_tracing or frame.hide(): - return - - space = self.space - - # Tracing cases - if event == 'call': - w_callback = self.w_tracefunc - else: - w_callback = frame.w_f_trace - - if w_callback is not None and event != "leaveframe": - if operr is not None: - w_arg = space.newtuple([operr.w_type, operr.w_value, - space.wrap(operr.application_traceback)]) - - frame.fast2locals() - self.is_tracing += 1 - try: - try: - w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg) - if space.is_w(w_result, space.w_None): - frame.w_f_trace = None - else: - frame.w_f_trace = w_result - except: - self.settrace(space.w_None) - frame.w_f_trace = None - raise - finally: - self.is_tracing -= 1 - frame.locals2fast() - space.frame_trace_action.fire() - - # Profile cases - if self.profilefunc is not None: - if event not in ['leaveframe', 'call', 'c_call', - 'c_return', 'c_exception']: - return - - last_exception = None - if event == 'leaveframe': - last_exception = frame.last_exception - event = 'return' - - assert self.is_tracing == 0 - self.is_tracing += 1 - try: - try: - self.profilefunc(space, self.w_profilefuncarg, - frame, event, w_arg) - except: - self.profilefunc = None - self.w_profilefuncarg = None - raise - - finally: - frame.last_exception = last_exception - self.is_tracing -= 1 - - def _freeze_(self): - raise Exception("ExecutionContext instances should not be seen during" - " translation. Now is a good time to inspect the" - " traceback and see where this one comes from :-)") - - -class AbstractActionFlag: - """This holds the global 'action flag'. It is a single bitfield - integer, with bits corresponding to AsyncAction objects that need to - be immediately triggered. The correspondance from bits to - AsyncAction instances is built at translation time. We can quickly - check if there is anything at all to do by checking if any of the - relevant bits is set. If threads are enabled, they consume the 20 - lower bits to hold a counter incremented at each bytecode, to know - when to release the GIL. - """ - def __init__(self): - self._periodic_actions = [] - self._nonperiodic_actions = [] - self.unused_bits = self.FREE_BITS[:] - self.has_bytecode_counter = False - self.interesting_bits = 0 - self._rebuild_action_dispatcher() - - def fire(self, action): - """Request for the action to be run before the next opcode. - The action must have been registered at space initalization time.""" - ticker = self.get() - self.set(ticker | action.bitmask) - - def register_action(self, action): - "NOT_RPYTHON" - assert isinstance(action, AsyncAction) - if action.bitmask == 0: - while True: - action.bitmask = self.unused_bits.pop(0) - if not (action.bitmask & self.interesting_bits): - break - self.interesting_bits |= action.bitmask - if action.bitmask & self.BYTECODE_COUNTER_OVERFLOW_BIT: - assert action.bitmask == self.BYTECODE_COUNTER_OVERFLOW_BIT - self._periodic_actions.append(action) - self.has_bytecode_counter = True - self.force_tick_counter() - else: - self._nonperiodic_actions.append((action, action.bitmask)) - self._rebuild_action_dispatcher() - - def setcheckinterval(self, space, interval): - if interval < self.CHECK_INTERVAL_MIN: - interval = self.CHECK_INTERVAL_MIN - elif interval > self.CHECK_INTERVAL_MAX: - interval = self.CHECK_INTERVAL_MAX - space.sys.checkinterval = interval - self.force_tick_counter() - - def force_tick_counter(self): - # force the tick counter to a valid value -- this actually forces - # it to reach BYTECODE_COUNTER_OVERFLOW_BIT at the next opcode. - ticker = self.get() - ticker &= ~ self.BYTECODE_COUNTER_OVERFLOW_BIT - ticker |= self.BYTECODE_COUNTER_MASK - self.set(ticker) - - def _rebuild_action_dispatcher(self): - periodic_actions = unrolling_iterable(self._periodic_actions) - nonperiodic_actions = unrolling_iterable(self._nonperiodic_actions) - has_bytecode_counter = self.has_bytecode_counter - - def action_dispatcher(ec, frame): - # periodic actions - if has_bytecode_counter: - ticker = self.get() - if ticker & self.BYTECODE_COUNTER_OVERFLOW_BIT: - # We must run the periodic actions now, but first - # reset the bytecode counter (the following line - # works by assuming that we just overflowed the - # counter, i.e. BYTECODE_COUNTER_OVERFLOW_BIT is - # set but none of the BYTECODE_COUNTER_MASK bits - # are). - ticker -= ec.space.sys.checkinterval - self.set(ticker) - for action in periodic_actions: - action.perform(ec, frame) - - # nonperiodic actions - for action, bitmask in nonperiodic_actions: - ticker = self.get() - if ticker & bitmask: - self.set(ticker & ~ bitmask) - action.perform(ec, frame) - - action_dispatcher._dont_inline_ = True - self.action_dispatcher = action_dispatcher - - # Bits reserved for the bytecode counter, if used - BYTECODE_COUNTER_MASK = (1 << 20) - 1 - BYTECODE_COUNTER_OVERFLOW_BIT = (1 << 20) - - # Free bits - FREE_BITS = [1 << _b for _b in range(21, LONG_BIT-1)] - - # The acceptable range of values for sys.checkinterval, so that - # the bytecode_counter fits in 20 bits - CHECK_INTERVAL_MIN = 1 - CHECK_INTERVAL_MAX = BYTECODE_COUNTER_OVERFLOW_BIT - - -class ActionFlag(AbstractActionFlag): - """The normal class for space.actionflag. The signal module provides - a different one.""" - _flags = 0 - - def get(self): - return self._flags - - def set(self, value): - self._flags = value - - -class AsyncAction(object): - """Abstract base class for actions that must be performed - asynchronously with regular bytecode execution, but that still need - to occur between two opcodes, not at a completely random time. - """ - bitmask = 0 # means 'please choose one bit automatically' - - def __init__(self, space): - self.space = space - - def fire(self): - """Request for the action to be run before the next opcode. - The action must have been registered at space initalization time.""" - self.space.actionflag.fire(self) - - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a portential thread switch). - Don't call this if threads are not enabled. - """ - from pypy.module.thread.gil import spacestate - spacestate.set_actionflag_bit_after_thread_switch |= self.bitmask - - def perform(self, executioncontext, frame): - """To be overridden.""" - - -class PeriodicAsyncAction(AsyncAction): - """Abstract base class for actions that occur automatically - every sys.checkinterval bytecodes. - """ - bitmask = ActionFlag.BYTECODE_COUNTER_OVERFLOW_BIT - - -class UserDelAction(AsyncAction): - """An action that invokes all pending app-level __del__() method. - This is done as an action instead of immediately when the - interp-level __del__() is invoked, because the latter can occur more - or less anywhere in the middle of code that might not be happy with - random app-level code mutating data structures under its feet. - """ - - def __init__(self, space): - AsyncAction.__init__(self, space) - self.dying_objects_w = [] - self.finalizers_lock_count = 0 - - def register_dying_object(self, w_obj): - self.dying_objects_w.append(w_obj) - self.fire() - - def perform(self, executioncontext, frame): - if self.finalizers_lock_count > 0: - return - # Each call to perform() first grabs the self.dying_objects_w - # and replaces it with an empty list. We do this to try to - # avoid too deep recursions of the kind of __del__ being called - # while in the middle of another __del__ call. - pending_w = self.dying_objects_w - self.dying_objects_w = [] - space = self.space - for w_obj in pending_w: - try: - space.userdel(w_obj) - except OperationError, e: - e.write_unraisable(space, 'method __del__ of ', w_obj) - e.clear(space) # break up reference cycles - # finally, this calls the interp-level destructor for the - # cases where there is both an app-level and a built-in __del__. - w_obj._call_builtin_destructor() - - -class FrameTraceAction(AsyncAction): - """An action that calls the local trace functions (w_f_trace).""" - - def perform(self, executioncontext, frame): - if frame.w_f_trace is None or executioncontext.is_tracing: - return - code = frame.pycode - if frame.instr_lb <= frame.last_instr < frame.instr_ub: - if frame.last_instr <= frame.instr_prev: - # We jumped backwards in the same line. - executioncontext._trace(frame, 'line', self.space.w_None) - else: - size = len(code.co_lnotab) / 2 - addr = 0 - line = code.co_firstlineno - p = 0 - lineno = code.co_lnotab - while size > 0: - c = ord(lineno[p]) - if (addr + c) > frame.last_instr: - break - addr += c - if c: - frame.instr_lb = addr - - line += ord(lineno[p + 1]) - p += 2 - size -= 1 - - if size > 0: - while True: - size -= 1 - if size < 0: - break - addr += ord(lineno[p]) - if ord(lineno[p + 1]): - break - p += 2 - frame.instr_ub = addr - else: - frame.instr_ub = sys.maxint - - if frame.instr_lb == frame.last_instr: # At start of line! - frame.f_lineno = line - executioncontext._trace(frame, 'line', self.space.w_None) - - frame.instr_prev = frame.last_instr - self.space.frame_trace_action.fire() # continue tracing Modified: pypy/branch/avm/pypy/interpreter/interactive.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/interactive.py (original) +++ pypy/branch/avm/pypy/interpreter/interactive.py Thu Nov 5 20:27:32 2009 @@ -1,229 +1,72 @@ -from pypy.interpreter import error -from pypy.interpreter import baseobjspace, module, main +import autopath + +from pypy.interpreter import executioncontext, pyframe, baseobjspace import sys import code -import time - - -class Completer: - """ Stolen mostly from CPython's rlcompleter.py """ - def __init__(self, space, w_globals): - self.space = space - self.w_globals = w_globals - - def complete(self, text, state): - if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) - try: - return self.matches[state] - - except IndexError: - return None - - def global_matches(self, text): - import keyword - w_res = self.space.call_method(self.w_globals, "keys") - namespace_keys = self.space.unwrap(w_res) - w_res = self.space.call_method(self.space.builtin.getdict(), "keys") - builtin_keys = self.space.unwrap(w_res) - - matches = [] - n = len(text) - - for l in [namespace_keys, builtin_keys, keyword.kwlist]: - for word in l: - if word[:n] == text and word != "__builtins__": - matches.append(word) - - return matches - - def attr_matches(self, text): - import re - m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text) - if not m: - return - - expr, attr = m.group(1, 3) - s = self.space - w_obj = s.eval(expr, self.w_globals, self.w_globals) - words = self.get_words(w_obj) - - w_clz = s.getattr(w_obj, s.wrap("__class__")) - words += self.get_class_members(w_clz) - - matches = [] - n = len(attr) - for word in words: - if word[:n] == attr and word != "__builtins__": - matches.append("%s.%s" % (expr, word)) - - return matches - - def get_words(self, w_clz): - s = self.space - w_dir_func = s.builtin.get("dir") - w_res = s.call_function(w_dir_func, w_clz) - return s.unwrap(w_res) - - def get_class_members(self, w_clz): - s = self.space - words = self.get_words(w_clz) - try: - w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.viewiterable(w_bases) - - except error.OperationError: - return words +import linecache - for w_clz in bases_w: - words += self.get_class_members(w_clz) - - return words class PyPyConsole(code.InteractiveConsole): - def __init__(self, objspace, verbose=0, completer=False): + def __init__(self, objspace): code.InteractiveConsole.__init__(self) self.space = objspace - self.verbose = verbose - space = self.space - self.console_compiler_flags = 0 - - mainmodule = main.ensure__main__(space) - self.w_globals = mainmodule.w_dict - space.setitem(self.w_globals, space.wrap('__builtins__'), space.builtin) - if completer: - self.enable_command_line_completer() - - # forbidden: - #space.exec_("__pytrace__ = 0", self.w_globals, self.w_globals) - space.setitem(self.w_globals, space.wrap('__pytrace__'),space.wrap(0)) - self.tracelevel = 0 - self.console_locals = {} - - def enable_command_line_completer(self): - try: - import readline - # Keep here to save windoze tears - readline.set_completer(Completer(self.space, self.w_globals).complete) - readline.parse_and_bind("tab: complete") - readline.set_history_length(25000) - - try: - readline.read_history_file() - except IOError: - pass # guess it doesn't exit - - import atexit - atexit.register(readline.write_history_file) - except: - pass + self.ec = executioncontext.ExecutionContext(self.space) + self.w_globals = self.ec.make_standard_w_globals() + self.space.setitem(self.w_globals, + self.space.wrap("__name__"), + self.space.wrap("__main__")) def interact(self, banner=None): - #banner = "Python %s in pypy\n%s / %s" % ( - # sys.version, self.__class__.__name__, - # self.space.__class__.__name__) - w_sys = self.space.sys - major, minor, micro, _, _ = self.space.unwrap(self.space.sys.get('pypy_version_info')) - elapsed = time.time() - self.space._starttime - banner = "PyPy %d.%d.%d in %r on top of Python %s (startuptime: %.2f secs)" % ( - major, minor, micro, self.space, sys.version.split()[0], elapsed) + if banner is None: + banner = "Python %s in pypy\n%s / %s" % ( + sys.version, self.__class__.__name__, + self.space.__class__.__name__) code.InteractiveConsole.interact(self, banner) def raw_input(self, prompt=""): # add a character to the PyPy prompt so that you know where you # are when you debug it with "python -i py.py" - try: - return code.InteractiveConsole.raw_input(self, prompt[0] + prompt) - except KeyboardInterrupt: - # fires into an interpreter-level console - print - banner = ("Python %s on %s\n" % (sys.version, sys.platform) + - "*** Entering interpreter-level console ***") - local = self.console_locals - # don't copy attributes that look like names that came - # from self.w_globals (itself the main offender) as they - # would then get copied back into the applevel namespace. - local.update(dict([(k,v) for (k, v) in self.__dict__.iteritems() - if not k.startswith('w_')])) - del local['locals'] - for w_name in self.space.unpackiterable(self.w_globals): - local['w_' + self.space.str_w(w_name)] = ( - self.space.getitem(self.w_globals, w_name)) - code.interact(banner=banner, local=local) - # copy back 'w_' names - for name in local: - if name.startswith('w_'): - self.space.setitem(self.w_globals, - self.space.wrap(name[2:]), - local[name]) - print '*** Leaving interpreter-level console ***' - raise + return code.InteractiveConsole.raw_input(self, prompt[0] + prompt) def runcode(self, code): - raise NotImplementedError - - def runsource(self, source, ignored_filename="", symbol="single"): - # the following hacked file name is recognized specially by error.py - hacked_filename = '\n' + source - compiler = self.space.getexecutioncontext().compiler - - # CPython 2.6 turns console input into unicode - if isinstance(source, unicode): - source = source.encode(sys.stdin.encoding) - - def doit(): - # compile the provided input - code = compiler.compile_command(source, hacked_filename, symbol, - self.console_compiler_flags) - if code is None: - raise IncompleteInput - self.console_compiler_flags |= compiler.getcodeflags(code) - - # execute it - self.settrace() + # 'code' is a CPython code object + from pypy.interpreter.pycode import PyCode + pycode = PyCode()._from_code(code) + try: + pycode.exec_code(self.space, self.w_globals, self.w_globals) + except baseobjspace.OperationError, operationerr: + # XXX insert exception info into the application-level sys.last_xxx + operationerr.print_detailed_traceback(self.space) + else: try: - code.exec_code(self.space, self.w_globals, self.w_globals) - finally: - if self.tracelevel: - self.space.unsettrace() - self.checktrace() + if sys.stdout.softspace: + print + except AttributeError: + # Don't crash if user defined stdout doesn't have softspace + pass - # run doit() in an exception-catching box + def runsource(self, source, ignored_filename="", symbol="single"): + hacked_filename = '\n'+source try: - main.run_toplevel(self.space, doit, verbose=self.verbose) - except IncompleteInput: - return 1 - else: + code = self.compile(source, hacked_filename, symbol) + except (OverflowError, SyntaxError, ValueError): + self.showsyntaxerror(self.filename) return 0 + if code is None: + return 1 + self.runcode(code) + return 0 - def settrace(self): - if self.tracelevel: - self.space.settrace() - - def checktrace(self): - from pypy.objspace import trace - - s = self.space - - # Did we modify __pytrace__ - tracelevel = s.int_w(s.getitem(self.w_globals, - s.wrap("__pytrace__"))) - - if self.tracelevel > 0 and tracelevel == 0: - s.reset_trace() - print "Tracing disabled" - - if self.tracelevel == 0 and tracelevel > 0: - trace.create_trace_space(s) - self.space.unsettrace() - print "Tracing enabled" - - self.tracelevel = tracelevel - - -class IncompleteInput(Exception): - pass - +if __name__ == '__main__': + try: + import readline + except ImportError: + pass + + from pypy.tool import option + from pypy.tool import testit + args = option.process_options(option.get_standard_options(), + option.Options) + objspace = option.objspace() + con = PyPyConsole(objspace) + con.interact() Modified: pypy/branch/avm/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/pyframe.py (original) +++ pypy/branch/avm/pypy/interpreter/pyframe.py Thu Nov 5 20:27:32 2009 @@ -1,604 +1,299 @@ """ PyFrame class implementation with the interpreter main loop. """ -from pypy.tool.pairtype import extendabletype -from pypy.interpreter import eval, baseobjspace, pycode -from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +from pypy.interpreter import eval, baseobjspace, gateway +from pypy.interpreter.miscutils import Stack from pypy.interpreter.error import OperationError from pypy.interpreter import pytraceback -import opcode -from pypy.rlib.objectmodel import we_are_translated, instantiate -from pypy.rlib.jit import we_are_jitted, hint -from pypy.rlib.debug import make_sure_not_resized - -# Define some opcodes used -g = globals() -for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY -POP_BLOCK END_FINALLY'''.split(): - g[op] = opcode.opmap[op] -HAVE_ARGUMENT = opcode.HAVE_ARGUMENT -class PyFrame(eval.Frame): + +class PyFrame(eval.Frame, baseobjspace.Wrappable): """Represents a frame for a regular Python function that needs to be interpreted. - See also pyopcode.PyStandardFrame and nestedscope.PyNestedScopeFrame. + See also pyopcode.PyStandardFrame and pynestedscope.PyNestedScopeFrame. Public fields: * 'space' is the object space this frame is running in * 'code' is the PyCode object this frame runs * 'w_locals' is the locals dictionary to use * 'w_globals' is the attached globals dictionary - * 'builtin' is the attached built-in module - * 'valuestack_w', 'blockstack', control the interpretation + * 'w_builtins' is the attached built-ins dictionary + * 'valuestack', 'blockstack', 'next_instr' control the interpretation """ - __metaclass__ = extendabletype - - frame_finished_execution = False - last_instr = -1 - last_exception = None - f_back = None - w_f_trace = None - # For tracing - instr_lb = 0 - instr_ub = -1 - instr_prev = -1 - is_being_profiled = False - def __init__(self, space, code, w_globals, closure): - #self = hint(self, access_directly=True) - assert isinstance(code, pycode.PyCode) - self.pycode = code - eval.Frame.__init__(self, space, w_globals, code.co_nlocals) - self.valuestack_w = [None] * code.co_stacksize - self.valuestackdepth = 0 - self.blockstack = [] - if space.config.objspace.honor__builtins__: - self.builtin = space.builtin.pick_builtin(w_globals) + eval.Frame.__init__(self, space, code, w_globals, code.co_nlocals) + self.valuestack = Stack() + self.blockstack = Stack() + self.last_exception = None + self.next_instr = 0 + self.w_builtins = self.space.w_builtins # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. # class bodies only have CO_NEWLOCALS. - self.initialize_frame_scopes(closure) - self.fastlocals_w = [None]*self.numlocals - make_sure_not_resized(self.fastlocals_w) - self.f_lineno = self.pycode.co_firstlineno - - def get_builtin(self): - if self.space.config.objspace.honor__builtins__: - return self.builtin - else: - return self.space.builtin - - def initialize_frame_scopes(self, closure): - # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. - # class bodies only have CO_NEWLOCALS. - # CO_NEWLOCALS: make a locals dict unless optimized is also set - # CO_OPTIMIZED: no locals dict needed at all - # NB: this method is overridden in nestedscope.py - flags = self.pycode.co_flags - if flags & pycode.CO_OPTIMIZED: - return - if flags & pycode.CO_NEWLOCALS: - self.w_locals = self.space.newdict() - else: - assert self.w_globals is not None - self.w_locals = self.w_globals - - def run(self): - """Start this frame's execution.""" - if self.pycode.co_flags & pycode.CO_GENERATOR: - from pypy.interpreter.generator import GeneratorIterator - return self.space.wrap(GeneratorIterator(self)) - else: - return self.execute_frame() - - def execute_generator_frame(self, w_inputvalue, ex=False): - # opcode semantic change in CPython 2.5: we must pass an input value - # when resuming a generator, which goes into the value stack. - # It's not working because the value of magic must be changed in PyCode - if self.pycode.magic >= 0xa0df294 and self.last_instr != -1 and not ex: - self.pushvalue(w_inputvalue) - return self.execute_frame() - - def execute_frame(self): - """Execute this frame. Main entry point to the interpreter.""" - from pypy.rlib import rstack - # the following 'assert' is an annotation hint: it hides from - # the annotator all methods that are defined in PyFrame but - # overridden in the FrameClass subclass of PyFrame. - assert isinstance(self, self.space.FrameClass) - executioncontext = self.space.getexecutioncontext() - executioncontext.enter(self) - try: - executioncontext.call_trace(self) - # Execution starts just after the last_instr. Initially, - # last_instr is -1. After a generator suspends it points to - # the YIELD_VALUE instruction. - next_instr = self.last_instr + 1 - try: - w_exitvalue = self.dispatch(self.pycode, next_instr, - executioncontext) - rstack.resume_point("execute_frame", self, executioncontext, - returns=w_exitvalue) - except Exception: - executioncontext.return_trace(self, self.space.w_None) - raise - executioncontext.return_trace(self, w_exitvalue) - # on exit, we try to release self.last_exception -- breaks an - # obvious reference cycle, so it helps refcounting implementations - self.last_exception = None - finally: - executioncontext.leave(self) - return w_exitvalue - execute_frame.insert_stack_check_here = True - - # stack manipulation helpers - def pushvalue(self, w_object): - depth = self.valuestackdepth - self.valuestack_w[depth] = w_object - self.valuestackdepth = depth + 1 - - def popvalue(self): - depth = self.valuestackdepth - 1 - assert depth >= 0, "pop from empty value stack" - w_object = self.valuestack_w[depth] - self.valuestack_w[depth] = None - self.valuestackdepth = depth - return w_object - - def popstrdictvalues(self, n): - dic_w = {} - while True: - n -= 1 - if n < 0: - break - w_value = self.popvalue() - w_key = self.popvalue() - key = self.space.str_w(w_key) - dic_w[key] = w_value - return dic_w - - # we need two popvalues that return different data types: - # one in case we want list another in case of tuple - def _new_popvalues(): - def popvalues(self, n): - values_w = [None] * n - while True: - n -= 1 - if n < 0: - break - values_w[n] = self.popvalue() - return values_w - return popvalues - popvalues = _new_popvalues() - popvalues_mutable = _new_popvalues() - del _new_popvalues - - def peekvalues(self, n): - values_w = [None] * n - base = self.valuestackdepth - n - assert base >= 0 - while True: - n -= 1 - if n < 0: - break - values_w[n] = self.valuestack_w[base+n] - return values_w - - def dropvalues(self, n): - finaldepth = self.valuestackdepth - n - assert finaldepth >= 0, "stack underflow in dropvalues()" - while True: - n -= 1 - if n < 0: - break - self.valuestack_w[finaldepth+n] = None - self.valuestackdepth = finaldepth - - def pushrevvalues(self, n, values_w): # n should be len(values_w) - while True: - n -= 1 - if n < 0: - break - self.pushvalue(values_w[n]) - - def dupvalues(self, n): - delta = n-1 - while True: - n -= 1 - if n < 0: - break - w_value = self.peekvalue(delta) - self.pushvalue(w_value) - - def peekvalue(self, index_from_top=0): - index = self.valuestackdepth + ~index_from_top - assert index >= 0, "peek past the bottom of the stack" - return self.valuestack_w[index] - - def settopvalue(self, w_object, index_from_top=0): - index = self.valuestackdepth + ~index_from_top - assert index >= 0, "settop past the bottom of the stack" - self.valuestack_w[index] = w_object - - def dropvaluesuntil(self, finaldepth): - depth = self.valuestackdepth - 1 - while depth >= finaldepth: - self.valuestack_w[depth] = None - depth -= 1 - self.valuestackdepth = finaldepth - - def savevaluestack(self): - return self.valuestack_w[:self.valuestackdepth] - - def restorevaluestack(self, items_w): - assert None not in items_w - self.valuestack_w[:len(items_w)] = items_w - self.dropvaluesuntil(len(items_w)) - - def make_arguments(self, nargs): - if we_are_jitted(): - return Arguments(self.space, self.peekvalues(nargs)) - else: - return ArgumentsFromValuestack(self.space, self, nargs) - - def descr__reduce__(self, space): - from pypy.interpreter.mixedmodule import MixedModule - from pypy.module._pickle_support import maker # helper fns - w_mod = space.getbuiltinmodule('_pickle_support') - mod = space.interp_w(MixedModule, w_mod) - new_inst = mod.get('frame_new') - w = space.wrap - nt = space.newtuple - - cells = self._getcells() - if cells is None: - w_cells = space.w_None - else: - w_cells = space.newlist([space.wrap(cell) for cell in cells]) - - if self.w_f_trace is None: - f_lineno = self.get_last_lineno() - else: - f_lineno = self.f_lineno - - values_w = self.valuestack_w[0:self.valuestackdepth] - w_valuestack = maker.slp_into_tuple_with_nulls(space, values_w) - - w_blockstack = nt([block._get_state_(space) for block in self.blockstack]) - w_fastlocals = maker.slp_into_tuple_with_nulls(space, self.fastlocals_w) - if self.last_exception is None: - w_exc_value = space.w_None - w_tb = space.w_None - else: - w_exc_value = self.last_exception.w_value - w_tb = w(self.last_exception.application_traceback) - - tup_state = [ - w(self.f_back), - w(self.get_builtin()), - w(self.pycode), - w_valuestack, - w_blockstack, - w_exc_value, # last_exception - w_tb, # - self.w_globals, - w(self.last_instr), - w(self.frame_finished_execution), - w(f_lineno), - w_fastlocals, - space.w_None, #XXX placeholder for f_locals - - #f_restricted requires no additional data! - space.w_None, ## self.w_f_trace, ignore for now + if code.dictscope_needed(): + self.w_locals = space.newdict([]) # set to None by Frame.__init__ - w(self.instr_lb), #do we need these three (that are for tracing) - w(self.instr_ub), - w(self.instr_prev), - w_cells, - ] - - return nt([new_inst, nt([]), nt(tup_state)]) - - def descr__setstate__(self, space, w_args): - from pypy.module._pickle_support import maker # helper fns - from pypy.interpreter.pycode import PyCode - from pypy.interpreter.module import Module - args_w = space.unpackiterable(w_args) - w_f_back, w_builtin, w_pycode, w_valuestack, w_blockstack, w_exc_value, w_tb,\ - w_globals, w_last_instr, w_finished, w_f_lineno, w_fastlocals, w_f_locals, \ - w_f_trace, w_instr_lb, w_instr_ub, w_instr_prev, w_cells = args_w - - new_frame = self - pycode = space.interp_w(PyCode, w_pycode) - - if space.is_w(w_cells, space.w_None): - closure = None - cellvars = [] - else: - from pypy.interpreter.nestedscope import Cell - cells_w = space.unpackiterable(w_cells) - cells = [space.interp_w(Cell, w_cell) for w_cell in cells_w] - ncellvars = len(pycode.co_cellvars) - cellvars = cells[:ncellvars] - closure = cells[ncellvars:] - - # do not use the instance's __init__ but the base's, because we set - # everything like cells from here - PyFrame.__init__(self, space, pycode, w_globals, closure) - new_frame.f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True) - new_frame.builtin = space.interp_w(Module, w_builtin) - new_frame.blockstack = [unpickle_block(space, w_blk) - for w_blk in space.unpackiterable(w_blockstack)] - values_w = maker.slp_from_tuple_with_nulls(space, w_valuestack) - for w_value in values_w: - new_frame.pushvalue(w_value) - if space.is_w(w_exc_value, space.w_None): - new_frame.last_exception = None - else: - from pypy.interpreter.pytraceback import PyTraceback - tb = space.interp_w(PyTraceback, w_tb) - new_frame.last_exception = OperationError(space.type(w_exc_value), - w_exc_value, tb - ) - new_frame.last_instr = space.int_w(w_last_instr) - new_frame.frame_finished_execution = space.is_true(w_finished) - new_frame.f_lineno = space.int_w(w_f_lineno) - new_frame.fastlocals_w = maker.slp_from_tuple_with_nulls(space, w_fastlocals) - - if space.is_w(w_f_trace, space.w_None): - new_frame.w_f_trace = None - else: - new_frame.w_f_trace = w_f_trace - - new_frame.instr_lb = space.int_w(w_instr_lb) #the three for tracing - new_frame.instr_ub = space.int_w(w_instr_ub) - new_frame.instr_prev = space.int_w(w_instr_prev) - - self._setcellvars(cellvars) - space.frame_trace_action.fire() - - def hide(self): - return self.pycode.hidden_applevel - - def getcode(self): - return hint(self.pycode, promote=True) - - def getfastscope(self): - "Get the fast locals as a list." - return self.fastlocals_w - - def setfastscope(self, scope_w): - """Initialize the fast locals from a list of values, - where the order is according to self.pycode.signature().""" - scope_len = len(scope_w) - if scope_len > len(self.fastlocals_w): - raise ValueError, "new fastscope is longer than the allocated area" - self.fastlocals_w[:scope_len] = scope_w - self.init_cells() - - def init_cells(self): - """Initialize cellvars from self.fastlocals_w - This is overridden in nestedscope.py""" - pass - def getclosure(self): return None - def _getcells(self): - return None - - def _setcellvars(self, cellvars): - pass - - ### line numbers ### - - # for f*_f_* unwrapping through unwrap_spec in typedef.py - - def fget_f_lineno(space, self): - "Returns the line number of the instruction currently being executed." - if self.w_f_trace is None: - return space.wrap(self.get_last_lineno()) - else: - return space.wrap(self.f_lineno) - - def fset_f_lineno(space, self, w_new_lineno): - "Returns the line number of the instruction currently being executed." + def eval(self, executioncontext): + "Interpreter main loop!" try: - new_lineno = space.int_w(w_new_lineno) - except OperationError, e: - raise OperationError(space.w_ValueError, - space.wrap("lineno must be an integer")) + while True: + try: + executioncontext.bytecode_trace(self) + last_instr = self.next_instr + try: + # fetch and dispatch the next opcode + # dispatch() is abstract, see pyopcode. + self.dispatch() + except OperationError, e: + pytraceback.record_application_traceback( + self.space, e, self, last_instr) + # convert an OperationError into a control flow + # exception + import sys + tb = sys.exc_info()[2] + raise SApplicationException(e, tb) + # XXX some other exceptions could be caught here too, + # like KeyboardInterrupt + + except ControlFlowException, ctlflowexc: + # we have a reason to change the control flow + # (typically unroll the stack) + ctlflowexc.action(self, last_instr, executioncontext) - if self.w_f_trace is None: - raise OperationError(space.w_ValueError, - space.wrap("f_lineo can only be set by a trace function.")) - - if new_lineno < self.pycode.co_firstlineno: - raise OperationError(space.w_ValueError, - space.wrap("line %d comes before the current code." % new_lineno)) - code = self.pycode.co_code - addr = 0 - line = self.pycode.co_firstlineno - new_lasti = -1 - offset = 0 - lnotab = self.pycode.co_lnotab - for offset in xrange(0, len(lnotab), 2): - addr += ord(lnotab[offset]) - line += ord(lnotab[offset + 1]) - if line >= new_lineno: - new_lasti = addr - new_lineno = line + except ExitFrame, e: + # leave that frame + w_exitvalue = e.args[0] + return w_exitvalue + + ### exception stack ### + + def clean_exceptionstack(self): + # remove all exceptions that can no longer be re-raised + # because the current valuestack is no longer deep enough + # to hold the corresponding information + while self.exceptionstack: + ctlflowexc, valuestackdepth = self.exceptionstack.top() + if valuestackdepth <= self.valuestack.depth(): break + self.exceptionstack.pop() + + ### application level visible attributes ### + def app_visible(self): + def makedict(**kw): return kw + space = self.space + d = makedict( + f_code = space.wrap(self.code), + f_locals = self.getdictscope(), + f_globals = self.w_globals, + f_builtins = self.w_builtins, + # XXX f_lasti, f_back, f_exc*, f_restricted need to do pypy_getattr directly + ) + return d.items() + +### Frame Blocks ### + +class FrameBlock: + + """Abstract base class for frame blocks from the blockstack, + used by the SETUP_XXX and POP_BLOCK opcodes.""" + + def __init__(self, frame, handlerposition): + self.handlerposition = handlerposition + self.valuestackdepth = frame.valuestack.depth() + + def __eq__(self, other): + return (self.__class__ is other.__class__ and + self.handlerposition == other.handlerposition and + self.valuestackdepth == other.valuestackdepth) + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash((self.handlerposition, self.valuestackdepth)) + + def cleanupstack(self, frame): + for i in range(self.valuestackdepth, frame.valuestack.depth()): + frame.valuestack.pop() + + def cleanup(self, frame): + "Clean up a frame when we normally exit the block." + self.cleanupstack(frame) + + def unroll(self, frame, unroller): + "Clean up a frame when we abnormally exit the block." + self.cleanupstack(frame) + return False # continue to unroll + + +class LoopBlock(FrameBlock): + """A loop block. Stores the end-of-loop pointer in case of 'break'.""" + + def unroll(self, frame, unroller): + if isinstance(unroller, SContinueLoop): + # re-push the loop block without cleaning up the value stack, + # and jump to the beginning of the loop, stored in the + # exception's argument + frame.blockstack.push(self) + jump_to = unroller.args[0] + frame.next_instr = jump_to + return True # stop unrolling + self.cleanupstack(frame) + if isinstance(unroller, SBreakLoop): + # jump to the end of the loop + frame.next_instr = self.handlerposition + return True # stop unrolling + return False + + +class ExceptBlock(FrameBlock): + """An try:except: block. Stores the position of the exception handler.""" + + def unroll(self, frame, unroller): + self.cleanupstack(frame) + if isinstance(unroller, SApplicationException): + # push the exception to the value stack for inspection by the + # exception handler (the code after the except:) + operationerr = unroller.args[0] + w_type = operationerr.w_type + w_value = operationerr.w_value + if frame.space.full_exceptions: + w_normalized = normalize_exception(frame.space, w_type, w_value) + w_type, w_value = frame.space.unpacktuple(w_normalized, 2) + # the stack setup is slightly different than in CPython: + # instead of the traceback, we store the unroller object, + # wrapped. + frame.valuestack.push(frame.space.wrap(unroller)) + frame.valuestack.push(w_value) + frame.valuestack.push(w_type) + frame.next_instr = self.handlerposition # jump to the handler + return True # stop unrolling + return False + +def app_normalize_exception(etype, evalue): + # XXX should really be defined as a method on OperationError, + # but this is not so easy because OperationError cannot be + # at the same time an old-style subclass of Exception and a + # new-style subclass of Wrappable :-( + # moreover, try importing gateway from errors.py and you'll see :-( + + # mistakes here usually show up as infinite recursion, which is fun. + if isinstance(evalue, etype): + return etype, evalue + if isinstance(etype, type) and issubclass(etype, Exception): + if evalue is None: + evalue = () + elif not isinstance(evalue, tuple): + evalue = (evalue,) + evalue = etype(*evalue) + else: + raise Exception, "?!" # XXX + return etype, evalue +normalize_exception = gateway.app2interp(app_normalize_exception) + + +class FinallyBlock(FrameBlock): + """A try:finally: block. Stores the position of the exception handler.""" + + def cleanup(self, frame): + # upon normal entry into the finally: part, the standard Python + # bytecode pushes a single None for END_FINALLY. In our case we + # always push three values into the stack: the wrapped ctlflowexc, + # the exception value and the exception type (which are all None + # here). + self.cleanupstack(frame) + # one None already pushed by the bytecode + frame.valuestack.push(frame.space.w_None) + frame.valuestack.push(frame.space.w_None) + + def unroll(self, frame, unroller): + # any abnormal reason for unrolling a finally: triggers the end of + # the block unrolling and the entering the finally: handler. + # see comments in cleanup(). + self.cleanupstack(frame) + frame.valuestack.push(frame.space.wrap(unroller)) + frame.valuestack.push(frame.space.w_None) + frame.valuestack.push(frame.space.w_None) + frame.next_instr = self.handlerposition # jump to the handler + return True # stop unrolling + + +### Internal exceptions that change the control flow ### +### and (typically) unroll the block stack ### + +class ControlFlowException(Exception): + """Abstract base class for interpreter-level exceptions that + instruct the interpreter to change the control flow and the + block stack. + + The concrete subclasses correspond to the various values WHY_XXX + values of the why_code enumeration in ceval.c: + + WHY_NOT, OK, not this one :-) + WHY_EXCEPTION, SApplicationException + WHY_RERAISE, we don't think this is needed + WHY_RETURN, SReturnValue + WHY_BREAK, SBreakLoop + WHY_CONTINUE, SContinueLoop + WHY_YIELD SYieldValue - if new_lasti == -1: - raise OperationError(space.w_ValueError, - space.wrap("line %d comes after the current code." % new_lineno)) - - # Don't jump to a line with an except in it. - if ord(code[new_lasti]) in (DUP_TOP, POP_TOP): - raise OperationError(space.w_ValueError, - space.wrap("can't jump to 'except' line as there's no exception")) - - # Don't jump into or out of a finally block. - f_lasti_setup_addr = -1 - new_lasti_setup_addr = -1 - blockstack = [] - addr = 0 - while addr < len(code): - op = ord(code[addr]) - if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY): - blockstack.append([addr, False]) - elif op == POP_BLOCK: - setup_op = ord(code[blockstack[-1][0]]) - if setup_op == SETUP_FINALLY: - blockstack[-1][1] = True - else: - blockstack.pop() - elif op == END_FINALLY: - if len(blockstack) > 0: - setup_op = ord(code[blockstack[-1][0]]) - if setup_op == SETUP_FINALLY: - blockstack.pop() - - if addr == new_lasti or addr == self.last_instr: - for ii in range(len(blockstack)): - setup_addr, in_finally = blockstack[~ii] - if in_finally: - if addr == new_lasti: - new_lasti_setup_addr = setup_addr - if addr == self.last_instr: - f_lasti_setup_addr = setup_addr - break - - if op >= HAVE_ARGUMENT: - addr += 3 - else: - addr += 1 - - assert len(blockstack) == 0 - - if new_lasti_setup_addr != f_lasti_setup_addr: - raise OperationError(space.w_ValueError, - space.wrap("can't jump into or out of a 'finally' block %d -> %d" % - (f_lasti_setup_addr, new_lasti_setup_addr))) - - if new_lasti < self.last_instr: - min_addr = new_lasti - max_addr = self.last_instr - else: - min_addr = self.last_instr - max_addr = new_lasti - - delta_iblock = min_delta_iblock = 0 - addr = min_addr - while addr < max_addr: - op = ord(code[addr]) - - if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY): - delta_iblock += 1 - elif op == POP_BLOCK: - delta_iblock -= 1 - if delta_iblock < min_delta_iblock: - min_delta_iblock = delta_iblock - - if op >= opcode.HAVE_ARGUMENT: - addr += 3 - else: - addr += 1 - - f_iblock = len(self.blockstack) - min_iblock = f_iblock + min_delta_iblock - if new_lasti > self.last_instr: - new_iblock = f_iblock + delta_iblock + """ + def action(self, frame, last_instr, executioncontext): + "Default unroller implementation." + while not frame.blockstack.empty(): + block = frame.blockstack.pop() + if block.unroll(frame, self): + break else: - new_iblock = f_iblock - delta_iblock + self.emptystack(frame) - if new_iblock > min_iblock: - raise OperationError(space.w_ValueError, - space.wrap("can't jump into the middle of a block")) - - while f_iblock > new_iblock: - block = self.blockstack.pop() - block.cleanup(self) - f_iblock -= 1 - - self.f_lineno = new_lineno - self.last_instr = new_lasti - - def get_last_lineno(self): - "Returns the line number of the instruction currently being executed." - return pytraceback.offset2lineno(self.pycode, self.last_instr) - - def fget_f_builtins(space, self): - return self.get_builtin().getdict() - - def fget_f_back(space, self): - return self.space.wrap(self.f_back) + def emptystack(self, frame): + "Default behavior when the block stack is exhausted." + # could occur e.g. when a BREAK_LOOP is not actually within a loop + raise BytecodeCorruption, "block stack exhausted" + +class SApplicationException(ControlFlowException): + """Unroll the stack because of an application-level exception + (i.e. an OperationException).""" + + def action(self, frame, last_instr, executioncontext): + e = self.args[0] + frame.last_exception = e + executioncontext.exception_trace(e) + + ControlFlowException.action(self, frame, + last_instr, executioncontext) + + def emptystack(self, frame): + # propagate the exception to the caller + if len(self.args) == 2: + operationerr, tb = self.args + raise operationerr.__class__, operationerr, tb + else: + operationerr = self.args[0] + raise operationerr + +class SBreakLoop(ControlFlowException): + """Signals a 'break' statement.""" + +class SContinueLoop(ControlFlowException): + """Signals a 'continue' statement. + Argument is the bytecode position of the beginning of the loop.""" + +class SReturnValue(ControlFlowException): + """Signals a 'return' statement. + Argument is the wrapped object to return.""" + def emptystack(self, frame): + w_returnvalue = self.args[0] + raise ExitFrame(w_returnvalue) + +class ExitFrame(Exception): + """Signals the end of the frame execution. + The argument is the returned or yielded value, already wrapped.""" - def fget_f_lasti(space, self): - return self.space.wrap(self.last_instr) - - def fget_f_trace(space, self): - return self.w_f_trace - - def fset_f_trace(space, self, w_trace): - if space.is_w(w_trace, space.w_None): - self.w_f_trace = None - else: - self.w_f_trace = w_trace - self.f_lineno = self.get_last_lineno() - space.frame_trace_action.fire() - - def fdel_f_trace(space, self): - self.w_f_trace = None - - def fget_f_exc_type(space, self): - if self.last_exception is not None: - f = self.f_back - while f is not None and f.last_exception is None: - f = f.f_back - if f is not None: - return f.last_exception.w_type - return space.w_None - - def fget_f_exc_value(space, self): - if self.last_exception is not None: - f = self.f_back - while f is not None and f.last_exception is None: - f = f.f_back - if f is not None: - return f.last_exception.w_value - return space.w_None - - def fget_f_exc_traceback(space, self): - if self.last_exception is not None: - f = self.f_back - while f is not None and f.last_exception is None: - f = f.f_back - if f is not None: - return space.wrap(f.last_exception.application_traceback) - return space.w_None - - def fget_f_restricted(space, self): - if space.config.objspace.honor__builtins__: - return space.wrap(self.builtin is not space.builtin) - return space.w_False - -# ____________________________________________________________ - -def get_block_class(opname): - # select the appropriate kind of block - from pypy.interpreter.pyopcode import block_classes - return block_classes[opname] - -def unpickle_block(space, w_tup): - w_opname, w_handlerposition, w_valuestackdepth = space.unpackiterable(w_tup) - opname = space.str_w(w_opname) - handlerposition = space.int_w(w_handlerposition) - valuestackdepth = space.int_w(w_valuestackdepth) - assert valuestackdepth >= 0 - blk = instantiate(get_block_class(opname)) - blk.handlerposition = handlerposition - blk.valuestackdepth = valuestackdepth - return blk +class BytecodeCorruption(ValueError): + """Detected bytecode corruption. Never caught; it's an error.""" Modified: pypy/branch/avm/pypy/interpreter/test/test_interpreter.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/test/test_interpreter.py (original) +++ pypy/branch/avm/pypy/interpreter/test/test_interpreter.py Thu Nov 5 20:27:32 2009 @@ -1,81 +1,75 @@ -import py -import sys +import autopath +from pypy.tool import testit -class TestInterpreter: - from pypy.interpreter.pycompiler import CPythonCompiler as CompilerClass +class TestInterpreter(testit.TestCase): def codetest(self, source, functionname, args): """Compile and run the given code string, and then call its function named by 'functionname' with arguments 'args'.""" - from pypy.interpreter import baseobjspace + from pypy.interpreter import baseobjspace, executioncontext from pypy.interpreter import pyframe, gateway, module space = self.space - source = str(py.code.Source(source).strip()) + '\n' - + compile = space.builtin.compile w = space.wrap - w_code = space.builtin.call('compile', - w(source), w(''), w('exec'), w(0), w(0)) + w_code = compile(w(source), w(''), w('exec'), w(0), w(0)) + + ec = executioncontext.ExecutionContext(space) tempmodule = module.Module(space, w("__temp__")) w_glob = tempmodule.w_dict - space.setitem(w_glob, w("__builtins__"), space.builtin) + space.setitem(w_glob, w("__builtins__"), space.w_builtins) code = space.unwrap(w_code) code.exec_code(space, w_glob, w_glob) - wrappedargs = [w(a) for a in args] + wrappedargs = w(args) wrappedfunc = space.getitem(w_glob, w(functionname)) + wrappedkwds = space.newdict([]) try: - w_output = space.call_function(wrappedfunc, *wrappedargs) + w_output = space.call(wrappedfunc, wrappedargs, wrappedkwds) except baseobjspace.OperationError, e: #e.print_detailed_traceback(space) return '<<<%s>>>' % e.errorstr(space) else: return space.unwrap(w_output) - def setup_method(self, arg): - ec = self.space.getexecutioncontext() - self.saved_compiler = ec.compiler - ec.compiler = self.CompilerClass(self.space) - - def teardown_method(self, arg): - ec = self.space.getexecutioncontext() - ec.compiler = self.saved_compiler + def setUp(self): + self.space = testit.objspace() def test_exception_trivial(self): - x = self.codetest('''\ - def f(): - try: - raise Exception() - except Exception, e: - return 1 - return 2 - ''', 'f', []) - assert x == 1 + x = self.codetest(''' +def f(): + try: + raise Exception() + except Exception, e: + return 1 + return 2 +''', 'f', []) + self.assertEquals(x, 1) def test_exception(self): x = self.codetest(''' - def f(): - try: - raise Exception, 1 - except Exception, e: - return e.args[0] - ''', 'f', []) - assert x == 1 +def f(): + try: + raise Exception, 1 + except Exception, e: + return e.args[0] +''', 'f', []) + self.assertEquals(x, 1) def test_finally(self): code = ''' - def f(a): - try: - if a: - raise Exception - a = -12 - finally: - return a - ''' - assert self.codetest(code, 'f', [0]) == -12 - assert self.codetest(code, 'f', [1]) == 1 +def f(a): + try: + if a: + raise Exception + a = -12 + finally: + return a +''' + self.assertEquals(self.codetest(code, 'f', [0]), -12) + self.assertEquals(self.codetest(code, 'f', [1]), 1) ## def test_raise(self): ## x = self.codetest(''' @@ -86,206 +80,137 @@ def test_except2(self): x = self.codetest(''' - def f(): - try: - z = 0 - try: - "x"+1 - except TypeError, e: - z = 5 - raise e - except TypeError: - return z - ''', 'f', []) - assert x == 5 +def f(): + try: + z = 0 + try: + "x"+1 + except TypeError, e: + z = 5 + raise e + except TypeError: + return z +''', 'f', []) + self.assertEquals(x, 5) def test_except3(self): code = ''' - def f(v): - z = 0 - try: - z = 1//v - except ZeroDivisionError, e: - z = "infinite result" - return z - ''' - assert self.codetest(code, 'f', [2]) == 0 - assert self.codetest(code, 'f', [0]) == "infinite result" +def f(v): + z = 0 + try: + z = 1//v + except ZeroDivisionError, e: + z = "infinite result" + return z +''' + self.assertEquals(self.codetest(code, 'f', [2]), 0) + self.assertEquals(self.codetest(code, 'f', [0]), "infinite result") ess = "TypeError: unsupported operand type" res = self.codetest(code, 'f', ['x']) - assert res.find(ess) >= 0 + self.failUnless(res.find(ess) >= 0) # the following (original) test was a bit too strict...: # self.assertEquals(self.codetest(code, 'f', ['x']), "<<>>") def test_break(self): code = ''' - def f(n): - total = 0 - for i in range(n): - try: - if i == 4: - break - finally: - total += i - return total - ''' - assert self.codetest(code, 'f', [4]) == 1+2+3 - assert self.codetest(code, 'f', [9]) == 1+2+3+4 +def f(n): + total = 0 + for i in range(n): + try: + if i == 4: + break + finally: + total += i + return total +''' + self.assertEquals(self.codetest(code, 'f', [4]), 1+2+3) + self.assertEquals(self.codetest(code, 'f', [9]), 1+2+3+4) def test_continue(self): code = ''' - def f(n): - total = 0 - for i in range(n): - try: - if i == 4: - continue - finally: - total += 100 - total += i - return total - ''' - assert self.codetest(code, 'f', [4]) == 1+2+3+400 - assert self.codetest(code, 'f', [9]) == ( +def f(n): + total = 0 + for i in range(n): + try: + if i == 4: + continue + finally: + total += 100 + total += i + return total +''' + self.assertEquals(self.codetest(code, 'f', [4]), 1+2+3+400) + self.assertEquals(self.codetest(code, 'f', [9]), 1+2+3 + 5+6+7+8+900) - def test_import(self): - # Regression test for a bug in PyFrame.IMPORT_NAME: when an - # import statement was executed in a function without a locals dict, a - # plain unwrapped None could be passed into space.call_function causing - # assertion errors later on. - real_call_function = self.space.call_function - def safe_call_function(w_obj, *arg_w): - for arg in arg_w: - assert arg is not None - return real_call_function(w_obj, *arg_w) - self.space.call_function = safe_call_function - code = ''' - def f(): - import sys - ''' - self.codetest(code, 'f', []) - - def test_extended_arg(self): - longexpr = 'x = x or ' + '-x' * 2500 - code = ''' - def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - while x: - x -= 1 # EXTENDED_ARG is for the JUMP_ABSOLUTE at the end of the loop - return x - ''' % ((longexpr,)*10) - assert self.codetest(code, 'f', [3]) == 0 - - def test_call_star_starstar(self): - code = '''\ - def f1(n): - return n*2 - def f38(n): - f = f1 - r = [ - f(n, *[]), - f(n), - apply(f, (n,)), - apply(f, [n]), - f(*(n,)), - f(*[n]), - f(n=n), - f(**{'n': n}), - apply(f, (n,), {}), - apply(f, [n], {}), - f(*(n,), **{}), - f(*[n], **{}), - f(n, **{}), - f(n, *[], **{}), - f(n=n, **{}), - f(n=n, *[], **{}), - f(*(n,), **{}), - f(*[n], **{}), - f(*[], **{'n':n}), - ] - return r - ''' - assert self.codetest(code, 'f38', [117]) == [234]*19 - - def test_star_arg(self): - code = ''' - def f(x, *y): - return y - def g(u, v): - return f(u, *v) - ''' - assert self.codetest(code, 'g', [12, ()]) == () - assert self.codetest(code, 'g', [12, (3,4)]) == (3,4) - assert self.codetest(code, 'g', [12, []]) == () - assert self.codetest(code, 'g', [12, [3,4]]) == (3,4) - assert self.codetest(code, 'g', [12, {}]) == () - assert self.codetest(code, 'g', [12, {3:1}]) == (3,) - - def test_closure(self): - code = ''' - def f(x, y): - def g(u, v): - return u - v + 7*x - return g - def callme(x, u, v): - return f(x, 123)(u, v) - ''' - assert self.codetest(code, 'callme', [1, 2, 3]) == 6 +class AppTestInterpreter(testit.AppTestCase): + def test_exception(self): + try: + raise Exception, 1 + except Exception, e: + self.assertEquals(e.args[0], 1) - def test_list_comprehension(self): - code = ''' - def f(): - return [dir() for i in [1]][0] - ''' - assert self.codetest(code, 'f', [])[0] == '_[1]' - - def test_import_statement(self): - for x in range(10): - import os - code = ''' - def f(): - for x in range(10): - import os - return os.name - ''' - assert self.codetest(code, 'f', []) == os.name + def test_trivial(self): + x = 42 + self.assertEquals(x, 42) + def test_raise(self): + def f(): + raise Exception + self.assertRaises(Exception, f) -class TestPyPyInterpreter(TestInterpreter): - """Runs the previous test with the pypy parser""" - from pypy.interpreter.pycompiler import PythonAstCompiler as CompilerClass + def test_exception(self): + try: + raise Exception + self.fail("exception failed to raise") + except: + pass + else: + self.fail("exception executing else clause!") - def test_extended_arg(self): - py.test.skip("expression too large for the recursive parser") + def test_raise2(self): + def f(r): + try: + raise r + except LookupError: + return 1 + self.assertRaises(Exception, f, Exception) + self.assertEquals(f(IndexError), 1) + def test_raise3(self): + try: + raise 1 + except TypeError: + pass + else: + self.fail("shouldn't be able to raise 1") -class AppTestInterpreter: - def test_trivial(self): - x = 42 - assert x == 42 + def test_raise_three_args(self): + import sys + try: + raise ValueError + except: + exc_type,exc_val,exc_tb = sys.exc_info() + try: + raise exc_type,exc_val,exc_tb + except: + exc_type2,exc_val2,exc_tb2 = sys.exc_info() + self.assertEquals(exc_type,exc_type2) + self.assertEquals(exc_val,exc_val2) + self.assertEquals(exc_tb,exc_tb2) def test_trivial_call(self): def f(): return 42 - assert f() == 42 + self.assertEquals(f(), 42) def test_trivial_call2(self): def f(): return 1 + 1 - assert f() == 2 + self.assertEquals(f(), 2) def test_print(self): import sys save = sys.stdout - class Out(object): + class Out: def __init__(self): self.args = [] def write(self, *args): @@ -294,10 +219,14 @@ try: sys.stdout = out print 10 - assert out.args == ['10','\n'] + self.assertEquals(out.args, ['10','\n']) finally: sys.stdout = save def test_identity(self): def f(x): return x - assert f(666) == 666 + self.assertEquals(f(666), 666) + + +if __name__ == '__main__': + testit.main() Modified: pypy/branch/avm/pypy/objspace/std/intobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/intobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/intobject.py Thu Nov 5 20:27:32 2009 @@ -1,153 +1,194 @@ from pypy.objspace.std.objspace import * -from pypy.objspace.std.noneobject import W_NoneObject -from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint -from pypy.rlib.rbigint import rbigint -from pypy.objspace.std.inttype import wrapint +from inttype import W_IntType +from noneobject import W_NoneObject +from restricted_int import r_int, LONG_BIT """ -In order to have the same behavior running -on CPython, and after RPython translation we use ovfcheck -from rarithmetic to explicitly check for overflows, -something CPython does not do anymore. +The implementation of integers is a bit difficult, +since integers are currently undergoing the change to turn +themselves into longs under overflow circumstances. +The restricted Python does not overflow or throws +exceptions. +The definitions in this file are fine, given that +restricted Python integers behave that way. +But for testing, the resticted stuff must be run +by CPython which has different behavior. +For that reason, I defined an r_int extension class +for native integers, which tries to behave as in +RPython, just for test purposes. """ class W_IntObject(W_Object): - __slots__ = 'intval' - - _immutable_ = True - - from pypy.objspace.std.inttype import int_typedef as typedef + statictype = W_IntType - def __init__(w_self, intval): - w_self.intval = intval + def __init__(w_self, space, intval): + W_Object.__init__(w_self, space) + w_self.intval = r_int(intval) def __repr__(w_self): """ representation for debugging purposes """ return "%s(%d)" % (w_self.__class__.__name__, w_self.intval) - def unwrap(w_self, space): - return int(w_self.intval) - registerimplementation(W_IntObject) -def int_w__Int(space, w_int1): - return int(w_int1.intval) - -def uint_w__Int(space, w_int1): - intval = w_int1.intval - if intval < 0: - raise OperationError(space.w_ValueError, - space.wrap("cannot convert negative integer to unsigned")) - else: - return r_uint(intval) +""" +XXX not implemented: +free list +FromString +FromUnicode +print +""" -def bigint_w__Int(space, w_int1): - return rbigint.fromint(w_int1.intval) +def unwrap__Int(space, w_int1): + return int(w_int1.intval) def repr__Int(space, w_int1): a = w_int1.intval - res = str(a) + res = "%ld" % a return space.wrap(res) str__Int = repr__Int -def declare_new_int_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_int1, w_int2): - i = w_int1.intval - j = w_int2.intval - return space.newbool(op(i, j)) - name = opname + "__Int_Int" - return func_with_new_name(f, name), name - -for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op) - globals()[name] = func +## deprecated +## we are going to support rich compare, only -def hash__Int(space, w_int1): - # unlike CPython, we don't special-case the value -1 in most of our - # hash functions, so there is not much sense special-casing it here either. - # Make sure this is consistent with the hash of floats and longs. - return int__Int(space, w_int1) - -# coerce -def coerce__Int_Int(space, w_int1, w_int2): - return space.newtuple([w_int1, w_int2]) +##def int_int_cmp(space, w_int1, w_int2): +## i = w_int1.intval +## j = w_int2.intval +## if i < j: +## ret = -1 +## elif i > j: +## ret = 1 +## else: +## ret = 0 +## return W_IntObject(space, ret) +## +##StdObjSpace.cmp.register(int_int_cmp, W_IntObject, W_IntObject) + +def lt__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i < j ) + +def le__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i <= j ) + +def eq__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i == j ) + +def ne__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i != j ) + +def gt__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i > j ) + +def ge__Int_Int(space, w_int1, w_int2): + i = w_int1.intval + j = w_int2.intval + return space.newbool( i >= j ) + +STRICT_HASH = True # temporary, to be put elsewhere or removed + +def _hash_strict(space, w_int1): + #/* XXX If this is changed, you also need to change the way + # Python's long, float and complex types are hashed. */ + x = w_int1.intval + if x == -1: + x = -2 + return W_IntObject(space, x) + +def _hash_liberal(space, w_int1): + # Armin: unlike CPython we have no need to special-case the value -1 + return w_int1 + +# Chris: I'm not yet convinced that we want to make hash() +# return different values that CPython does. +# So for the moment, both versions are here, +# and we might think of some config options +# or decide to drop compatibility (using pypy-dev). +def hash__Int(space, w_int1): + if STRICT_HASH: + return _hash_strict(space, w_int1) + else: + return _hash_liberal(space, w_int1) def add__Int_Int(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x + y) + z = x + y except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer addition")) - return wrapint(space, z) + return W_IntObject(space, z) def sub__Int_Int(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x - y) + z = x - y except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer substraction")) - return wrapint(space, z) + return W_IntObject(space, z) def mul__Int_Int(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x * y) + z = x * y except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer multiplication")) - return wrapint(space, z) + return W_IntObject(space, z) -def floordiv__Int_Int(space, w_int1, w_int2): +def _floordiv(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x // y) + z = x // y except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer division by zero")) except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer division")) - return wrapint(space, z) -div__Int_Int = floordiv__Int_Int + return W_IntObject(space, z) -def truediv__Int_Int(space, w_int1, w_int2): - x = float(w_int1.intval) - y = float(w_int2.intval) - if y == 0.0: - raise FailedToImplement(space.w_ZeroDivisionError, space.wrap("float division")) - return space.wrap(x / y) +def _truediv(space, w_int1, w_int2): + # cannot implement, since it gives floats + raise FailedToImplement(space.w_OverflowError, + space.wrap("integer division")) def mod__Int_Int(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x % y) + z = x % y except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer modulo by zero")) except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer modulo")) - return wrapint(space, z) + return W_IntObject(space, z) def divmod__Int_Int(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval try: - z = ovfcheck(x // y) + z = x // y except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer divmod by zero")) @@ -156,30 +197,41 @@ space.wrap("integer modulo")) # no overflow possible m = x % y - w = space.wrap - return space.newtuple([w(z), w(m)]) + return space.wrap((z,m)) + +def div__Int_Int(space, w_int1, w_int2): + # Select the proper div + if 1 / 2 == 1 // 2: + return _floordiv(space, w_int1, w_int2) + else: + return _truediv(space, w_int1, w_int2) +floordiv__Int_Int = _floordiv # helper for pow() -def _impl_int_int_pow(space, iv, iw, iz=0): +def _impl_int_int_pow(space, iv, iw, iz=None): if iw < 0: - if iz != 0: + if iz is not None: raise OperationError(space.w_TypeError, space.wrap("pow() 2nd argument " "cannot be negative when 3rd argument specified")) ## bounce it, since it always returns float raise FailedToImplement(space.w_ValueError, space.wrap("integer exponentiation")) + if iz is not None: + if iz == 0: + raise OperationError(space.w_ValueError, + space.wrap("pow() 3rd argument cannot be 0")) temp = iv ix = 1 try: while iw > 0: if iw & 1: - ix = ovfcheck(ix*temp) + ix = ix*temp iw >>= 1 #/* Shift exponent down by 1 bit */ if iw==0: break - temp = ovfcheck(temp*temp) #/* Square the value of temp */ + temp *= temp #/* Square the value of temp */ if iz: #/* If we did a multiplication, perform a modulo */ ix = ix % iz; @@ -189,31 +241,48 @@ except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer exponentiation")) - return wrapint(space, ix) + return ix +""" def pow__Int_Int_Int(space, w_int1, w_int2, w_int3): x = w_int1.intval y = w_int2.intval z = w_int3.intval - if z == 0: - raise OperationError(space.w_ValueError, - space.wrap("pow() 3rd argument cannot be 0")) - return _impl_int_int_pow(space, x, y, z) + ret = _impl_int_int_pow(space, x, y, z) + return W_IntObject(space, ret) +""" + +def pow__Int_Int_Int(space, w_int1, w_int2, w_int3): + x = w_int1.intval + y = w_int2.intval + z = w_int3.intval + ret = _impl_int_int_pow(space, x, y, z) + return W_IntObject(space, ret) def pow__Int_Int_None(space, w_int1, w_int2, w_int3): x = w_int1.intval y = w_int2.intval - return _impl_int_int_pow(space, x, y) + ret = _impl_int_int_pow(space, x, y) + return W_IntObject(space, ret) def neg__Int(space, w_int1): a = w_int1.intval try: - x = ovfcheck(-a) + x = -a except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer negation")) - return wrapint(space, x) + return W_IntObject(space, x) +# pos__Int is supposed to do nothing, unless it has +# a derived integer object, where it should return +# an exact one. +def pos__Int(space, w_int1): + #not sure if this should be done this way: + if w_int1.__class__ is W_IntObject: + return w_int1 + a = w_int1.intval + return W_IntObject(space, a) def abs__Int(space, w_int1): if w_int1.intval >= 0: @@ -221,13 +290,14 @@ else: return neg__Int(space, w_int1) -def nonzero__Int(space, w_int1): - return space.newbool(w_int1.intval != 0) +def is_true__Int(space, w_int1): + ''' note: this must return an UNWRAPPED bool!!! ''' + return w_int1.intval != 0 def invert__Int(space, w_int1): x = w_int1.intval a = ~x - return wrapint(space, a) + return W_IntObject(space, a) def lshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval @@ -236,16 +306,25 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return Int_pos(w_int1) if b >= LONG_BIT: raise FailedToImplement(space.w_OverflowError, space.wrap("integer left shift")) + ## + ## XXX please! have a look into pyport.h and see how to implement + ## the overflow checking, using macro Py_ARITHMETIC_RIGHT_SHIFT + ## we *assume* that the overflow checking is done correctly + ## in the code generator, which is not trivial! try: - c = ovfcheck_lshift(a, b) + c = a << b + ## the test in C code is + ## if (a != Py_ARITHMETIC_RIGHT_SHIFT(long, c, b)) { + ## if (PyErr_Warn(PyExc_FutureWarning, + # and so on except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer left shift")) - return wrapint(space, c) + return W_IntObject(space, c) def rshift__Int_Int(space, w_int1, w_int2): a = w_int1.intval @@ -254,46 +333,59 @@ raise OperationError(space.w_ValueError, space.wrap("negative shift count")) if a == 0 or b == 0: - return int__Int(space, w_int1) + return Int_pos(w_int1) if b >= LONG_BIT: if a < 0: a = -1 else: a = 0 else: + ## please look into pyport.h, how >> should be implemented! + ## a = Py_ARITHMETIC_RIGHT_SHIFT(long, a, b); a = a >> b - return wrapint(space, a) + return W_IntObject(space, a) def and__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval res = a & b - return wrapint(space, res) + return W_IntObject(space, res) def xor__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval res = a ^ b - return wrapint(space, res) + return W_IntObject(space, res) def or__Int_Int(space, w_int1, w_int2): a = w_int1.intval b = w_int2.intval res = a | b - return wrapint(space, res) + return W_IntObject(space, res) + +# coerce is not wanted +## +##static int +##coerce__Int(PyObject **pv, PyObject **pw) +##{ +## if (PyInt_Check(*pw)) { +## Py_INCREF(*pv); +## Py_INCREF(*pw); +## return 0; +## } +## return 1; /* Can't do it */ +##} -# int__Int is supposed to do nothing, unless it has -# a derived integer object, where it should return -# an exact one. def int__Int(space, w_int1): - if space.is_w(space.type(w_int1), space.w_int): - return w_int1 - a = w_int1.intval - return wrapint(space, a) -pos__Int = int__Int + return w_int1 -def index__Int(space, w_int1): - return int__Int(space, w_int1) +""" +# Not registered +def long__Int(space, w_int1): + a = w_int1.intval + x = long(a) ## XXX should this really be done so? + return space.newlong(x) +""" def float__Int(space, w_int1): a = w_int1.intval @@ -301,13 +393,33 @@ return space.newfloat(x) def oct__Int(space, w_int1): - return space.wrap(oct(w_int1.intval)) + x = w_int1.intval + if x < 0: + ## XXX what about this warning? + #if (PyErr_Warn(PyExc_FutureWarning, + # "hex()/oct() of negative int will return " + # "a signed string in Python 2.4 and up") < 0) + # return NULL; + pass + if x == 0: + ret = "0" + else: + ret = "0%lo" % x + return space.wrap(ret) def hex__Int(space, w_int1): - return space.wrap(hex(w_int1.intval)) - -def getnewargs__Int(space, w_int1): - return space.newtuple([wrapint(space, w_int1.intval)]) - + x = w_int1.intval + if x < 0: + ## XXX what about this warning? + #if (PyErr_Warn(PyExc_FutureWarning, + # "hex()/oct() of negative int will return " + # "a signed string in Python 2.4 and up") < 0) + # return NULL; + pass + if x == 0: + ret = "0" + else: + ret = "0x%lx" % x + return space.wrap(ret) register_all(vars()) Modified: pypy/branch/avm/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/listobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/listobject.py Thu Nov 5 20:27:32 2009 @@ -1,531 +1,536 @@ from pypy.objspace.std.objspace import * -from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.listtype import get_list_index -from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice - -from pypy.objspace.std import slicetype -from pypy.interpreter import gateway, baseobjspace -from pypy.rlib.listsort import TimSort +from listtype import W_ListType +from intobject import W_IntObject +from sliceobject import W_SliceObject +from tupleobject import W_TupleObject + +import slicetype +from pypy.interpreter import gateway +from restricted_int import r_int, r_uint + class W_ListObject(W_Object): - from pypy.objspace.std.listtype import list_typedef as typedef - - def __init__(w_self, wrappeditems): - w_self.wrappeditems = wrappeditems + statictype = W_ListType + + def __init__(w_self, space, wrappeditems): + W_Object.__init__(w_self, space) + w_self.ob_item = [] + w_self.ob_size = 0 + newlen = len(wrappeditems) + _list_resize(w_self, newlen) + w_self.ob_size = newlen + items = w_self.ob_item + p = newlen + while p: + p -= 1 + items[p] = wrappeditems[p] def __repr__(w_self): """ representation for debugging purposes """ - return "%s(%s)" % (w_self.__class__.__name__, w_self.wrappeditems) + reprlist = [repr(w_item) for w_item in w_self.ob_item[:w_self.ob_size]] + return "%s(%s)" % (w_self.__class__.__name__, ', '.join(reprlist)) - def unwrap(w_list, space): - items = [space.unwrap(w_item) for w_item in w_list.wrappeditems]# XXX generic mixed types unwrap - return list(items) - - def append(w_list, w_item): - w_list.wrappeditems.append(w_item) registerimplementation(W_ListObject) -EMPTY_LIST = W_ListObject([]) - -def init__List(space, w_list, __args__): - w_iterable, = __args__.parse('list', - (['sequence'], None, None), # signature - [EMPTY_LIST]) # default argument - # - # this is the old version of the loop at the end of this function: - # - # w_list.wrappeditems = space.unpackiterable(w_iterable) - # - # This is commented out to avoid assigning a new RPython list to - # 'wrappeditems', which defeats the W_FastSeqIterObject optimization. - # - items_w = w_list.wrappeditems - del items_w[:] - if w_iterable is not EMPTY_LIST: +def unwrap__List(space, w_list): + items = [space.unwrap(w_item) for w_item in w_list.ob_item[:w_list.ob_size]] + return list(items) + +def object_init__List(space, w_list, w_args, w_kwds): + if space.is_true(w_kwds): + raise OperationError(space.w_TypeError, + space.wrap("no keyword arguments expected")) + w_list.ob_size = 0 # XXX think about it later + args = space.unpackiterable(w_args) + if len(args) == 0: + pass # empty list + elif len(args) == 1: + w_iterable = args[0] w_iterator = space.iter(w_iterable) while True: try: w_item = space.next(w_iterator) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise + except NoValue: break # done - items_w.append(w_item) + _ins1(w_list, w_list.ob_size, w_item) + else: + raise OperationError(space.w_TypeError, + space.wrap("list() takes at most 1 argument")) def len__List(space, w_list): - result = len(w_list.wrappeditems) - return wrapint(space, result) + result = w_list.ob_size + return W_IntObject(space, result) -def getitem__List_ANY(space, w_list, w_index): - try: - return w_list.wrappeditems[get_list_index(space, w_index)] - except IndexError: +def getitem__List_Int(space, w_list, w_index): + items = w_list.ob_item + idx = w_index.intval + if idx < 0: + idx += w_list.ob_size + if idx < 0 or idx >= w_list.ob_size: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) + w_item = items[idx] + return w_item def getitem__List_Slice(space, w_list, w_slice): - # XXX consider to extend rlist's functionality? - length = len(w_list.wrappeditems) - start, stop, step, slicelength = w_slice.indices4(space, length) + items = w_list.ob_item + length = w_list.ob_size + start, stop, step, slicelength = slicetype.indices4(space, w_slice, length) assert slicelength >= 0 - if step == 1 and 0 <= start <= stop: - return W_ListObject(w_list.wrappeditems[start:stop]) - w_res = W_ListObject([None] * slicelength) - items_w = w_list.wrappeditems - subitems_w = w_res.wrappeditems + w_res = W_ListObject(space, []) + _list_resize(w_res, slicelength) + subitems = w_res.ob_item for i in range(slicelength): - subitems_w[i] = items_w[start] + subitems[i] = items[start] start += step + w_res.ob_size = slicelength return w_res -def getslice__List_ANY_ANY(space, w_list, w_start, w_stop): - length = len(w_list.wrappeditems) - start, stop = normalize_simple_slice(space, length, w_start, w_stop) - return W_ListObject(w_list.wrappeditems[start:stop]) - -def setslice__List_ANY_ANY_ANY(space, w_list, w_start, w_stop, w_sequence): - length = len(w_list.wrappeditems) - start, stop = normalize_simple_slice(space, length, w_start, w_stop) - _setitem_slice_helper(space, w_list, start, 1, stop-start, w_sequence) - -def delslice__List_ANY_ANY(space, w_list, w_start, w_stop): - length = len(w_list.wrappeditems) - start, stop = normalize_simple_slice(space, length, w_start, w_stop) - _delitem_slice_helper(space, w_list, start, 1, stop-start) - -def contains__List_ANY(space, w_list, w_obj): - # needs to be safe against eq_w() mutating the w_list behind our back - i = 0 - items_w = w_list.wrappeditems - while i < len(items_w): # intentionally always calling len! - if space.eq_w(items_w[i], w_obj): - return space.w_True - i += 1 - return space.w_False - def iter__List(space, w_list): - from pypy.objspace.std import iterobject - return iterobject.W_FastListIterObject(w_list, w_list.wrappeditems) + import iterobject + return iterobject.W_SeqIterObject(space, w_list) def add__List_List(space, w_list1, w_list2): - return W_ListObject(w_list1.wrappeditems + w_list2.wrappeditems) + w_res = W_ListObject(space, []) + newlen = w_list1.ob_size + w_list2.ob_size + _list_resize(w_res, newlen) + p = 0 + items = w_res.ob_item + src = w_list1.ob_item + for i in range(w_list1.ob_size): + items[p] = src[i] + p += 1 + src = w_list2.ob_item + for i in range(w_list2.ob_size): + items[p] = src[i] + p += 1 + w_res.ob_size = p + return w_res +def mul__List_Int(space, w_list, w_int): + w_res = W_ListObject(space, []) + times = w_int.intval + src = w_list.ob_item + size = w_list.ob_size + newlen = size * times # XXX check overflow + _list_resize(w_res, newlen) + items = w_res.ob_item + p = 0 + for _ in range(times): + for i in range(size): + items[p] = src[i] + p += 1 + w_res.ob_size = p + return w_res -def inplace_add__List_ANY(space, w_list1, w_iterable2): - list_extend__List_ANY(space, w_list1, w_iterable2) - return w_list1 - -def inplace_add__List_List(space, w_list1, w_list2): - list_extend__List_List(space, w_list1, w_list2) - return w_list1 - -def mul_list_times(space, w_list, w_times): - try: - times = space.getindex_w(w_times, space.w_OverflowError) - except OperationError, e: - if e.match(space, space.w_TypeError): - raise FailedToImplement - raise - return W_ListObject(w_list.wrappeditems * times) - -def mul__List_ANY(space, w_list, w_times): - return mul_list_times(space, w_list, w_times) - -def mul__ANY_List(space, w_times, w_list): - return mul_list_times(space, w_list, w_times) - -def inplace_mul__List_ANY(space, w_list, w_times): - try: - times = space.getindex_w(w_times, space.w_OverflowError) - except OperationError, e: - if e.match(space, space.w_TypeError): - raise FailedToImplement - raise - w_list.wrappeditems *= times - return w_list +def mul__Int_List(space, w_int, w_list): + return mul__List_Int(space, w_list, w_int) def eq__List_List(space, w_list1, w_list2): - # needs to be safe against eq_w() mutating the w_lists behind our back - items1_w = w_list1.wrappeditems - items2_w = w_list2.wrappeditems - return equal_wrappeditems(space, items1_w, items2_w) - -def equal_wrappeditems(space, items1_w, items2_w): - if len(items1_w) != len(items2_w): + items1 = w_list1.ob_item + items2 = w_list2.ob_item + if w_list1.ob_size != w_list2.ob_size: return space.w_False - i = 0 - while i < len(items1_w) and i < len(items2_w): - if not space.eq_w(items1_w[i], items2_w[i]): + for i in range(w_list1.ob_size): + if not space.is_true(space.eq(items1[i], items2[i])): return space.w_False - i += 1 return space.w_True -def lessthan_unwrappeditems(space, items1_w, items2_w): - # needs to be safe against eq_w() mutating the w_lists behind our back +def _min(a, b): + if a < b: + return a + return b + +def lt__List_List(space, w_list1, w_list2): + items1 = w_list1.ob_item + items2 = w_list2.ob_item + ncmp = _min(w_list1.ob_size, w_list2.ob_size) # Search for the first index where items are different - i = 0 - while i < len(items1_w) and i < len(items2_w): - w_item1 = items1_w[i] - w_item2 = items2_w[i] - if not space.eq_w(w_item1, w_item2): - return space.lt(w_item1, w_item2) - i += 1 + for p in range(ncmp): + if not space.is_true(space.eq(items1[p], items2[p])): + return space.lt(items1[p], items2[p]) # No more items to compare -- compare sizes - return space.newbool(len(items1_w) < len(items2_w)) + return space.newbool(w_list1.ob_size < w_list2.ob_size) -def greaterthan_unwrappeditems(space, items1_w, items2_w): - # needs to be safe against eq_w() mutating the w_lists behind our back +def gt__List_List(space, w_list1, w_list2): + items1 = w_list1.ob_item + items2 = w_list2.ob_item + ncmp = _min(w_list1.ob_size, w_list2.ob_size) # Search for the first index where items are different - i = 0 - while i < len(items1_w) and i < len(items2_w): - w_item1 = items1_w[i] - w_item2 = items2_w[i] - if not space.eq_w(w_item1, w_item2): - return space.gt(w_item1, w_item2) - i += 1 + for p in range(ncmp): + if not space.is_true(space.eq(items1[p], items2[p])): + return space.gt(items1[p], items2[p]) # No more items to compare -- compare sizes - return space.newbool(len(items1_w) > len(items2_w)) - -def lt__List_List(space, w_list1, w_list2): - return lessthan_unwrappeditems(space, w_list1.wrappeditems, - w_list2.wrappeditems) + return space.newbool(w_list1.ob_size > w_list2.ob_size) -def gt__List_List(space, w_list1, w_list2): - return greaterthan_unwrappeditems(space, w_list1.wrappeditems, - w_list2.wrappeditems) +# upto here, lists are nearly identical to tuples, despite the +# fact that we now support over-allocation! -def delitem__List_ANY(space, w_list, w_idx): - idx = get_list_index(space, w_idx) - try: - del w_list.wrappeditems[idx] - except IndexError: +def delitem__List_Int(space, w_list, w_idx): + i = w_idx.intval + if i < 0: + i += w_list.ob_size + if i < 0 or i >= w_list.ob_size: raise OperationError(space.w_IndexError, space.wrap("list deletion index out of range")) + _del_slice(w_list, i, i+1) return space.w_None - def delitem__List_Slice(space, w_list, w_slice): - start, stop, step, slicelength = w_slice.indices4(space, - len(w_list.wrappeditems)) - _delitem_slice_helper(space, w_list, start, step, slicelength) - -def _delitem_slice_helper(space, w_list, start, step, slicelength): - if slicelength==0: - return - - if step < 0: - start = start + step * (slicelength-1) - step = -step - + start, stop, step, slicelength = slicetype.indices4(space, w_slice, w_list.ob_size) if step == 1: - assert start >= 0 - assert slicelength >= 0 - del w_list.wrappeditems[start:start+slicelength] - else: - items = w_list.wrappeditems - n = len(items) - i = start - - for discard in range(1, slicelength): - j = i+1 - i += step - while j < i: - items[j-discard] = items[j] - j += 1 - - j = i+1 - while j < n: - items[j-slicelength] = items[j] - j += 1 - start = n - slicelength - assert start >= 0 # annotator hint - del items[start:] - -def setitem__List_ANY_ANY(space, w_list, w_index, w_any): - idx = get_list_index(space, w_index) - try: - w_list.wrappeditems[idx] = w_any - except IndexError: + return _setitem_slice_helper(space, w_list, w_slice, [], 0) + + # The current code starts from the top, to simplify + # coding. A later optimization could be to start from + # the bottom, which would reduce the list motion. + # A further later optimization would be to special-case + # a step of -1, because this version will perform a LOT + # of extra motion for this case. Anybody with a real-life + # use-case for this is welcome to write the special case. + r = range(start, stop, step) + if step > 0: + r.reverse() + for i in r: + _del_slice(w_list, i, i+1) + return space.w_None + +def setitem__List_Int_ANY(space, w_list, w_index, w_any): + items = w_list.ob_item + idx = w_index.intval + if idx < 0: + idx += w_list.ob_size + if idx < 0 or idx >= w_list.ob_size: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) + items[idx] = w_any return space.w_None -def setitem__List_Slice_ANY(space, w_list, w_slice, w_iterable): - oldsize = len(w_list.wrappeditems) - start, stop, step, slicelength = w_slice.indices4(space, oldsize) - _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable) - -def _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable): - if isinstance(w_iterable, W_ListObject): - sequence2 = w_iterable.wrappeditems - else: - sequence2 = space.unpackiterable(w_iterable) +def setitem__List_Slice_List(space, w_list, w_slice, w_list2): + return _setitem_slice_helper(space, w_list, w_slice, w_list2.ob_item, w_list2.ob_size) + +def setitem__List_Slice_Tuple(space, w_list, w_slice, w_tuple): + t = w_tuple.wrappeditems + return _setitem_slice_helper(space, w_list, w_slice, t, len(t)) +def _setitem_slice_helper(space, w_list, w_slice, sequence2, len2): + start, stop, step, slicelength = slicetype.indices4(space, w_slice, w_list.ob_size) assert slicelength >= 0 - items = w_list.wrappeditems - oldsize = len(items) - len2 = len(sequence2) + if step == 1: # Support list resizing for non-extended slices - delta = slicelength - len2 - if delta < 0: - delta = -delta - newsize = oldsize + delta - # XXX support this in rlist! - items += [None] * delta - lim = start+len2 - i = newsize - 1 - while i >= lim: - items[i] = items[i-delta] - i -= 1 - elif start >= 0: - del items[start:start+delta] - else: - assert delta==0 + oldsize = w_list.ob_size + delta = len2 - slicelength + newsize = oldsize + delta + _list_resize(w_list, newsize) + w_list.ob_size = newsize + r = range(stop+delta, newsize) + if delta > 0: + r.reverse() + items = w_list.ob_item + for i in r: + items[i] = items[i-delta] elif len2 != slicelength: # No resize for extended slices raise OperationError(space.w_ValueError, space.wrap("attempt to " "assign sequence of size %d to extended slice of size %d" % (len2,slicelength))) + r = range(len2) + items = w_list.ob_item if sequence2 is items: if step > 0: # Always copy starting from the right to avoid # having to make a shallow copy in the case where # the source and destination lists are the same list. - i = len2 - 1 - start += i*step - while i >= 0: - items[start] = sequence2[i] - start -= step - i -= 1 - return + r.reverse() else: # Make a shallow copy to more easily handle the reversal case sequence2 = list(sequence2) - for i in range(len2): - items[start] = sequence2[i] - start += step - -app = gateway.applevel(""" - def listrepr(currently_in_repr, l): - 'The app-level part of repr().' - list_id = id(l) - if list_id in currently_in_repr: - return '[...]' - currently_in_repr[list_id] = 1 - try: - return "[" + ", ".join([repr(x) for x in l]) + ']' - finally: - try: - del currently_in_repr[list_id] - except: - pass -""", filename=__file__) - -listrepr = app.interphook("listrepr") + for i in r: + items[start+i*step] = sequence2[i] + return space.w_None def repr__List(space, w_list): - if len(w_list.wrappeditems) == 0: - return space.wrap('[]') - ec = space.getexecutioncontext() - w_currently_in_repr = ec._py_repr - if w_currently_in_repr is None: - w_currently_in_repr = ec._py_repr = space.newdict() - return listrepr(space, w_currently_in_repr, w_list) - -def list_insert__List_ANY_ANY(space, w_list, w_where, w_any): - where = space.int_w(w_where) - length = len(w_list.wrappeditems) + w = space.wrap + a = space.add + reprs_w = map(space.repr, space.unpackiterable(w_list)) + from pypy.objspace.std.stringtype import W_StringType + w_bm = space.getattr(space.wrap(', '), space.wrap('join')) + return a(a(w('['), space.call_function(w_bm, space.newlist(reprs_w))), w(']')) + return space.newstring([]) + +def hash__List(space,w_list): + raise OperationError(space.w_TypeError,space.wrap("list objects are unhashable")) + +# adapted C code +def _roundupsize(n): + nbits = r_uint(0) + n2 = n >> 5 + +## /* Round up: +## * If n < 256, to a multiple of 8. +## * If n < 2048, to a multiple of 64. +## * If n < 16384, to a multiple of 512. +## * If n < 131072, to a multiple of 4096. +## * If n < 1048576, to a multiple of 32768. +## * If n < 8388608, to a multiple of 262144. +## * If n < 67108864, to a multiple of 2097152. +## * If n < 536870912, to a multiple of 16777216. +## * ... +## * If n < 2**(5+3*i), to a multiple of 2**(3*i). +## * +## * This over-allocates proportional to the list size, making room +## * for additional growth. The over-allocation is mild, but is +## * enough to give linear-time amortized behavior over a long +## * sequence of appends() in the presence of a poorly-performing +## * system realloc() (which is a reality, e.g., across all flavors +## * of Windows, with Win9x behavior being particularly bad -- and +## * we've still got address space fragmentation problems on Win9x +## * even with this scheme, although it requires much longer lists to +## * provoke them than it used to). +## */ + while 1: + n2 >>= 3 + nbits += 3 + if not n2 : + break + return ((n >> nbits) + 1) << nbits + +# before we have real arrays, +# we use lists, allocated to fixed size. +# XXX memory overflow is ignored here. +# See listobject.c for reference. + +for_later = """ +#define NRESIZE(var, type, nitems) \ +do { \ + size_t _new_size = _roundupsize(nitems); \ + if (_new_size <= ((~(size_t)0) / sizeof(type))) \ + PyMem_RESIZE(var, type, _new_size); \ + else \ + var = NULL; \ +} while (0) +""" + +def _list_resize(w_list, newlen): + if newlen > len(w_list.ob_item): + true_size = _roundupsize(newlen) + old_items = w_list.ob_item + w_list.ob_item = items = [None] * true_size + for p in range(len(old_items)): + items[p] = old_items[p] + +def _ins1(w_list, where, w_any): + _list_resize(w_list, w_list.ob_size+1) + size = w_list.ob_size + items = w_list.ob_item if where < 0: - where += length - if where < 0: - where = 0 - elif where > length: - where = length - w_list.wrappeditems.insert(where, w_any) - return space.w_None + where += size + if where < 0: + where = 0 + if (where > size): + where = size + for i in range(size, where, -1): + items[i] = items[i-1] + items[where] = w_any + w_list.ob_size += 1 -def list_append__List_ANY(space, w_list, w_any): - w_list.wrappeditems.append(w_any) +def list_insert__List_Int_ANY(space, w_list, w_where, w_any): + _ins1(w_list, w_where.intval, w_any) return space.w_None -def list_extend__List_List(space, w_list, w_other): - w_list.wrappeditems += w_other.wrappeditems +def list_append__List_ANY(space, w_list, w_any): + _ins1(w_list, w_list.ob_size, w_any) return space.w_None def list_extend__List_ANY(space, w_list, w_any): - w_list.wrappeditems += space.unpackiterable(w_any) + lis = space.unpackiterable(w_any) + newlen = w_list.ob_size + len(lis) + _list_resize(w_list, newlen) + d = w_list.ob_size + items = w_list.ob_item + for i in range(len(lis)): + items[d+i] = lis[i] + w_list.ob_size = newlen return space.w_None +def _del_slice(w_list, ilow, ihigh): + """ similar to the deletion part of list_ass_slice in CPython """ + if ilow < 0: + ilow = 0 + elif ilow > w_list.ob_size: + ilow = w_list.ob_size + if ihigh < ilow: + ihigh = ilow + elif ihigh > w_list.ob_size: + ihigh = w_list.ob_size + items = w_list.ob_item + d = ihigh-ilow + # XXX this is done by CPython to hold the elements + # to be deleted. I have no idea how to express + # this here, but we need to be aware when we write + # a compiler. + # recycle = [items[i] for i in range(ilow, ihigh)] + for i in range(ilow, w_list.ob_size - d): + items[i] = items[i+d] + items[i+d] = None + w_list.ob_size -= d + # note that the default value will come back wrapped!!! -def list_pop__List_ANY(space, w_list, w_idx=-1): - items = w_list.wrappeditems - if len(items)== 0: +def list_pop__List_Int(space, w_list, w_idx=-1): + if w_list.ob_size == 0: raise OperationError(space.w_IndexError, space.wrap("pop from empty list")) - idx = space.int_w(w_idx) - try: - return items.pop(idx) - except IndexError: + i = w_idx.intval + if i < 0: + i += w_list.ob_size + if i < 0 or i >= w_list.ob_size: raise OperationError(space.w_IndexError, space.wrap("pop index out of range")) + w_res = w_list.ob_item[i] + _del_slice(w_list, i, i+1) + return w_res def list_remove__List_ANY(space, w_list, w_any): - # needs to be safe against eq_w() mutating the w_list behind our back - items = w_list.wrappeditems - i = 0 - while i < len(items): - if space.eq_w(items[i], w_any): - if i < len(items): # if this is wrong the list was changed - del items[i] + eq = space.eq + items = w_list.ob_item + for i in range(w_list.ob_size): + cmp = eq(items[i], w_any) + if space.is_true(cmp): + _del_slice(w_list, i, i+1) return space.w_None - i += 1 - raise OperationError(space.w_ValueError, + raise OperationError(space.w_IndexError, space.wrap("list.remove(x): x not in list")) -def list_index__List_ANY_ANY_ANY(space, w_list, w_any, w_start, w_stop): - # needs to be safe against eq_w() mutating the w_list behind our back - items = w_list.wrappeditems - size = len(items) - i = slicetype.adapt_bound(space, size, w_start) - stop = slicetype.adapt_bound(space, size, w_stop) - while i < stop and i < len(items): - if space.eq_w(items[i], w_any): +def list_index__List_ANY_Int_Int(space, w_list, w_any, w_start, w_stop): + eq = space.eq + items = w_list.ob_item + size = w_list.ob_size + start = space.unwrap(w_start) + if start < 0: + start += size + start = min(max(0,start),size) + stop = space.unwrap(w_stop) + if stop < 0: + stop += size + stop = min(max(start,stop),size) + + for i in range(start,stop): + cmp = eq(items[i], w_any) + if space.is_true(cmp): return space.wrap(i) - i += 1 raise OperationError(space.w_ValueError, space.wrap("list.index(x): x not in list")) def list_count__List_ANY(space, w_list, w_any): - # needs to be safe against eq_w() mutating the w_list behind our back - count = 0 - i = 0 - items = w_list.wrappeditems - while i < len(items): - if space.eq_w(items[i], w_any): + eq = space.eq + items = w_list.ob_item + count = r_int(0) + for i in range(w_list.ob_size): + cmp = eq(items[i], w_any) + if space.is_true(cmp): count += 1 - i += 1 return space.wrap(count) +# Reverse a slice of a list in place, from lo up to (exclusive) hi. +# (also used in sort, later) + +def _reverse_slice(lis, lo, hi): + hi -= 1 + while lo < hi: + t = lis[lo] + lis[lo] = lis[hi] + lis[hi] = t + lo += 1 + hi -= 1 + def list_reverse__List(space, w_list): - w_list.wrappeditems.reverse() + if w_list.ob_size > 1: + _reverse_slice(w_list.ob_item, 0, w_list.ob_size) return space.w_None -# ____________________________________________________________ -# Sorting + -# Reverse a slice of a list in place, from lo up to (exclusive) hi. -# (used in sort) +# Python Quicksort Written by Magnus Lie Hetland +# http://www.hetland.org/python/quicksort.html -class KeyContainer(baseobjspace.W_Root): - def __init__(self, w_key, w_item): - self.w_key = w_key - self.w_item = w_item - -# NOTE: all the subclasses of TimSort should inherit from a common subclass, -# so make sure that only SimpleSort inherits directly from TimSort. -# This is necessary to hide the parent method TimSort.lt() from the -# annotator. -class SimpleSort(TimSort): - def lt(self, a, b): - space = self.space - return space.is_true(space.lt(a, b)) - -class CustomCompareSort(SimpleSort): - def lt(self, a, b): - space = self.space - w_cmp = self.w_cmp - w_result = space.call_function(w_cmp, a, b) - try: - result = space.int_w(w_result) - except OperationError, e: - if e.match(space, space.w_TypeError): +# NOTE: we cannot yet detect that a user comparision +# function modifies the list in-place. The +# CPython sort() should be studied to learn how +# to implement this functionality. + +def _partition(list, start, end, lt): + pivot = list[end] # Partition around the last value + bottom = start-1 # Start outside the area to be partitioned + top = end # Ditto + + done = 0 + while not done: # Until all elements are partitioned... + + while not done: # Until we find an out of place element... + bottom = bottom+1 # ... move the bottom up. + + if bottom == top: # If we hit the top... + done = 1 # ... we are done. + break + + if lt(pivot, list[bottom]): # Is the bottom out of place? + list[top] = list[bottom] # Then put it at the top... + break # ... and start searching from the top. + + while not done: # Until we find an out of place element... + top = top-1 # ... move the top down. + + if top == bottom: # If we hit the bottom... + done = 1 # ... we are done. + break + + if lt(list[top], pivot): # Is the top out of place? + list[bottom] = list[top] # Then put it at the bottom... + break # ...and start searching from the bottom. + + list[top] = pivot # Put the pivot in its place. + return top # Return the split point + + +def _quicksort(list, start, end, lt): + if start < end: # If there are two or more elements... + split = _partition(list, start, end, lt) # ... partition the sublist... + _quicksort(list, start, split-1, lt) # ... and sort both halves. + _quicksort(list, split+1, end, lt) + +def list_sort__List_ANY(space, w_list, w_cmp): + if w_cmp is space.w_None: + def lt(a,b): + return space.is_true(space.lt(a,b)) + else: + def lt(a,b): + result = space.unwrap(space.call_function(w_cmp, a, b)) + if not isinstance(result,int): raise OperationError(space.w_TypeError, - space.wrap("comparison function must return int")) - raise - return result < 0 - -class CustomKeySort(SimpleSort): - def lt(self, a, b): - assert isinstance(a, KeyContainer) - assert isinstance(b, KeyContainer) - space = self.space - return space.is_true(space.lt(a.w_key, b.w_key)) - -class CustomKeyCompareSort(CustomCompareSort): - def lt(self, a, b): - assert isinstance(a, KeyContainer) - assert isinstance(b, KeyContainer) - return CustomCompareSort.lt(self, a.w_key, b.w_key) - -def list_sort__List_ANY_ANY_ANY(space, w_list, w_cmp, w_keyfunc, w_reverse): - has_cmp = not space.is_w(w_cmp, space.w_None) - has_key = not space.is_w(w_keyfunc, space.w_None) - has_reverse = space.is_true(w_reverse) - - # create and setup a TimSort instance - if has_cmp: - if has_key: - sorterclass = CustomKeyCompareSort - else: - sorterclass = CustomCompareSort - else: - if has_key: - sorterclass = CustomKeySort - else: - sorterclass = SimpleSort - items = w_list.wrappeditems - sorter = sorterclass(items, len(items)) - sorter.space = space - sorter.w_cmp = w_cmp - - try: - # The list is temporarily made empty, so that mutations performed - # by comparison functions can't affect the slice of memory we're - # sorting (allowing mutations during sorting is an IndexError or - # core-dump factory, since wrappeditems may change). - w_list.wrappeditems = [] - - # wrap each item in a KeyContainer if needed - if has_key: - for i in range(sorter.listlength): - w_item = sorter.list[i] - w_key = space.call_function(w_keyfunc, w_item) - sorter.list[i] = KeyContainer(w_key, w_item) - - # Reverse sort stability achieved by initially reversing the list, - # applying a stable forward sort, then reversing the final result. - if has_reverse: - sorter.list.reverse() - - # perform the sort - sorter.sort() - - # reverse again - if has_reverse: - sorter.list.reverse() - - finally: - # unwrap each item if needed - if has_key: - for i in range(sorter.listlength): - w_obj = sorter.list[i] - if isinstance(w_obj, KeyContainer): - sorter.list[i] = w_obj.w_item - - # check if the user mucked with the list during the sort - mucked = len(w_list.wrappeditems) > 0 - - # put the items back into the list - w_list.wrappeditems = sorter.list - - if mucked: - raise OperationError(space.w_ValueError, - space.wrap("list modified during sort")) + space.wrap("comparison function must return int")) + return result < 0 + # XXX Basic quicksort implementation + # XXX this is not stable !! + _quicksort(w_list.ob_item, 0, w_list.ob_size-1, lt) return space.w_None -from pypy.objspace.std import listtype -register_all(vars(), listtype) +""" +static PyMethodDef list_methods[] = { + {"append", (PyCFunction)listappend, METH_O, append_doc}, + {"insert", (PyCFunction)listinsert, METH_VARARGS, insert_doc}, + {"extend", (PyCFunction)listextend, METH_O, extend_doc}, + {"pop", (PyCFunction)listpop, METH_VARARGS, pop_doc}, + {"remove", (PyCFunction)listremove, METH_O, remove_doc}, + {"index", (PyCFunction)listindex, METH_O, index_doc}, + {"count", (PyCFunction)listcount, METH_O, count_doc}, + {"reverse", (PyCFunction)listreverse, METH_NOARGS, reverse_doc}, + {"sort", (PyCFunction)listsort, METH_VARARGS, sort_doc}, + {NULL, NULL} /* sentinel */ +}; +""" + +register_all(vars(), W_ListType) Modified: pypy/branch/avm/pypy/objspace/std/noneobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/noneobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/noneobject.py Thu Nov 5 20:27:32 2009 @@ -5,19 +5,17 @@ """ from pypy.objspace.std.objspace import * +from nonetype import W_NoneType class W_NoneObject(W_Object): - from pypy.objspace.std.nonetype import none_typedef as typedef - - def unwrap(w_self, space): - return None - + statictype = W_NoneType registerimplementation(W_NoneObject) -W_NoneObject.w_None = W_NoneObject() +def unwrap__None(space, w_none): + return None -def nonzero__None(space, w_none): - return space.w_False +def is_true__None(space, w_none): + return False def repr__None(space, w_none): return space.wrap('None') Modified: pypy/branch/avm/pypy/objspace/std/sliceobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/sliceobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/sliceobject.py Thu Nov 5 20:27:32 2009 @@ -6,135 +6,37 @@ """ from pypy.objspace.std.objspace import * -from pypy.interpreter import gateway -from pypy.objspace.std.slicetype import _Eval_SliceIndex +from slicetype import W_SliceType class W_SliceObject(W_Object): - from pypy.objspace.std.slicetype import slice_typedef as typedef + statictype = W_SliceType - def __init__(w_self, w_start, w_stop, w_step): - assert w_start is not None - assert w_stop is not None - assert w_step is not None + def __init__(w_self, space, w_start, w_stop, w_step): + W_Object.__init__(w_self, space) w_self.w_start = w_start w_self.w_stop = w_stop w_self.w_step = w_step - def unwrap(w_slice, space): - return slice(space.unwrap(w_slice.w_start), space.unwrap(w_slice.w_stop), space.unwrap(w_slice.w_step)) +registerimplementation(W_SliceObject) - def indices3(w_slice, space, length): - if space.is_w(w_slice.w_step, space.w_None): - step = 1 - else: - step = _Eval_SliceIndex(space, w_slice.w_step) - if step == 0: - raise OperationError(space.w_ValueError, - space.wrap("slice step cannot be zero")) - if space.is_w(w_slice.w_start, space.w_None): - if step < 0: - start = length - 1 - else: - start = 0 + +def getattribute__Slice_ANY(space, w_slice, w_attr): + if space.is_true(space.eq(w_attr, space.wrap('start'))): + if w_slice.w_start is None: + return space.w_None else: - start = _Eval_SliceIndex(space, w_slice.w_start) - if start < 0: - start += length - if start < 0: - if step < 0: - start = -1 - else: - start = 0 - elif start >= length: - if step < 0: - start = length - 1 - else: - start = length - if space.is_w(w_slice.w_stop, space.w_None): - if step < 0: - stop = -1 - else: - stop = length + return w_slice.w_start + if space.is_true(space.eq(w_attr, space.wrap('stop'))): + if w_slice.w_stop is None: + return space.w_None else: - stop = _Eval_SliceIndex(space, w_slice.w_stop) - if stop < 0: - stop += length - if stop < 0: - stop =-1 - elif stop > length: - stop = length - return start, stop, step - - def indices4(w_slice, space, length): - start, stop, step = w_slice.indices3(space, length) - if (step < 0 and stop >= start) or (step > 0 and start >= stop): - slicelength = 0 - elif step < 0: - slicelength = (stop - start + 1) / step + 1 + return w_slice.w_stop + if space.is_true(space.eq(w_attr, space.wrap('step'))): + if w_slice.w_step is None: + return space.w_None else: - slicelength = (stop - start - 1) / step + 1 - return start, stop, step, slicelength + return w_slice.w_step + raise FailedToImplement(space.w_AttributeError) -registerimplementation(W_SliceObject) - - -def normalize_simple_slice(space, length, w_start, w_stop): - """Helper for the {get,set,del}slice multimethod implementations.""" - # this returns a pair (start, stop) which is usable for slicing - # a sequence of the given length in the most friendly way, i.e. - # guaranteeing that 0 <= start <= stop <= length. - start = space.int_w(w_start) - stop = space.int_w(w_stop) - assert length >= 0 - if start < 0: - start = 0 - if stop < start: - stop = start - if stop > length: - stop = length - if start > length: - start = length - return start, stop - - -repr__Slice = gateway.applevel(""" - def repr__Slice(aslice): - return 'slice(%r, %r, %r)' % (aslice.start, aslice.stop, aslice.step) -""", filename=__file__).interphook("repr__Slice") - -def eq__Slice_Slice(space, w_slice1, w_slice2): - # We need this because CPython considers that slice1 == slice1 - # is *always* True (e.g. even if slice1 was built with non-comparable - # parameters - if space.is_w(w_slice1, w_slice2): - return space.w_True - if space.eq_w(w_slice1.w_start, w_slice2.w_start) and \ - space.eq_w(w_slice1.w_stop, w_slice2.w_stop) and \ - space.eq_w(w_slice1.w_step, w_slice2.w_step): - return space.w_True - else: - return space.w_False - -def lt__Slice_Slice(space, w_slice1, w_slice2): - if space.is_w(w_slice1, w_slice2): - return space.w_False # see comments in eq__Slice_Slice() - if space.eq_w(w_slice1.w_start, w_slice2.w_start): - if space.eq_w(w_slice1.w_stop, w_slice2.w_stop): - return space.lt(w_slice1.w_step, w_slice2.w_step) - else: - return space.lt(w_slice1.w_stop, w_slice2.w_stop) - else: - return space.lt(w_slice1.w_start, w_slice2.w_start) - -# indices impl - -def slice_indices__Slice_ANY(space, w_slice, w_length): - length = space.getindex_w(w_length, space.w_OverflowError) - start, stop, step = w_slice.indices3(space, length) - return space.newtuple([space.wrap(start), space.wrap(stop), - space.wrap(step)]) - -# register all methods -from pypy.objspace.std import slicetype -register_all(vars(), slicetype) +register_all(vars()) Modified: pypy/branch/avm/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/stringobject.py Thu Nov 5 20:27:32 2009 @@ -1,190 +1,211 @@ -# -*- coding: latin-1 -*- +# -*- Coding: Latin-1 -*- +""" +stringobject.py + +Synopsis of implemented methods (* marks work in progress) + +Py PyPy + + def _is_generic(w_self, fun): + def mod__String_ANY(space, w_str, w_item):def mod__String_Tuple(space, w_str, w_tuple):def mod_str_tuple(space, w_format, w_args): + def ord__String(space, w_str): + def string_richcompare(space, w_str1, w_str2, op): + def unwrap__String(space, w_str): +__add__ def add__String_String(space, w_left, w_right): +__class__ +__contains__ +__delattr__ +__doc__ +__eq__ def eq__String_String(space, w_str1, w_str2): +__ge__ def ge__String_String(space, w_str1, w_str2): +__getattribute__ +__getitem__ def getitem__String_Int(space, w_str, w_int): def getitem__String_Slice(space, w_str, w_slice): +__getslice__ +__gt__ def gt__String_String(space, w_str1, w_str2): +__hash__ def hash__String(space, w_str): +__init__ +__le__ def le__String_String(space, w_str1, w_str2): +__len__ def len__String(space, w_str): +__lt__ def lt__String_String(space, w_str1, w_str2): +__mul__ +__ne__ def ne__String_String(space, w_str1, w_str2): +__new__ +__reduce__ +__repr__ def repr__String(space, w_str): +__rmul__ +__setattr__ +__str__ def str__String(space, w_str): +capitalize def str_capitalize__String(space, w_self): +center def str_center__String_Int(space, w_self): +count def str_count__String_String_Int_Int(space, w_self): [optional arguments not supported now] +decode !Unicode not supported now +encode !Unicode not supported now +endswith str_endswith__String_String [optional arguments not supported now] +expandtabs str_expandtabs__String_Int +find OK +index OK +isalnum def str_isalnum__String(space, w_self): def _isalnum(ch): +isalpha def str_isalpha__String(space, w_self): def _isalpha(ch): +isdigit def str_isdigit__String(space, w_self): def _isdigit(ch): +islower def str_islower__String(space, w_self): def _islower(ch): +isspace def str_isspace__String(space, w_self): def _isspace(ch): +istitle def str_istitle(space, w_self): +isupper def str_isupper__String(space, w_self): def _isupper(ch): +join def str_join__String_ANY(space, w_self, w_list): +ljust def str_ljust__String_ANY(space, w_self, w_arg): +lower OK +lstrip def str_lstrip__String_String(space, w_self, w_chars): +replace OK +rfind OK +rindex OK +rjust def str_rjust__String_ANY(space, w_self, w_arg): +rstrip def str_rstrip__String_String(space, w_self, w_chars): +split def str_split__String_None_Int(space, w_self, w_none, w_maxsplit=-1):def str_split__String_String_Int(space, w_self, w_by, w_maxsplit=-1): +splitlines def str_splitlines__String_String(space, w_self, w_keepends): +startswith str_startswith__String_String [optional arguments not supported now] +strip def str_strip__String_String(space, w_self, w_chars): +swapcase OK +title def str_title__String(space, w_self): +translate OK +upper def str_upper__String(space, w_self): +zfill OK +""" from pypy.objspace.std.objspace import * from pypy.interpreter import gateway -from pypy.rlib.rarithmetic import ovfcheck, _hash_string -from pypy.rlib.objectmodel import we_are_translated -from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice -from pypy.objspace.std import slicetype -from pypy.objspace.std.listobject import W_ListObject -from pypy.objspace.std.noneobject import W_NoneObject -from pypy.objspace.std.tupleobject import W_TupleObject -from pypy.rlib.rstring import StringBuilder -from pypy.interpreter.buffer import StringBuffer +from stringtype import W_StringType +from intobject import W_IntObject +from sliceobject import W_SliceObject +import slicetype +from listobject import W_ListObject +from noneobject import W_NoneObject +from tupleobject import W_TupleObject -from pypy.objspace.std.stringtype import sliced, joined, wrapstr, wrapchar, \ - stringendswith, stringstartswith, joined2 +# XXX consider reimplementing _value to be a list of characters +# instead of a plain string -from pypy.objspace.std.formatting import mod_format class W_StringObject(W_Object): - from pypy.objspace.std.stringtype import str_typedef as typedef + statictype = W_StringType - _immutable_ = True - def __init__(w_self, str): + def __init__(w_self, space, str): + W_Object.__init__(w_self, space) w_self._value = str def __repr__(w_self): """ representation for debugging purposes """ return "%s(%r)" % (w_self.__class__.__name__, w_self._value) - def unwrap(w_self, space): - return w_self._value registerimplementation(W_StringObject) -W_StringObject.EMPTY = W_StringObject('') -W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)] -del i +def _isspace(ch): + return ord(ch) in (9, 10, 11, 12, 13, 32) -def _decode_ascii(space, s): - try: - return s.decode("ascii") - except UnicodeDecodeError: - for i in range(len(s)): - if ord(s[i]) > 127: - raise OperationError( - space.w_UnicodeDecodeError, - space.newtuple([ - space.wrap('ascii'), - space.wrap(s), - space.wrap(i), - space.wrap(i+1), - space.wrap("ordinal not in range(128)")])) - assert False, "unreachable" - -def unicode_w__String(space, w_self): - # XXX should this use the default encoding? - return _decode_ascii(space, w_self._value) - -def _is_generic(space, w_self, fun): - v = w_self._value +def _isdigit(ch): + o = ord(ch) + return o >= 48 and o <= 57 + +def _isalpha(ch): + o = ord(ch) + return (o>=97 and o<=122) or (o>=65 and o<=90) + +def _isalnum(ch): + o = ord(ch) + return (o>=97 and o<=122) \ + or (o>=65 and o<=90) \ + or (o>=48 and o<=57) + +def _isupper(ch): + o = ord(ch) + return (o>=65 and o<=90) + +def _islower(ch): + o = ord(ch) + return (o>=97 and o<=122) + +def _is_generic(w_self, fun): + space = w_self.space + v = space.unwrap(w_self) if len(v) == 0: return space.w_False if len(v) == 1: c = v[0] return space.newbool(fun(c)) else: + res = 1 for idx in range(len(v)): if not fun(v[idx]): return space.w_False return space.w_True -_is_generic._annspecialcase_ = "specialize:arg(2)" - -def _upper(ch): - if ch.islower(): - o = ord(ch) - 32 - return chr(o) - else: - return ch - -def _lower(ch): - if ch.isupper(): - o = ord(ch) + 32 - return chr(o) - else: - return ch - -_isspace = lambda c: c.isspace() -_isdigit = lambda c: c.isdigit() -_isalpha = lambda c: c.isalpha() -_isalnum = lambda c: c.isalnum() def str_isspace__String(space, w_self): - return _is_generic(space, w_self, _isspace) + return _is_generic(w_self, _isspace) def str_isdigit__String(space, w_self): - return _is_generic(space, w_self, _isdigit) + return _is_generic(w_self, _isdigit) def str_isalpha__String(space, w_self): - return _is_generic(space, w_self, _isalpha) + return _is_generic(w_self, _isalpha) def str_isalnum__String(space, w_self): - return _is_generic(space, w_self, _isalnum) + return _is_generic(w_self, _isalnum) def str_isupper__String(space, w_self): - """Return True if all cased characters in S are uppercase and there is -at least one cased character in S, False otherwise.""" - v = w_self._value - if len(v) == 1: - c = v[0] - return space.newbool(c.isupper()) - cased = False - for idx in range(len(v)): - if v[idx].islower(): - return space.w_False - elif not cased and v[idx].isupper(): - cased = True - return space.newbool(cased) + return _is_generic(w_self, _isupper) def str_islower__String(space, w_self): - """Return True if all cased characters in S are lowercase and there is -at least one cased character in S, False otherwise.""" - v = w_self._value - if len(v) == 1: - c = v[0] - return space.newbool(c.islower()) - cased = False - for idx in range(len(v)): - if v[idx].isupper(): - return space.w_False - elif not cased and v[idx].islower(): - cased = True - return space.newbool(cased) + return _is_generic(w_self, _islower) def str_istitle__String(space, w_self): - """Return True if S is a titlecased string and there is at least one -character in S, i.e. uppercase characters may only follow uncased -characters and lowercase characters only cased ones. Return False -otherwise.""" - input = w_self._value - cased = False - previous_is_cased = False + input = space.unwrap(w_self) + prev_letter='!' for pos in range(0, len(input)): ch = input[pos] - if ch.isupper(): - if previous_is_cased: - return space.w_False - previous_is_cased = True - cased = True - elif ch.islower(): - if not previous_is_cased: - return space.w_False - cased = True - else: - previous_is_cased = False + if ch.isalpha(): + if (prev_letter.isalpha() and ch.isupper()) or \ + (not prev_letter.isalpha() and ch.islower()): + return space.w_False + prev_letter = ch - return space.newbool(cased) + return space.w_True def str_upper__String(space, w_self): - self = w_self._value + self = space.unwrap(w_self) res = [' '] * len(self) for i in range(len(self)): ch = self[i] - res[i] = _upper(ch) + if _islower(ch): + o = ord(ch) - 32 + res[i] = chr(o) + else: + res[i] = ch return space.wrap("".join(res)) def str_lower__String(space, w_self): - self = w_self._value + self = space.unwrap(w_self) res = [' '] * len(self) for i in range(len(self)): ch = self[i] - res[i] = _lower(ch) + if _isupper(ch): + o = ord(ch) + 32 + res[i] = chr(o) + else: + res[i] = ch return space.wrap("".join(res)) def str_swapcase__String(space, w_self): - self = w_self._value + self = space.unwrap(w_self) res = [' '] * len(self) for i in range(len(self)): ch = self[i] - if ch.isupper(): + if _isupper(ch): o = ord(ch) + 32 res[i] = chr(o) - elif ch.islower(): + elif _islower(ch): o = ord(ch) - 32 res[i] = chr(o) else: @@ -194,11 +215,11 @@ def str_capitalize__String(space, w_self): - input = w_self._value + input = space.unwrap(w_self) buffer = [' '] * len(input) if len(input) > 0: ch = input[0] - if ch.islower(): + if _islower(ch): o = ord(ch) - 32 buffer[0] = chr(o) else: @@ -206,7 +227,7 @@ for i in range(1, len(input)): ch = input[i] - if ch.isupper(): + if _isupper(ch): o = ord(ch) + 32 buffer[i] = chr(o) else: @@ -215,330 +236,305 @@ return space.wrap("".join(buffer)) def str_title__String(space, w_self): - input = w_self._value + input = space.unwrap(w_self) buffer = [' '] * len(input) prev_letter=' ' for pos in range(0, len(input)): ch = input[pos] if not prev_letter.isalpha(): - buffer[pos] = _upper(ch) + buffer[pos] = ch.upper() else: - buffer[pos] = _lower(ch) + buffer[pos] = ch.lower() prev_letter = buffer[pos] return space.wrap("".join(buffer)) -def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1): - maxsplit = space.int_w(w_maxsplit) - res_w = [] - value = w_self._value - length = len(value) - i = 0 - while True: - # find the beginning of the next word - while i < length: - if not value[i].isspace(): - break # found - i += 1 - else: - break # end of string, finished - - # find the end of the word - if maxsplit == 0: - j = length # take all the rest of the string - else: - j = i + 1 - while j < length and not value[j].isspace(): - j += 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[i:j] - res_w.append(sliced(space, value, i, j, w_self)) - - # continue to look from the character following the space after the word - i = j + 1 - - return space.newlist(res_w) - -def str_split__String_String_ANY(space, w_self, w_by, w_maxsplit=-1): - maxsplit = space.int_w(w_maxsplit) - value = w_self._value - by = w_by._value +def str_split__String_None_Int(space, w_self, w_none, w_maxsplit=-1): + res = [] + inword = 0 + u = space.unwrap + value = u(w_self) + maxsplit = u(w_maxsplit) + pos = 0 + + for ch in value: + if ch.isspace(): + if inword: + inword = 0 + else: + if inword: + res[-1] += ch + else: + if maxsplit > -1: + if maxsplit == 0: + res.append(value[pos:]) + break + maxsplit = maxsplit - 1 + res.append(ch) + inword = 1 + pos = pos + 1 + + for i in range(len(res)): + res[i] = W_StringObject(space, res[i]) + return W_ListObject(space, res) + +def str_split__String_String_Int(space, w_self, w_by, w_maxsplit=-1): + u = space.unwrap + res = [] + start = 0 + value = u(w_self) + by = u(w_by) bylen = len(by) - if bylen == 0: - raise OperationError(space.w_ValueError, space.wrap("empty separator")) + maxsplit = u(w_maxsplit) - res_w = [] - start = 0 - while maxsplit != 0: - next = value.find(by, start) + #if maxsplit is default, then you have no limit + #of the length of the resulting array + if maxsplit == -1: + splitcount = 1 + else: + splitcount = maxsplit + + while splitcount: + next = _find(value, by, start, len(value), 1) + #next = value.find(by, start) #of course we cannot use + #the find method, if next < 0: + res.append(value[start:]) + start = len(value) + 1 break - res_w.append(sliced(space, value, start, next, w_self)) + res.append(value[start:next]) start = next + bylen - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - res_w.append(sliced(space, value, start, len(value), w_self)) - return space.newlist(res_w) - -def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1): - maxsplit = space.int_w(w_maxsplit) - res_w = [] - value = w_self._value - i = len(value)-1 - while True: - # starting from the end, find the end of the next word - while i >= 0: - if not value[i].isspace(): - break # found - i -= 1 - else: - break # end of string, finished - - # find the start of the word - # (more precisely, 'j' will be the space character before the word) - if maxsplit == 0: - j = -1 # take all the rest of the string - else: - j = i - 1 - while j >= 0 and not value[j].isspace(): - j -= 1 - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - # the word is value[j+1:i+1] - j1 = j + 1 - assert j1 >= 0 - res_w.append(sliced(space, value, j1, i+1, w_self)) - - # continue to look from the character before the space before the word - i = j - 1 - - res_w.reverse() - return space.newlist(res_w) - -def make_rsplit_with_delim(funcname, sliced): - from pypy.tool.sourcetools import func_with_new_name - - def fn(space, w_self, w_by, w_maxsplit=-1): - maxsplit = space.int_w(w_maxsplit) - res_w = [] - value = w_self._value - end = len(value) - by = w_by._value - bylen = len(by) - if bylen == 0: - raise OperationError(space.w_ValueError, space.wrap("empty separator")) - - while maxsplit != 0: - next = value.rfind(by, 0, end) - if next < 0: - break - res_w.append(sliced(space, value, next+bylen, end, w_self)) - end = next - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - res_w.append(sliced(space, value, 0, end, w_self)) - res_w.reverse() - return space.newlist(res_w) - - return func_with_new_name(fn, funcname) - -str_rsplit__String_String_ANY = make_rsplit_with_delim('str_rsplit__String_String_ANY', - sliced) + #decrese the counter only then, when + #we don't have default maxsplit + if maxsplit > -1: + splitcount = splitcount - 1 + + if start < len(value): + res.append(value[start:]) + + for i in range(len(res)): + res[i] = W_StringObject(w_self.space, res[i]) + return W_ListObject(w_self.space, res) def str_join__String_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) - str_w = space.str_w - if list_w: - self = w_self._value + u = space.unwrap + list = space.unpackiterable(w_list) + if list: + self = u(w_self) + firstelem = 1 listlen = 0 - reslen = 0 - l = [] - for i in range(len(list_w)): - w_s = list_w[i] - if not space.is_true(space.isinstance(w_s, space.w_str)): - if space.is_true(space.isinstance(w_s, space.w_unicode)): - w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", space.newlist(list_w)) - raise OperationError( - space.w_TypeError, - space.wrap("sequence item %d: expected string, %s " - "found" % (i, - space.type(w_s).getname(space, '?')))) - l.append(space.str_w(w_s)) - return space.wrap(self.join(l)) + reslen = 0 + #compute the length of the resulting string + for w_item in list: + reslen = reslen + len(u(w_item)) + listlen = listlen + 1 + + reslen = reslen + (listlen - 1) * len(self) + + #allocate the string buffer + res = [' '] * reslen + + pos = 0 + #fill in the string buffer + for w_item in list: + item = u(w_item) + if firstelem: + for i in range(len(item)): + res[i+pos] = item[i] + firstelem = 0 + pos = pos + len(item) + else: + for i in range(len(self)): + res[i+pos] = self[i] + pos = pos + len(self) + + for i in range(len(item)): + res[i+pos] = item[i] + pos = pos + len(item) + + return space.wrap("".join(res)) else: - return W_StringObject.EMPTY + return space.wrap("") + + +def str_rjust__String_ANY(space, w_self, w_arg): + u = space.unwrap -def str_rjust__String_ANY_ANY(space, w_self, w_arg, w_fillchar): - u_arg = space.int_w(w_arg) - u_self = w_self._value - fillchar = space.str_w(w_fillchar) - if len(fillchar) != 1: - raise OperationError(space.w_TypeError, - space.wrap("rjust() argument 2 must be a single character")) + u_arg = u(w_arg) + u_self = u(w_self) d = u_arg - len(u_self) if d>0: - fillchar = fillchar[0] # annotator hint: it's a single character - u_self = d * fillchar + u_self + u_self = d * ' ' + u_self return space.wrap(u_self) -def str_ljust__String_ANY_ANY(space, w_self, w_arg, w_fillchar): - u_self = w_self._value - u_arg = space.int_w(w_arg) - fillchar = space.str_w(w_fillchar) - if len(fillchar) != 1: - raise OperationError(space.w_TypeError, - space.wrap("ljust() argument 2 must be a single character")) +def str_ljust__String_ANY(space, w_self, w_arg): + u = space.unwrap + + u_self = u(w_self) + u_arg = u(w_arg) d = u_arg - len(u_self) if d>0: - fillchar = fillchar[0] # annotator hint: it's a single character - u_self += d * fillchar + u_self += d * ' ' return space.wrap(u_self) -def _convert_idx_params(space, w_self, w_sub, w_start, w_end, upper_bound=False): - self = w_self._value - sub = w_sub._value - if upper_bound: - start = slicetype.adapt_bound(space, len(self), w_start) - end = slicetype.adapt_bound(space, len(self), w_end) - else: - start = slicetype.adapt_lower_bound(space, len(self), w_start) - end = slicetype.adapt_lower_bound(space, len(self), w_end) +def _convert_idx_params(space, w_self, w_sub, w_start, w_end): + u = space.unwrap + start = u(w_start) + end = u(w_end) + self = u(w_self) + sub = u(w_sub) + if start is None: + start = 0 + if end is None: + end = len(self) + return (self, sub, start, end) -_convert_idx_params._annspecialcase_ = 'specialize:arg(5)' -def contains__String_String(space, w_self, w_sub): - self = w_self._value - sub = w_sub._value - return space.newbool(self.find(sub) >= 0) -def str_find__String_String_ANY_ANY(space, w_self, w_sub, w_start, w_end): +def str_find__String_String_ANY_ANY(space, w_self, w_sub, w_start=None, w_end=None): + (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - res = self.find(sub, start, end) + res = _find(self, sub, start, end, 1) return space.wrap(res) -def str_rfind__String_String_ANY_ANY(space, w_self, w_sub, w_start, w_end): +def str_rfind__String_String_ANY_ANY(space, w_self, w_sub, w_start=None, w_end=None): + (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - res = self.rfind(sub, start, end) + res = _find(self, sub, start, end, -1) return space.wrap(res) -def str_partition__String_String(space, w_self, w_sub): - self = w_self._value - sub = w_sub._value - if not sub: - raise OperationError(space.w_ValueError, - space.wrap("empty separator")) - pos = self.find(sub) - if pos == -1: - return space.newtuple([w_self, space.wrap(''), space.wrap('')]) - else: - return space.newtuple([sliced(space, self, 0, pos, w_self), - w_sub, - sliced(space, self, pos+len(sub), len(self), - w_self)]) - -def str_rpartition__String_String(space, w_self, w_sub): - self = w_self._value - sub = w_sub._value - if not sub: - raise OperationError(space.w_ValueError, - space.wrap("empty separator")) - pos = self.rfind(sub) - if pos == -1: - return space.newtuple([space.wrap(''), space.wrap(''), w_self]) - else: - return space.newtuple([sliced(space, self, 0, pos, w_self), - w_sub, - sliced(space, self, pos+len(sub), len(self), w_self)]) - +def str_index__String_String_ANY_ANY(space, w_self, w_sub, w_start=None, w_end=None): -def str_index__String_String_ANY_ANY(space, w_self, w_sub, w_start, w_end): (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - res = self.find(sub, start, end) - if res < 0: + res = _find(self, sub, start, end, 1) + + if res == -1: raise OperationError(space.w_ValueError, space.wrap("substring not found in string.index")) return space.wrap(res) -def str_rindex__String_String_ANY_ANY(space, w_self, w_sub, w_start, w_end): +def str_rindex__String_String_ANY_ANY(space, w_self, w_sub, w_start=None, w_end=None): + (self, sub, start, end) = _convert_idx_params(space, w_self, w_sub, w_start, w_end) - res = self.rfind(sub, start, end) - if res < 0: + res = _find(self, sub, start, end, -1) + if res == -1: raise OperationError(space.w_ValueError, space.wrap("substring not found in string.rindex")) return space.wrap(res) -def _string_replace(space, input, sub, by, maxsplit): - if maxsplit == 0: - return space.wrap(input) + +def str_replace__String_String_String_Int(space, w_self, w_sub, w_by, w_maxsplit=-1): + u = space.unwrap + + input = u(w_self) + sub = u(w_sub) + by = u(w_by) + maxsplit = u(w_maxsplit) #I don't use it now #print "from replace, input: %s, sub: %s, by: %s" % (input, sub, by) - if not sub: - upper = len(input) - if maxsplit > 0 and maxsplit < upper + 2: - upper = maxsplit - 1 - assert upper >= 0 - substrings_w = [""] - for i in range(upper): - c = input[i] - substrings_w.append(c) - substrings_w.append(input[upper:]) - else: + #what do we have to replace? + startidx = 0 + endidx = len(input) + indices = [] + foundidx = _find(input, sub, startidx, endidx, 1) + while foundidx > -1 and (maxsplit == -1 or maxsplit > 0): + indices.append(foundidx) + if len(sub) == 0: + #so that we go forward, even if sub is empty + startidx = foundidx + 1 + else: + startidx = foundidx + len(sub) + foundidx = _find(input, sub, startidx, endidx, 1) + if maxsplit != -1: + maxsplit = maxsplit - 1 + indiceslen = len(indices) + buf = [' '] * (len(input) - indiceslen * len(sub) + indiceslen * len(by)) + startidx = 0 + + #ok, so do it + bufpos = 0 + for i in range(indiceslen): + for j in range(startidx, indices[i]): + buf[bufpos] = input[j] + bufpos = bufpos + 1 + + for j in range(len(by)): + buf[bufpos] = by[j] + bufpos = bufpos + 1 + + startidx = indices[i] + len(sub) + + for j in range(startidx, len(input)): + buf[bufpos] = input[j] + bufpos = bufpos + 1 + return space.wrap("".join(buf)) + +def _find(self, sub, start, end, dir): + + length = len(self) + + #adjust_indicies + if (end > length): + end = length + elif (end < 0): + end += length + if (end < 0): + end = 0 + if (start < 0): + start += length + if (start < 0): start = 0 - sublen = len(sub) - substrings_w = [] - while maxsplit != 0: - next = input.find(sub, start) - if next < 0: - break - substrings_w.append(input[start:next]) - start = next + sublen - maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - - substrings_w.append(input[start:]) + if dir > 0: + if len(sub) == 0 and start < end: + return start + + end = end - len(sub) + 1 + + for i in range(start, end): + match = 1 + for idx in range(len(sub)): + if sub[idx] != self[idx+i]: + match = 0 + break + if match: + return i + return -1 + else: + if len(sub) == 0 and start < end: + return end + + end = end - len(sub) + + for j in range(end, start-1, -1): + match = 1 + for idx in range(len(sub)): + if sub[idx] != self[idx+j]: + match = 0 + break + if match: + return j + return -1 - try: - # XXX conservative estimate. If your strings are that close - # to overflowing, bad luck. - one = ovfcheck(len(substrings_w) * len(by)) - ovfcheck(one + len(input)) - except OverflowError: - raise OperationError( - space.w_OverflowError, - space.wrap("replace string is too long")) - - return space.wrap(by.join(substrings_w)) - - -def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit): - return _string_replace(space, w_self._value, space.buffer_w(w_sub).as_str(), - space.buffer_w(w_by).as_str(), - space.int_w(w_maxsplit)) - -def str_replace__String_String_String_ANY(space, w_self, w_sub, w_by, w_maxsplit=-1): - input = w_self._value - sub = w_sub._value - by = w_by._value - maxsplit = space.int_w(w_maxsplit) - return _string_replace(space, input, sub, by, maxsplit) def _strip(space, w_self, w_chars, left, right): "internal function called by str_xstrip methods" - u_self = w_self._value - u_chars = w_chars._value + u_self = space.unwrap(w_self) + u_chars = space.unwrap(w_chars) + + if u_self == None or u_chars == None: + return w_self lpos = 0 rpos = len(u_self) @@ -549,113 +545,98 @@ lpos += 1 if right: - while rpos > lpos and u_self[rpos - 1] in u_chars: + while rpos > 0 and u_self[rpos - 1] in u_chars: rpos -= 1 - assert rpos >= lpos # annotator hint, don't remove - return sliced(space, u_self, lpos, rpos, w_self) + return space.wrap(u_self[lpos:rpos]) -def _strip_none(space, w_self, left, right): - "internal function called by str_xstrip methods" - u_self = w_self._value - - lpos = 0 - rpos = len(u_self) - - if left: - #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) - while lpos < rpos and u_self[lpos].isspace(): - lpos += 1 - - if right: - while rpos > lpos and u_self[rpos - 1].isspace(): - rpos -= 1 - - assert rpos >= lpos # annotator hint, don't remove - return sliced(space, u_self, lpos, rpos, w_self) def str_strip__String_String(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=1, right=1) -def str_strip__String_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=1) def str_rstrip__String_String(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=0, right=1) -def str_rstrip__String_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=0, right=1) - def str_lstrip__String_String(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=1, right=0) + -def str_lstrip__String_None(space, w_self, w_chars): - return _strip_none(space, w_self, left=1, right=0) - - - -def str_center__String_ANY_ANY(space, w_self, w_arg, w_fillchar): - u_self = w_self._value - u_arg = space.int_w(w_arg) - fillchar = space.str_w(w_fillchar) - if len(fillchar) != 1: - raise OperationError(space.w_TypeError, - space.wrap("center() argument 2 must be a single character")) +def str_center__String_Int(space, w_self, w_arg): + u_self = space.unwrap(w_self) + u_arg = space.unwrap(w_arg) d = u_arg - len(u_self) if d>0: - offset = d//2 + (d & u_arg & 1) - fillchar = fillchar[0] # annotator hint: it's a single character - u_centered = offset * fillchar + u_self + (d - offset) * fillchar + offset = d//2 + u_centered = offset * ' ' + u_self + (d - offset) * ' ' else: u_centered = u_self - return wrapstr(space, u_centered) - + return W_StringObject(space, u_centered) + + def str_count__String_String_ANY_ANY(space, w_self, w_arg, w_start, w_end): - u_self, u_arg, u_start, u_end = _convert_idx_params(space, w_self, w_arg, - w_start, w_end) - return wrapint(space, u_self.count(u_arg, u_start, u_end)) - -def str_endswith__String_String_ANY_ANY(space, w_self, w_suffix, w_start, w_end): - (u_self, suffix, start, end) = _convert_idx_params(space, w_self, - w_suffix, w_start, - w_end, True) - return space.newbool(stringendswith(u_self, suffix, start, end)) - -def str_endswith__String_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): - (u_self, _, start, end) = _convert_idx_params(space, w_self, - space.wrap(''), w_start, - w_end, True) - for w_suffix in space.viewiterable(w_suffixes): - if space.is_true(space.isinstance(w_suffix, space.w_unicode)): - w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "endswith", w_suffixes, w_start, - w_end) - suffix = space.str_w(w_suffix) - if stringendswith(u_self, suffix, start, end): - return space.w_True - return space.w_False + u_self = space.unwrap(w_self) + u_arg = space.unwrap(w_arg) + u_start = space.unwrap(w_start) + u_end = space.unwrap(w_end) + + + if u_end == None: + u_end = len(u_self) + elif u_end < 0: + u_end += len(u_self) + + if u_start == None: u_start = 0 + + area = u_self [u_start:u_end] + + count = 0 + + pos = -1 + while 1: + pos = _find(area, u_arg, pos+1, u_end, 1) + #pos = area.find(u_arg, pos+1, u_end) + if pos == -1: + break + count += 1 + + return W_IntObject(space, count) -def str_startswith__String_String_ANY_ANY(space, w_self, w_prefix, w_start, w_end): - (u_self, prefix, start, end) = _convert_idx_params(space, w_self, - w_prefix, w_start, - w_end, True) - return space.newbool(stringstartswith(u_self, prefix, start, end)) - -def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): - (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), - w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): - if space.is_true(space.isinstance(w_prefix, space.w_unicode)): - w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "startswith", w_prefixes, w_start, - w_end) - prefix = space.str_w(w_prefix) - if stringstartswith(u_self, prefix, start, end): - return space.w_True - return space.w_False + +#[optional arguments not supported now] +def str_endswith__String_String(space, w_self, w_end): + u_self = space.unwrap(w_self) + u_end = space.unwrap(w_end) + + found = 0 + if u_end: + endlen = len(u_end) + if endlen <= len(u_self): + found = (u_end == u_self[-endlen:]) + else: + found = 1 + + return W_IntObject(space, found) + + +#[optional arguments not supported now] +def str_startswith__String_String(space, w_self, w_start): + u_self = space.unwrap(w_self) + u_start = space.unwrap(w_start) + + found = 0 + if u_start: + startlen = len(u_start) + if startlen <= len(u_self): + found = (u_start == u_self[:startlen]) + else: + found = 1 + + return W_IntObject(space, found) + def _tabindent(u_token, u_tabsize): "calculates distance behind the token to the next tabstop" @@ -683,13 +664,13 @@ return distance -def str_expandtabs__String_ANY(space, w_self, w_tabsize): - u_self = w_self._value - u_tabsize = space.int_w(w_tabsize) +def str_expandtabs__String_Int(space, w_self, w_tabsize): + u_self = space.unwrap(w_self) + u_tabsize = space.unwrap(w_tabsize) u_expanded = "" if u_self: - split = u_self.split("\t") + split = u_self.split("\t") #XXX use pypy split u_expanded =oldtoken = split.pop(0) for token in split: @@ -697,40 +678,37 @@ u_expanded += " " * _tabindent(oldtoken,u_tabsize) + token oldtoken = token - return wrapstr(space, u_expanded) + return W_StringObject(space, u_expanded) -def str_splitlines__String_ANY(space, w_self, w_keepends): - u_keepends = space.int_w(w_keepends) # truth value, but type checked - data = w_self._value - selflen = len(data) - strs_w = [] - i = j = 0 - while i < selflen: - # Find a line and append it - while i < selflen and data[i] != '\n' and data[i] != '\r': - i += 1 - # Skip the line break reading CRLF as one line break - eol = i - i += 1 - if i < selflen and data[i-1] == '\r' and data[i] == '\n': - i += 1 - if u_keepends: - eol = i - strs_w.append(sliced(space, data, j, eol, w_self)) - j = i - - if j < selflen: - strs_w.append(sliced(space, data, j, len(data), w_self)) - return space.newlist(strs_w) - -def str_zfill__String_ANY(space, w_self, w_width): - input = w_self._value - width = space.int_w(w_width) +def str_splitlines__String_Int(space, w_self, w_keepends): + u_self = space.unwrap(w_self) + u_keepends = space.unwrap(w_keepends) + selflen = len(u_self) + + L = [] + pos = 0 + while 1: + oldpos = pos + pos = _find(u_self, '\n', pos, selflen, 1) + 1 + if pos > oldpos: + w_item = space.wrap(u_self[oldpos:pos]) + if not u_keepends: + w_item = _strip(space, w_item, W_StringObject(space,'\n'), left=0, right=1) + L.append(w_item) + else: + break + return W_ListObject(space, L) + +def str_zfill__String_Int(space, w_self, w_width): + u = space.unwrap + input = u(w_self) + width = u(w_width) if len(input) >= width: - # cannot return w_self, in case it is a subclass of str - return space.wrap(input) + return w_self + + b = width - len(input) buf = [' '] * width if len(input) > 0 and (input[0] == '+' or input[0] == '-'): @@ -749,256 +727,285 @@ start = start + 1 return space.wrap("".join(buf)) + + +def app_str_translate__String_String_String(s, table, deletechars=''): + """charfilter - unicode handling is not implemented + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, + which must be a string of length 256""" + + if len(table) < 256: + raise ValueError("translation table must be 256 characters long") -def str_w__String(space, w_str): + L = [ table[ord(s[i])] for i in range(len(s)) if s[i] not in deletechars ] + return ''.join(L) + +str_translate__String_String_String = gateway.app2interp(app_str_translate__String_String_String) + + +def unwrap__String(space, w_str): return w_str._value def hash__String(space, w_str): - s = w_str._value - if we_are_translated(): - x = hash(s) # to use the hash cache in rpython strings + return W_IntObject(space, hash(space.unwrap(w_str))) + + +EQ = 1 +LE = 2 +GE = 3 +GT = 4 +LT = 5 +NE = 6 + + +def string_richcompare(space, w_str1, w_str2, op): + u = space.unwrap + str1 = u(w_str1) + str2 = u(w_str2) + + if space.is_true(space.is_(w_str1, w_str2)): + if op == EQ or op == LE or op == GE: + return space.w_True + elif op == GT or op == LT or op == NE: + return space.w_False + if 0: + pass else: - x = _hash_string(s) # to make sure we get the same hash as rpython - # (otherwise translation will freeze W_DictObjects where we can't find - # the keys any more!) - return wrapint(space, x) + if op == EQ: + if len(str1) == len(str2): + for i in range(len(str1)): + if ord(str1[i]) != ord(str2[i]): + return space.w_False + return space.w_True + else: + return space.w_False + else: + if len(str1) > len(str2): + min_len = len(str2) + else: + min_len = len(str1) + + c = 0 + idx = 0 + if (min_len > 0): + while (c == 0) and (idx < min_len): + c = ord(str1[idx]) - ord(str2[idx]) + idx = idx + 1 + else: + c = 0 + + if (c == 0): + if len(str1) < len(str2): + c = -1 + elif len(str1) > len(str2): + c = 1 + else: + c = 0 + + if op == LT: + return space.newbool(c < 0) + elif op == LE: + return space.newbool(c <= 0) + elif op == NE: + return space.newbool(c != 0) + elif op == GT: + return space.newbool(c > 0) + elif op == GE: + return space.newbool(c >= 0) + else: + return NotImplemented def lt__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 < s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, LT) def le__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 <= s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, LE) def eq__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 == s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, EQ) def ne__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 != s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, NE) def gt__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 > s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, GT) def ge__String_String(space, w_str1, w_str2): - s1 = w_str1._value - s2 = w_str2._value - if s1 >= s2: - return space.w_True - else: - return space.w_False + return string_richcompare(space, w_str1, w_str2, GE) -def getitem__String_ANY(space, w_str, w_index): - ival = space.getindex_w(w_index, space.w_IndexError, "string index") - str = w_str._value - slen = len(str) +def getitem__String_Int(space, w_str, w_int): + u = space.unwrap + ival = w_int.intval + str = u(w_str) + slen = len(u(w_str)) if ival < 0: ival += slen if ival < 0 or ival >= slen: exc = space.call_function(space.w_IndexError, space.wrap("string index out of range")) raise OperationError(space.w_IndexError, exc) - return wrapchar(space, str[ival]) + return W_StringObject(space, str[ival]) def getitem__String_Slice(space, w_str, w_slice): + # XXX this is really too slow for slices with no step argument w = space.wrap - s = w_str._value - length = len(s) - start, stop, step, sl = w_slice.indices4(space, length) - if sl == 0: - return W_StringObject.EMPTY - elif step == 1: - assert start >= 0 and stop >= 0 - return sliced(space, s, start, stop, w_str) - else: - str = "".join([s[start + i*step] for i in range(sl)]) - return wrapstr(space, str) + length = len(w_str._value) + start, stop, step, sl = slicetype.indices4(space, w_slice, length) + r = [space.getitem(w_str, w(start + i*step)) for i in range(sl)] + w_r = space.newlist(r) + w_empty = space.newstring([]) + return str_join__String_ANY(space, w_empty, w_r) + +def mul__String_Int(space, w_str, w_mul): + u = space.unwrap + input = u(w_str) + mul = u(w_mul) + + buffer = [' '] * (mul*len(input)) + + pos = 0 + for i in range(mul): + for j in range(len(input)): + buffer[pos] = input[j] + pos = pos + 1 -def getslice__String_ANY_ANY(space, w_str, w_start, w_stop): - s = w_str._value - start, stop = normalize_simple_slice(space, len(s), w_start, w_stop) - if start == stop: - return W_StringObject.EMPTY - else: - return sliced(space, s, start, stop, w_str) - -def mul_string_times(space, w_str, w_times): - try: - mul = space.getindex_w(w_times, space.w_OverflowError) - except OperationError, e: - if e.match(space, space.w_TypeError): - raise FailedToImplement - raise - if mul <= 0: - return W_StringObject.EMPTY - input = w_str._value - input_len = len(input) - try: - buflen = ovfcheck(mul * input_len) - except OverflowError: - raise OperationError( - space.w_OverflowError, - space.wrap("repeated string is too long: %d %d" % (input_len, mul))) - # XXX maybe only do this when input has a big length - return joined(space, [input] * mul) - -def mul__String_ANY(space, w_str, w_times): - return mul_string_times(space, w_str, w_times) - -def mul__ANY_String(space, w_times, w_str): - return mul_string_times(space, w_str, w_times) + return space.wrap("".join(buffer)) def add__String_String(space, w_left, w_right): - right = w_right._value - left = w_left._value - return joined2(space, left, right) + u = space.unwrap + right = u(w_right) + left = u(w_left) + buf = [' '] * (len(left) + len(right)) + for i in range(len(left)): + buf[i] = left[i] + for i in range(len(right)): + buf[i+len(left)] = right[i] + return space.wrap("".join(buf)) + +def mod_str_tuple(space, w_format, w_args): + # XXX implement me + format = space.unwrap(w_format) + args = space.unwrap(w_args) + try: + s = format % args + except TypeError, e: + raise OperationError(space.w_TypeError, space.wrap(str(e))) + except ValueError, e: + raise OperationError(space.w_ValueError, space.wrap(str(e))) + return space.wrap(s) def len__String(space, w_str): - return space.wrap(len(w_str._value)) + return space.wrap(len(space.unwrap(w_str))) def str__String(space, w_str): - if type(w_str) is W_StringObject: - return w_str - return wrapstr(space, w_str._value) - -def iter__String(space, w_list): - from pypy.objspace.std import iterobject - return iterobject.W_SeqIterObject(w_list) - -def ord__String(space, w_str): - u_str = w_str._value - if len(u_str) != 1: - raise OperationError( - space.w_TypeError, - space.wrap("ord() expected a character, but string " - "of length %d found"%(len(w_str._value),))) - return space.wrap(ord(u_str)) + return w_str -def getnewargs__String(space, w_str): - return space.newtuple([wrapstr(space, w_str._value)]) -def repr__String(space, w_str): - s = w_str._value +def iter__String(space, w_list): + import iterobject + return iterobject.W_SeqIterObject(space, w_list) - buf = StringBuilder(50) +def app_repr__String(s): quote = "'" if quote in s and '"' not in s: quote = '"' - buf.append(quote) - startslice = 0 + repr = quote - for i in range(len(s)): - c = s[i] - use_bs_char = False # character quoted by backspace - - if c == '\\' or c == quote: - bs_char = c - use_bs_char = True - elif c == '\t': - bs_char = 't' - use_bs_char = True - elif c == '\r': - bs_char = 'r' - use_bs_char = True - elif c == '\n': - bs_char = 'n' - use_bs_char = True + for c in s: + if c == '\\' or c == quote: repr += '\\'+c + elif c == '\t': repr += '\\t' + elif c == '\r': repr += '\\r' + elif c == '\n': repr += '\\n' elif not '\x20' <= c < '\x7f': n = ord(c) - if i != startslice: - buf.append_slice(s, startslice, i) - startslice = i + 1 - buf.append('\\x') - buf.append("0123456789abcdef"[n>>4]) - buf.append("0123456789abcdef"[n&0xF]) - - if use_bs_char: - if i != startslice: - buf.append_slice(s, startslice, i) - startslice = i + 1 - buf.append('\\') - buf.append(bs_char) + repr += '\\x'+"0123456789abcdef"[n>>4]+"0123456789abcdef"[n&0xF] + else: + repr += c - if len(s) != startslice: - buf.append_slice(s, startslice, len(s)) + repr += quote - buf.append(quote) + return repr - return space.wrap(buf.build()) +repr__String = gateway.app2interp(app_repr__String) - -def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''): - """charfilter - unicode handling is not implemented - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, - which must be a string of length 256""" +def ord__String(space, w_str): + return space.wrap(ord(space.unwrap(w_str))) + +def mod__String_ANY(space, w_str, w_item): + return mod__String_Tuple(space, w_str, space.newtuple([w_item])) + +def app_mod__String_ANY(format, values): + pieces = [] + start = 0 + state = 0 + i = 0 + index = -1 + len_format = len(format) + while i < len_format: + c = format[i] + if state == 0: + # just copy constant-pieces of the format + if c=='%': + pieces.append(format[start:i]) + state = 1 + else: + if c=='%': + pieces.append('%') + else: + if c == '(': + # read name + j = format.find(')', i+1) + if j == -1: + raise ValueError, "incomplete format string" + if index >= 0: + raise TypeError, "format string mismatch" + name = format[i+1:j] + value = values[name] + index = -2 + i = j+1 + c = format[i] + else: + index += 1 + if index < 0: + raise TypeError, "format string mismatch" + elif index == 0 and not isinstance(values, tuple): + values = tuple([values]) + try: + value = values[index] + except IndexError: + raise TypeError, "not enough arguments for format string" + + if c=='s': + pieces.append(str(value)) + elif c=='d': + pieces.append(str(int(value))) + elif c=='x': + pieces.append(hex(int(value))) + elif c=='r': + pieces.append(repr(value)) + else: + raise ValueError, "unsupported format character '%s' (%x) at index %d" % ( + c, ord(c), i) + state = 0 + start = i+1 + i += 1 + + if state == 1: + raise ValueError, "incomplete format" + if index >= 0 and index < len(values) - 1: + raise TypeError, 'not all arguments converted during string formatting' + pieces.append(format[start:]) + return ''.join(pieces) + +mod__String_ANY = gateway.app2interp(app_mod__String_ANY) + +# register all methods +register_all(vars(), W_StringType) + - # XXX CPython accepts buffers, too, not sure what we should do - table = space.str_w(w_table) - if len(table) != 256: - raise OperationError( - space.w_ValueError, - space.wrap("translation table must be 256 characters long")) - - string = w_string._value - chars = [] - for char in string: - w_char = W_StringObject.PREBUILT[ord(char)] - if not space.is_true(space.contains(w_deletechars, w_char)): - chars.append(table[ord(char)]) - return W_StringObject(''.join(chars)) - -def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): - from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ - unicode_from_string, decode_object - encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - if encoding is None and errors is None: - return unicode_from_string(space, w_string) - return decode_object(space, w_string, encoding, errors) - -def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None): - from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \ - encode_object - encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors) - return encode_object(space, w_string, encoding, errors) - -# CPython's logic for deciding if ""%values is -# an error (1 value, 0 %-formatters) or not -# (values is of a mapping type) -def mod__String_ANY(space, w_format, w_values): - return mod_format(space, w_format, w_values, do_unicode=False) - -def buffer__String(space, w_string): - from pypy.interpreter.buffer import StringBuffer - return space.wrap(StringBuffer(w_string._value)) - -# register all methods -from pypy.objspace.std import stringtype -register_all(vars(), stringtype) Modified: pypy/branch/avm/pypy/objspace/std/test/test_multimethod.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/test/test_multimethod.py (original) +++ pypy/branch/avm/pypy/objspace/std/test/test_multimethod.py Thu Nov 5 20:27:32 2009 @@ -1,250 +1,149 @@ -from py.test import raises +import autopath -from pypy.objspace.std import multimethod -from pypy.objspace.std.multimethod import FailedToImplement +from pypy.objspace.std.multimethod import * +from pypy.tool import testit +BoundMultiMethod.ASSERT_BASE_TYPE = object -class W_Root(object): - pass -class W_IntObject(W_Root): - pass - -class W_BoolObject(W_Root): - pass - -class W_StringObject(W_Root): - pass - -def delegate_b2i(space, w_x): - assert isinstance(w_x, W_BoolObject) - return W_IntObject() - -def add__Int_Int(space, w_x, w_y): - assert space == 'space' - assert isinstance(w_x, W_IntObject) - assert isinstance(w_y, W_IntObject) - return 'fine' - - -class TestMultiMethod1: - Installer = multimethod.InstallerVersion1 - - def setup_class(cls): - cls.prev_installer = multimethod.Installer - multimethod.Installer = cls.Installer - add = multimethod.MultiMethodTable(2, root_class=W_Root, - argnames_before=['space']) - add.register(add__Int_Int, W_IntObject, W_IntObject) - typeorder = { - W_IntObject: [(W_IntObject, None), (W_Root, None)], - W_BoolObject: [(W_BoolObject, None), (W_IntObject, delegate_b2i), - (W_Root, None)], - W_StringObject: [(W_StringObject, None), (W_Root, None)], - } - cls.typeorder = typeorder - cls.add = add - cls.add1 = staticmethod(add.install('__add', [typeorder, typeorder])) - - def teardown_class(cls): - multimethod.Installer = cls.prev_installer - - def test_simple(self): - space = 'space' - w_x = W_IntObject() - w_y = W_IntObject() - assert self.add1(space, w_x, w_y) == 'fine' - - def test_failtoimplement(self): - space = 'space' - w_x = W_IntObject() - w_s = W_StringObject() - raises(FailedToImplement, "self.add1(space, w_x, w_s)") - raises(FailedToImplement, "self.add1(space, w_s, w_x)") - - def test_delegate(self): - space = 'space' - w_x = W_IntObject() - w_s = W_StringObject() - w_b = W_BoolObject() - assert self.add1(space, w_x, w_b) == 'fine' - assert self.add1(space, w_b, w_x) == 'fine' - assert self.add1(space, w_b, w_b) == 'fine' - raises(FailedToImplement, "self.add1(space, w_b, w_s)") - raises(FailedToImplement, "self.add1(space, w_s, w_b)") - - def test_not_baked(self): - typeorder = self.typeorder - add2 = self.add.install('__add2', [typeorder, typeorder], - baked_perform_call=False) - assert add2[0] == ['space', 'arg0', 'arg1'] - if multimethod.Installer is multimethod.InstallerVersion1: - assert add2[1] == 'arg0.__add2(space, arg1)' - assert isinstance(add2[2], dict) - assert not add2[3] - - def test_empty(self): - add3_installer = multimethod.Installer(self.add, '__add3', [{},{}]) - assert add3_installer.is_empty() - if multimethod.Installer is multimethod.InstallerVersion1: - assert len(add3_installer.to_install) == 1 - assert add3_installer.to_install[0][0] is None - - def test_empty_direct(self): - assert not self.add.install_if_not_empty('__add4', [{},{}]) - - def test_empty_not_baked(self): - add5_installer = multimethod.Installer(self.add, '__add5', [{},{}], - baked_perform_call=False) - assert add5_installer.is_empty() - if multimethod.Installer is multimethod.InstallerVersion1: - assert len(add5_installer.to_install) == 0 - add5 = add5_installer.install() - assert add5[0] == ['space', 'arg0', 'arg1'] - assert add5[1] == 'raiseFailedToImplement()' - assert isinstance(add5[2], dict) - assert add5[3] - - def test_mmdispatcher(self): - typeorder = self.typeorder - add2 = multimethod.MMDispatcher(self.add, [typeorder, typeorder]) - space = 'space' - w_x = W_IntObject() - w_s = W_StringObject() - w_b1 = W_BoolObject() - w_b2 = W_BoolObject() - assert add2(space, w_x, w_b1) == 'fine' - assert add2(space, w_b2, w_x) == 'fine' - assert add2(space, w_b1, w_b2) == 'fine' - raises(FailedToImplement, "add2(space, w_b2, w_s)") - raises(FailedToImplement, "add2(space, w_s, w_b1)") - - def test_forbidden_subclasses(self): - mul = multimethod.MultiMethodTable(2, root_class=W_Root, - argnames_before=['space']) - class UserW_StringObject(W_StringObject): - pass - def mul__Int_String(space, w_x, w_y): - assert space == 'space' - assert isinstance(w_x, W_IntObject) - assert isinstance(w_y, W_StringObject) - return 'fine' - mul.register(mul__Int_String, W_IntObject, W_StringObject) - - mul1 = mul.install('__mul1', [self.typeorder, self.typeorder]) - assert mul1('space', W_IntObject(), W_StringObject()) == 'fine' - assert mul1('space', W_IntObject(), UserW_StringObject()) == 'fine' - - ext_typeorder = self.typeorder.copy() - ext_typeorder[UserW_StringObject] = [] - mul2 = mul.install('__mul2', [ext_typeorder, ext_typeorder]) - assert mul2('space', W_IntObject(), W_StringObject()) == 'fine' - raises(FailedToImplement, - mul2, 'baz', W_IntObject(), UserW_StringObject()) - - def test_more_forbidden_subclasses(self): - mul = multimethod.MultiMethodTable(2, root_class=W_Root, - argnames_before=['space']) - class UserW_StringObject(W_StringObject): - pass - def mul__String_String(space, w_x, w_y): - assert space == 'space' - assert isinstance(w_x, W_StringObject) - assert isinstance(w_y, W_StringObject) - return 'fine' - mul.register(mul__String_String, W_StringObject, W_StringObject) - - ext_typeorder = {W_StringObject: [(W_StringObject, None)], - UserW_StringObject: []} - mul2 = mul.install('__mul2', [ext_typeorder, ext_typeorder]) - assert mul2('space', W_StringObject(), W_StringObject()) == 'fine' - raises(FailedToImplement, - mul2, 'baz', W_StringObject(), UserW_StringObject()) - raises(FailedToImplement, - mul2, 'baz', UserW_StringObject(), W_StringObject()) - raises(FailedToImplement, - mul2, 'baz', UserW_StringObject(), UserW_StringObject()) - - def test_ANY(self): - setattr = multimethod.MultiMethodTable(3, root_class=W_Root, - argnames_before=['space']) - def setattr__Int_ANY_ANY(space, w_x, w_y, w_z): - assert space == 'space' - assert isinstance(w_x, W_IntObject) - assert isinstance(w_y, W_Root) - assert isinstance(w_z, W_Root) - return w_y.__class__.__name__ + w_z.__class__.__name__ - setattr.register(setattr__Int_ANY_ANY, W_IntObject, W_Root, W_Root) - setattr1 = setattr.install('__setattr1', [self.typeorder]*3) - for cls1 in self.typeorder: - for cls2 in self.typeorder: - assert setattr1('space', W_IntObject(), cls1(), cls2()) == ( - cls1.__name__ + cls2.__name__) - - def test_all_cases(self): - import random - space = 'space' - w_x = W_IntObject() - w_x.expected = [W_IntObject, W_Root] - w_s = W_StringObject() - w_s.expected = [W_StringObject, W_Root] - w_b = W_BoolObject() - w_b.expected = [W_BoolObject, W_IntObject, W_Root] - - def test(indices): - sub = multimethod.MultiMethodTable(2, root_class=W_Root, - argnames_before=['space']) - def addimpl(cls1, cls2): - token = random.random() - def sub__cls1_cls2(space, w_x, w_y): - assert space == 'space' - assert isinstance(w_x, cls1) - assert isinstance(w_y, cls2) - return token - sub.register(sub__cls1_cls2, cls1, cls2) - return token - - def check(w1, w2): - try: - res = sub1(space, w1, w2) - except FailedToImplement: - res = FailedToImplement - for cls1 in w1.expected: - for cls2 in w2.expected: - if (cls1, cls2) in expected: - assert res == expected[cls1, cls2] - return - else: - assert res is FailedToImplement - - random.shuffle(indices) - expected = {} - for index in indices: - cls1, cls2 = choices[index] - token = addimpl(cls1, cls2) - expected[cls1, cls2] = token - - typeorder = self.typeorder - sub1 = sub.install('__sub', [typeorder, typeorder]) - for w1 in [w_x, w_s, w_b]: - for w2 in [w_x, w_s, w_b]: - check(w1, w2) - - classes = [W_Root, W_StringObject, W_IntObject, W_BoolObject] - choices = [(cls1, cls2) for cls1 in classes - for cls2 in classes] - # each choice is a pair of classes which can be implemented or - # not by the multimethod 'sub'. Test all combinations that - # involve at most three implemented choices. - for i in range(len(choices)): - test([i]) - for j in range(i+1, len(choices)): - test([i, j]) - for k in range(j+1, len(choices)): - test([i, j, k]) - #for l in range(k+1, len(choices)): -- for a 4th choice - # test([i, j, k, l]) -- (takes a while) +class X: + def __init__(self, value): + self.value = value + def __repr__(self): + return '' % self.value + +def from_y_to_x(space, yinstance): + return X(yinstance) + +from_y_to_x.result_class = X +from_y_to_x.priority = 2 + +def from_x_to_str(space, xinstance): + #if xinstance.value: + return w('!' + repr(xinstance.value)) + #else: + # return [] + +from_x_to_str.result_class = str +from_x_to_str.priority = 2 + + +class Y: + def __init__(self, value): + self.value = value + def __repr__(self): + return '' % self.value + def __nonzero__(self): + return self.value != 666 + + +def add_x_x(space, x1, x2): + return "add_x_x", x1, x2 + +def add_x_y(space, x1, y2): + if x1.value < 0: + raise FailedToImplement(ValueError, 'not good') + return "add_x_y", x1, y2 + +def add_y_y(space, y1, y2): + return "add_y_y", y1, y2 + +def add_string_string(space, x, y): + return "add_string_string", x, y + +def add_int_string(space, x, y): + return "add_int_string", x, y + +def add_int_any(space, y1, o2): + return "add_int_any", y1, o2 + +class FakeObjSpace: + add = MultiMethod('+', 2, []) + add.register(add_x_x, X, X) + add.register(add_x_y, X, Y) + add.register(add_y_y, Y, Y) + add.register(add_string_string, str, str) + add.register(add_int_string, int, str) + add.register(add_int_any, int, object) + + delegate = DelegateMultiMethod() + delegate.register(from_y_to_x, Y) + delegate.register(from_x_to_str, X) + + def wrap(self, x): + return '' % (x,) + w_TypeError = 'w_TypeError' + +##def w(x, cache={}): +## if type(x) in cache: +## Stub = cache[type(x)] +## else: +## Stub = type(type(x))('%s_stub' % type(x).__name__, (type(x),), {}) +## Stub.dispatchclass = Stub +## cache[type(x)] = Stub +## return Stub(x) + +##X.dispatchclass = X +##Y.dispatchclass = Y + +def w(x): + return x + + +class TestMultiMethod(testit.TestCase): + def setUp(self): + # only run when testing stdobjectspace + #XXX removed: testit.objspace('std') + self.space = FakeObjSpace() + + def test_non_delegate(self): + space = self.space + + r = space.add(X(2), X(5)) + self.assertEquals(repr(r), "('add_x_x', , )") + + r = space.add(X(3), Y(4)) + self.assertEquals(repr(r), "('add_x_y', , )") + + r = space.add(Y(0), Y(20)) + self.assertEquals(repr(r), "('add_y_y', , )") + + r = space.add(w(-3), w([7,6,5])) + self.assertEquals(repr(r), "('add_int_any', -3, [7, 6, 5])") + + r = space.add(w(5), w("test")) + self.assertEquals(repr(r), "('add_int_string', 5, 'test')") + + r = space.add(w("x"), w("y")) + self.assertEquals(repr(r), "('add_string_string', 'x', 'y')") + + def test_delegate_y_to_x(self): + space = self.space + r = space.add(Y(-1), X(7)) + self.assertEquals(repr(r), "('add_x_x', >, )") + + r = space.add(Y(1), X(7)) + self.assertEquals(repr(r), "('add_x_x', >, )") + + r = space.add(X(-3), Y(20)) + self.assertEquals(repr(r), "('add_x_x', , >)") + + def test_no_operation_defined(self): + space = self.space + self.assertRaises(OperationError, space.add, w([3]), w(4)) + self.assertRaises(OperationError, space.add, w(3.0), w('bla')) + #self.assertRaises(OperationError, space.add, X(0), w("spam")) + #self.assertRaises(OperationError, space.add, Y(666), w("egg")) + + def test_delegate_x_to_str(self): + space = self.space + r = space.add(X(42), w("spam")) + self.assertEquals(repr(r), "('add_string_string', '!42', 'spam')") + r = space.add(Y(20), w("egg")) + self.assertEquals(repr(r), "('add_string_string', '!', 'egg')") -class TestMultiMethod2(TestMultiMethod1): - Installer = multimethod.InstallerVersion2 + + +if __name__ == '__main__': + testit.main() From arigo at codespeak.net Thu Nov 5 20:32:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 20:32:19 +0100 (CET) Subject: [pypy-svn] r69003 - in pypy/branch/gc-dump-malloc/pypy: rpython/memory rpython/memory/gc translator/c/src Message-ID: <20091105193219.8C458168417@codespeak.net> Author: arigo Date: Thu Nov 5 20:32:18 2009 New Revision: 69003 Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/generation.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/hybrid.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h Log: Following tradition pioneered by fijal, also record the total size of objects, not just the count. Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/generation.py Thu Nov 5 20:32:18 2009 @@ -215,8 +215,8 @@ result = self.collect_nursery() llarena.arena_reserve(result, totalsize) # GCFLAG_NO_YOUNG_PTRS is never set on young objs - self.init_gc_object(result, typeid, flags=0) (result + size_gc_header + offset_to_length).signed[0] = length + self.init_gc_object(result, typeid, flags=0) self.nursery_free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/hybrid.py Thu Nov 5 20:32:18 2009 @@ -168,8 +168,8 @@ if raw_malloc_usage(totalsize) <= self.nursery_top - result: llarena.arena_reserve(result, totalsize) # GCFLAG_NO_YOUNG_PTRS is never set on young objs - self.init_gc_object(result, typeid, flags=0) (result + size_gc_header + offset_to_length).signed[0] = length + self.init_gc_object(result, typeid, flags=0) self.nursery_free = result + llarena.round_up_for_allocation( totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, @@ -201,8 +201,8 @@ else: result = self.malloc_varsize_collecting_nursery(totalsize) flags = self.GCFLAGS_FOR_NEW_YOUNG_OBJECTS - self.init_gc_object(result, typeid, flags) (result + size_gc_header + offset_to_length).signed[0] = length + self.init_gc_object(result, typeid, flags) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) malloc_varsize_slowpath._dont_inline_ = True Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gc/semispace.py Thu Nov 5 20:32:18 2009 @@ -114,8 +114,8 @@ raise memoryError result = self.obtain_free_space(totalsize) llarena.arena_reserve(result, totalsize) - self.init_gc_object(result, typeid16) (result + size_gc_header + offset_to_length).signed[0] = length + self.init_gc_object(result, typeid16) self.free = result + llarena.round_up_for_allocation(totalsize) return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF) @@ -428,7 +428,10 @@ def init_gc_object(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) hdr.tid = self.combine(typeid16, flags) - self.count_allocation(typeid16) + # + size_gc_header = self.gcheaderbuilder.size_gc_header + size = self.get_size(addr + size_gc_header) + self.count_allocation(typeid16, size) def init_gc_object_immortal(self, addr, typeid16, flags=0): hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR)) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Thu Nov 5 20:32:18 2009 @@ -21,6 +21,7 @@ # structure describing the layout of a typeid TYPE_INFO = lltype.Struct("type_info", ("counter", lltype.Signed), + ("counter_size", lltype.Signed), ("infobits", lltype.Signed), # combination of the T_xxx consts ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), @@ -94,8 +95,10 @@ return weakptr_offset return -1 - def q_count_allocation(self, typeid): - self.get(typeid).counter += 1 + def q_count_allocation(self, typeid, size): + p = self.get(typeid) + p.counter += 1 + p.counter_size += llmemory.raw_malloc_usage(size) def set_query_functions(self, gc): gc.set_query_functions( Modified: pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h (original) +++ pypy/branch/gc-dump-malloc/pypy/translator/c/src/main.h Thu Nov 5 20:32:18 2009 @@ -17,8 +17,11 @@ void dump_group_info(const char *typename, long *counter) { - if (*counter) - fprintf(stderr, "|%12lu %s\n", (unsigned long)*counter, typename); + if (counter[0]) + fprintf(stderr, "|%12lu %12lu %s\n", + (unsigned long)counter[0], + (unsigned long)counter[1], + typename); } int main(int argc, char *argv[]) From magcius at codespeak.net Thu Nov 5 20:41:31 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 5 Nov 2009 20:41:31 +0100 (CET) Subject: [pypy-svn] r69004 - in pypy/branch/avm/pypy: annotation annotation/test config config/test doc/config interpreter module/__builtin__ module/__builtin__/test module/_ast module/_stackless module/_weakref/test module/dyngram module/parser module/posix module/pypyjit module/pypyjit/test module/pypyjit/test/loops module/recparser module/signal module/symbol module/sys module/token objspace objspace/flow objspace/std rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/ootypesystem rpython/test tool/pytest translator translator/backendopt translator/c translator/c/gcc translator/c/test translator/cli translator/goal translator/jvm translator/oosupport translator/oosupport/test_template translator/test Message-ID: <20091105194131.DEAEE318137@codespeak.net> Author: magcius Date: Thu Nov 5 20:41:26 2009 New Revision: 69004 Added: pypy/branch/avm/pypy/doc/config/translation.backendopt.really_remove_asserts.txt - copied unchanged from r67353, pypy/branch/pyjitpl5/pypy/doc/config/translation.backendopt.really_remove_asserts.txt pypy/branch/avm/pypy/module/_ast/ - copied from r67376, pypy/branch/newtrunk/pypy/module/_ast/ pypy/branch/avm/pypy/module/parser/ - copied from r67376, pypy/branch/newtrunk/pypy/module/parser/ pypy/branch/avm/pypy/module/pypyjit/test/loops/ - copied from r67353, pypy/branch/pyjitpl5/pypy/module/pypyjit/test/loops/ pypy/branch/avm/pypy/module/token/ - copied from r67376, pypy/branch/newtrunk/pypy/module/token/ Removed: pypy/branch/avm/pypy/module/dyngram/ pypy/branch/avm/pypy/module/recparser/ pypy/branch/avm/pypy/rpython/lltypesystem/rvirtualizable.py pypy/branch/avm/pypy/rpython/lltypesystem/test/test_rvirtualizable.py Modified: pypy/branch/avm/pypy/annotation/specialize.py pypy/branch/avm/pypy/annotation/test/test_annrpython.py pypy/branch/avm/pypy/annotation/unaryop.py pypy/branch/avm/pypy/config/config.py pypy/branch/avm/pypy/config/pypyoption.py pypy/branch/avm/pypy/config/test/test_config.py pypy/branch/avm/pypy/config/translationoption.py pypy/branch/avm/pypy/interpreter/baseobjspace.py pypy/branch/avm/pypy/interpreter/executioncontext.py pypy/branch/avm/pypy/interpreter/gateway.py pypy/branch/avm/pypy/interpreter/pycode.py pypy/branch/avm/pypy/interpreter/pyframe.py pypy/branch/avm/pypy/interpreter/typedef.py pypy/branch/avm/pypy/module/__builtin__/compiling.py pypy/branch/avm/pypy/module/__builtin__/test/test_builtin.py pypy/branch/avm/pypy/module/_stackless/interp_clonable.py pypy/branch/avm/pypy/module/_stackless/interp_coroutine.py pypy/branch/avm/pypy/module/_stackless/interp_greenlet.py pypy/branch/avm/pypy/module/_weakref/test/test_weakref.py pypy/branch/avm/pypy/module/posix/__init__.py pypy/branch/avm/pypy/module/pypyjit/interp_jit.py pypy/branch/avm/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/avm/pypy/module/signal/interp_signal.py pypy/branch/avm/pypy/module/symbol/__init__.py pypy/branch/avm/pypy/module/sys/version.py pypy/branch/avm/pypy/module/sys/vm.py pypy/branch/avm/pypy/objspace/flow/flowcontext.py pypy/branch/avm/pypy/objspace/std/celldict.py pypy/branch/avm/pypy/objspace/std/objspace.py pypy/branch/avm/pypy/objspace/std/typeobject.py pypy/branch/avm/pypy/objspace/taint.py pypy/branch/avm/pypy/rlib/jit.py pypy/branch/avm/pypy/rlib/test/test_jit.py pypy/branch/avm/pypy/rpython/llinterp.py pypy/branch/avm/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/avm/pypy/rpython/lltypesystem/lloperation.py pypy/branch/avm/pypy/rpython/lltypesystem/lltype.py pypy/branch/avm/pypy/rpython/lltypesystem/opimpl.py pypy/branch/avm/pypy/rpython/lltypesystem/rclass.py pypy/branch/avm/pypy/rpython/lltypesystem/rvirtualizable2.py pypy/branch/avm/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/avm/pypy/rpython/ootypesystem/rclass.py pypy/branch/avm/pypy/rpython/rclass.py pypy/branch/avm/pypy/rpython/rpbc.py pypy/branch/avm/pypy/rpython/test/test_rclass.py pypy/branch/avm/pypy/tool/pytest/appsupport.py pypy/branch/avm/pypy/translator/backendopt/inline.py pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py pypy/branch/avm/pypy/translator/c/genc.py pypy/branch/avm/pypy/translator/c/test/test_symbolic.py pypy/branch/avm/pypy/translator/cli/opcodes.py pypy/branch/avm/pypy/translator/driver.py pypy/branch/avm/pypy/translator/goal/translate.py pypy/branch/avm/pypy/translator/interactive.py pypy/branch/avm/pypy/translator/jvm/opcodes.py pypy/branch/avm/pypy/translator/oosupport/metavm.py pypy/branch/avm/pypy/translator/oosupport/test_template/operations.py pypy/branch/avm/pypy/translator/test/test_driver.py Log: Merge from trunk Modified: pypy/branch/avm/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/avm/pypy/annotation/specialize.py (original) +++ pypy/branch/avm/pypy/annotation/specialize.py Thu Nov 5 20:41:26 2009 @@ -59,18 +59,33 @@ # first flatten the *args args_s, key, name_suffix, builder = flatten_star_args(funcdesc, args_s) # two versions: a regular one and one for instances with 'access_directly' - for s_obj in args_s: + jit_look_inside = getattr(funcdesc.pyobj, '_look_inside_me_', True) + # change args_s in place, "official" interface + access_directly = False + for i, s_obj in enumerate(args_s): if (isinstance(s_obj, annmodel.SomeInstance) and 'access_directly' in s_obj.flags): - key = (AccessDirect, key) - name_suffix += '_AccessDirect' - break + if jit_look_inside: + access_directly = True + key = (AccessDirect, key) + name_suffix += '_AccessDirect' + break + else: + new_flags = s_obj.flags.copy() + del new_flags['access_directly'] + new_s_obj = annmodel.SomeInstance(s_obj.classdef, s_obj.can_be_None, + flags = new_flags) + args_s[i] = new_s_obj + # done if name_suffix: alt_name = '%s%s' % (funcdesc.name, name_suffix) else: alt_name = None - return funcdesc.cachedgraph(key, alt_name=alt_name, builder=builder) + graph = funcdesc.cachedgraph(key, alt_name=alt_name, builder=builder) + if access_directly: + graph.access_directly = True + return graph class AccessDirect(object): """marker for specialization: set when any arguments is a SomeInstance Modified: pypy/branch/avm/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/avm/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/avm/pypy/annotation/test/test_annrpython.py Thu Nov 5 20:41:26 2009 @@ -2746,11 +2746,10 @@ assert isinstance(s, annmodel.SomeInteger) def test_instance_with_flags(self): - py.test.skip("not supported any more") from pypy.rlib.jit import hint class A: - _virtualizable_ = True + _virtualizable2_ = [] class B(A): def meth(self): return self @@ -3106,6 +3105,15 @@ s = a.build_types(f, [int]) assert s.const == 0 + def test_hash(self): + class A(object): + pass + def f(): + return hash(A()) + hash(None) + a = self.RPythonAnnotator() + s = a.build_types(f, []) + assert s.knowntype == int + def g(n): return [0,1,2,n] Modified: pypy/branch/avm/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/avm/pypy/annotation/unaryop.py (original) +++ pypy/branch/avm/pypy/annotation/unaryop.py Thu Nov 5 20:41:26 2009 @@ -683,6 +683,13 @@ else: return SomeObject() # len() on a pbc? no chance + def hash(pbc): + if pbc.isNone(): + # only supports hash(None) as part of hash() + return SomeInteger() + else: + return SomeObject.hash(pbc) + class __extend__(SomeGenericCallable): def call(self, args): bookkeeper = getbookkeeper() Modified: pypy/branch/avm/pypy/config/config.py ============================================================================== --- pypy/branch/avm/pypy/config/config.py (original) +++ pypy/branch/avm/pypy/config/config.py Thu Nov 5 20:41:26 2009 @@ -259,7 +259,11 @@ for path, reqvalue in self._requires.get(value, []): toplevel = config._cfgimpl_get_toplevel() homeconfig, name = toplevel._cfgimpl_get_home_by_path(path) - homeconfig.setoption(name, reqvalue, who) + if who == 'default': + who2 = 'default' + else: + who2 = 'required' + homeconfig.setoption(name, reqvalue, who2) for path, reqvalue in self._suggests.get(value, []): toplevel = config._cfgimpl_get_toplevel() homeconfig, name = toplevel._cfgimpl_get_home_by_path(path) @@ -303,7 +307,11 @@ for path, reqvalue in self._requires: toplevel = config._cfgimpl_get_toplevel() homeconfig, name = toplevel._cfgimpl_get_home_by_path(path) - homeconfig.setoption(name, reqvalue, "required") + if who == 'default': + who2 = 'default' + else: + who2 = 'required' + homeconfig.setoption(name, reqvalue, who2) if value and self._suggests is not None: for path, reqvalue in self._suggests: toplevel = config._cfgimpl_get_toplevel() Modified: pypy/branch/avm/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/avm/pypy/config/pypyoption.py (original) +++ pypy/branch/avm/pypy/config/pypyoption.py Thu Nov 5 20:41:26 2009 @@ -356,7 +356,7 @@ backend = config.translation.backend # all the good optimizations for PyPy should be listed here - if level in ['2', '3']: + if level in ['2', '3', 'jit']: config.objspace.opcodes.suggest(CALL_LIKELY_BUILTIN=True) config.objspace.opcodes.suggest(CALL_METHOD=True) config.objspace.std.suggest(withmultidict=True) @@ -392,6 +392,10 @@ if type_system == 'ootype': config.objspace.std.suggest(multimethods="doubledispatch") + # extra optimizations with the JIT + if level == 'jit': + config.objspace.std.suggest(withsharingdict=True) + def enable_allworkingmodules(config): if config.translation.type_system == 'ootype': Modified: pypy/branch/avm/pypy/config/test/test_config.py ============================================================================== --- pypy/branch/avm/pypy/config/test/test_config.py (original) +++ pypy/branch/avm/pypy/config/test/test_config.py Thu Nov 5 20:41:26 2009 @@ -342,7 +342,19 @@ config.set(backend='cli') assert config.backend == 'cli' assert config.type_system == 'oo' - + +def test_overrides_require_as_default_boolopt(): + descr = OptionDescription("test", "", [ + BoolOption("backend", "", default=False, + requires=[('type_system', True)]), + BoolOption("type_system", "", default=False) + ]) + config = Config(descr, backend=True) + config.set(backend=False) + config.set(type_system=False) + assert config.backend == False + assert config.type_system == False + def test_overrides_dont_change_user_options(): descr = OptionDescription("test", "", [ BoolOption("b", "", default=False)]) @@ -502,6 +514,41 @@ assert not c.t3 assert not c.t2 +def test_suggests_can_fail_choiceopt(): + # this is what occurs in "./translate.py --gcrootfinder=asmgcc --jit" + # with --jit suggesting the boehm gc, but --gcrootfinder requiring the + # framework gctransformer. + descr = OptionDescription("test", '', [ + ChoiceOption("t1", "", ["a", "b"], default="a"), + ChoiceOption("t2", "", ["c", "d"], default="c", + requires={"d": [("t3", "f")]}), + ChoiceOption("t3", "", ["e", "f"], default="e"), + ChoiceOption("opt", "", ["g", "h"], default="g", + suggests={"h": [("t1", "b"), ("t2", "d")]}) + ]) + c = Config(descr) + assert c.t1 == 'a' + assert c.t2 == 'c' + assert c.t3 == 'e' + assert c.opt == 'g' + c.opt = "h" + assert c.opt == 'h' + assert c.t1 == 'b' + assert c.t2 == 'd' + assert c.t3 == 'f' + # does not crash + c.t2 = 'c' + assert c.t2 == 'c' + + c = Config(descr) + c.t3 = 'e' + assert c.t3 == 'e' + # does not crash + c.opt = 'h' + assert c.opt == 'h' + assert c.t3 == 'e' + assert c.t2 == 'c' + def test_choice_suggests(): descr = OptionDescription("test", '', [ Modified: pypy/branch/avm/pypy/config/translationoption.py ============================================================================== --- pypy/branch/avm/pypy/config/translationoption.py (original) +++ pypy/branch/avm/pypy/config/translationoption.py Thu Nov 5 20:41:26 2009 @@ -17,10 +17,6 @@ 'distutils', ] -def check_have_jit(config): - import pypy.jit # check out branch/pyjitpl5 instead of trunk! - - translation_optiondescription = OptionDescription( "translation", "Translation Options", [ BoolOption("stackless", "enable stackless features during compilation", @@ -34,6 +30,7 @@ ("translation.backendopt.heap2stack", False), ("translation.backendopt.clever_malloc_removal", False), ("translation.list_comprehension_operations", False), + ("translation.gc", "boehm"), # it's not really used, but some jit code expects a value here ] }), ChoiceOption("backend", "Backend to use for code generation", @@ -85,11 +82,7 @@ "asmgcc": [("translation.gctransformer", "framework"), ("translation.backend", "c"), ("translation.thread", False)], - }, - suggests={ - "shadowstack": [("translation.gc", "generation")], - "asmgcc": [("translation.gc", "generation")], - }), + }), # other noticeable options BoolOption("thread", "enable use of threading primitives", @@ -101,12 +94,19 @@ BoolOption("rweakref", "The backend supports RPython-level weakrefs", default=True), - # JIT generation + # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", - default=False, cmdline="--jit", - validator=check_have_jit, - requires=[("translation.gc", "boehm"), + default=False, + requires=[("translation.thread", False)], + suggests=[("translation.gc", "boehm"), # for now ("translation.list_comprehension_operations", True)]), + ChoiceOption("jit_backend", "choose the backend for the JIT", + ["auto", "x86", "llvm"], + default="auto", cmdline="--jit-backend"), + ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", + ["off", "profile", "steps", "detailed"], + default="steps", # XXX for now + cmdline="--jit-debug"), # misc BoolOption("verbose", "Print extra information", default=False), @@ -234,6 +234,10 @@ "Remove operations that look like 'raise AssertionError', " "which lets the C optimizer remove the asserts", default=False), + BoolOption("really_remove_asserts", + "Really remove operations that look like 'raise AssertionError', " + "without relying on the C compiler", + default=False), BoolOption("stack_optimization", "Tranform graphs in SSI form into graphs tailored for " @@ -294,7 +298,7 @@ # ____________________________________________________________ -OPT_LEVELS = ['0', '1', 'size', 'mem', '2', '3'] +OPT_LEVELS = ['0', '1', 'size', 'mem', '2', '3', 'jit'] DEFAULT_OPT_LEVEL = '2' OPT_TABLE_DOC = { @@ -304,6 +308,7 @@ 'mem': 'Optimize for run-time memory usage and use a memory-saving GC.', '2': 'Enable most optimizations and use a high-performance GC.', '3': 'Enable all optimizations and use a high-performance GC.', + 'jit': 'Enable the JIT.', } OPT_TABLE = { @@ -314,6 +319,7 @@ 'mem': 'markcompact lowinline remove_asserts', '2': 'hybrid extraopts', '3': 'hybrid extraopts remove_asserts', + 'jit': 'boehm extraopts jit', # XXX boehm for now, fix me } def set_opt_level(config, level): @@ -333,9 +339,9 @@ gc = words.pop(0) # set the GC (only meaningful with lltype) - if config.translation.sandbox and gc == 'hybrid': - gc = 'generation' - config.translation.suggest(gc=gc) + # but only set it if it wasn't already suggested to be something else + if config.translation._cfgimpl_value_owners['gc'] != 'suggested': + config.translation.suggest(gc=gc) # set the backendopts for word in words: @@ -348,6 +354,8 @@ config.translation.backendopt.suggest(remove_asserts=True) elif word == 'extraopts': config.translation.suggest(withsmallfuncsets=5) + elif word == 'jit': + config.translation.suggest(jit=True) else: raise ValueError(word) Modified: pypy/branch/avm/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/avm/pypy/interpreter/baseobjspace.py Thu Nov 5 20:41:26 2009 @@ -1,130 +1,595 @@ -from pypy.interpreter.executioncontext import ExecutionContext +from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag +from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction from pypy.interpreter.error import OperationError -from pypy.interpreter.miscutils import Stack, getthreadlocals -import pypy.module +from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack +from pypy.interpreter.pycompiler import CPythonCompiler, PythonAstCompiler +from pypy.interpreter.miscutils import ThreadLocals +from pypy.tool.cache import Cache +from pypy.tool.uid import HUGEVAL_BYTES +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib.timer import DummyTimer, Timer +from pypy.rlib.jit import we_are_jitted, dont_look_inside +import os, sys + +__all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] + + +class W_Root(object): + """This is the abstract root class of all wrapped objects that live + in a 'normal' object space like StdObjSpace.""" + __slots__ = () + _settled_ = True + + def getdict(self): + return None + + def getdictvalue_w(self, space, attr): + return self.getdictvalue(space, space.wrap(attr)) + + def getdictvalue(self, space, w_attr): + w_dict = self.getdict() + if w_dict is not None: + return space.finditem(w_dict, w_attr) + return None + + def getdictvalue_attr_is_in_class(self, space, w_attr): + return self.getdictvalue(space, w_attr) + + def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + w_dict = self.getdict() + if w_dict is not None: + space.set_str_keyed_item(w_dict, w_attr, w_value, shadows_type) + return True + return False -__all__ = ['ObjSpace', 'OperationError', 'NoValue'] + def deldictvalue(self, space, w_name): + w_dict = self.getdict() + if w_dict is not None: + try: + space.delitem(w_dict, w_name) + return True + except OperationError, ex: + if not ex.match(space, space.w_KeyError): + raise + return False + def setdict(self, space, w_dict): + typename = space.type(self).getname(space, '?') + raise OperationError(space.w_TypeError, + space.wrap("attribute '__dict__' of %s objects " + "is not writable" % typename)) + + # to be used directly only by space.type implementations + def getclass(self, space): + return space.gettypeobject(self.typedef) + + def setclass(self, space, w_subtype): + raise OperationError(space.w_TypeError, + space.wrap("__class__ assignment: only for heap types")) -class Wrappable(object): - """A subclass of Wrappable is an internal, interpreter-level class - that can nevertheless be exposed at application-level by space.wrap().""" + def user_setup(self, space, w_subtype): + assert False, "only for interp-level user subclasses from typedef.py" - def get_wdict(self): - space = self.space + def getname(self, space, default): try: - return self.w_dict - except AttributeError: - w_dict = self.w_dict = space.newdict([]) - for name,w_value in self.app_visible(): - space.setitem(w_dict,space.wrap(name),w_value) - return w_dict + return space.str_w(space.getattr(self, space.wrap('__name__'))) + except OperationError, e: + if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError): + return default + raise + + def getaddrstring(self, space): + # XXX slowish + w_id = space.id(self) + w_4 = space.wrap(4) + w_0x0F = space.wrap(0x0F) + i = 2 * HUGEVAL_BYTES + addrstring = [' '] * i + while True: + n = space.int_w(space.and_(w_id, w_0x0F)) + n += ord('0') + if n > ord('9'): + n += (ord('a') - ord('9') - 1) + i -= 1 + addrstring[i] = chr(n) + if i == 0: + break + w_id = space.rshift(w_id, w_4) + return ''.join(addrstring) + + def getrepr(self, space, info, moreinfo=''): + addrstring = self.getaddrstring(space) + return space.wrap("<%s at 0x%s%s>" % (info, addrstring, + moreinfo)) - def app_visible(self): - """ returns [(name,w_value)...] for application-level visible attributes """ + def getslotvalue(self, index): raise NotImplementedError - def pypy_getattr(self, w_name): - space = self.space - w_dict = self.get_wdict() - try: - return space.getitem(w_dict,w_name) - except OperationError,e: - if not e.match(space,space.w_KeyError): - raise - raise OperationError(space.w_AttributeError,w_name) + def setslotvalue(self, index, w_val): + raise NotImplementedError + def descr_call_mismatch(self, space, opname, RequiredClass, args): + if RequiredClass is None: + classname = '?' + else: + classname = wrappable_class_name(RequiredClass) + msg = "'%s' object expected, got '%s' instead" % ( + classname, self.getclass(space).getname(space, '?')) + raise OperationError(space.w_TypeError, space.wrap(msg)) + + # used by _weakref implemenation + + def getweakref(self): + return None + + def setweakref(self, space, weakreflifeline): + typename = space.type(self).getname(space, '?') + raise OperationError(space.w_TypeError, space.wrap( + "cannot create weak reference to '%s' object" % typename)) + + def clear_all_weakrefs(self): + """Call this at the beginning of interp-level __del__() methods + in subclasses. It ensures that weakrefs (if any) are cleared + before the object is further destroyed. + """ + lifeline = self.getweakref() + if lifeline is not None: + # Clear all weakrefs to this object before we proceed with + # the destruction of the object. We detach the lifeline + # first: if the code following before_del() calls the + # app-level, e.g. a user-defined __del__(), and this code + # tries to use weakrefs again, it won't reuse the broken + # (already-cleared) weakrefs from this lifeline. + self.setweakref(lifeline.space, None) + lifeline.clear_all_weakrefs() + + __already_enqueued_for_destruction = False + + def _enqueue_for_destruction(self, space): + """Put the object in the destructor queue of the space. + At a later, safe point in time, UserDelAction will use + space.userdel() to call the object's app-level __del__ method. + """ + # this function always resurect the object, so when + # running on top of CPython we must manually ensure that + # we enqueue it only once + if not we_are_translated(): + if self.__already_enqueued_for_destruction: + return + self.__already_enqueued_for_destruction = True + self.clear_all_weakrefs() + space.user_del_action.register_dying_object(self) -class NoValue(Exception): - """Raised to signal absence of value, e.g. in the iterator accessing - method 'op.next()' of object spaces.""" + def _call_builtin_destructor(self): + pass # method overridden in typedef.py -class ObjSpace: +class Wrappable(W_Root): + """A subclass of Wrappable is an internal, interpreter-level class + that can nevertheless be exposed at application-level by space.wrap().""" + __slots__ = () + _settled_ = True + + def __spacebind__(self, space): + return self + +class InternalSpaceCache(Cache): + """A generic cache for an object space. Arbitrary information can + be attached to the space by defining a function or class 'f' which + can be called as 'f(space)'. Its result is stored in this + ObjSpaceCache. + """ + def __init__(self, space): + Cache.__init__(self) + self.space = space + def _build(self, callable): + return callable(self.space) + +class SpaceCache(Cache): + """A base class for all our concrete caches.""" + def __init__(self, space): + Cache.__init__(self) + self.space = space + def _build(self, key): + val = self.space.enter_cache_building_mode() + try: + return self.build(key) + finally: + self.space.leave_cache_building_mode(val) + def _ready(self, result): + val = self.space.enter_cache_building_mode() + try: + return self.ready(result) + finally: + self.space.leave_cache_building_mode(val) + def ready(self, result): + pass + +class UnpackValueError(ValueError): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class DescrMismatch(Exception): + pass + +def wrappable_class_name(Class): + try: + return Class.typedef.name + except AttributeError: + return 'internal subclass of %s' % (Class.__name__,) +wrappable_class_name._annspecialcase_ = 'specialize:memo' + +# ____________________________________________________________ + +class ObjSpace(object): """Base class for the interpreter-level implementations of object spaces. - http://codespeak.net/moin/pypy/moin.cgi/ObjectSpace""" - + http://codespeak.net/pypy/dist/pypy/doc/objspace.html""" + full_exceptions = True # full support for exceptions (normalization & more) - def __init__(self): - "Basic initialization of objects." + def __init__(self, config=None): + "NOT_RPYTHON: Basic initialization of objects." + self.fromcache = InternalSpaceCache(self).getorbuild + self.threadlocals = ThreadLocals() + # set recursion limit + # sets all the internal descriptors + if config is None: + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=False) + self.config = config + + # import extra modules for side-effects + import pypy.interpreter.nestedscope # register *_DEREF bytecodes + + self.interned_strings = {} + self.actionflag = ActionFlag() # changed by the signal module + self.user_del_action = UserDelAction(self) + self.frame_trace_action = FrameTraceAction(self) + self.actionflag.register_action(self.user_del_action) + self.actionflag.register_action(self.frame_trace_action) + + from pypy.interpreter.pyframe import PyFrame + self.FrameClass = PyFrame # can be overridden to a subclass + + if self.config.objspace.logbytecodes: + self.bytecodecounts = [0] * 256 + self.bytecodetransitioncount = {} + + if self.config.objspace.timing: + self.timer = Timer() + else: + self.timer = DummyTimer() + self.initialize() + def startup(self): + # To be called before using the space + + # Initialize all builtin modules + from pypy.interpreter.module import Module + for w_modname in self.unpackiterable( + self.sys.get('builtin_module_names')): + modname = self.str_w(w_modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): + import time + self.timer.start("startup " + modname) + mod.startup(self) + self.timer.stop("startup " + modname) + + def finish(self): + w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') + if w_exitfunc is not None: + self.call_function(w_exitfunc) + from pypy.interpreter.module import Module + for w_modname in self.unpackiterable( + self.sys.get('builtin_module_names')): + modname = self.str_w(w_modname) + mod = self.interpclass_w(self.getbuiltinmodule(modname)) + if isinstance(mod, Module): + mod.shutdown(self) + if self.config.objspace.std.withdictmeasurement: + from pypy.objspace.std.dictmultiobject import report + report() + if self.config.objspace.logbytecodes: + self.reportbytecodecounts() + if self.config.objspace.std.logspaceoptypes: + for s in self.FrameClass._space_op_types: + print s + + def reportbytecodecounts(self): + os.write(2, "Starting bytecode report.\n") + fd = os.open('bytecode.txt', os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0644) + os.write(fd, "bytecodecounts = {\n") + for opcode in range(len(self.bytecodecounts)): + count = self.bytecodecounts[opcode] + if not count: + continue + os.write(fd, " %s: %s,\n" % (opcode, count)) + os.write(fd, "}\n") + os.write(fd, "bytecodetransitioncount = {\n") + for opcode, probs in self.bytecodetransitioncount.iteritems(): + os.write(fd, " %s: {\n" % (opcode, )) + for nextcode, count in probs.iteritems(): + os.write(fd, " %s: %s,\n" % (nextcode, count)) + os.write(fd, " },\n") + os.write(fd, "}\n") + os.close(fd) + os.write(2, "Reporting done.\n") + + def __repr__(self): + try: + return self._this_space_repr_ + except AttributeError: + return self.__class__.__name__ + + def setbuiltinmodule(self, importname): + """NOT_RPYTHON. load a lazy pypy/module and put it into sys.modules""" + import sys + + fullname = "pypy.module.%s" % importname + + Module = __import__(fullname, + None, None, ["Module"]).Module + if Module.applevel_name is not None: + name = Module.applevel_name + else: + name = importname + + w_name = self.wrap(name) + w_mod = self.wrap(Module(self, w_name)) + w_modules = self.sys.get('modules') + self.setitem(w_modules, w_name, w_mod) + return name + + def getbuiltinmodule(self, name): + w_name = self.wrap(name) + w_modules = self.sys.get('modules') + return self.getitem(w_modules, w_name) + + def get_builtinmodule_to_install(self): + """NOT_RPYTHON""" + try: + return self._builtinmodule_list + except AttributeError: + pass + + modules = [] + + # You can enable more modules by specifying --usemodules=xxx,yyy + for name, value in self.config.objspace.usemodules: + if value and name not in modules: + modules.append(name) + + # a bit of custom logic: time2 or rctime take precedence over time + # XXX this could probably be done as a "requires" in the config + if ('time2' in modules or 'rctime' in modules) and 'time' in modules: + modules.remove('time') + + import pypy + if not self.config.objspace.nofaking: + for modname in self.ALL_BUILTIN_MODULES: + if not (os.path.exists( + os.path.join(os.path.dirname(pypy.__file__), + 'lib', modname+'.py'))): + modules.append('faked+'+modname) + + self._builtinmodule_list = modules + return self._builtinmodule_list + + ALL_BUILTIN_MODULES = [ + 'posix', 'nt', 'os2', 'mac', 'ce', 'riscos', + 'math', 'array', 'select', + '_random', '_sre', 'time', '_socket', 'errno', + 'unicodedata', + 'parser', 'fcntl', '_codecs', 'binascii' + ] + def make_builtins(self): - # initializing builtins may require creating a frame which in - # turn already accesses space.w_builtins, provide a dummy one ... - self.w_builtins = self.newdict([]) - - assert not hasattr(self, 'builtin') - if not hasattr(self, 'sys'): - self.make_sys() - - from pypy.interpreter.extmodule import BuiltinModule - - # the builtins are iteratively initialized - self.builtin = BuiltinModule(self, '__builtin__', self.w_builtins) - self.w_builtin = self.wrap(self.builtin) + "NOT_RPYTHON: only for initializing the space." + + from pypy.module.sys import Module + w_name = self.wrap('sys') + self.sys = Module(self, w_name) + w_modules = self.sys.get('modules') + self.setitem(w_modules, w_name, self.wrap(self.sys)) + + from pypy.module.__builtin__ import Module + w_name = self.wrap('__builtin__') + self.builtin = Module(self, w_name) + w_builtin = self.wrap(self.builtin) + self.setitem(w_modules, w_name, w_builtin) + self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin) + + bootstrap_modules = ['sys', '__builtin__', 'exceptions'] + installed_builtin_modules = bootstrap_modules[:] # initialize with "bootstrap types" from objspace (e.g. w_None) for name, value in self.__dict__.items(): - if name.startswith('w_'): + if name.startswith('w_') and not name.endswith('Type'): name = name[2:] - if name.startswith('builtin') or name.startswith('sys'): - continue #print "setitem: space instance %-20s into builtins" % name - self.setitem(self.w_builtins, self.wrap(name), value) - - self.sys.setbuiltinmodule(self.w_builtin, '__builtin__') + self.setitem(self.builtin.w_dict, self.wrap(name), value) - def make_sys(self): - from pypy.interpreter.extmodule import BuiltinModule - assert not hasattr(self, 'sys') - self.sys = BuiltinModule(self, 'sys') - self.w_sys = self.wrap(self.sys) - self.sys.setbuiltinmodule(self.w_sys, 'sys') - - def get_builtin_module(self, name): - if name not in self.sys.builtin_modules: - return None - module = self.sys.builtin_modules[name] - if module is None: - from pypy.interpreter.extmodule import BuiltinModule - module = BuiltinModule(self, name) - self.sys.builtin_modules[name] = module - w_module = self.wrap(module) - self.sys.setbuiltinmodule(w_module, name) - return w_module + # install mixed and faked modules and set builtin_module_names on sys + for mixedname in self.get_builtinmodule_to_install(): + if (mixedname not in bootstrap_modules + and not mixedname.startswith('faked+')): + self.install_mixedmodule(mixedname, installed_builtin_modules) + for mixedname in self.get_builtinmodule_to_install(): + if mixedname.startswith('faked+'): + modname = mixedname[6:] + self.install_faked_module(modname, installed_builtin_modules) + + installed_builtin_modules.sort() + w_builtin_module_names = self.newtuple( + [self.wrap(fn) for fn in installed_builtin_modules]) + + # force this value into the dict without unlazyfying everything + self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'), + w_builtin_module_names) + + def install_mixedmodule(self, mixedname, installed_builtin_modules): + """NOT_RPYTHON""" + modname = self.setbuiltinmodule(mixedname) + if modname: + assert modname not in installed_builtin_modules, ( + "duplicate interp-level module enabled for the " + "app-level module %r" % (modname,)) + installed_builtin_modules.append(modname) + + def load_cpython_module(self, modname): + "NOT_RPYTHON. Steal a module from CPython." + cpy_module = __import__(modname, {}, {}, ['*']) + return cpy_module + + def install_faked_module(self, modname, installed_builtin_modules): + """NOT_RPYTHON""" + if modname in installed_builtin_modules: + return + try: + module = self.load_cpython_module(modname) + except ImportError: + return + else: + w_modules = self.sys.get('modules') + self.setitem(w_modules, self.wrap(modname), self.wrap(module)) + installed_builtin_modules.append(modname) + + def setup_builtin_modules(self): + "NOT_RPYTHON: only for initializing the space." + from pypy.interpreter.module import Module + for w_modname in self.unpackiterable(self.sys.get('builtin_module_names')): + modname = self.unwrap(w_modname) + mod = self.getbuiltinmodule(modname) + if isinstance(mod, Module): + mod.setup_after_space_initialization() def initialize(self): - """Abstract method that should put some minimal content into the - w_builtins.""" + """NOT_RPYTHON: Abstract method that should put some minimal + content into the w_builtins.""" + + def enter_cache_building_mode(self): + "hook for the flow object space" + def leave_cache_building_mode(self, val): + "hook for the flow object space" def getexecutioncontext(self): "Return what we consider to be the active execution context." - ec = getthreadlocals().executioncontext #it's allways None (dec. 2003) + # Important: the annotator must not see a prebuilt ExecutionContext: + # you should not see frames while you translate + # so we make sure that the threadlocals never *have* an + # ExecutionContext during translation. + if self.config.translating and not we_are_translated(): + assert self.threadlocals.getvalue() is None, ( + "threadlocals got an ExecutionContext during translation!") + try: + return self._ec_during_translation + except AttributeError: + ec = self.createexecutioncontext() + self._ec_during_translation = ec + return ec + # normal case follows. The 'thread' module installs a real + # thread-local object in self.threadlocals, so this builds + # and caches a new ec in each thread. + ec = self.threadlocals.getvalue() if ec is None: ec = self.createexecutioncontext() + self.threadlocals.setvalue(ec) return ec + def _freeze_(self): + return True + def createexecutioncontext(self): "Factory function for execution contexts." return ExecutionContext(self) + def createcompiler(self): + "Factory function creating a compiler object." + # XXX simple selection logic for now + try: + return self.default_compiler + except AttributeError: + if self.config.objspace.compiler == 'cpython': + compiler = CPythonCompiler(self) + elif self.config.objspace.compiler == 'ast': + compiler = PythonAstCompiler(self) + else: + raise ValueError('unknown --compiler option value: %r' % ( + self.config.objspace.compiler,)) + self.default_compiler = compiler + return compiler + + def createframe(self, code, w_globals, closure=None): + "Create an empty PyFrame suitable for this code object." + return self.FrameClass(self, code, w_globals, closure) + + def allocate_lock(self): + """Return an interp-level Lock object if threads are enabled, + and a dummy object if they are not.""" + if self.config.objspace.usemodules.thread: + # we use a sub-function to avoid putting the 'import' statement + # here, where the flow space would see it even if thread=False + return self.__allocate_lock() + else: + return dummy_lock + + def __allocate_lock(self): + from pypy.module.thread.ll_thread import allocate_lock, error + try: + return allocate_lock() + except error: + raise OperationError(self.w_RuntimeError, + self.wrap("out of resources")) + # Following is a friendly interface to common object space operations # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. - def is_(self, w_x, w_y): - "'x is y'." - w_id_x = self.id(w_x) - w_id_y = self.id(w_y) - return self.eq(w_id_x, w_id_y) - - def unwrapdefault(self, w_value, default): - if w_value is None or w_value == self.w_None: - return default - else: - return self.unwrap(w_value) + #def is_(self, w_x, w_y): -- not really useful. Must be subclassed + # "'x is y'." + # w_id_x = self.id(w_x) + # w_id_y = self.id(w_y) + # return self.eq(w_id_x, w_id_y) + + def not_(self, w_obj): + return self.wrap(not self.is_true(w_obj)) + + def eq_w(self, w_obj1, w_obj2): + """shortcut for space.is_true(space.eq(w_obj1, w_obj2))""" + return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2)) + + def is_w(self, w_obj1, w_obj2): + """shortcut for space.is_true(space.is_(w_obj1, w_obj2))""" + return self.is_true(self.is_(w_obj1, w_obj2)) + + def hash_w(self, w_obj): + """shortcut for space.int_w(space.hash(w_obj))""" + return self.int_w(self.hash(w_obj)) + + def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): + return self.setitem(w_obj, w_key, w_value) + + def finditem(self, w_obj, w_key): + try: + return self.getitem(w_obj, w_key) + except OperationError, e: + if e.match(self, self.w_KeyError): + return None + raise + + def findattr(self, w_object, w_name): + try: + return self.getattr(w_object, w_name) + except OperationError, e: + # a PyPy extension: let SystemExit and KeyboardInterrupt go through + if e.async(self): + raise + return None def newbool(self, b): if b: @@ -132,78 +597,464 @@ else: return self.w_False - def unpackiterable(self, w_iterable, expected_length=None): + def new_interned_w_str(self, w_s): + s = self.str_w(w_s) + try: + return self.interned_strings[s] + except KeyError: + pass + self.interned_strings[s] = w_s + return w_s + + def new_interned_str(self, s): + try: + return self.interned_strings[s] + except KeyError: + pass + w_s = self.interned_strings[s] = self.wrap(s) + return w_s + + def interpclass_w(space, w_obj): + """ + If w_obj is a wrapped internal interpreter class instance unwrap to it, + otherwise return None. (Can be overridden in specific spaces; you + should generally use the helper space.interp_w() instead.) + """ + if isinstance(w_obj, Wrappable): + return w_obj + return None + + def descr_self_interp_w(self, RequiredClass, w_obj): + obj = self.interpclass_w(w_obj) + if not isinstance(obj, RequiredClass): + raise DescrMismatch() + return obj + descr_self_interp_w._annspecialcase_ = 'specialize:arg(1)' + + def interp_w(self, RequiredClass, w_obj, can_be_None=False): + """ + Unwrap w_obj, checking that it is an instance of the required internal + interpreter class (a subclass of Wrappable). + """ + assert RequiredClass is not None + if can_be_None and self.is_w(w_obj, self.w_None): + return None + obj = self.interpclass_w(w_obj) + if not isinstance(obj, RequiredClass): # or obj is None + msg = "'%s' object expected, got '%s' instead" % ( + wrappable_class_name(RequiredClass), + w_obj.getclass(self).getname(self, '?')) + raise OperationError(self.w_TypeError, self.wrap(msg)) + return obj + interp_w._annspecialcase_ = 'specialize:arg(1)' + + def unpackiterable(self, w_iterable, expected_length=-1): """Unpack an iterable object into a real (interpreter-level) list. - Raise a real ValueError if the length is wrong.""" + Raise a real (subclass of) ValueError if the length is wrong.""" w_iterator = self.iter(w_iterable) items = [] while True: try: w_item = self.next(w_iterator) - except NoValue: + except OperationError, e: + if not e.match(self, self.w_StopIteration): + raise break # done - if expected_length is not None and len(items) == expected_length: - raise ValueError, "too many values to unpack" + if expected_length != -1 and len(items) == expected_length: + raise UnpackValueError("too many values to unpack") items.append(w_item) - if expected_length is not None and len(items) < expected_length: + if expected_length != -1 and len(items) < expected_length: i = len(items) if i == 1: plural = "" else: plural = "s" - raise ValueError, "need more than %d value%s to unpack" % (i, plural) + raise UnpackValueError("need more than %d value%s to unpack" % + (i, plural)) return items - def unpacktuple(self, w_tuple, expected_length=None): - """Same as unpackiterable(), but only for tuples. - Only use for bootstrapping or performance reasons.""" - tuple_length = self.unwrap(self.len(w_tuple)) - if expected_length is not None and tuple_length != expected_length: - raise ValueError, "got a tuple of length %d instead of %d" % ( - tuple_length, expected_length) - items = [ - self.getitem(w_tuple, self.wrap(i)) for i in range(tuple_length)] - return items + def viewiterable(self, w_iterable, expected_length=-1): + """ More or less the same as unpackiterable, but does not return + a copy. Please don't modify the result + """ + return make_sure_not_resized(self.unpackiterable(w_iterable, + expected_length)[:]) def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" - check_list = [w_check_class] - while check_list: - w_item = check_list.pop() - # Match identical items. - if self.is_true(self.is_(w_exc_type, w_item)): - return True + if self.is_w(w_exc_type, w_check_class): + return True # fast path (also here to handle string exceptions) + try: + return self.abstract_issubclass_w(w_exc_type, w_check_class) + except OperationError, e: + if e.match(self, self.w_TypeError): # string exceptions maybe + return False + raise + + def call_obj_args(self, w_callable, w_obj, args): + if not self.config.objspace.disable_call_speedhacks: + # XXX start of hack for performance + from pypy.interpreter.function import Function + if isinstance(w_callable, Function): + return w_callable.call_obj_args(w_obj, args) + # XXX end of hack for performance + return self.call_args(w_callable, args.prepend(w_obj)) + + def call(self, w_callable, w_args, w_kwds=None): + args = Arguments.frompacked(self, w_args, w_kwds) + return self.call_args(w_callable, args) + + def call_function(self, w_func, *args_w): + nargs = len(args_w) # used for pruning funccall versions + if not self.config.objspace.disable_call_speedhacks and nargs < 5: + # XXX start of hack for performance + from pypy.interpreter.function import Function, Method + if isinstance(w_func, Method): + w_inst = w_func.w_instance + if w_inst is not None: + if nargs < 4: + func = w_func.w_function + if isinstance(func, Function): + return func.funccall(w_inst, *args_w) + elif args_w and ( + self.abstract_isinstance_w(args_w[0], w_func.w_class)): + w_func = w_func.w_function + + if isinstance(w_func, Function): + return w_func.funccall(*args_w) + # XXX end of hack for performance + + args = Arguments(self, list(args_w)) + return self.call_args(w_func, args) + + def call_valuestack(self, w_func, nargs, frame): + from pypy.interpreter.function import Function, Method, is_builtin_code + if (not we_are_jitted() and frame.is_being_profiled and + is_builtin_code(w_func)): + # XXX: this code is copied&pasted :-( from the slow path below + # call_valuestack(). + args = frame.make_arguments(nargs) try: - # Match subclasses. - if self.is_true(self.issubtype(w_exc_type, w_item)): - return True - except OperationError: - # Assume that this is a TypeError: w_item not a type, - # and assume that w_item is then actually a tuple. - exclst = self.unpackiterable(w_item) - check_list.extend(exclst) - return False + return self.call_args_and_c_profile(frame, w_func, args) + finally: + if isinstance(args, ArgumentsFromValuestack): + args.frame = None + + if not self.config.objspace.disable_call_speedhacks: + # XXX start of hack for performance + if isinstance(w_func, Method): + w_inst = w_func.w_instance + if w_inst is not None: + w_func = w_func.w_function + # reuse callable stack place for w_inst + frame.settopvalue(w_inst, nargs) + nargs += 1 + elif nargs > 0 and ( + self.abstract_isinstance_w(frame.peekvalue(nargs-1), # :-( + w_func.w_class)): + w_func = w_func.w_function + + if isinstance(w_func, Function): + return w_func.funccall_valuestack(nargs, frame) + # XXX end of hack for performance - def call_function(self, w_func, *args_w, **kw_w): - w_kw = self.newdict([(self.wrap(k), w_v) for k, w_v in kw_w.iteritems()]) - return self.call(w_func, self.newtuple(list(args_w)), w_kw) + args = frame.make_arguments(nargs) + try: + return self.call_args(w_func, args) + finally: + if isinstance(args, ArgumentsFromValuestack): + args.frame = None + + @dont_look_inside + def call_args_and_c_profile(self, frame, w_func, args): + ec = self.getexecutioncontext() + ec.c_call_trace(frame, w_func) + try: + w_res = self.call_args(w_func, args) + except OperationError, e: + ec.c_exception_trace(frame, e.w_value) + raise + ec.c_return_trace(frame, w_func) + return w_res - def call_method(self, w_obj, methname, *arg_w, **kw_w): + def call_method(self, w_obj, methname, *arg_w): w_meth = self.getattr(w_obj, self.wrap(methname)) - return self.call_function(w_meth, *arg_w, **kw_w) + return self.call_function(w_meth, *arg_w) + + def lookup(self, w_obj, name): + w_type = self.type(w_obj) + w_mro = self.getattr(w_type, self.wrap("__mro__")) + for w_supertype in self.unpackiterable(w_mro): + w_value = w_supertype.getdictvalue_w(self, name) + if w_value is not None: + return w_value + return None + + def is_oldstyle_instance(self, w_obj): + # xxx hack hack hack + from pypy.module.__builtin__.interp_classobj import W_InstanceObject + obj = self.interpclass_w(w_obj) + return obj is not None and isinstance(obj, W_InstanceObject) + + def callable(self, w_obj): + if self.lookup(w_obj, "__call__") is not None: + if self.is_oldstyle_instance(w_obj): + # ugly old style class special treatment, but well ... + try: + self.getattr(w_obj, self.wrap("__call__")) + return self.w_True + except OperationError, e: + if not e.match(self, self.w_AttributeError): + raise + return self.w_False + else: + return self.w_True + return self.w_False def isinstance(self, w_obj, w_type): w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + def abstract_issubclass_w(self, w_cls1, w_cls2): + # Equivalent to 'issubclass(cls1, cls2)'. The code below only works + # for the simple case (new-style class, new-style class). + # This method is patched with the full logic by the __builtin__ + # module when it is loaded. + return self.is_true(self.issubtype(w_cls1, w_cls2)) + + def abstract_isinstance_w(self, w_obj, w_cls): + # Equivalent to 'isinstance(obj, cls)'. The code below only works + # for the simple case (new-style instance, new-style class). + # This method is patched with the full logic by the __builtin__ + # module when it is loaded. + return self.is_true(self.isinstance(w_obj, w_cls)) + + def abstract_isclass_w(self, w_obj): + # Equivalent to 'isinstance(obj, type)'. The code below only works + # for the simple case (new-style instance without special stuff). + # This method is patched with the full logic by the __builtin__ + # module when it is loaded. + return self.is_true(self.isinstance(w_obj, self.w_type)) + + def abstract_getclass(self, w_obj): + # Equivalent to 'obj.__class__'. The code below only works + # for the simple case (new-style instance without special stuff). + # This method is patched with the full logic by the __builtin__ + # module when it is loaded. + return self.type(w_obj) + + def eval(self, expression, w_globals, w_locals, hidden_applevel=False): + "NOT_RPYTHON: For internal debugging." + import types + from pypy.interpreter.pycode import PyCode + if isinstance(expression, str): + expression = compile(expression, '?', 'eval') + if isinstance(expression, types.CodeType): + expression = PyCode._from_code(self, expression, + hidden_applevel=hidden_applevel) + if not isinstance(expression, PyCode): + raise TypeError, 'space.eval(): expected a string, code or PyCode object' + return expression.exec_code(self, w_globals, w_locals) + + def exec_(self, statement, w_globals, w_locals, hidden_applevel=False): + "NOT_RPYTHON: For internal debugging." + import types + from pypy.interpreter.pycode import PyCode + if isinstance(statement, str): + statement = compile(statement, '?', 'exec') + if isinstance(statement, types.CodeType): + statement = PyCode._from_code(self, statement, + hidden_applevel=hidden_applevel) + if not isinstance(statement, PyCode): + raise TypeError, 'space.exec_(): expected a string, code or PyCode object' + w_key = self.wrap('__builtins__') + if not self.is_true(self.contains(w_globals, w_key)): + self.setitem(w_globals, w_key, self.wrap(self.builtin)) + return statement.exec_code(self, w_globals, w_locals) + + def appexec(self, posargs_w, source): + """ return value from executing given source at applevel. + EXPERIMENTAL. The source must look like + '''(x, y): + do_stuff... + return result + ''' + """ + w_func = self.fromcache(AppExecCache).getorbuild(source) + args = Arguments(self, list(posargs_w)) + return self.call_args(w_func, args) + appexec._annspecialcase_ = 'specialize:arg(2)' + + def decode_index(self, w_index_or_slice, seqlength): + """Helper for custom sequence implementations + -> (index, 0, 0) or + (start, stop, step) + """ + if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): + w_indices = self.call_method(w_index_or_slice, "indices", + self.wrap(seqlength)) + w_start, w_stop, w_step = self.unpackiterable(w_indices, 3) + start = self.int_w(w_start) + stop = self.int_w(w_stop) + step = self.int_w(w_step) + if step == 0: + raise OperationError(self.w_ValueError, + self.wrap("slice step cannot be zero")) + if start < 0: + start = 0 + if stop < start: + stop = start + assert stop <= seqlength + else: + start = self.int_w(w_index_or_slice) + if start < 0: + start += seqlength + if not (0 <= start < seqlength): + raise OperationError(self.w_IndexError, + self.wrap("index out of range")) + stop = 0 + step = 0 + return start, stop, step + + def getindex_w(self, w_obj, w_exception, objdescr=None): + """Return w_obj.__index__() as an RPython int. + If w_exception is None, silently clamp in case of overflow; + else raise w_exception. + """ + try: + w_index = self.index(w_obj) + except OperationError, err: + if objdescr is None or not err.match(self, self.w_TypeError): + raise + msg = "%s must be an integer, not %s" % ( + objdescr, self.type(w_obj).getname(self, '?')) + raise OperationError(self.w_TypeError, self.wrap(msg)) + try: + index = self.int_w(w_index) + except OperationError, err: + if not err.match(self, self.w_OverflowError): + raise + if not w_exception: + # w_index should be a long object, but can't be sure of that + if self.is_true(self.lt(w_index, self.wrap(0))): + return -sys.maxint-1 + else: + return sys.maxint + else: + raise OperationError( + w_exception, self.wrap( + "cannot fit '%s' into an index-sized " + "integer" % self.type(w_obj).getname(self, '?'))) + else: + return index + + def r_longlong_w(self, w_obj): + bigint = self.bigint_w(w_obj) + try: + return bigint.tolonglong() + except OverflowError: + raise OperationError(self.w_OverflowError, + self.wrap('integer too large')) + + def r_ulonglong_w(self, w_obj): + bigint = self.bigint_w(w_obj) + try: + return bigint.toulonglong() + except OverflowError: + raise OperationError(self.w_OverflowError, + self.wrap('integer too large')) + except ValueError: + raise OperationError(self.w_ValueError, + self.wrap('cannot convert negative integer ' + 'to unsigned int')) + + def buffer_w(self, w_obj): + # returns a Buffer instance + from pypy.interpreter.buffer import Buffer + w_buffer = self.buffer(w_obj) + return self.interp_w(Buffer, w_buffer) + + def rwbuffer_w(self, w_obj): + # returns a RWBuffer instance + from pypy.interpreter.buffer import RWBuffer + buffer = self.buffer_w(w_obj) + if not isinstance(buffer, RWBuffer): + raise OperationError(self.w_TypeError, + self.wrap('read-write buffer expected')) + return buffer + + def bufferstr_w(self, w_obj): + # Directly returns an interp-level str. Note that if w_obj is a + # unicode string, this is different from str_w(buffer(w_obj)): + # indeed, the latter returns a string with the raw bytes from + # the underlying unicode buffer, but bufferstr_w() just converts + # the unicode to an ascii string. This inconsistency is kind of + # needed because CPython has the same issue. (Well, it's + # unclear if there is any use at all for getting the bytes in + # the unicode buffer.) + try: + return self.str_w(w_obj) + except OperationError, e: + if not e.match(self, self.w_TypeError): + raise + buffer = self.buffer_w(w_obj) + return buffer.as_str() + + def bool_w(self, w_obj): + # Unwraps a bool, also accepting an int for compatibility. + # This is here mostly just for gateway.int_unwrapping_space_method(). + return bool(self.int_w(w_obj)) + + def nonnegint_w(self, w_obj): + # Like space.int_w(), but raises an app-level ValueError if + # the integer is negative. Mostly here for gateway.py. + value = self.int_w(w_obj) + if value < 0: + raise OperationError(self.w_ValueError, + self.wrap("expected a non-negative integer")) + return value + + def warn(self, msg, w_warningcls): + self.appexec([self.wrap(msg), w_warningcls], """(msg, warningcls): + import warnings + warnings.warn(msg, warningcls, stacklevel=2) + """) + + def resolve_target(self, w_obj): + """ A space method that can be used by special object spaces (like + thunk) to replace an object by another. """ + return w_obj + + +class AppExecCache(SpaceCache): + def build(cache, source): + """ NOT_RPYTHON """ + space = cache.space + # XXX will change once we have our own compiler + import py + source = source.lstrip() + assert source.startswith('('), "incorrect header in:\n%s" % (source,) + source = py.code.Source("def anonymous%s\n" % source) + w_glob = space.newdict() + space.exec_(source.compile(), w_glob, w_glob) + return space.getitem(w_glob, space.wrap('anonymous')) + +class DummyLock(object): + def acquire(self, flag): + return True + def release(self): + pass + def _freeze_(self): + return True +dummy_lock = DummyLock() ## Table describing the regular part of the interface of object spaces, ## namely all methods which only take w_ arguments and return a w_ result -## (if any). XXX Maybe we should say that these methods must be accessed -## as 'space.op.xxx()' instead of directly 'space.xxx()'. +## (if any). Note: keep in sync with pypy.objspace.flow.operation.Table. ObjSpace.MethodTable = [ # method name # symbol # number of arguments # special method name(s) + ('is_', 'is', 2, []), ('id', 'id', 1, []), ('type', 'type', 1, []), ('issubtype', 'issubtype', 2, []), # not for old-style classes @@ -217,13 +1068,15 @@ ('getitem', 'getitem', 2, ['__getitem__']), ('setitem', 'setitem', 3, ['__setitem__']), ('delitem', 'delitem', 2, ['__delitem__']), + ('getslice', 'getslice', 3, ['__getslice__']), + ('setslice', 'setslice', 4, ['__setslice__']), + ('delslice', 'delslice', 3, ['__delslice__']), ('pos', 'pos', 1, ['__pos__']), ('neg', 'neg', 1, ['__neg__']), - ('not_', 'not', 1, []), + ('nonzero', 'truth', 1, ['__nonzero__']), ('abs' , 'abs', 1, ['__abs__']), ('hex', 'hex', 1, ['__hex__']), ('oct', 'oct', 1, ['__oct__']), - ('round', 'round', 2, []), ('ord', 'ord', 1, []), ('invert', '~', 1, ['__invert__']), ('add', '+', 2, ['__add__', '__radd__']), @@ -241,7 +1094,9 @@ ('or_', '|', 2, ['__or__', '__ror__']), ('xor', '^', 2, ['__xor__', '__rxor__']), ('int', 'int', 1, ['__int__']), + ('index', 'index', 1, ['__index__']), ('float', 'float', 1, ['__float__']), + ('long', 'long', 1, ['__long__']), ('inplace_add', '+=', 2, ['__iadd__']), ('inplace_sub', '-=', 2, ['__isub__']), ('inplace_mul', '*=', 2, ['__imul__']), @@ -261,12 +1116,17 @@ ('ne', '!=', 2, ['__ne__', '__ne__']), ('gt', '>', 2, ['__gt__', '__lt__']), ('ge', '>=', 2, ['__ge__', '__le__']), + ('cmp', 'cmp', 2, ['__cmp__']), # rich cmps preferred + ('coerce', 'coerce', 2, ['__coerce__', '__coerce__']), ('contains', 'contains', 2, ['__contains__']), ('iter', 'iter', 1, ['__iter__']), - ('call', 'call', 3, ['__call__']), + ('next', 'next', 1, ['next']), +# ('call', 'call', 3, ['__call__']), ('get', 'get', 3, ['__get__']), ('set', 'set', 3, ['__set__']), ('delete', 'delete', 2, ['__delete__']), + ('userdel', 'del', 1, ['__del__']), + ('buffer', 'buffer', 1, ['__buffer__']), # see buffer.py ] ObjSpace.BuiltinModuleTable = [ @@ -319,13 +1179,38 @@ ## Irregular part of the interface: # -# wrap(x) -> w_x -# unwrap(w_x) -> x -# is_true(w_x) -> True or False -# newtuple([w_1, w_2,...]) -> w_tuple -# newlist([w_1, w_2,...]) -> w_list -# newstring([w_1, w_2,...]) -> w_string from ascii numbers (bytes) -# newdict([(w_key,w_value),...]) -> w_dict -#newslice(w_start,w_stop,w_step) -> w_slice (any argument may be a real None) -# next(w_iter) -> w_value or raise NoValue -# +# wrap(x) -> w_x +# str_w(w_str) -> str +# int_w(w_ival or w_long_ival) -> ival +# float_w(w_floatval) -> floatval +# uint_w(w_ival or w_long_ival) -> r_uint_val (unsigned int value) +# bigint_w(w_ival or w_long_ival) -> rbigint +#interpclass_w(w_interpclass_inst or w_obj) -> interpclass_inst|w_obj +# unwrap(w_x) -> x +# is_true(w_x) -> True or False +# newtuple([w_1, w_2,...]) -> w_tuple +# newlist([w_1, w_2,...]) -> w_list +# newdict() -> empty w_dict +# newslice(w_start,w_stop,w_step) -> w_slice +# call_args(w_obj,Arguments()) -> w_result + +ObjSpace.IrregularOpTable = [ + 'wrap', + 'str_w', + 'int_w', + 'float_w', + 'uint_w', + 'bigint_w', + 'unicode_w', + 'interpclass_w', + 'unwrap', + 'is_true', + 'is_w', + 'newtuple', + 'newlist', + 'newdict', + 'newslice', + 'call_args', + 'marshal_w', + ] + Modified: pypy/branch/avm/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/avm/pypy/interpreter/executioncontext.py Thu Nov 5 20:41:26 2009 @@ -1,52 +1,565 @@ -from pypy.interpreter.miscutils import getthreadlocals, Stack +import sys +from pypy.interpreter.miscutils import Stack +from pypy.interpreter.error import OperationError +from pypy.rlib.rarithmetic import LONG_BIT +from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.jit import we_are_jitted +from pypy.rlib import jit + +def app_profile_call(space, w_callable, frame, event, w_arg): + space.call_function(w_callable, + space.wrap(frame), + space.wrap(event), w_arg) class ExecutionContext: """An ExecutionContext holds the state of an execution thread in the Python interpreter.""" - + def __init__(self, space): - # Note that self.framestack only contains PyFrames self.space = space - self.framestack = Stack() + # 'some_frame' points to any frame from this thread's frame stack + # (although in general it should point to the top one). + self.some_frame = None + self.framestackdepth = 0 + # tracing: space.frame_trace_action.fire() must be called to ensure + # that tracing occurs whenever self.w_tracefunc or self.is_tracing + # is modified. + self.w_tracefunc = None + self.is_tracing = 0 + self.compiler = space.createcompiler() + self.profilefunc = None + self.w_profilefuncarg = None + + def gettopframe(self): + frame = self.some_frame + if frame is not None: + while frame.f_forward is not None: + frame = frame.f_forward + return frame + + def gettopframe_nohidden(self): + frame = self.gettopframe() + while frame and frame.hide(): + frame = frame.f_back + return frame + + def getnextframe_nohidden(frame): + frame = frame.f_back + while frame and frame.hide(): + frame = frame.f_back + return frame + getnextframe_nohidden = staticmethod(getnextframe_nohidden) def enter(self, frame): - locals = getthreadlocals() - self.framestack.push(frame) - previous_ec = locals.executioncontext - locals.executioncontext = self - return previous_ec - - def leave(self, previous_ec): - locals = getthreadlocals() - locals.executioncontext = previous_ec - self.framestack.pop() - - def get_w_builtins(self): - if self.framestack.empty(): - return self.space.w_builtins + if self.framestackdepth > self.space.sys.recursionlimit: + raise OperationError(self.space.w_RuntimeError, + self.space.wrap("maximum recursion depth exceeded")) + self.framestackdepth += 1 + # + curtopframe = self.gettopframe() + frame.f_back = curtopframe + if curtopframe is not None: + curtopframe.f_forward = frame + if not we_are_jitted(): + self.some_frame = frame + + def leave(self, frame): + if self.profilefunc: + self._trace(frame, 'leaveframe', self.space.w_None) + + #assert frame is self.gettopframe() --- slowish + f_back = frame.f_back + if f_back is not None: + f_back.f_forward = None + if not we_are_jitted() or self.some_frame is frame: + self.some_frame = f_back + self.framestackdepth -= 1 + + if self.w_tracefunc is not None and not frame.hide(): + self.space.frame_trace_action.fire() + + + class Subcontext(object): + # coroutine: subcontext support + + def __init__(self): + self.topframe = None + self.framestackdepth = 0 + self.w_tracefunc = None + self.profilefunc = None + self.w_profilefuncarg = None + self.is_tracing = 0 + + def enter(self, ec): + ec.some_frame = self.topframe + ec.framestackdepth = self.framestackdepth + ec.w_tracefunc = self.w_tracefunc + ec.profilefunc = self.profilefunc + ec.w_profilefuncarg = self.w_profilefuncarg + ec.is_tracing = self.is_tracing + ec.space.frame_trace_action.fire() + + def leave(self, ec): + self.topframe = ec.gettopframe() + self.framestackdepth = ec.framestackdepth + self.w_tracefunc = ec.w_tracefunc + self.profilefunc = ec.profilefunc + self.w_profilefuncarg = ec.w_profilefuncarg + self.is_tracing = ec.is_tracing + + def clear_framestack(self): + self.topframe = None + self.framestackdepth = 0 + + # the following interface is for pickling and unpickling + def getstate(self, space): + # XXX we could just save the top frame, which brings + # the whole frame stack, but right now we get the whole stack + items = [space.wrap(f) for f in self.getframestack()] + return space.newtuple(items) + + def setstate(self, space, w_state): + from pypy.interpreter.pyframe import PyFrame + frames_w = space.unpackiterable(w_state) + if len(frames_w) > 0: + self.topframe = space.interp_w(PyFrame, frames_w[-1]) + else: + self.topframe = None + self.framestackdepth = len(frames_w) + + def getframestack(self): + index = self.framestackdepth + lst = [None] * index + f = self.topframe + while index > 0: + index -= 1 + lst[index] = f + f = f.f_back + assert f is None + return lst + # coroutine: I think this is all, folks! + + + def get_builtin(self): + frame = self.gettopframe_nohidden() + if frame is not None: + return frame.builtin else: - return self.framestack.top().w_builtins + return self.space.builtin + # XXX this one should probably be dropped in favor of a module def make_standard_w_globals(self): "Create a new empty 'globals' dictionary." w_key = self.space.wrap("__builtins__") - w_value = self.get_w_builtins() - w_globals = self.space.newdict([(w_key, w_value)]) + w_value = self.space.wrap(self.get_builtin()) + w_globals = self.space.newdict() + space.setitem(w_globals, w_key, w_value) return w_globals + @jit.dont_look_inside + def c_call_trace(self, frame, w_func): + "Profile the call of a builtin function" + if self.profilefunc is None: + frame.is_being_profiled = False + else: + self._trace(frame, 'c_call', w_func) + + @jit.dont_look_inside + def c_return_trace(self, frame, w_retval): + "Profile the return from a builtin function" + if self.profilefunc is None: + frame.is_being_profiled = False + else: + self._trace(frame, 'c_return', w_retval) + + @jit.dont_look_inside + def c_exception_trace(self, frame, w_exc): + "Profile function called upon OperationError." + if self.profilefunc is None: + frame.is_being_profiled = False + else: + self._trace(frame, 'c_exception', w_exc) + + @jit.dont_look_inside + def call_trace(self, frame): + "Trace the call of a function" + if self.w_tracefunc is not None or self.profilefunc is not None: + self._trace(frame, 'call', self.space.w_None) + if self.profilefunc: + frame.is_being_profiled = True + + @jit.dont_look_inside + def return_trace(self, frame, w_retval): + "Trace the return from a function" + if self.w_tracefunc is not None: + self._trace(frame, 'return', w_retval) + def bytecode_trace(self, frame): "Trace function called before each bytecode." + # this is split into a fast path and a slower path that is + # not invoked every time bytecode_trace() is. + actionflag = self.space.actionflag + ticker = actionflag.get() + if actionflag.has_bytecode_counter: # this "if" is constant-folded + ticker += 1 + actionflag.set(ticker) + if ticker & actionflag.interesting_bits: # fast check + actionflag.action_dispatcher(self, frame) # slow path + bytecode_trace._always_inline_ = True - def exception_trace(self, operationerr): + @jit.dont_look_inside + def exception_trace(self, frame, operationerr): "Trace function called upon OperationError." operationerr.record_interpreter_traceback() + if self.w_tracefunc is not None: + self._trace(frame, 'exception', None, operationerr) #operationerr.print_detailed_traceback(self.space) - def sys_exc_info(self): + def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!! """Implements sys.exc_info(). Return an OperationError instance or None.""" - for i in range(self.framestack.depth()): - frame = self.framestack.top(i) + frame = self.gettopframe_nohidden() + while frame: if frame.last_exception is not None: return frame.last_exception + frame = self.getnextframe_nohidden(frame) return None + + def settrace(self, w_func): + """Set the global trace function.""" + if self.space.is_w(w_func, self.space.w_None): + self.w_tracefunc = None + else: + self.w_tracefunc = w_func + self.space.frame_trace_action.fire() + + def setprofile(self, w_func): + """Set the global trace function.""" + if self.space.is_w(w_func, self.space.w_None): + self.profilefunc = None + self.w_profilefuncarg = None + else: + self.setllprofile(app_profile_call, w_func) + + def setllprofile(self, func, w_arg): + self.profilefunc = func + if func is not None: + if w_arg is None: + raise ValueError("Cannot call setllprofile with real None") + frame = self.gettopframe_nohidden() + while frame: + frame.is_being_profiled = True + frame = self.getnextframe_nohidden(frame) + self.w_profilefuncarg = w_arg + + def call_tracing(self, w_func, w_args): + is_tracing = self.is_tracing + self.is_tracing = 0 + try: + self.space.frame_trace_action.fire() + return self.space.call(w_func, w_args) + finally: + self.is_tracing = is_tracing + + def _trace(self, frame, event, w_arg, operr=None): + if self.is_tracing or frame.hide(): + return + + space = self.space + + # Tracing cases + if event == 'call': + w_callback = self.w_tracefunc + else: + w_callback = frame.w_f_trace + + if w_callback is not None and event != "leaveframe": + if operr is not None: + w_arg = space.newtuple([operr.w_type, operr.w_value, + space.wrap(operr.application_traceback)]) + + frame.fast2locals() + self.is_tracing += 1 + try: + try: + w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg) + if space.is_w(w_result, space.w_None): + frame.w_f_trace = None + else: + frame.w_f_trace = w_result + except: + self.settrace(space.w_None) + frame.w_f_trace = None + raise + finally: + self.is_tracing -= 1 + frame.locals2fast() + space.frame_trace_action.fire() + + # Profile cases + if self.profilefunc is not None: + if event not in ['leaveframe', 'call', 'c_call', + 'c_return', 'c_exception']: + return + + last_exception = None + if event == 'leaveframe': + last_exception = frame.last_exception + event = 'return' + + assert self.is_tracing == 0 + self.is_tracing += 1 + try: + try: + self.profilefunc(space, self.w_profilefuncarg, + frame, event, w_arg) + except: + self.profilefunc = None + self.w_profilefuncarg = None + raise + + finally: + frame.last_exception = last_exception + self.is_tracing -= 1 + + def _freeze_(self): + raise Exception("ExecutionContext instances should not be seen during" + " translation. Now is a good time to inspect the" + " traceback and see where this one comes from :-)") + + +class AbstractActionFlag: + """This holds the global 'action flag'. It is a single bitfield + integer, with bits corresponding to AsyncAction objects that need to + be immediately triggered. The correspondance from bits to + AsyncAction instances is built at translation time. We can quickly + check if there is anything at all to do by checking if any of the + relevant bits is set. If threads are enabled, they consume the 20 + lower bits to hold a counter incremented at each bytecode, to know + when to release the GIL. + """ + def __init__(self): + self._periodic_actions = [] + self._nonperiodic_actions = [] + self.unused_bits = self.FREE_BITS[:] + self.has_bytecode_counter = False + self.interesting_bits = 0 + self._rebuild_action_dispatcher() + + def fire(self, action): + """Request for the action to be run before the next opcode. + The action must have been registered at space initalization time.""" + ticker = self.get() + self.set(ticker | action.bitmask) + + def register_action(self, action): + "NOT_RPYTHON" + assert isinstance(action, AsyncAction) + if action.bitmask == 0: + while True: + action.bitmask = self.unused_bits.pop(0) + if not (action.bitmask & self.interesting_bits): + break + self.interesting_bits |= action.bitmask + if action.bitmask & self.BYTECODE_COUNTER_OVERFLOW_BIT: + assert action.bitmask == self.BYTECODE_COUNTER_OVERFLOW_BIT + self._periodic_actions.append(action) + self.has_bytecode_counter = True + self.force_tick_counter() + else: + self._nonperiodic_actions.append((action, action.bitmask)) + self._rebuild_action_dispatcher() + + def setcheckinterval(self, space, interval): + if interval < self.CHECK_INTERVAL_MIN: + interval = self.CHECK_INTERVAL_MIN + elif interval > self.CHECK_INTERVAL_MAX: + interval = self.CHECK_INTERVAL_MAX + space.sys.checkinterval = interval + self.force_tick_counter() + + def force_tick_counter(self): + # force the tick counter to a valid value -- this actually forces + # it to reach BYTECODE_COUNTER_OVERFLOW_BIT at the next opcode. + ticker = self.get() + ticker &= ~ self.BYTECODE_COUNTER_OVERFLOW_BIT + ticker |= self.BYTECODE_COUNTER_MASK + self.set(ticker) + + def _rebuild_action_dispatcher(self): + periodic_actions = unrolling_iterable(self._periodic_actions) + nonperiodic_actions = unrolling_iterable(self._nonperiodic_actions) + has_bytecode_counter = self.has_bytecode_counter + + @jit.dont_look_inside + def action_dispatcher(ec, frame): + # periodic actions + if has_bytecode_counter: + ticker = self.get() + if ticker & self.BYTECODE_COUNTER_OVERFLOW_BIT: + # We must run the periodic actions now, but first + # reset the bytecode counter (the following line + # works by assuming that we just overflowed the + # counter, i.e. BYTECODE_COUNTER_OVERFLOW_BIT is + # set but none of the BYTECODE_COUNTER_MASK bits + # are). + ticker -= ec.space.sys.checkinterval + self.set(ticker) + for action in periodic_actions: + action.perform(ec, frame) + + # nonperiodic actions + for action, bitmask in nonperiodic_actions: + ticker = self.get() + if ticker & bitmask: + self.set(ticker & ~ bitmask) + action.perform(ec, frame) + + action_dispatcher._dont_inline_ = True + self.action_dispatcher = action_dispatcher + + # Bits reserved for the bytecode counter, if used + BYTECODE_COUNTER_MASK = (1 << 20) - 1 + BYTECODE_COUNTER_OVERFLOW_BIT = (1 << 20) + + # Free bits + FREE_BITS = [1 << _b for _b in range(21, LONG_BIT-1)] + + # The acceptable range of values for sys.checkinterval, so that + # the bytecode_counter fits in 20 bits + CHECK_INTERVAL_MIN = 1 + CHECK_INTERVAL_MAX = BYTECODE_COUNTER_OVERFLOW_BIT + + +class ActionFlag(AbstractActionFlag): + """The normal class for space.actionflag. The signal module provides + a different one.""" + _flags = 0 + + def get(self): + return self._flags + + def set(self, value): + self._flags = value + + +class AsyncAction(object): + """Abstract base class for actions that must be performed + asynchronously with regular bytecode execution, but that still need + to occur between two opcodes, not at a completely random time. + """ + bitmask = 0 # means 'please choose one bit automatically' + + def __init__(self, space): + self.space = space + + def fire(self): + """Request for the action to be run before the next opcode. + The action must have been registered at space initalization time.""" + self.space.actionflag.fire(self) + + def fire_after_thread_switch(self): + """Bit of a hack: fire() the action but only the next time the GIL + is released and re-acquired (i.e. after a portential thread switch). + Don't call this if threads are not enabled. + """ + from pypy.module.thread.gil import spacestate + spacestate.set_actionflag_bit_after_thread_switch |= self.bitmask + + def perform(self, executioncontext, frame): + """To be overridden.""" + + +class PeriodicAsyncAction(AsyncAction): + """Abstract base class for actions that occur automatically + every sys.checkinterval bytecodes. + """ + bitmask = ActionFlag.BYTECODE_COUNTER_OVERFLOW_BIT + + +class UserDelAction(AsyncAction): + """An action that invokes all pending app-level __del__() method. + This is done as an action instead of immediately when the + interp-level __del__() is invoked, because the latter can occur more + or less anywhere in the middle of code that might not be happy with + random app-level code mutating data structures under its feet. + """ + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.dying_objects_w = [] + self.finalizers_lock_count = 0 + + def register_dying_object(self, w_obj): + self.dying_objects_w.append(w_obj) + self.fire() + + def perform(self, executioncontext, frame): + if self.finalizers_lock_count > 0: + return + # Each call to perform() first grabs the self.dying_objects_w + # and replaces it with an empty list. We do this to try to + # avoid too deep recursions of the kind of __del__ being called + # while in the middle of another __del__ call. + pending_w = self.dying_objects_w + self.dying_objects_w = [] + space = self.space + for w_obj in pending_w: + try: + space.userdel(w_obj) + except OperationError, e: + e.write_unraisable(space, 'method __del__ of ', w_obj) + e.clear(space) # break up reference cycles + # finally, this calls the interp-level destructor for the + # cases where there is both an app-level and a built-in __del__. + w_obj._call_builtin_destructor() + + +class FrameTraceAction(AsyncAction): + """An action that calls the local trace functions (w_f_trace).""" + + def perform(self, executioncontext, frame): + if frame.w_f_trace is None or executioncontext.is_tracing: + return + code = frame.pycode + if frame.instr_lb <= frame.last_instr < frame.instr_ub: + if frame.last_instr <= frame.instr_prev: + # We jumped backwards in the same line. + executioncontext._trace(frame, 'line', self.space.w_None) + else: + size = len(code.co_lnotab) / 2 + addr = 0 + line = code.co_firstlineno + p = 0 + lineno = code.co_lnotab + while size > 0: + c = ord(lineno[p]) + if (addr + c) > frame.last_instr: + break + addr += c + if c: + frame.instr_lb = addr + + line += ord(lineno[p + 1]) + p += 2 + size -= 1 + + if size > 0: + while True: + size -= 1 + if size < 0: + break + addr += ord(lineno[p]) + if ord(lineno[p + 1]): + break + p += 2 + frame.instr_ub = addr + else: + frame.instr_ub = sys.maxint + + if frame.instr_lb == frame.last_instr: # At start of line! + frame.f_lineno = line + executioncontext._trace(frame, 'line', self.space.w_None) + + frame.instr_prev = frame.last_instr + self.space.frame_trace_action.fire() # continue tracing Modified: pypy/branch/avm/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/gateway.py (original) +++ pypy/branch/avm/pypy/interpreter/gateway.py Thu Nov 5 20:41:26 2009 @@ -535,6 +535,7 @@ # (verbose) performance hack below class BuiltinCodePassThroughArguments0(BuiltinCode): + _immutable_ = True def funcrun(self, func, args): space = func.space @@ -559,6 +560,7 @@ return w_result class BuiltinCodePassThroughArguments1(BuiltinCode): + _immutable_ = True fast_natural_arity = -1 def funcrun_obj(self, func, w_obj, args): @@ -584,10 +586,10 @@ return w_result class BuiltinCode0(BuiltinCode): + _immutable_ = True fast_natural_arity = 0 def fastcall_0(self, space, w_func): - #self = hint(self, deepfreeze=True) try: w_result = self.fastfunc_0(space) except KeyboardInterrupt: @@ -604,10 +606,10 @@ return w_result class BuiltinCode1(BuiltinCode): + _immutable_ = True fast_natural_arity = 1 def fastcall_1(self, space, w_func, w1): - #self = hint(self, deepfreeze=True) try: w_result = self.fastfunc_1(space, w1) except KeyboardInterrupt: @@ -629,10 +631,10 @@ return w_result class BuiltinCode2(BuiltinCode): + _immutable_ = True fast_natural_arity = 2 def fastcall_2(self, space, w_func, w1, w2): - #self = hint(self, deepfreeze=True) try: w_result = self.fastfunc_2(space, w1, w2) except KeyboardInterrupt: @@ -654,10 +656,10 @@ return w_result class BuiltinCode3(BuiltinCode): + _immutable_ = True fast_natural_arity = 3 def fastcall_3(self, space, func, w1, w2, w3): - #self = hint(self, deepfreeze=True) try: w_result = self.fastfunc_3(space, w1, w2, w3) except KeyboardInterrupt: @@ -679,10 +681,10 @@ return w_result class BuiltinCode4(BuiltinCode): + _immutable_ = True fast_natural_arity = 4 def fastcall_4(self, space, func, w1, w2, w3, w4): - #self = hint(self, deepfreeze=True) try: w_result = self.fastfunc_4(space, w1, w2, w3, w4) except KeyboardInterrupt: Modified: pypy/branch/avm/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/pycode.py (original) +++ pypy/branch/avm/pypy/interpreter/pycode.py Thu Nov 5 20:41:26 2009 @@ -12,6 +12,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified +from pypy.rlib import jit # helper @@ -53,6 +54,8 @@ class PyCode(eval.Code): "CPython-style code objects." _immutable_ = True + _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]", + "co_freevars[*]", "co_cellvars[*]"] def __init__(self, space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, @@ -180,6 +183,7 @@ self.fast_natural_arity = PyCode.FLATPYCALL | self.co_argcount + @jit.dont_look_inside def funcrun(self, func, args): frame = self.space.createframe(self, func.w_func_globals, func.closure) @@ -191,6 +195,7 @@ frame.init_cells() return frame.run() + @jit.dont_look_inside def funcrun_obj(self, func, w_obj, args): frame = self.space.createframe(self, func.w_func_globals, func.closure) Modified: pypy/branch/avm/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/pyframe.py (original) +++ pypy/branch/avm/pypy/interpreter/pyframe.py Thu Nov 5 20:41:26 2009 @@ -1,13 +1,25 @@ """ PyFrame class implementation with the interpreter main loop. """ -from pypy.interpreter import eval, baseobjspace, gateway -from pypy.interpreter.miscutils import Stack +from pypy.tool.pairtype import extendabletype +from pypy.interpreter import eval, baseobjspace, pycode +from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack from pypy.interpreter.error import OperationError from pypy.interpreter import pytraceback +import opcode +from pypy.rlib.objectmodel import we_are_translated, instantiate +from pypy.rlib.jit import we_are_jitted, hint +from pypy.rlib.debug import make_sure_not_resized +from pypy.rlib import jit + +# Define some opcodes used +g = globals() +for op in '''DUP_TOP POP_TOP SETUP_LOOP SETUP_EXCEPT SETUP_FINALLY +POP_BLOCK END_FINALLY'''.split(): + g[op] = opcode.opmap[op] +HAVE_ARGUMENT = opcode.HAVE_ARGUMENT - -class PyFrame(eval.Frame, baseobjspace.Wrappable): +class PyFrame(eval.Frame): """Represents a frame for a regular Python function that needs to be interpreted. @@ -18,282 +30,590 @@ * 'code' is the PyCode object this frame runs * 'w_locals' is the locals dictionary to use * 'w_globals' is the attached globals dictionary - * 'w_builtins' is the attached built-ins dictionary - * 'valuestack', 'blockstack', 'next_instr' control the interpretation + * 'builtin' is the attached built-in module + * 'valuestack_w', 'blockstack', control the interpretation """ + __metaclass__ = extendabletype + + frame_finished_execution = False + last_instr = -1 + last_exception = None + f_back = None # these two should be modified together + f_forward = None # they make a doubly-linked list + w_f_trace = None + # For tracing + instr_lb = 0 + instr_ub = -1 + instr_prev = -1 + is_being_profiled = False + def __init__(self, space, code, w_globals, closure): - eval.Frame.__init__(self, space, code, w_globals, code.co_nlocals) - self.valuestack = Stack() - self.blockstack = Stack() - self.last_exception = None - self.next_instr = 0 - self.w_builtins = self.space.w_builtins + self = hint(self, access_directly=True, fresh_virtualizable=True) + assert isinstance(code, pycode.PyCode) + self.pycode = code + eval.Frame.__init__(self, space, w_globals, code.co_nlocals) + self.valuestack_w = [None] * code.co_stacksize + self.valuestackdepth = 0 + self.blockstack = [] + if space.config.objspace.honor__builtins__: + self.builtin = space.builtin.pick_builtin(w_globals) # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. # class bodies only have CO_NEWLOCALS. - if code.dictscope_needed(): - self.w_locals = space.newdict([]) # set to None by Frame.__init__ + self.initialize_frame_scopes(closure, code) + self.fastlocals_w = [None]*self.numlocals + make_sure_not_resized(self.fastlocals_w) + self.f_lineno = code.co_firstlineno + + def get_builtin(self): + if self.space.config.objspace.honor__builtins__: + return self.builtin + else: + return self.space.builtin + + def initialize_frame_scopes(self, closure, code): + # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. + # class bodies only have CO_NEWLOCALS. + # CO_NEWLOCALS: make a locals dict unless optimized is also set + # CO_OPTIMIZED: no locals dict needed at all + # NB: this method is overridden in nestedscope.py + flags = code.co_flags + if flags & pycode.CO_OPTIMIZED: + return + if flags & pycode.CO_NEWLOCALS: + self.w_locals = self.space.newdict() + else: + assert self.w_globals is not None + self.w_locals = self.w_globals - def getclosure(self): - return None + def run(self): + """Start this frame's execution.""" + if self.getcode().co_flags & pycode.CO_GENERATOR: + from pypy.interpreter.generator import GeneratorIterator + return self.space.wrap(GeneratorIterator(self)) + else: + return self.execute_frame() - def eval(self, executioncontext): - "Interpreter main loop!" + def execute_generator_frame(self, w_inputvalue, ex=False): + # opcode semantic change in CPython 2.5: we must pass an input value + # when resuming a generator, which goes into the value stack. + # It's not working because the value of magic must be changed in PyCode + if self.pycode.magic >= 0xa0df294 and self.last_instr != -1 and not ex: + self.pushvalue(w_inputvalue) + return self.execute_frame() + + def execute_frame(self): + """Execute this frame. Main entry point to the interpreter.""" + from pypy.rlib import rstack + # the following 'assert' is an annotation hint: it hides from + # the annotator all methods that are defined in PyFrame but + # overridden in the FrameClass subclass of PyFrame. + assert isinstance(self, self.space.FrameClass) + executioncontext = self.space.getexecutioncontext() + executioncontext.enter(self) try: + if not we_are_jitted(): + executioncontext.call_trace(self) + # Execution starts just after the last_instr. Initially, + # last_instr is -1. After a generator suspends it points to + # the YIELD_VALUE instruction. + next_instr = self.last_instr + 1 + try: + w_exitvalue = self.dispatch(self.pycode, next_instr, + executioncontext) + rstack.resume_point("execute_frame", self, executioncontext, + returns=w_exitvalue) + except Exception: + if not we_are_jitted(): + executioncontext.return_trace(self, self.space.w_None) + raise + if not we_are_jitted(): + executioncontext.return_trace(self, w_exitvalue) + # on exit, we try to release self.last_exception -- breaks an + # obvious reference cycle, so it helps refcounting implementations + self.last_exception = None + finally: + executioncontext.leave(self) + return w_exitvalue + execute_frame.insert_stack_check_here = True + + # stack manipulation helpers + def pushvalue(self, w_object): + depth = self.valuestackdepth + self.valuestack_w[depth] = w_object + self.valuestackdepth = hint(depth + 1, promote=True) + + def popvalue(self): + depth = self.valuestackdepth - 1 + assert depth >= 0, "pop from empty value stack" + w_object = self.valuestack_w[depth] + self.valuestack_w[depth] = None + self.valuestackdepth = hint(depth, promote=True) + return w_object + + def popstrdictvalues(self, n): + dic_w = {} + while True: + n -= 1 + if n < 0: + break + w_value = self.popvalue() + w_key = self.popvalue() + key = self.space.str_w(w_key) + dic_w[key] = w_value + return dic_w + + # we need two popvalues that return different data types: + # one in case we want list another in case of tuple + def _new_popvalues(): + def popvalues(self, n): + values_w = [None] * n while True: - try: - executioncontext.bytecode_trace(self) - last_instr = self.next_instr - try: - # fetch and dispatch the next opcode - # dispatch() is abstract, see pyopcode. - self.dispatch() - except OperationError, e: - pytraceback.record_application_traceback( - self.space, e, self, last_instr) - # convert an OperationError into a control flow - # exception - import sys - tb = sys.exc_info()[2] - raise SApplicationException(e, tb) - # XXX some other exceptions could be caught here too, - # like KeyboardInterrupt - - except ControlFlowException, ctlflowexc: - # we have a reason to change the control flow - # (typically unroll the stack) - ctlflowexc.action(self, last_instr, executioncontext) - - except ExitFrame, e: - # leave that frame - w_exitvalue = e.args[0] - return w_exitvalue - - ### exception stack ### - - def clean_exceptionstack(self): - # remove all exceptions that can no longer be re-raised - # because the current valuestack is no longer deep enough - # to hold the corresponding information - while self.exceptionstack: - ctlflowexc, valuestackdepth = self.exceptionstack.top() - if valuestackdepth <= self.valuestack.depth(): + n -= 1 + if n < 0: + break + values_w[n] = self.popvalue() + return values_w + return popvalues + popvalues = _new_popvalues() + popvalues_mutable = _new_popvalues() + del _new_popvalues + + def peekvalues(self, n): + values_w = [None] * n + base = self.valuestackdepth - n + assert base >= 0 + while True: + n -= 1 + if n < 0: break - self.exceptionstack.pop() - - ### application level visible attributes ### - def app_visible(self): - def makedict(**kw): return kw - space = self.space - d = makedict( - f_code = space.wrap(self.code), - f_locals = self.getdictscope(), - f_globals = self.w_globals, - f_builtins = self.w_builtins, - # XXX f_lasti, f_back, f_exc*, f_restricted need to do pypy_getattr directly - ) - return d.items() - -### Frame Blocks ### - -class FrameBlock: - - """Abstract base class for frame blocks from the blockstack, - used by the SETUP_XXX and POP_BLOCK opcodes.""" - - def __init__(self, frame, handlerposition): - self.handlerposition = handlerposition - self.valuestackdepth = frame.valuestack.depth() - - def __eq__(self, other): - return (self.__class__ is other.__class__ and - self.handlerposition == other.handlerposition and - self.valuestackdepth == other.valuestackdepth) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash((self.handlerposition, self.valuestackdepth)) - - def cleanupstack(self, frame): - for i in range(self.valuestackdepth, frame.valuestack.depth()): - frame.valuestack.pop() - - def cleanup(self, frame): - "Clean up a frame when we normally exit the block." - self.cleanupstack(frame) - - def unroll(self, frame, unroller): - "Clean up a frame when we abnormally exit the block." - self.cleanupstack(frame) - return False # continue to unroll - - -class LoopBlock(FrameBlock): - """A loop block. Stores the end-of-loop pointer in case of 'break'.""" - - def unroll(self, frame, unroller): - if isinstance(unroller, SContinueLoop): - # re-push the loop block without cleaning up the value stack, - # and jump to the beginning of the loop, stored in the - # exception's argument - frame.blockstack.push(self) - jump_to = unroller.args[0] - frame.next_instr = jump_to - return True # stop unrolling - self.cleanupstack(frame) - if isinstance(unroller, SBreakLoop): - # jump to the end of the loop - frame.next_instr = self.handlerposition - return True # stop unrolling - return False - - -class ExceptBlock(FrameBlock): - """An try:except: block. Stores the position of the exception handler.""" - - def unroll(self, frame, unroller): - self.cleanupstack(frame) - if isinstance(unroller, SApplicationException): - # push the exception to the value stack for inspection by the - # exception handler (the code after the except:) - operationerr = unroller.args[0] - w_type = operationerr.w_type - w_value = operationerr.w_value - if frame.space.full_exceptions: - w_normalized = normalize_exception(frame.space, w_type, w_value) - w_type, w_value = frame.space.unpacktuple(w_normalized, 2) - # the stack setup is slightly different than in CPython: - # instead of the traceback, we store the unroller object, - # wrapped. - frame.valuestack.push(frame.space.wrap(unroller)) - frame.valuestack.push(w_value) - frame.valuestack.push(w_type) - frame.next_instr = self.handlerposition # jump to the handler - return True # stop unrolling - return False - -def app_normalize_exception(etype, evalue): - # XXX should really be defined as a method on OperationError, - # but this is not so easy because OperationError cannot be - # at the same time an old-style subclass of Exception and a - # new-style subclass of Wrappable :-( - # moreover, try importing gateway from errors.py and you'll see :-( + values_w[n] = self.valuestack_w[base+n] + return values_w + + def dropvalues(self, n): + n = hint(n, promote=True) + finaldepth = self.valuestackdepth - n + assert finaldepth >= 0, "stack underflow in dropvalues()" + while True: + n -= 1 + if n < 0: + break + self.valuestack_w[finaldepth+n] = None + self.valuestackdepth = finaldepth + + def pushrevvalues(self, n, values_w): # n should be len(values_w) + while True: + n -= 1 + if n < 0: + break + self.pushvalue(values_w[n]) + + def dupvalues(self, n): + delta = n-1 + while True: + n -= 1 + if n < 0: + break + w_value = self.peekvalue(delta) + self.pushvalue(w_value) + + def peekvalue(self, index_from_top=0): + index_from_top = hint(index_from_top, promote=True) + index = self.valuestackdepth + ~index_from_top + assert index >= 0, "peek past the bottom of the stack" + return self.valuestack_w[index] + + def settopvalue(self, w_object, index_from_top=0): + index_from_top = hint(index_from_top, promote=True) + index = self.valuestackdepth + ~index_from_top + assert index >= 0, "settop past the bottom of the stack" + self.valuestack_w[index] = w_object + + def dropvaluesuntil(self, finaldepth): + depth = self.valuestackdepth - 1 + finaldepth = hint(finaldepth, promote=True) + while depth >= finaldepth: + self.valuestack_w[depth] = None + depth -= 1 + self.valuestackdepth = finaldepth + + def savevaluestack(self): + return self.valuestack_w[:self.valuestackdepth] + + def restorevaluestack(self, items_w): + assert None not in items_w + self.valuestack_w[:len(items_w)] = items_w + self.dropvaluesuntil(len(items_w)) + + def make_arguments(self, nargs): + if we_are_jitted(): + return Arguments(self.space, self.peekvalues(nargs)) + else: + return ArgumentsFromValuestack(self.space, self, nargs) + + @jit.dont_look_inside + def descr__reduce__(self, space): + from pypy.interpreter.mixedmodule import MixedModule + from pypy.module._pickle_support import maker # helper fns + w_mod = space.getbuiltinmodule('_pickle_support') + mod = space.interp_w(MixedModule, w_mod) + new_inst = mod.get('frame_new') + w = space.wrap + nt = space.newtuple + + cells = self._getcells() + if cells is None: + w_cells = space.w_None + else: + w_cells = space.newlist([space.wrap(cell) for cell in cells]) + + if self.w_f_trace is None: + f_lineno = self.get_last_lineno() + else: + f_lineno = self.f_lineno + + values_w = self.valuestack_w[0:self.valuestackdepth] + w_valuestack = maker.slp_into_tuple_with_nulls(space, values_w) + + w_blockstack = nt([block._get_state_(space) for block in self.blockstack]) + w_fastlocals = maker.slp_into_tuple_with_nulls(space, self.fastlocals_w) + if self.last_exception is None: + w_exc_value = space.w_None + w_tb = space.w_None + else: + w_exc_value = self.last_exception.w_value + w_tb = w(self.last_exception.application_traceback) + + tup_state = [ + w(self.f_back), + w(self.get_builtin()), + w(self.pycode), + w_valuestack, + w_blockstack, + w_exc_value, # last_exception + w_tb, # + self.w_globals, + w(self.last_instr), + w(self.frame_finished_execution), + w(f_lineno), + w_fastlocals, + space.w_None, #XXX placeholder for f_locals + + #f_restricted requires no additional data! + space.w_None, ## self.w_f_trace, ignore for now + + w(self.instr_lb), #do we need these three (that are for tracing) + w(self.instr_ub), + w(self.instr_prev), + w_cells, + ] + + return nt([new_inst, nt([]), nt(tup_state)]) + + @jit.dont_look_inside + def descr__setstate__(self, space, w_args): + from pypy.module._pickle_support import maker # helper fns + from pypy.interpreter.pycode import PyCode + from pypy.interpreter.module import Module + args_w = space.unpackiterable(w_args) + w_f_back, w_builtin, w_pycode, w_valuestack, w_blockstack, w_exc_value, w_tb,\ + w_globals, w_last_instr, w_finished, w_f_lineno, w_fastlocals, w_f_locals, \ + w_f_trace, w_instr_lb, w_instr_ub, w_instr_prev, w_cells = args_w + + new_frame = self + pycode = space.interp_w(PyCode, w_pycode) + + if space.is_w(w_cells, space.w_None): + closure = None + cellvars = [] + else: + from pypy.interpreter.nestedscope import Cell + cells_w = space.unpackiterable(w_cells) + cells = [space.interp_w(Cell, w_cell) for w_cell in cells_w] + ncellvars = len(pycode.co_cellvars) + cellvars = cells[:ncellvars] + closure = cells[ncellvars:] + + # do not use the instance's __init__ but the base's, because we set + # everything like cells from here + PyFrame.__init__(self, space, pycode, w_globals, closure) + new_frame.f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True) + new_frame.builtin = space.interp_w(Module, w_builtin) + new_frame.blockstack = [unpickle_block(space, w_blk) + for w_blk in space.unpackiterable(w_blockstack)] + values_w = maker.slp_from_tuple_with_nulls(space, w_valuestack) + for w_value in values_w: + new_frame.pushvalue(w_value) + if space.is_w(w_exc_value, space.w_None): + new_frame.last_exception = None + else: + from pypy.interpreter.pytraceback import PyTraceback + tb = space.interp_w(PyTraceback, w_tb) + new_frame.last_exception = OperationError(space.type(w_exc_value), + w_exc_value, tb + ) + new_frame.last_instr = space.int_w(w_last_instr) + new_frame.frame_finished_execution = space.is_true(w_finished) + new_frame.f_lineno = space.int_w(w_f_lineno) + new_frame.fastlocals_w = maker.slp_from_tuple_with_nulls(space, w_fastlocals) + + if space.is_w(w_f_trace, space.w_None): + new_frame.w_f_trace = None + else: + new_frame.w_f_trace = w_f_trace + + new_frame.instr_lb = space.int_w(w_instr_lb) #the three for tracing + new_frame.instr_ub = space.int_w(w_instr_ub) + new_frame.instr_prev = space.int_w(w_instr_prev) + + self._setcellvars(cellvars) + space.frame_trace_action.fire() + + def hide(self): + return self.pycode.hidden_applevel + + def getcode(self): + return hint(self.pycode, promote=True) + + @jit.dont_look_inside + def getfastscope(self): + "Get the fast locals as a list." + return self.fastlocals_w + + def setfastscope(self, scope_w): + """Initialize the fast locals from a list of values, + where the order is according to self.pycode.signature().""" + scope_len = len(scope_w) + if scope_len > len(self.fastlocals_w): + raise ValueError, "new fastscope is longer than the allocated area" + # don't assign directly to 'fastlocals_w[:scope_len]' to be + # virtualizable-friendly + for i in range(scope_len): + self.fastlocals_w[i] = scope_w[i] + self.init_cells() + + def init_cells(self): + """Initialize cellvars from self.fastlocals_w + This is overridden in nestedscope.py""" + pass - # mistakes here usually show up as infinite recursion, which is fun. - if isinstance(evalue, etype): - return etype, evalue - if isinstance(etype, type) and issubclass(etype, Exception): - if evalue is None: - evalue = () - elif not isinstance(evalue, tuple): - evalue = (evalue,) - evalue = etype(*evalue) - else: - raise Exception, "?!" # XXX - return etype, evalue -normalize_exception = gateway.app2interp(app_normalize_exception) - - -class FinallyBlock(FrameBlock): - """A try:finally: block. Stores the position of the exception handler.""" - - def cleanup(self, frame): - # upon normal entry into the finally: part, the standard Python - # bytecode pushes a single None for END_FINALLY. In our case we - # always push three values into the stack: the wrapped ctlflowexc, - # the exception value and the exception type (which are all None - # here). - self.cleanupstack(frame) - # one None already pushed by the bytecode - frame.valuestack.push(frame.space.w_None) - frame.valuestack.push(frame.space.w_None) - - def unroll(self, frame, unroller): - # any abnormal reason for unrolling a finally: triggers the end of - # the block unrolling and the entering the finally: handler. - # see comments in cleanup(). - self.cleanupstack(frame) - frame.valuestack.push(frame.space.wrap(unroller)) - frame.valuestack.push(frame.space.w_None) - frame.valuestack.push(frame.space.w_None) - frame.next_instr = self.handlerposition # jump to the handler - return True # stop unrolling - - -### Internal exceptions that change the control flow ### -### and (typically) unroll the block stack ### - -class ControlFlowException(Exception): - """Abstract base class for interpreter-level exceptions that - instruct the interpreter to change the control flow and the - block stack. - - The concrete subclasses correspond to the various values WHY_XXX - values of the why_code enumeration in ceval.c: - - WHY_NOT, OK, not this one :-) - WHY_EXCEPTION, SApplicationException - WHY_RERAISE, we don't think this is needed - WHY_RETURN, SReturnValue - WHY_BREAK, SBreakLoop - WHY_CONTINUE, SContinueLoop - WHY_YIELD SYieldValue + def getclosure(self): + return None - """ - def action(self, frame, last_instr, executioncontext): - "Default unroller implementation." - while not frame.blockstack.empty(): - block = frame.blockstack.pop() - if block.unroll(frame, self): + def _getcells(self): + return None + + def _setcellvars(self, cellvars): + pass + + ### line numbers ### + + # for f*_f_* unwrapping through unwrap_spec in typedef.py + + def fget_f_lineno(space, self): + "Returns the line number of the instruction currently being executed." + if self.w_f_trace is None: + return space.wrap(self.get_last_lineno()) + else: + return space.wrap(self.f_lineno) + + def fset_f_lineno(space, self, w_new_lineno): + "Returns the line number of the instruction currently being executed." + try: + new_lineno = space.int_w(w_new_lineno) + except OperationError, e: + raise OperationError(space.w_ValueError, + space.wrap("lineno must be an integer")) + + if self.w_f_trace is None: + raise OperationError(space.w_ValueError, + space.wrap("f_lineo can only be set by a trace function.")) + + if new_lineno < self.pycode.co_firstlineno: + raise OperationError(space.w_ValueError, + space.wrap("line %d comes before the current code." % new_lineno)) + code = self.pycode.co_code + addr = 0 + line = self.pycode.co_firstlineno + new_lasti = -1 + offset = 0 + lnotab = self.pycode.co_lnotab + for offset in xrange(0, len(lnotab), 2): + addr += ord(lnotab[offset]) + line += ord(lnotab[offset + 1]) + if line >= new_lineno: + new_lasti = addr + new_lineno = line break + + if new_lasti == -1: + raise OperationError(space.w_ValueError, + space.wrap("line %d comes after the current code." % new_lineno)) + + # Don't jump to a line with an except in it. + if ord(code[new_lasti]) in (DUP_TOP, POP_TOP): + raise OperationError(space.w_ValueError, + space.wrap("can't jump to 'except' line as there's no exception")) + + # Don't jump into or out of a finally block. + f_lasti_setup_addr = -1 + new_lasti_setup_addr = -1 + blockstack = [] + addr = 0 + while addr < len(code): + op = ord(code[addr]) + if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY): + blockstack.append([addr, False]) + elif op == POP_BLOCK: + setup_op = ord(code[blockstack[-1][0]]) + if setup_op == SETUP_FINALLY: + blockstack[-1][1] = True + else: + blockstack.pop() + elif op == END_FINALLY: + if len(blockstack) > 0: + setup_op = ord(code[blockstack[-1][0]]) + if setup_op == SETUP_FINALLY: + blockstack.pop() + + if addr == new_lasti or addr == self.last_instr: + for ii in range(len(blockstack)): + setup_addr, in_finally = blockstack[~ii] + if in_finally: + if addr == new_lasti: + new_lasti_setup_addr = setup_addr + if addr == self.last_instr: + f_lasti_setup_addr = setup_addr + break + + if op >= HAVE_ARGUMENT: + addr += 3 + else: + addr += 1 + + assert len(blockstack) == 0 + + if new_lasti_setup_addr != f_lasti_setup_addr: + raise OperationError(space.w_ValueError, + space.wrap("can't jump into or out of a 'finally' block %d -> %d" % + (f_lasti_setup_addr, new_lasti_setup_addr))) + + if new_lasti < self.last_instr: + min_addr = new_lasti + max_addr = self.last_instr + else: + min_addr = self.last_instr + max_addr = new_lasti + + delta_iblock = min_delta_iblock = 0 + addr = min_addr + while addr < max_addr: + op = ord(code[addr]) + + if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY): + delta_iblock += 1 + elif op == POP_BLOCK: + delta_iblock -= 1 + if delta_iblock < min_delta_iblock: + min_delta_iblock = delta_iblock + + if op >= opcode.HAVE_ARGUMENT: + addr += 3 + else: + addr += 1 + + f_iblock = len(self.blockstack) + min_iblock = f_iblock + min_delta_iblock + if new_lasti > self.last_instr: + new_iblock = f_iblock + delta_iblock else: - self.emptystack(frame) + new_iblock = f_iblock - delta_iblock - def emptystack(self, frame): - "Default behavior when the block stack is exhausted." - # could occur e.g. when a BREAK_LOOP is not actually within a loop - raise BytecodeCorruption, "block stack exhausted" - -class SApplicationException(ControlFlowException): - """Unroll the stack because of an application-level exception - (i.e. an OperationException).""" - - def action(self, frame, last_instr, executioncontext): - e = self.args[0] - frame.last_exception = e - executioncontext.exception_trace(e) - - ControlFlowException.action(self, frame, - last_instr, executioncontext) - - def emptystack(self, frame): - # propagate the exception to the caller - if len(self.args) == 2: - operationerr, tb = self.args - raise operationerr.__class__, operationerr, tb - else: - operationerr = self.args[0] - raise operationerr - -class SBreakLoop(ControlFlowException): - """Signals a 'break' statement.""" - -class SContinueLoop(ControlFlowException): - """Signals a 'continue' statement. - Argument is the bytecode position of the beginning of the loop.""" - -class SReturnValue(ControlFlowException): - """Signals a 'return' statement. - Argument is the wrapped object to return.""" - def emptystack(self, frame): - w_returnvalue = self.args[0] - raise ExitFrame(w_returnvalue) - -class ExitFrame(Exception): - """Signals the end of the frame execution. - The argument is the returned or yielded value, already wrapped.""" + if new_iblock > min_iblock: + raise OperationError(space.w_ValueError, + space.wrap("can't jump into the middle of a block")) + + while f_iblock > new_iblock: + block = self.blockstack.pop() + block.cleanup(self) + f_iblock -= 1 + + self.f_lineno = new_lineno + self.last_instr = new_lasti + + def get_last_lineno(self): + "Returns the line number of the instruction currently being executed." + return pytraceback.offset2lineno(self.pycode, self.last_instr) + + def fget_f_builtins(space, self): + return self.get_builtin().getdict() + + def fget_f_back(space, self): + return self.space.wrap(self.f_back) -class BytecodeCorruption(ValueError): - """Detected bytecode corruption. Never caught; it's an error.""" + def fget_f_lasti(space, self): + return self.space.wrap(self.last_instr) + + def fget_f_trace(space, self): + return self.w_f_trace + + def fset_f_trace(space, self, w_trace): + if space.is_w(w_trace, space.w_None): + self.w_f_trace = None + else: + self.w_f_trace = w_trace + self.f_lineno = self.get_last_lineno() + space.frame_trace_action.fire() + + def fdel_f_trace(space, self): + self.w_f_trace = None + + def fget_f_exc_type(space, self): + if self.last_exception is not None: + f = self.f_back + while f is not None and f.last_exception is None: + f = f.f_back + if f is not None: + return f.last_exception.w_type + return space.w_None + + def fget_f_exc_value(space, self): + if self.last_exception is not None: + f = self.f_back + while f is not None and f.last_exception is None: + f = f.f_back + if f is not None: + return f.last_exception.w_value + return space.w_None + + def fget_f_exc_traceback(space, self): + if self.last_exception is not None: + f = self.f_back + while f is not None and f.last_exception is None: + f = f.f_back + if f is not None: + return space.wrap(f.last_exception.application_traceback) + return space.w_None + + def fget_f_restricted(space, self): + if space.config.objspace.honor__builtins__: + return space.wrap(self.builtin is not space.builtin) + return space.w_False + +# ____________________________________________________________ + +def get_block_class(opname): + # select the appropriate kind of block + from pypy.interpreter.pyopcode import block_classes + return block_classes[opname] + +def unpickle_block(space, w_tup): + w_opname, w_handlerposition, w_valuestackdepth = space.unpackiterable(w_tup) + opname = space.str_w(w_opname) + handlerposition = space.int_w(w_handlerposition) + valuestackdepth = space.int_w(w_valuestackdepth) + assert valuestackdepth >= 0 + blk = instantiate(get_block_class(opname)) + blk.handlerposition = handlerposition + blk.valuestackdepth = valuestackdepth + return blk Modified: pypy/branch/avm/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/typedef.py (original) +++ pypy/branch/avm/pypy/interpreter/typedef.py Thu Nov 5 20:41:26 2009 @@ -11,6 +11,7 @@ from pypy.tool.sourcetools import compile2, func_with_new_name from pypy.rlib.objectmodel import instantiate from pypy.rlib.rarithmetic import intmask +from pypy.rlib.jit import hint class TypeDef: def __init__(self, __name, __base=None, **rawdict): @@ -225,7 +226,7 @@ if "user" in features: # generic feature needed by all subcls class Proto(object): def getclass(self, space): - return self.w__class__ + return hint(self.w__class__, promote=True) def setclass(self, space, w_subtype): # only used by descr_set___class__ Modified: pypy/branch/avm/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/avm/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/avm/pypy/module/__builtin__/compiling.py Thu Nov 5 20:41:26 2009 @@ -4,7 +4,8 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError +from pypy.interpreter.astcompiler import consts, ast from pypy.interpreter.gateway import NoneNotWrapped def compile(space, w_source, filename, mode, flags=0, dont_inherit=0): @@ -20,22 +21,30 @@ compile; if absent or zero these statements do influence the compilation, in addition to any features explicitly specified. """ - if space.is_true(space.isinstance(w_source, space.w_unicode)): - # hack: encode the unicode string as UTF-8 and attach - # a BOM at the start - w_source = space.call_method(w_source, 'encode', space.wrap('utf-8')) - str_ = space.str_w(w_source) - str_ = '\xEF\xBB\xBF' + str_ + + ast_node = None + w_ast_type = space.gettypeobject(ast.AST.typedef) + str_ = None + if space.is_true(space.isinstance(w_source, w_ast_type)): + ast_node = space.interp_w(ast.mod, w_source) + ast_node.sync_app_attrs(space) + elif space.is_true(space.isinstance(w_source, space.w_unicode)): + w_utf_8_source = space.call_method(w_source, "encode", + space.wrap("utf-8")) + str_ = space.str_w(w_utf_8_source) + # This flag tells the parser to reject any coding cookies it sees. + flags |= consts.PyCF_SOURCE_IS_UTF8 else: str_ = space.str_w(w_source) ec = space.getexecutioncontext() + if flags & ~(ec.compiler.compiler_flags | consts.PyCF_AST_ONLY | + consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8): + raise OperationError(space.w_ValueError, + space.wrap("compile() unrecognized flags")) if not dont_inherit: - try: - caller = ec.framestack.top() - except IndexError: - pass - else: + caller = ec.gettopframe_nohidden() + if caller: flags |= ec.compiler.getcodeflags(caller.getcode()) if mode not in ('exec', 'eval', 'single'): @@ -43,7 +52,14 @@ space.wrap("compile() arg 3 must be 'exec' " "or 'eval' or 'single'")) - code = ec.compiler.compile(str_, filename, mode, flags) + if ast_node is None: + if flags & consts.PyCF_AST_ONLY: + mod = ec.compiler.compile_to_ast(str_, filename, mode, flags) + return space.wrap(mod) + else: + code = ec.compiler.compile(str_, filename, mode, flags) + else: + code = ec.compiler.compile_ast(ast_node, filename, mode, flags) return space.wrap(code) # compile.unwrap_spec = [ObjSpace,W_Root,str,str,int,int] @@ -70,11 +86,7 @@ raise OperationError(space.w_TypeError, w('eval() arg 1 must be a string or code object')) - try: - caller = space.getexecutioncontext().framestack.top() - except IndexError: - caller = None - + caller = space.getexecutioncontext().gettopframe_nohidden() if w_globals is None or space.is_w(w_globals, space.w_None): if caller is None: w_globals = w_locals = space.newdict() Modified: pypy/branch/avm/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/branch/avm/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/branch/avm/pypy/module/__builtin__/test/test_builtin.py Thu Nov 5 20:41:26 2009 @@ -400,6 +400,7 @@ raises(SyntaxError, compile, '-', '?', 'eval') raises(ValueError, compile, '"\\xt"', '?', 'eval') raises(ValueError, compile, '1+2', '?', 'maybenot') + raises(ValueError, compile, "\n", "", "exec", 0xff) raises(TypeError, compile, '1+2', 12, 34) def test_unicode_compile(self): Modified: pypy/branch/avm/pypy/module/_stackless/interp_clonable.py ============================================================================== --- pypy/branch/avm/pypy/module/_stackless/interp_clonable.py (original) +++ pypy/branch/avm/pypy/module/_stackless/interp_clonable.py Thu Nov 5 20:41:26 2009 @@ -65,7 +65,7 @@ class AppClonableCoState(AppCoState): def post_install(self): self.current = self.main = AppClonableCoroutine(self.space, state=self) - self.main.subctx.framestack = None # wack + self.main.subctx.clear_framestack() # wack def post_install(module): makeStaticMethod(module, 'clonable', 'getcurrent') Modified: pypy/branch/avm/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/avm/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/avm/pypy/module/_stackless/interp_coroutine.py Thu Nov 5 20:41:26 2009 @@ -183,7 +183,7 @@ w_parent = self.w_getmain(space) self.parent = space.interp_w(AppCoroutine, w_parent) ec = self.space.getexecutioncontext() - self.subctx.setstate(self.space, w_state) + self.subctx.setstate(space, w_state) self.reconstruct_framechain() if space.is_w(w_thunk, space.w_None): self.thunk = None @@ -199,7 +199,7 @@ def reconstruct_framechain(self): from pypy.interpreter.pyframe import PyFrame from pypy.rlib.rstack import resume_state_create - if self.subctx.framestack.empty(): + if self.subctx.topframe is None: self.frame = None return @@ -213,7 +213,7 @@ # ("appthunk", costate, returns=w_result) appthunk_frame = resume_state_create(_bind_frame, "appthunk", costate) chain = appthunk_frame - for frame in self.subctx.framestack.items: + for frame in self.subctx.getframestack(): assert isinstance(frame, PyFrame) # ("execute_frame", self, executioncontext, returns=w_exitvalue) chain = resume_state_create(chain, "execute_frame", frame, ec) @@ -270,11 +270,15 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - if self.subctx.framestack is not None: - items = [space.wrap(item) for item in self.subctx.framestack.items] - return space.newtuple(items) - else: - return space.newtuple([]) + index = self.subctx.framestackdepth + items = [None] * index + f = self.subctx.topframe + while index > 0: + index -= 1 + items[index] = space.wrap(f) + f = f.f_back + assert f is None + return space.newtuple(items) def makeStaticMethod(module, classname, funcname): space = module.space @@ -333,7 +337,7 @@ def post_install(self): self.current = self.main = AppCoroutine(self.space, state=self) - self.main.subctx.framestack = None # wack + self.main.subctx.clear_framestack() # wack def return_main(space): return AppCoroutine._get_state(space).main Modified: pypy/branch/avm/pypy/module/_stackless/interp_greenlet.py ============================================================================== --- pypy/branch/avm/pypy/module/_stackless/interp_greenlet.py (original) +++ pypy/branch/avm/pypy/module/_stackless/interp_greenlet.py Thu Nov 5 20:41:26 2009 @@ -52,7 +52,7 @@ self.active = is_main self.subctx = space.getexecutioncontext().Subcontext() if is_main: - self.subctx.framestack = None # wack + self.subctx.clear_framestack() # wack else: self.bind(GreenletThunk(self)) @@ -191,10 +191,7 @@ if not self.active or self.costate.current is self: f = None else: - try: - f = self.subctx.framestack.top(0) - except IndexError: - f = None + f = self.subctx.topframe return space.wrap(f) def get(space, name): Modified: pypy/branch/avm/pypy/module/_weakref/test/test_weakref.py ============================================================================== --- pypy/branch/avm/pypy/module/_weakref/test/test_weakref.py (original) +++ pypy/branch/avm/pypy/module/_weakref/test/test_weakref.py Thu Nov 5 20:41:26 2009 @@ -231,6 +231,12 @@ del g gc.collect() assert w() is None + g = f(10) + w = _weakref.ref(g) + assert list(g) == range(10) + del g + gc.collect() + assert w() is None def test_weakref_subclass_with_del(self): import _weakref, gc Modified: pypy/branch/avm/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/avm/pypy/module/posix/__init__.py (original) +++ pypy/branch/avm/pypy/module/posix/__init__.py Thu Nov 5 20:41:26 2009 @@ -124,7 +124,7 @@ backend = space.config.translation.backend # the Win32 urandom implementation isn't going to translate on JVM or CLI # so we have to remove it - if (backend == 'cli' or backend == 'jvm') and 'urandom' in self.interpleveldefs: + if backend == 'cli' or backend == 'jvm': del self.interpleveldefs['urandom'] MixedModule.__init__(self, space, w_name) Modified: pypy/branch/avm/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/avm/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/avm/pypy/module/pypyjit/interp_jit.py Thu Nov 5 20:41:26 2009 @@ -6,7 +6,7 @@ import sys from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.jit import JitDriver, hint, we_are_jitted import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, Arguments @@ -15,9 +15,41 @@ from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.function import Function from pypy.interpreter.pyopcode import ExitFrame - -PyFrame._virtualizable2_ = True -PyFrame._always_virtual_ = ['valuestack_w', 'fastlocals_w'] +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance +from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT +from opcode import opmap +from pypy.rlib.objectmodel import we_are_translated + +PyFrame._virtualizable2_ = ['last_instr', 'pycode', + 'valuestackdepth', 'valuestack_w[*]', + 'fastlocals_w[*]', 'f_forward', + ] + +JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] + +def can_inline(next_instr, bytecode): + if we_are_translated(): + bytecode = cast_base_ptr_to_instance(PyCode, bytecode) + co_code = bytecode.co_code + next_instr = 0 + while next_instr < len(co_code): + opcode = ord(co_code[next_instr]) + next_instr += 1 + if opcode >= HAVE_ARGUMENT: + next_instr += 2 + while opcode == opcodedesc.EXTENDED_ARG.index: + opcode = ord(co_code[next_instr]) + next_instr += 3 + if opcode == JUMP_ABSOLUTE: + return False + return True + +def get_printable_location(next_instr, bytecode): + from pypy.tool.stdlib_opcode import opcode_method_names + if we_are_translated(): + bytecode = cast_base_ptr_to_instance(PyCode, bytecode) + name = opcode_method_names[ord(bytecode.co_code[next_instr])] + return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] @@ -32,63 +64,36 @@ ## blockstack = frame.blockstack ## return (valuestackdepth, blockstack) -pypyjitdriver = PyPyJitDriver() +pypyjitdriver = PyPyJitDriver(can_inline = can_inline, + get_printable_location = get_printable_location) class __extend__(PyFrame): def dispatch(self, pycode, next_instr, ec): + self = hint(self, access_directly=True) next_instr = r_uint(next_instr) try: while True: - pypyjitdriver.jit_merge_point( - frame=self, ec=ec, next_instr=next_instr, pycode=pycode) + pypyjitdriver.jit_merge_point(ec=ec, + frame=self, next_instr=next_instr, pycode=pycode) co_code = pycode.co_code self.valuestackdepth = hint(self.valuestackdepth, promote=True) next_instr = self.handle_bytecode(co_code, next_instr, ec) except ExitFrame: return self.popvalue() - def JUMP_ABSOLUTE(f, jumpto, next_instr, *ignored): - ec = f.space.getexecutioncontext() + def JUMP_ABSOLUTE(f, jumpto, _, ec=None): + if we_are_jitted(): + f.last_instr = intmask(jumpto) + ec.bytecode_trace(f) + jumpto = r_uint(f.last_instr) pypyjitdriver.can_enter_jit(frame=f, ec=ec, next_instr=jumpto, pycode=f.getcode()) return jumpto -##class __extend__(Function): -## __metaclass__ = extendabletype - -## def getcode(self): -## # if the self is a compile time constant and if its code -## # is a BuiltinCode => grab and return its code as a constant -## if _is_early_constant(self): -## from pypy.interpreter.gateway import BuiltinCode -## code = hint(self, deepfreeze=True).code -## if not isinstance(code, BuiltinCode): code = self.code -## else: -## code = self.code -## return code - - # ____________________________________________________________ # -# Public interface - -def jit_startup(space, argv): - # save the app-level sys.executable in JITInfo, where the machine - # code backend can fish for it. A bit hackish. - from pypy.jit.backend.hlinfo import highleveljitinfo - highleveljitinfo.sys_executable = argv[0] - - # recognize the option --jit PARAM=VALUE,PARAM=VALUE... - # if it is at the beginning. A bit ad-hoc. - if len(argv) > 2 and argv[1] == '--jit': - argv.pop(1) - try: - pypyjitdriver.set_user_param(argv.pop(1)) - except ValueError: - from pypy.rlib.debug import debug_print - debug_print("WARNING: invalid --jit parameters string") - +# Public interface def set_param(space, args): '''Configure the tunable JIT parameters. Modified: pypy/branch/avm/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/avm/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/avm/pypy/module/pypyjit/test/test_pypy_c.py Thu Nov 5 20:41:26 2009 @@ -1,8 +1,20 @@ from pypy.conftest import gettestobjspace, option from pypy.tool.udir import udir +from pypy.jit.backend import loopparser import py import sys, os +def run_source(sourcefile, pypy_executable, tmpdir): + logfilepath = tmpdir.join(sourcefile.basename[:-3] + '.log') + if sys.platform.startswith('win'): + py.test.skip("XXX this is not Windows-friendly") + + result = py.process.cmdexec('PYPYJITLOG="%s" "%s" "%s"' % ( + logfilepath, pypy_executable, sourcefile)) + assert result + assert logfilepath.check() + return result, logfilepath + '.ops' + class PyPyCJITTests(object): def run_source(self, source, testcases): source = py.code.Source(source) @@ -14,8 +26,7 @@ # some support code... print >> f, py.code.Source(""" import sys, pypyjit - pypyjit.enable() - pypyjit.setthreshold(3) + pypyjit.set_param(threshold=3) def check(args, expected): for i in range(3): @@ -134,5 +145,34 @@ cls.counter = 0 cls.pypy_c = option.pypy_c - - + def run_and_compare(self, sourcefile): + fname = py.magic.autopath().dirpath().join('loops', sourcefile) + pypy_out, log = run_source(fname, self.pypy_c, self.tmpdir) + cpy_out = py.process.cmdexec('"%s" "%s"' % ( + sys.executable, fname)) + assert pypy_out == cpy_out + parser = loopparser.Parser() + loops = parser.parse(log) + if option.view: + from pypy.jit.metainterp.graphpage import display_loops + display_loops(loops) + return loops + + def assert_no_op(self, loop, opname): + for operation in loop.iter_operations(): + assert operation.opname != opname + + def test_trivial_add(self): + loops = self.run_and_compare('simple_add.py') + for loop in loops: + # naive way if finding the relevant loop to inspect + if isinstance(loop.operations[0], loopparser.ByteCodeRef): + self.assert_no_op(loop, 'call') + break + else: + assert False + + def test_dict_lookup(self): + py.test.skip('should remove dict lookups') + loops = self.run_and_compare('dict_lookup.py') + self.assert_no_op(loops[1], 'getfield_gc') Modified: pypy/branch/avm/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/avm/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/avm/pypy/module/signal/interp_signal.py Thu Nov 5 20:41:26 2009 @@ -97,10 +97,7 @@ # invoke the app-level handler space = self.space ec = space.getexecutioncontext() - try: - w_frame = ec.framestack.top() - except IndexError: - w_frame = space.w_None + w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) def report_pending_signals(self): Modified: pypy/branch/avm/pypy/module/symbol/__init__.py ============================================================================== --- pypy/branch/avm/pypy/module/symbol/__init__.py (original) +++ pypy/branch/avm/pypy/module/symbol/__init__.py Thu Nov 5 20:41:26 2009 @@ -5,10 +5,7 @@ """ from pypy.interpreter.mixedmodule import MixedModule - -# Forward imports so they run at startup time -import pypy.interpreter.pyparser.pythonlexer -import pypy.interpreter.pyparser.pythonparse +from pypy.interpreter.pyparser import pygram class Module(MixedModule): @@ -17,23 +14,11 @@ interpleveldefs = {} # see below -# Export the values from our custom symbol module. -# Skip negative values (the corresponding symbols are not visible in -# pure Python). -sym_name = {} - -def _init_symbols(grammar_version): - global sym_name - +def _init_symbols(): sym_name = {} - from pypy.interpreter.pyparser.pythonparse import make_pyparser - parser = make_pyparser(grammar_version) - - for name, val in parser.symbols.items(): - if val >= 0: - Module.interpleveldefs[name] = 'space.wrap(%d)' % val - sym_name[val] = name + for name, val in pygram.python_grammar.symbol_ids.iteritems(): + Module.interpleveldefs[name] = 'space.wrap(%d)' % val + sym_name[val] = name Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,) -# This is very evil -_init_symbols('2.5') +_init_symbols() Modified: pypy/branch/avm/pypy/module/sys/version.py ============================================================================== --- pypy/branch/avm/pypy/module/sys/version.py (original) +++ pypy/branch/avm/pypy/module/sys/version.py Thu Nov 5 20:41:26 2009 @@ -13,7 +13,7 @@ TRIM_URL_UP_TO = 'svn/pypy/' SVN_URL = """$HeadURL$"""[10:-28] -REV = """68301""" +REV = """$LastChangedRevision$"""[22:-2] import pypy Modified: pypy/branch/avm/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/avm/pypy/module/sys/vm.py (original) +++ pypy/branch/avm/pypy/module/sys/vm.py Thu Nov 5 20:41:26 2009 @@ -26,14 +26,19 @@ This function should be used for internal and specialized purposes only.""" depth = space.int_w(w_depth) - try: - f = space.getexecutioncontext().framestack.top(depth) - except IndexError: - raise OperationError(space.w_ValueError, - space.wrap("call stack is not deep enough")) - except ValueError: + if depth < 0: raise OperationError(space.w_ValueError, space.wrap("frame index must not be negative")) + ec = space.getexecutioncontext() + f = ec.gettopframe_nohidden() + while True: + if f is None: + raise OperationError(space.w_ValueError, + space.wrap("call stack is not deep enough")) + if depth == 0: + break + depth -= 1 + f = ec.getnextframe_nohidden(f) return space.wrap(f) # directly from the C code in ceval.c, might be moved somewhere else. Modified: pypy/branch/avm/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/avm/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/avm/pypy/objspace/flow/flowcontext.py Thu Nov 5 20:41:26 2009 @@ -259,7 +259,8 @@ except StopFlowing: continue # restarting a dead SpamBlock try: - self.framestack.push(frame) + old_frame = self.some_frame + self.some_frame = frame self.crnt_frame = frame try: w_result = frame.dispatch(frame.pycode, @@ -267,7 +268,7 @@ self) finally: self.crnt_frame = None - self.framestack.pop() + self.some_frame = old_frame except OperationThatShouldNotBePropagatedError, e: raise Exception( Modified: pypy/branch/avm/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/celldict.py (original) +++ pypy/branch/avm/pypy/objspace/std/celldict.py Thu Nov 5 20:41:26 2009 @@ -1,6 +1,7 @@ from pypy.objspace.std.dictmultiobject import DictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash +from pypy.rlib import jit class ModuleCell(object): def __init__(self, w_value=None): @@ -21,13 +22,13 @@ self.unshadowed_builtins = {} def getcell(self, key, make_new=True): - try: - return self.content[key] - except KeyError: - if not make_new: - raise - result = self.content[key] = ModuleCell() - return result + res = self.content.get(key, None) + if res is not None: + return res + if not make_new: + return None + result = self.content[key] = ModuleCell() + return result def add_unshadowed_builtin(self, name, builtin_impl): assert isinstance(builtin_impl, ModuleDictImplementation) @@ -66,6 +67,8 @@ if space.is_w(w_key_type, space.w_str): key = space.str_w(w_key) cell = self.getcell(key, False) + if cell is None: + raise KeyError cell.invalidate() del self.content[key] return self @@ -81,10 +84,10 @@ space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - try: - return self.getcell(space.str_w(w_lookup), False).w_value - except KeyError: + res = self.getcell(space.str_w(w_lookup), False) + if res is None: return None + return res.w_value elif _is_sane_hash(space, w_lookup_type): return None else: @@ -186,9 +189,10 @@ def getcache_slow(self, space, code, w_globals, implementation): state = space.fromcache(State) if not isinstance(implementation, ModuleDictImplementation): - missing_length = max(len(code.co_names_w) - len(state.always_invalid_cache), 0) - state.always_invalid_cache.extend([state.invalidcell] * missing_length) cache = state.always_invalid_cache + if len(code.co_names_w) > len(cache): + cache = [state.invalidcell] * len(code.co_names_w) + state.always_invalid_cache = cache else: cache = [state.invalidcell] * len(code.co_names_w) self.cache = cache @@ -202,8 +206,7 @@ def get_global_cache(space, code, w_globals): from pypy.interpreter.pycode import PyCode - if not isinstance(code, PyCode): - return [] + assert isinstance(code, PyCode) holder = code.globalcacheholder return holder.getcache(space, code, w_globals) @@ -224,12 +227,10 @@ def find_cell_from_dict(implementation, name): if isinstance(implementation, ModuleDictImplementation): - try: - return implementation.getcell(name, False) - except KeyError: - return None + return implementation.getcell(name, False) return None + at jit.dont_look_inside def load_global_fill_cache(f, nameindex): name = f.space.str_w(f.getname_w(nameindex)) implementation = getimplementation(f.w_globals) Modified: pypy/branch/avm/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/objspace.py (original) +++ pypy/branch/avm/pypy/objspace/std/objspace.py Thu Nov 5 20:41:26 2009 @@ -168,9 +168,11 @@ def call_likely_builtin(f, w_function, nargs): if isinstance(w_function, function.Function): executioncontext = self.getexecutioncontext() - executioncontext.c_call_trace(f, w_function) + if not we_are_jitted(): + executioncontext.c_call_trace(f, w_function) res = w_function.funccall_valuestack(nargs, f) - executioncontext.c_return_trace(f, w_function) + if not we_are_jitted(): + executioncontext.c_return_trace(f, w_function) return res args = f.make_arguments(nargs) try: Modified: pypy/branch/avm/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/avm/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/avm/pypy/objspace/std/typeobject.py Thu Nov 5 20:41:26 2009 @@ -50,6 +50,8 @@ lazyloaders = {} # can be overridden by specific instances version_tag = None + _immutable_fields_ = ["__flags__"] + uses_object_getattribute = False # ^^^ for config.objspace.std.getattributeshortcut # (False is a conservative default, fixed during real usage) @@ -186,9 +188,17 @@ space = w_self.space assert space.config.objspace.std.withmethodcache version_tag = w_self.version_tag + version_tag = hint(version_tag, promote=True) if version_tag is None: tup = w_self._lookup_where(name) return tup + w_self = hint(w_self, promote=True) + name = hint(name, promote=True) + return w_self._pure_lookup_where_with_method_cache(name, version_tag) + + @purefunction + def _pure_lookup_where_with_method_cache(w_self, name, version_tag): + space = w_self.space SHIFT = r_uint.BITS - space.config.objspace.std.methodcachesizeexp version_tag_as_int = current_object_addr_as_int(version_tag) # ^^^Note: if the version_tag object is moved by a moving GC, the @@ -257,7 +267,6 @@ raise UnwrapError(w_self) def is_heaptype(w_self): - w_self = hint(w_self, deepfreeze=True) return w_self.__flags__&_HEAPTYPE def get_module(w_self): @@ -520,11 +529,8 @@ # initialize __module__ in the dict (user-defined types only) if '__module__' not in w_self.dict_w: space = w_self.space - try: - caller = space.getexecutioncontext().framestack.top() - except IndexError: - pass - else: + caller = space.getexecutioncontext().gettopframe_nohidden() + if caller is not None: w_globals = caller.w_globals w_name = space.finditem(w_globals, space.wrap('__name__')) if w_name is not None: Modified: pypy/branch/avm/pypy/objspace/taint.py ============================================================================== --- pypy/branch/avm/pypy/objspace/taint.py (original) +++ pypy/branch/avm/pypy/objspace/taint.py Thu Nov 5 20:41:26 2009 @@ -43,15 +43,11 @@ def record_debug_info(self): ec = self.space.getexecutioncontext() - try: - frame = ec.framestack.top() - except IndexError: - pass - else: - if isinstance(frame, PyFrame): - self.filename = frame.pycode.co_filename - self.codename = frame.pycode.co_name - self.codeline = frame.get_last_lineno() + frame = ec.gettopframe_nohidden() + if isinstance(frame, PyFrame): # and, in particular, frame != None + self.filename = frame.pycode.co_filename + self.codename = frame.pycode.co_name + self.codeline = frame.get_last_lineno() if get_debug_level(self.space) > 0: self.debug_dump() @@ -177,15 +173,11 @@ filename = '?' codename = '?' codeline = 0 - try: - frame = ec.framestack.top() - except IndexError: - pass - else: - if isinstance(frame, PyFrame): - filename = frame.pycode.co_filename - codename = frame.pycode.co_name - codeline = frame.get_last_lineno() + frame = ec.gettopframe_nohidden() + if isinstance(frame, PyFrame): # and, in particular, frame != None + filename = frame.pycode.co_filename + codename = frame.pycode.co_name + codeline = frame.get_last_lineno() os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n %s\n' % ( filename, codeline, codename, operr.errorstr(space))) Modified: pypy/branch/avm/pypy/rlib/jit.py ============================================================================== --- pypy/branch/avm/pypy/rlib/jit.py (original) +++ pypy/branch/avm/pypy/rlib/jit.py Thu Nov 5 20:41:26 2009 @@ -15,6 +15,23 @@ def compute_result_annotation(self, s_x, **kwds_s): from pypy.annotation import model as annmodel s_x = annmodel.not_const(s_x) + access_directly = 's_access_directly' in kwds_s + fresh_virtualizable = 's_fresh_virtualizable' in kwds_s + if access_directly or fresh_virtualizable: + assert access_directly, "lone fresh_virtualizable hint" + if isinstance(s_x, annmodel.SomeInstance): + from pypy.objspace.flow.model import Constant + classdesc = s_x.classdef.classdesc + virtualizable = classdesc.read_attribute('_virtualizable2_', + Constant(None)).value + if virtualizable is not None: + flags = s_x.flags.copy() + flags['access_directly'] = True + if fresh_virtualizable: + flags['fresh_virtualizable'] = True + s_x = annmodel.SomeInstance(s_x.classdef, + s_x.can_be_None, + flags) return s_x def specialize_call(self, hop, **kwds_i): @@ -49,29 +66,9 @@ def specialize_call(self, hop): from pypy.rpython.lltypesystem import lltype + hop.exception_cannot_occur() return hop.inputconst(lltype.Signed, _we_are_jitted) -def _is_early_constant(x): - return False - -class Entry(ExtRegistryEntry): - _about_ = _is_early_constant - - def compute_result_annotation(self, s_value): - from pypy.annotation import model as annmodel - s = annmodel.SomeBool() - if s_value.is_constant(): - s.const = True - return s - - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - if hop.s_result.is_constant(): - assert hop.s_result.const - return hop.inputconst(lltype.Bool, True) - v, = hop.inputargs(hop.args_r[0]) - return hop.genop('is_early_constant', [v], resulttype=lltype.Bool) - # ____________________________________________________________ # User interface for the hotpath JIT policy Modified: pypy/branch/avm/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/branch/avm/pypy/rlib/test/test_jit.py (original) +++ pypy/branch/avm/pypy/rlib/test/test_jit.py Thu Nov 5 20:41:26 2009 @@ -1,5 +1,5 @@ import py -from pypy.rlib.jit import hint, _is_early_constant +from pypy.rlib.jit import hint, we_are_jitted from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin @@ -11,21 +11,14 @@ res = self.interpret(f, []) assert res == 5 - def test_is_early_constant(self): + def test_we_are_jitted(self): def f(x): - if _is_early_constant(x): - return 42 - return 0 - - assert f(3) == 0 - res = self.interpret(f, [5]) - assert res == 0 - - def g(): - return f(88) - - res = self.interpret(g, []) - assert res == 42 - - + try: + if we_are_jitted(): + return x + return x + 1 + except Exception: + return 5 + res = self.interpret(f, [4]) + assert res == 5 Modified: pypy/branch/avm/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/avm/pypy/rpython/llinterp.py (original) +++ pypy/branch/avm/pypy/rpython/llinterp.py Thu Nov 5 20:41:26 2009 @@ -572,9 +572,6 @@ def op_hint(self, x, hints): return x - def op_is_early_constant(self, x): - return False - def op_resume_point(self, *args): pass Modified: pypy/branch/avm/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/ll2ctypes.py Thu Nov 5 20:41:26 2009 @@ -13,19 +13,19 @@ from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.tool.uid import fixid from pypy.tool.tls import tlsobject -from pypy.rlib.rarithmetic import r_uint, r_singlefloat +from pypy.rlib.rarithmetic import r_uint, r_singlefloat, intmask from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter, LLException from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress -from pypy.translator.platform import platform def uaddressof(obj): return fixid(ctypes.addressof(obj)) _ctypes_cache = {} _eci_cache = {} +_parent_cache = {} def _setup_ctypes_cache(): from pypy.rpython.lltypesystem import rffi @@ -109,6 +109,8 @@ assert max_n >= 0 ITEM = A.OF ctypes_item = get_ctypes_type(ITEM, delayed_builders) + MAX_SIZE = sys.maxint/64 + PtrType = ctypes.POINTER(MAX_SIZE * ctypes_item) class CArray(ctypes.Structure): if not A._hints.get('nolength'): @@ -128,7 +130,7 @@ _malloc = classmethod(_malloc) def _indexable(self, index): - PtrType = ctypes.POINTER((index+1) * ctypes_item) + assert index + 1 < MAX_SIZE p = ctypes.cast(ctypes.pointer(self.items), PtrType) return p.contents @@ -245,6 +247,9 @@ if isinstance(FIELDTYPE, lltype.Struct): csubstruct = getattr(cstruct, field_name) convert_struct(field_value, csubstruct) + subcontainer = getattr(container, field_name) + substorage = subcontainer._storage + update_parent_cache(substorage, subcontainer) elif field_name == STRUCT._arrayfld: # inlined var-sized part csubarray = getattr(cstruct, field_name) convert_array(field_value, csubarray) @@ -299,8 +304,11 @@ FIELDTYPE = getattr(STRUCT, field_name) if isinstance(FIELDTYPE, lltype.ContainerType): if isinstance(FIELDTYPE, lltype.Struct): - struct_use_ctypes_storage(getattr(container, field_name), - getattr(ctypes_storage, field_name)) + struct_container = getattr(container, field_name) + struct_storage = getattr(ctypes_storage, field_name) + struct_use_ctypes_storage(struct_container, struct_storage) + struct_container._setparentstructure(container, field_name) + update_parent_cache(ctypes_storage, struct_container) elif isinstance(FIELDTYPE, lltype.Array): assert FIELDTYPE._hints.get('nolength', False) == False arraycontainer = _array_of_known_length(FIELDTYPE) @@ -478,6 +486,7 @@ _all_callbacks = {} _all_callbacks_results = [] _callback2obj = {} +_callback_exc_info = None # this is just another hack that passes around references to applevel types # disguised as base_ptr_lltype @@ -517,13 +526,15 @@ if isinstance(T, lltype.Ptr): if not llobj: # NULL pointer + if T == llmemory.GCREF: + return ctypes.c_void_p(0) return get_ctypes_type(T)() if T is base_ptr_lltype(): return new_opaque_object(llobj) if T == llmemory.GCREF: - if isinstance(llobj, _llgcref): - return ctypes.c_void_p(llobj.intval) + if isinstance(llobj._obj, _llgcopaque): + return ctypes.c_void_p(llobj._obj.intval) container = llobj._obj.container T = lltype.Ptr(lltype.typeOf(container)) # otherwise it came from integer and we want a c_void_p with @@ -542,7 +553,7 @@ v1voidlist = [(i, getattr(container, '_void' + str(i), None)) for i in range(len(T.TO.ARGS)) if T.TO.ARGS[i] is lltype.Void] - def callback(*cargs): + def callback_internal(*cargs): cargs = list(cargs) for v1 in v1voidlist: cargs.insert(v1[0], v1[1]) @@ -562,8 +573,16 @@ except LLException, lle: llinterp._store_exception(lle) return 0 + #except: + # import pdb + # pdb.set_trace() else: - llres = container._callable(*llargs) + try: + llres = container._callable(*llargs) + except LLException, lle: + llinterp = LLInterpreter.current_interpreter + llinterp._store_exception(lle) + return 0 assert lltype.typeOf(llres) == T.TO.RESULT if T.TO.RESULT is lltype.Void: return None @@ -575,14 +594,16 @@ return 0 return res - if getattr(conftest.option, 'usepdb', False): - callback_original = callback - def callback(*cargs): - try: - return callback_original(*cargs) - except: - import pdb, sys; pdb.post_mortem(sys.exc_traceback) - raise + def callback(*cargs): + try: + return callback_internal(*cargs) + except: + import sys + #if conftest.option.usepdb: + # import pdb; pdb.post_mortem(sys.exc_traceback) + global _callback_exc_info + _callback_exc_info = sys.exc_info() + raise if isinstance(T.TO.RESULT, lltype.Ptr): TMod = lltype.Ptr(lltype.FuncType(T.TO.ARGS, @@ -621,20 +642,18 @@ raise NotImplementedError(T) container._ctypes_storage_was_allocated() storage = container._storage + if lltype.parentlink(container)[0] is not None: + update_parent_cache(storage, container) p = ctypes.pointer(storage) if index: p = ctypes.cast(p, ctypes.c_void_p) p = ctypes.c_void_p(p.value + index) c_tp = get_ctypes_type(T.TO) storage._normalized_ctype = c_tp - if normalize and getattr(T.TO, '_arrayfld', None): - # XXX doesn't cache - c_tp = build_ctypes_struct(T.TO, [], - len(getattr(storage, T.TO._arrayfld).items)) - # make sure you cache according to the len() above! - p = ctypes.cast(p, ctypes.POINTER(c_tp)) - elif normalize and hasattr(storage, '_normalized_ctype'): + if normalize and hasattr(storage, '_normalized_ctype'): p = ctypes.cast(p, ctypes.POINTER(storage._normalized_ctype)) + if lltype.typeOf(llobj) == llmemory.GCREF: + p = ctypes.cast(p, ctypes.c_void_p) return p if isinstance(llobj, Symbolic): @@ -682,6 +701,9 @@ ctypes.cast(cobj, ctypes_instance))) container = lltype._struct(T.TO) struct_use_ctypes_storage(container, cobj.contents) + addr = ctypes.addressof(cobj.contents) + if addr in _parent_cache: + setparentstructure(container, _parent_cache[addr]) elif isinstance(T.TO, lltype.Array): if T.TO._hints.get('nolength', False): container = _array_of_unknown_length(T.TO) @@ -699,8 +721,7 @@ _callable=_callable) elif isinstance(T.TO, lltype.OpaqueType): if T == llmemory.GCREF: - # XXX obscure hack - return _llgcref(cobj) + container = _llgcopaque(cobj) else: container = lltype._opaque(T.TO) else: @@ -717,6 +738,9 @@ llobj = unichr(cobj) elif T is lltype.Signed: llobj = cobj + elif T is lltype.Bool: + assert cobj == True or cobj == False # 0 and 1 work too + llobj = bool(cobj) elif T is lltype.SingleFloat: if isinstance(cobj, ctypes.c_float): cobj = cobj.value @@ -783,6 +807,33 @@ # ____________________________________________ +# xxx from ctypes.util, this code is a useful fallback on darwin too +if sys.platform == 'darwin': + # Andreas Degert's find function using gcc + import re, tempfile, errno + + def _findLib_gcc_fallback(name): + expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) + fdout, ccout = tempfile.mkstemp() + os.close(fdout) + cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; else CC=cc; fi;' \ + '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + try: + f = os.popen(cmd) + trace = f.read() + f.close() + finally: + try: + os.unlink(ccout) + except OSError, e: + if e.errno != errno.ENOENT: + raise + res = re.search(expr, trace) + if not res: + return None + return res.group(0) +else: + _findLib_gcc_fallback = lambda name: None def get_ctypes_callable(funcptr, calling_conv): if not ctypes: @@ -798,11 +849,14 @@ old_eci = funcptr._obj.compilation_info funcname = funcptr._obj._name - try: - eci = _eci_cache[old_eci] - except KeyError: - eci = old_eci.compile_shared_lib() - _eci_cache[old_eci] = eci + if hasattr(old_eci, '_with_ctypes'): + eci = old_eci._with_ctypes + else: + try: + eci = _eci_cache[old_eci] + except KeyError: + eci = old_eci.compile_shared_lib() + _eci_cache[old_eci] = eci libraries = list(eci.libraries + eci.frameworks) @@ -817,26 +871,27 @@ not_found = [] for libname in libraries: libpath = None - ext = platform.so_ext - prefixes = platform.so_prefixes for dir in eci.library_dirs: - if libpath: + # xxx untested directly, what about 'lib' prefix + if sys.platform == "win32": + tryfile = os.path.join(dir, libname + '.dll') + elif sys.platform == "darwin": + tryfile = os.path.join(dir, libname + '.dylib') + else: + tryfile = os.path.join(dir, libname + '.so') + if os.path.isfile(tryfile): + libpath = tryfile break - for prefix in prefixes: - tryfile = os.path.join(dir, prefix + libname + '.' + ext) - if os.path.isfile(tryfile): - libpath = tryfile - break if not libpath: libpath = ctypes.util.find_library(libname) + if not libpath: + libpath = _findLib_gcc_fallback(libname) if not libpath and os.path.isabs(libname): libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') - # on ie slackware there was need for RTLD_GLOBAL here. - # this breaks a lot of things, since passing RTLD_GLOBAL - # creates symbol conflicts on C level. - clib = dllclass.LoadLibrary(libpath) + # urgh, cannot pass the flag to dllclass.LoadLibrary + clib = dllclass._dlltype(libpath, ctypes.RTLD_GLOBAL) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break @@ -858,6 +913,8 @@ place = 'library %r' % (libraries[0],) else: place = 'any of the libraries %r' % (libraries,) + if not_found: + place += ' (did not find %r)' % (not_found,) raise NotImplementedError("function %r not found in %s" % ( funcname, place)) @@ -898,6 +955,7 @@ if FUNCTYPE.ARGS[i] is lltype.Void: void_arguments.append(i) def invoke_via_ctypes(*argvalues): + global _callback_exc_info cargs = [] for i in range(len(FUNCTYPE.ARGS)): if i not in void_arguments: @@ -905,9 +963,14 @@ if i in container_arguments: cvalue = cvalue.contents cargs.append(cvalue) + _callback_exc_info = None _restore_c_errno() cres = cfunc(*cargs) _save_c_errno() + if _callback_exc_info: + etype, evalue, etb = _callback_exc_info + _callback_exc_info = None + raise etype, evalue, etb return ctypes2lltype(RESULT, cres) return invoke_via_ctypes @@ -916,7 +979,7 @@ if not isinstance(RESTYPE, lltype.LowLevelType): raise TypeError("rffi.cast() first arg should be a TYPE") if isinstance(value, llmemory.fakeaddress): - value = value.ptr + value = value.ptr or 0 TYPE1 = lltype.typeOf(value) cvalue = lltype2ctypes(value) cresulttype = get_ctypes_type(RESTYPE) @@ -1000,11 +1063,11 @@ _TYPE = llmemory.Address def __new__(cls, void_p): + if isinstance(void_p, (int, long)): + void_p = ctypes.c_void_p(void_p) self = long.__new__(cls, void_p.value) self.void_p = void_p - self.intval = void_p.value - if self.intval > sys.maxint: - self.intval = int(self.intval - 2*(sys.maxint + 1)) + self.intval = intmask(void_p.value) return self def _cast_to_ptr(self, TP): @@ -1014,36 +1077,41 @@ return '<_lladdress %s>' % (self.void_p,) def __eq__(self, other): - return cast_adr_to_int(other) == self.intval + if not isinstance(other, (int, long)): + other = cast_adr_to_int(other) + return intmask(other) == self.intval def __ne__(self, other): return not self == other -class _llgcref(object): - _TYPE = llmemory.GCREF +class _llgcopaque(lltype._container): + _TYPE = llmemory.GCREF.TO + _name = "_llgcopaque" def __init__(self, void_p): - self.intval = void_p.value + if isinstance(void_p, (int, long)): + self.intval = intmask(void_p) + else: + self.intval = intmask(void_p.value) def __eq__(self, other): - if isinstance(other, _llgcref): + if isinstance(other, _llgcopaque): return self.intval == other.intval - return force_cast(lltype.Signed, other) == self.intval + if other.container._storage in (None, True): + return False + return force_cast(lltype.Signed, other._as_ptr()) == self.intval def __ne__(self, other): return not self == other - def __nonzero__(self): - return bool(self.intval) - def _cast_to_ptr(self, PTRTYPE): - return force_cast(PTRTYPE, self.intval) + return force_cast(PTRTYPE, self.intval) - def _cast_to_int(self): - return self.intval +## def _cast_to_int(self): +## return self.intval - def _cast_to_adr(self): - return _lladdress(ctypes.c_void_p(self.intval)) +## def _cast_to_adr(self): +## return _lladdress(self.intval) def cast_adr_to_int(addr): if isinstance(addr, llmemory.fakeaddress): @@ -1074,6 +1142,46 @@ return hop.genop('cast_adr_to_int', [adr], resulttype = lltype.Signed) +# ------------------------------------------------------------ + +def parentchain(container): + current = container + links = [] + while True: + link = lltype.parentlink(current) + if link[0] is None: + try: + addr = ctypes.addressof(container._storage) + actual = _parent_cache[addr] + if len(links) < len(actual): + return actual + except KeyError: + pass + return links + links.append(link) + current = link[0] + +def update_parent_cache(storage, container): + chain = parentchain(container) + addr = ctypes.addressof(storage) + try: + current = _parent_cache[addr] + if len(chain) > len(current): + _parent_cache[addr] = chain + except KeyError: + _parent_cache[addr] = chain + +def setparentstructure(container, chain): + TP = lltype.typeOf(container) + current = container + for i, elem in enumerate(chain): + if lltype.typeOf(elem[0]) == TP: + chain = chain[i + 1:] + break + for elem in chain: + current._setparentstructure(*elem) + current = elem[0] + # ____________________________________________________________ # errno Modified: pypy/branch/avm/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/lloperation.py Thu Nov 5 20:41:26 2009 @@ -464,7 +464,6 @@ 'keepalive': LLOp(), 'same_as': LLOp(canfold=True), 'hint': LLOp(), - 'is_early_constant': LLOp(sideeffects=False), 'check_no_more_arg': LLOp(canraise=(Exception,)), 'check_self_nonzero': LLOp(canraise=(Exception,)), 'decode_arg': LLOp(canraise=(Exception,)), Modified: pypy/branch/avm/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/lltype.py Thu Nov 5 20:41:26 2009 @@ -798,8 +798,8 @@ "%s to %s" % (CURTYPE, PTRTYPE)) if (isinstance(CURTYPE.TO, OpaqueType) and not isinstance(PTRTYPE.TO, OpaqueType)): - if hasattr(ptr, '_cast_to_ptr'): - return ptr._cast_to_ptr(PTRTYPE) + if hasattr(ptr._obj, '_cast_to_ptr'): + return ptr._obj._cast_to_ptr(PTRTYPE) if not ptr: return nullptr(PTRTYPE.TO) try: @@ -1696,7 +1696,7 @@ def __eq__(self, other): if self.__class__ is not other.__class__: - return False + return NotImplemented if hasattr(self, 'container') and hasattr(other, 'container'): obj1 = self.container._normalizedcontainer() obj2 = other.container._normalizedcontainer() @@ -1705,6 +1705,8 @@ return self is other def __ne__(self, other): + if self.__class__ is not other.__class__: + return NotImplemented return not (self == other) def __hash__(self): Modified: pypy/branch/avm/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/opimpl.py Thu Nov 5 20:41:26 2009 @@ -386,7 +386,7 @@ print arg, print -def op_promote_virtualizable(object, fieldname): +def op_promote_virtualizable(object, fieldname, flags): pass # XXX should do something # ____________________________________________________________ Modified: pypy/branch/avm/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/rclass.py Thu Nov 5 20:41:26 2009 @@ -8,7 +8,8 @@ AbstractInstanceRepr,\ MissingRTypeAttribute,\ getclassrepr, getinstancerepr,\ - get_type_repr, rtype_new_instance + get_type_repr, rtype_new_instance, \ + FieldListAccessor from pypy.rpython.lltypesystem.lltype import \ Ptr, Struct, GcStruct, malloc, \ cast_pointer, cast_ptr_to_int, castable, nullptr, \ @@ -19,7 +20,6 @@ from pypy.rpython.robject import PyObjRepr, pyobj_repr from pypy.rpython.extregistry import ExtRegistryEntry from pypy.annotation import model as annmodel -from pypy.rlib.objectmodel import UnboxedValue from pypy.rlib.rarithmetic import intmask # @@ -351,6 +351,15 @@ if '_immutable_' in self.classdef.classdesc.classdict: hints = hints.copy() hints['immutable'] = True + if '_immutable_fields_' in self.classdef.classdesc.classdict: + hints = hints.copy() + self.immutable_field_list = self.classdef.classdesc.classdict['_immutable_fields_'].value + accessor = FieldListAccessor() + hints['immutable_fields'] = accessor + if ('_hash_cache_' in fields or + '_hash_cache_' in self.rbase.allinstancefields): + adtmeths = adtmeths.copy() + adtmeths['gethash'] = self.get_ll_hash_function() object_type = MkStruct(self.classdef.name, ('super', self.rbase.object_type), hints=hints, @@ -365,6 +374,10 @@ attachRuntimeTypeInfo(self.object_type) def _setup_repr_final(self): + hints = self.object_type._hints + if "immutable_fields" in hints: + accessor = hints["immutable_fields"] + self._parse_field_list(self.immutable_field_list, accessor) if self.gcflavor == 'gc': if (self.classdef is not None and self.classdef.classdesc.lookup('__del__') is not None): @@ -391,6 +404,9 @@ def common_repr(self): # -> object or nongcobject reprs return getinstancerepr(self.rtyper, None, self.gcflavor) + def _get_field(self, attr): + return self.fields[attr] + def null_instance(self): return nullptr(self.object_type) @@ -469,6 +485,7 @@ cname = inputconst(Void, mangled_name) if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) + self.hook_access_field(vinst, cname, llops, flags) return llops.genop('getfield', [vinst, cname], resulttype=r) else: if self.classdef is None: @@ -484,6 +501,7 @@ cname = inputconst(Void, mangled_name) if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) + self.hook_access_field(vinst, cname, llops, flags) llops.genop('setfield', [vinst, cname, vvalue]) else: if self.classdef is None: @@ -491,6 +509,9 @@ self.rbase.setfield(vinst, attr, vvalue, llops, force_cast=True, flags=flags) + def hook_access_field(self, vinst, cname, llops, flags): + pass # for virtualizables; see rvirtualizable2.py + def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" flavor = self.gcflavor @@ -606,41 +627,6 @@ return hop.gendirectcall(ll_isinstance, v_obj, v_cls) -def buildinstancerepr(rtyper, classdef, gcflavor='gc'): - if classdef is None: - unboxed = [] - virtualizable = False - virtualizable2 = False - else: - unboxed = [subdef for subdef in classdef.getallsubdefs() - if subdef.classdesc.pyobj is not None and - issubclass(subdef.classdesc.pyobj, UnboxedValue)] - virtualizable = classdef.classdesc.read_attribute('_virtualizable_', - Constant(False)).value - virtualizable2 = classdef.classdesc.read_attribute('_virtualizable2_', - Constant(False)).value - if virtualizable: - assert len(unboxed) == 0 - assert gcflavor == 'gc' - from pypy.rpython.lltypesystem import rvirtualizable - return rvirtualizable.VirtualizableInstanceRepr(rtyper, classdef) - elif virtualizable2: - assert len(unboxed) == 0 - assert gcflavor == 'gc' - from pypy.rpython.lltypesystem import rvirtualizable2 - return rvirtualizable2.Virtualizable2InstanceRepr(rtyper, classdef) - elif len(unboxed) == 0: - return InstanceRepr(rtyper, classdef, gcflavor) - else: - # the UnboxedValue class and its parent classes need a - # special repr for their instances - if len(unboxed) != 1: - raise TyperError("%r has several UnboxedValue subclasses" % ( - classdef,)) - assert gcflavor == 'gc' - from pypy.rpython.lltypesystem import rtagged - return rtagged.TaggedInstanceRepr(rtyper, classdef, unboxed[0]) - class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): Modified: pypy/branch/avm/pypy/rpython/lltypesystem/rvirtualizable2.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/rvirtualizable2.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/rvirtualizable2.py Thu Nov 5 20:41:26 2009 @@ -1,93 +1,18 @@ -import py from pypy.rpython.rmodel import inputconst from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.lltypesystem.rclass import InstanceRepr -from pypy.rpython.lltypesystem.rvirtualizable import VABLERTIPTR +from pypy.rpython.lltypesystem.rclass import InstanceRepr, OBJECTPTR +from pypy.rpython.rvirtualizable2 import AbstractVirtualizable2InstanceRepr +VABLERTIPTR = OBJECTPTR -class VirtualizableAccessor(object): +class Virtualizable2InstanceRepr(AbstractVirtualizable2InstanceRepr, InstanceRepr): - def initialize(self, STRUCT, redirected_fields, PARENT=None): - self.STRUCT = STRUCT - self.redirected_fields = redirected_fields - self.subaccessors = [] - if PARENT is None: - self.parent = None - else: - self.parent = PARENT.access - self.parent.subaccessors.append(self) - - def __repr__(self): - return '' % getattr(self, 'STRUCT', '?') - - def __getattr__(self, name): - if name.startswith('getset') and 'getsets' not in self.__dict__: - self.prepare_getsets() - return getattr(self, name) - else: - raise AttributeError("%s object has no attribute %r" % ( - self.__class__.__name__, name)) - - def prepare_getsets(self): - self.getsets = {} - STRUCT = self.STRUCT - for fieldname in self.redirected_fields: - FIELDTYPE = getattr(STRUCT, fieldname) - GETTER = lltype.FuncType([lltype.Ptr(STRUCT)], FIELDTYPE) - SETTER = lltype.FuncType([lltype.Ptr(STRUCT), FIELDTYPE], - lltype.Void) - VABLE_GETSET = lltype.Struct('vable_getset', - ('get', lltype.Ptr(GETTER)), - ('set', lltype.Ptr(SETTER)), - hints={'immutable': True}) - getset = lltype.malloc(VABLE_GETSET, flavor='raw', zero=False) - # as long as no valid pointer has been put in the structure - # by the JIT, accessing the fields should raise, in order - # to prevent constant-folding - py.test.raises(lltype.UninitializedMemoryAccess, "getset.get") - py.test.raises(lltype.UninitializedMemoryAccess, "getset.set") - self.getsets[fieldname] = getset - setattr(self, 'getset_' + fieldname, getset) - - def _freeze_(self): - return True - - -class Virtualizable2InstanceRepr(InstanceRepr): - - def __init__(self, rtyper, classdef): - InstanceRepr.__init__(self, rtyper, classdef) - classdesc = classdef.classdesc - if '_virtualizable2_' in classdesc.classdict: - basedesc = classdesc.basedesc - assert basedesc is None or basedesc.lookup('_virtualizable2_') is None - self.top_of_virtualizable_hierarchy = True - else: - self.top_of_virtualizable_hierarchy = False - try: - self.virtuals = tuple(classdesc.classdict['_always_virtual_'].value) - except KeyError: - self.virtuals = () - self.accessor = VirtualizableAccessor() - - def _setup_repr(self): + def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: llfields.append(('vable_base', llmemory.Address)) llfields.append(('vable_rti', VABLERTIPTR)) - InstanceRepr._setup_repr(self, llfields, - hints = {'virtualizable2': True, - 'virtuals' : self.virtuals}, - adtmeths = {'access': self.accessor}) - my_redirected_fields = [] - for _, (mangled_name, _) in self.fields.items(): - my_redirected_fields.append(mangled_name) - self.my_redirected_fields = dict.fromkeys(my_redirected_fields) - if self.top_of_virtualizable_hierarchy: - self.accessor.initialize(self.object_type, my_redirected_fields) - else: - self.accessor.initialize(self.object_type, my_redirected_fields, - self.rbase.lowleveltype.TO) + return llfields def set_vable(self, llops, vinst, force_cast=False): if self.top_of_virtualizable_hierarchy: @@ -98,36 +23,3 @@ llops.genop('setfield', [vinst, cname, vvalue]) else: self.rbase.set_vable(llops, vinst, force_cast=True) - - def new_instance(self, llops, classcallhop=None): - vptr = InstanceRepr.new_instance(self, llops, classcallhop) - self.set_vable(llops, vptr) - return vptr - - def getfield(self, vinst, attr, llops, force_cast=False, flags={}): - """Read the given attribute (or __class__ for the type) of 'vinst'.""" - if not flags.get('access_directly') and attr in self.fields: - mangled_name, r = self.fields[attr] - if mangled_name in self.my_redirected_fields: - if force_cast: - vinst = llops.genop('cast_pointer', [vinst], - resulttype=self) - c_name = inputconst(lltype.Void, mangled_name) - llops.genop('promote_virtualizable', [vinst, c_name]) - return llops.genop('getfield', [vinst, c_name], resulttype=r) - return InstanceRepr.getfield(self, vinst, attr, llops, force_cast) - - def setfield(self, vinst, attr, vvalue, llops, force_cast=False, - flags={}): - """Write the given attribute (or __class__ for the type) of 'vinst'.""" - if not flags.get('access_directly') and attr in self.fields: - mangled_name, r = self.fields[attr] - if mangled_name in self.my_redirected_fields: - if force_cast: - vinst = llops.genop('cast_pointer', [vinst], - resulttype=self) - c_name = inputconst(lltype.Void, mangled_name) - llops.genop('promote_virtualizable', [vinst, c_name]) - llops.genop('setfield', [vinst, c_name, vvalue]) - return - InstanceRepr.setfield(self, vinst, attr, vvalue, llops, force_cast) Modified: pypy/branch/avm/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/avm/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/avm/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Thu Nov 5 20:41:26 2009 @@ -15,7 +15,6 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.rtyper import RPythonTyper -from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -43,6 +42,8 @@ assert lltype2ctypes(rffi.r_ulong(-1)) == sys.maxint * 2 + 1 res = ctypes2lltype(lltype.Unsigned, sys.maxint * 2 + 1) assert (res, type(res)) == (rffi.r_ulong(-1), rffi.r_ulong) + assert ctypes2lltype(lltype.Bool, 0) is False + assert ctypes2lltype(lltype.Bool, 1) is True res = lltype2ctypes(llmemory.sizeof(lltype.Signed)) assert res == struct.calcsize("l") @@ -121,7 +122,10 @@ a.y[0] = 'x' a.y[1] = 'y' a.y[2] = 'z' - ac = lltype2ctypes(a) + # we need to pass normalize=False, otherwise 'ac' is returned of + # a normalized standard type, which complains about IndexError + # when doing 'ac.contents.y.items[2]'. + ac = lltype2ctypes(a, normalize=False) assert ac.contents.y.length == 3 assert ac.contents.y.items[2] == ord('z') lltype.free(a, flavor='raw') @@ -800,7 +804,7 @@ assert f() == 6 - def test_qsort(self): + def test_qsort_callback(self): TP = rffi.CArrayPtr(rffi.INT) a = lltype.malloc(TP.TO, 5, flavor='raw') a[0] = 5 @@ -994,47 +998,99 @@ assert v v2 = ctypes2lltype(llmemory.GCREF, ctypes.c_void_p(1235)) assert v2 != v + + def test_gcref_type(self): + NODE = lltype.GcStruct('NODE') + node = lltype.malloc(NODE) + ref = lltype.cast_opaque_ptr(llmemory.GCREF, node) + v = lltype2ctypes(ref) + assert isinstance(v, ctypes.c_void_p) + + def test_gcref_null(self): + ref = lltype.nullptr(llmemory.GCREF.TO) + v = lltype2ctypes(ref) + assert isinstance(v, ctypes.c_void_p) + + def test_cast_null_gcref(self): + ref = lltype.nullptr(llmemory.GCREF.TO) + value = rffi.cast(lltype.Signed, ref) + assert value == 0 + + def test_cast_null_fakeaddr(self): + ref = llmemory.NULL + value = rffi.cast(lltype.Signed, ref) + assert value == 0 + + def test_gcref_truth(self): + p0 = ctypes.c_void_p(0) + ref0 = ctypes2lltype(llmemory.GCREF, p0) + assert not ref0 + + p1234 = ctypes.c_void_p(1234) + ref1234 = ctypes2lltype(llmemory.GCREF, p1234) + assert p1234 + + def test_gcref_casts(self): + p0 = ctypes.c_void_p(0) + ref0 = ctypes2lltype(llmemory.GCREF, p0) + + assert lltype.cast_ptr_to_int(ref0) == 0 + assert llmemory.cast_ptr_to_adr(ref0) == llmemory.NULL + + NODE = lltype.GcStruct('NODE') + assert lltype.cast_opaque_ptr(lltype.Ptr(NODE), ref0) == lltype.nullptr(NODE) + + node = lltype.malloc(NODE) + ref1 = lltype.cast_opaque_ptr(llmemory.GCREF, node) + + intval = rffi.cast(lltype.Signed, node) + intval1 = rffi.cast(lltype.Signed, ref1) + + assert intval == intval1 + + ref2 = ctypes2lltype(llmemory.GCREF, intval1) + + assert lltype.cast_opaque_ptr(lltype.Ptr(NODE), ref2) == node -class TestPlatform(object): - def test_lib_on_libpaths(self): - from pypy.translator.platform import platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - tmpdir = udir.join('lib_on_libppaths') - tmpdir.ensure(dir=1) - c_file = tmpdir.join('c_file.c') - c_file.write('int f(int a, int b) { return (a + b); }') - eci = ExternalCompilationInfo(export_symbols=['f']) - so = platform.compile([c_file], eci, standalone=False) - eci = ExternalCompilationInfo( - libraries = ['c_file'], - library_dirs = [str(so.dirpath())] - ) - f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, - compilation_info=eci) - assert f(3, 4) == 7 - - def test_prefix(self): - - if sys.platform != 'linux2': - py.test.skip("Not supported") - - from pypy.translator.platform import platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - tmpdir = udir.join('lib_on_libppaths_prefix') - tmpdir.ensure(dir=1) - c_file = tmpdir.join('c_file.c') - c_file.write('int f(int a, int b) { return (a + b); }') - eci = ExternalCompilationInfo() - so = platform.compile([c_file], eci, standalone=False) - sopath = py.path.local(so) - sopath.move(sopath.dirpath().join('libc_file.so')) - eci = ExternalCompilationInfo( - libraries = ['c_file'], - library_dirs = [str(so.dirpath())] - ) - f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, - compilation_info=eci) - assert f(3, 4) == 7 + #addr = llmemory.cast_ptr_to_adr(ref1) + #assert llmemory.cast_adr_to_int(addr) == intval + + #assert lltype.cast_ptr_to_int(ref1) == intval + + def test_mixed_gcref_comparison(self): + NODE = lltype.GcStruct('NODE') + node = lltype.malloc(NODE) + ref1 = lltype.cast_opaque_ptr(llmemory.GCREF, node) + ref2 = rffi.cast(llmemory.GCREF, 123) + + assert ref1 != ref2 + assert not (ref1 == ref2) + + assert ref2 != ref1 + assert not (ref2 == ref1) + + assert node._obj._storage is True + + # forced! + rffi.cast(lltype.Signed, ref1) + assert node._obj._storage not in (True, None) + + assert ref1 != ref2 + assert not (ref1 == ref2) + + assert ref2 != ref1 + assert not (ref2 == ref1) + + def test_gcref_comparisons_back_and_forth(self): + NODE = lltype.GcStruct('NODE') + node = lltype.malloc(NODE) + ref1 = lltype.cast_opaque_ptr(llmemory.GCREF, node) + numb = rffi.cast(lltype.Signed, ref1) + ref2 = rffi.cast(llmemory.GCREF, numb) + assert ref1 == ref2 + assert ref2 == ref1 + assert not (ref1 != ref2) + assert not (ref2 != ref1) + + Modified: pypy/branch/avm/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/branch/avm/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/branch/avm/pypy/rpython/ootypesystem/rclass.py Thu Nov 5 20:41:26 2009 @@ -182,12 +182,18 @@ hints = classdef.classdesc.pyobj._rpython_hints else: hints = {} + if '_immutable_' in self.classdef.classdesc.classdict: + hints = hints.copy() + hints['immutable'] = True self.lowleveltype = ootype.Instance(classdef.name, b, {}, {}, _hints = hints) self.prebuiltinstances = {} # { id(x): (x, _ptr) } self.object_type = self.lowleveltype self.gcflavor = gcflavor - def _setup_repr(self): + def _setup_repr(self, llfields=None, hints=None): + if hints: + self.lowleveltype._hints.update(hints) + if self.classdef is None: self.allfields = {} self.allmethods = {} @@ -206,6 +212,9 @@ fields = {} fielddefaults = {} + + if llfields: + fields.update(dict(llfields)) selfattrs = self.classdef.attrs @@ -258,7 +267,6 @@ self.rbase = getinstancerepr(self.rtyper, self.classdef.basedef) self.rbase.setup() - methods = {} classattributes = {} baseInstance = self.lowleveltype._superclass classrepr = getclassrepr(self.rtyper, self.classdef) @@ -269,32 +277,6 @@ oovalue = classrepr.get_meta_instance() self.attach_class_attr_accessor('getmeta', oovalue) - for mangled, (name, s_value) in allmethods.iteritems(): - methdescs = s_value.descriptions - origin = dict([(methdesc.originclassdef, methdesc) for - methdesc in methdescs]) - if self.classdef in origin: - methdesc = origin[self.classdef] - else: - if name in selfattrs: - for superdef in self.classdef.getmro(): - if superdef in origin: - # put in methods - methdesc = origin[superdef] - break - else: - # abstract method - methdesc = None - else: - continue - - # get method implementation - from pypy.rpython.ootypesystem.rpbc import MethodImplementations - methimpls = MethodImplementations.get(self.rtyper, s_value) - m_impls = methimpls.get_impl(mangled, methdesc, - is_finalizer=name == "__del__") - - methods.update(m_impls) for classdef in self.classdef.getmro(): @@ -320,8 +302,6 @@ if not attrdef.s_value.is_constant(): classattributes[mangled] = attrdef.s_value, value - ootype.addMethods(self.lowleveltype, methods) - self.allfields = allfields self.allmethods = allmethods self.allclassattributes = allclassattributes @@ -336,6 +316,41 @@ ootype.addFields(self.lowleveltype, {mangled: (oot, impl)}) def _setup_repr_final(self): + if self.classdef is None: + return + + # we attach methods here and not in _setup(), because we want + # to be sure that all the reprs of the input arguments of all + # our methods have been computed at this point + methods = {} + selfattrs = self.classdef.attrs + for mangled, (name, s_value) in self.allmethods.iteritems(): + methdescs = s_value.descriptions + origin = dict([(methdesc.originclassdef, methdesc) for + methdesc in methdescs]) + if self.classdef in origin: + methdesc = origin[self.classdef] + else: + if name in selfattrs: + for superdef in self.classdef.getmro(): + if superdef in origin: + # put in methods + methdesc = origin[superdef] + break + else: + # abstract method + methdesc = None + else: + continue + # get method implementation + from pypy.rpython.ootypesystem.rpbc import MethodImplementations + methimpls = MethodImplementations.get(self.rtyper, s_value) + m_impls = methimpls.get_impl(mangled, methdesc, + is_finalizer=name == "__del__") + methods.update(m_impls) + ootype.addMethods(self.lowleveltype, methods) + + # step 3: provide accessor methods for class attributes that # are really overridden in subclasses. Must be done here # instead of _setup_repr to avoid recursion problems if class @@ -378,6 +393,10 @@ ootype.overrideDefaultForFields(self.lowleveltype, overridden_defaults) + def _get_field(self, attr): + mangled = mangle(attr, self.rtyper.getconfig()) + return mangled, self.allfields[mangled] + def attach_abstract_class_attr_accessor(self, mangled, attrtype): M = ootype.Meth([], attrtype) m = ootype.meth(M, _name=mangled, abstract=True) @@ -405,12 +424,10 @@ s_inst = hop.args_s[0] attr = hop.args_s[1].const mangled = mangle(attr, self.rtyper.getconfig()) - v_attr = hop.inputconst(ootype.Void, mangled) if mangled in self.allfields: # regular instance attributes - self.lowleveltype._check_field(mangled) - return hop.genop("oogetfield", [v_inst, v_attr], - resulttype = hop.r_result.lowleveltype) + return self.getfield(v_inst, attr, hop.llops, + flags=hop.args_s[0].flags) elif mangled in self.allmethods: # special case for methods: represented as their 'self' only # (see MethodsPBCRepr) @@ -449,17 +466,27 @@ self.lowleveltype._check_field(mangled) r_value = self.allfields[mangled] v_inst, _, v_newval = hop.inputargs(self, ootype.Void, r_value) - v_attr = hop.inputconst(ootype.Void, mangled) - return hop.genop('oosetfield', [v_inst, v_attr, v_newval]) + self.setfield(v_inst, attr, v_newval, hop.llops, + flags=hop.args_s[0].flags) + + def getfield(self, v_inst, attr, llops, flags={}): + mangled = mangle(attr, self.rtyper.getconfig()) + v_attr = inputconst(ootype.Void, mangled) + r_value = self.allfields[mangled] + self.lowleveltype._check_field(mangled) + self.hook_access_field(v_inst, v_attr, llops, flags) + return llops.genop('oogetfield', [v_inst, v_attr], + resulttype = r_value) - def setfield(self, vinst, attr, vvalue, llops): - # this method emulates behaviour from the corresponding - # lltypesystem one. It is referenced in some obscure corners - # like rtyping of OSError. + def setfield(self, vinst, attr, vvalue, llops, flags={}): mangled_name = mangle(attr, self.rtyper.getconfig()) cname = inputconst(ootype.Void, mangled_name) + self.hook_access_field(vinst, cname, llops, flags) llops.genop('oosetfield', [vinst, cname, vvalue]) + def hook_access_field(self, vinst, cname, llops, flags): + pass # for virtualizables; see rvirtualizable2.py + def rtype_is_true(self, hop): vinst, = hop.inputargs(self) return hop.genop('oononnull', [vinst], resulttype=ootype.Bool) @@ -521,8 +548,6 @@ if '_hash_cache_' in self.lowleveltype._allfields(): result._hash_cache_ = hash(value) -buildinstancerepr = InstanceRepr - class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): @@ -559,6 +584,8 @@ def ll_inst_hash(ins): + if not ins: + return 0 cached = ins._hash_cache_ if cached == 0: cached = ins._hash_cache_ = ootype.ooidentityhash(ins) Modified: pypy/branch/avm/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/avm/pypy/rpython/rclass.py (original) +++ pypy/branch/avm/pypy/rpython/rclass.py Thu Nov 5 20:41:26 2009 @@ -5,6 +5,20 @@ from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, getgcflavor + +class FieldListAccessor(object): + + def initialize(self, TYPE, fields): + self.TYPE = TYPE + self.fields = fields + + def __repr__(self): + return '' % getattr(self, 'TYPE', '?') + + def _freeze_(self): + return True + + def getclassrepr(rtyper, classdef): try: result = rtyper.class_reprs[classdef] @@ -22,13 +36,43 @@ try: result = rtyper.instance_reprs[classdef, flavor] except KeyError: - result = rtyper.type_system.rclass.buildinstancerepr( - rtyper, classdef, gcflavor=flavor) + result = buildinstancerepr(rtyper, classdef, gcflavor=flavor) rtyper.instance_reprs[classdef, flavor] = result rtyper.add_pendingsetup(result) return result + +def buildinstancerepr(rtyper, classdef, gcflavor='gc'): + from pypy.rlib.objectmodel import UnboxedValue + from pypy.objspace.flow.model import Constant + + if classdef is None: + unboxed = [] + virtualizable2 = False + else: + unboxed = [subdef for subdef in classdef.getallsubdefs() + if subdef.classdesc.pyobj is not None and + issubclass(subdef.classdesc.pyobj, UnboxedValue)] + virtualizable2 = classdef.classdesc.read_attribute('_virtualizable2_', + Constant(False)).value + if virtualizable2: + assert len(unboxed) == 0 + assert gcflavor == 'gc' + return rtyper.type_system.rvirtualizable2.Virtualizable2InstanceRepr(rtyper, classdef) + elif len(unboxed) > 0 and rtyper.type_system.name == 'lltypesystem': + # the UnboxedValue class and its parent classes need a + # special repr for their instances + if len(unboxed) != 1: + raise TyperError("%r has several UnboxedValue subclasses" % ( + classdef,)) + assert gcflavor == 'gc' + from pypy.rpython.lltypesystem import rtagged + return rtagged.TaggedInstanceRepr(rtyper, classdef, unboxed[0]) + else: + return rtyper.type_system.rclass.InstanceRepr(rtyper, classdef, gcflavor) + + class MissingRTypeAttribute(TyperError): pass @@ -122,6 +166,19 @@ def _setup_repr_final(self): pass + def _parse_field_list(self, fields, accessor): + with_suffix = {} + for name in fields: + if name.endswith('[*]'): + name = name[:-3] + suffix = '[*]' + else: + suffix = '' + mangled_name, r = self._get_field(name) + with_suffix[mangled_name] = suffix + accessor.initialize(self.object_type, with_suffix) + return with_suffix + def new_instance(self, llops, classcallhop=None): raise NotImplementedError Modified: pypy/branch/avm/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/avm/pypy/rpython/rpbc.py (original) +++ pypy/branch/avm/pypy/rpython/rpbc.py Thu Nov 5 20:41:26 2009 @@ -609,11 +609,17 @@ def ll_str(self, none): return llstr("None") + def get_ll_hash_function(self): + return ll_none_hash + rtype_simple_call = none_call rtype_call_args = none_call none_frozen_pbc_repr = NoneFrozenPBCRepr() +def ll_none_hash(_): + return 0 + class __extend__(pairtype(Repr, NoneFrozenPBCRepr)): Modified: pypy/branch/avm/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/avm/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/avm/pypy/rpython/test/test_rclass.py Thu Nov 5 20:41:26 2009 @@ -661,6 +661,25 @@ return a.revealconst(1) + b.revealconst(2) + a.revealconst(3) assert self.interpret(fn, []) == 3 + 8 + 9 + def test_hash_of_none(self): + class A: + pass + def fn(x): + if x: + obj = A() + else: + obj = None + return hash(obj) + res = self.interpret(fn, [0]) + assert res == 0 + + def test_hash_of_only_none(self): + def fn(): + obj = None + return hash(obj) + res = self.interpret(fn, []) + assert res == 0 + class TestLltype(BaseTestRclass, LLRtypeMixin): @@ -742,6 +761,21 @@ t, typer, graph = self.gengraph(f, [], backendopt=True) assert summary(graph) == {} + def test_immutable_fields(self): + class A(object): + _immutable_fields_ = ["x", "y[*]"] + + def __init__(self, x, y): + self.x = x + self.y = y + + def f(): + return A(3, []) + t, typer, graph = self.gengraph(f, []) + A_TYPE = graph.getreturnvar().concretetype + accessor = A_TYPE.TO._hints["immutable_fields"] + assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} + def test_immutable_inheritance(self): class I(object): def __init__(self, v): @@ -781,6 +815,41 @@ expected = hex(r_uint(xid)).lower().replace('l', '') assert expected in xstr + def test_hash_via_type(self): + from pypy.annotation import model as annmodel + from pypy.rpython import extregistry + from pypy.rpython.annlowlevel import cast_object_to_ptr + class X(object): pass + class Y(X): pass + class Z(Y): pass + + def my_gethash(z): + not_implemented + + def ll_my_gethash(ptr): + return ptr.gethash() + + class MyGetHashEntry(extregistry.ExtRegistryEntry): + _about_ = my_gethash + def compute_result_annotation(self, s_instance): + return annmodel.SomeInteger() + def specialize_call(self, hop): + [v_instance] = hop.inputargs(*hop.args_r) + return hop.gendirectcall(ll_my_gethash, v_instance) + + def f(n): + if n > 10: + z = Y() + got = -1 # path never used + else: + z = Z() + got = my_gethash(z) + expected = hash(z) # put the _hash_cache_ in the class Y + return got - expected + + res = self.interpret(f, [5]) + assert res == 0 + class TestOOtype(BaseTestRclass, OORtypeMixin): @@ -838,3 +907,65 @@ assert destra == destrc assert destrb is not None assert destra is not None + + def test_cast_object_instance(self): + A = ootype.Instance("Foo", ootype.ROOT) + + def fn_instance(): + a = ootype.new(A) + obj = ootype.cast_to_object(a) + a2 = ootype.cast_from_object(A, obj) + a3 = ootype.cast_from_object(ootype.ROOT, obj) + assert a is a2 + assert a is a3 + self.interpret(fn_instance, []) + + def test_cast_object_record(self): + B = ootype.Record({'x': ootype.Signed}) + + def fn_record(): + b = ootype.new(B) + b.x = 42 + obj = ootype.cast_to_object(b) + b2 = ootype.cast_from_object(B, obj) + assert b2.x == 42 + assert b is b2 + self.interpret(fn_record, []) + + def test_cast_object_null(self): + A = ootype.Instance("Foo", ootype.ROOT) + B = ootype.Record({'x': ootype.Signed}) + + def fn_null(): + a = ootype.null(A) + b = ootype.null(B) + obj1 = ootype.cast_to_object(a) + obj2 = ootype.cast_to_object(b) + assert obj1 == obj2 + assert ootype.cast_from_object(A, obj1) == a + assert ootype.cast_from_object(B, obj2) == b + self.interpret(fn_null, []) + + def test_cast_object_is_true(self): + A = ootype.Instance("Foo", ootype.ROOT) + def fn_is_true(flag): + if flag: + a = ootype.new(A) + else: + a = ootype.null(A) + obj = ootype.cast_to_object(a) + return bool(obj) + assert self.interpret(fn_is_true, [True]) is True + assert self.interpret(fn_is_true, [False]) is False + + def test_cast_object_mix_null(self): + A = ootype.Instance("Foo", ootype.ROOT) + def fn_mix_null(flag): + a = ootype.new(A) + obj = ootype.cast_to_object(a) + if flag: + return obj + else: + return ootype.NULL + res = self.interpret(fn_mix_null, [False]) + assert res is ootype.NULL Modified: pypy/branch/avm/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/avm/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/avm/pypy/tool/pytest/appsupport.py Thu Nov 5 20:41:26 2009 @@ -139,18 +139,16 @@ w_parent_init = space.getattr(w_BuiltinAssertionError, space.wrap('__init__')) space.call_args(w_parent_init, __args__.prepend(w_self)) - framestack = space.getexecutioncontext().framestack ## # Argh! we may see app-level helpers in the frame stack! ## # that's very probably very bad... -## if frame.code.co_name == 'normalize_exception': -## frame = framestack.top(1) +## ^^^the above comment may be outdated, but we are not sure # if the assertion provided a message, don't do magic args_w, kwargs_w = __args__.unpack() if args_w: w_msg = args_w[0] else: - frame = framestack.top(0) + frame = space.getexecutioncontext().gettopframe() runner = AppFrame(space, frame) try: source = runner.statement @@ -192,7 +190,7 @@ "after a string expression")) expr = space.unwrap(w_expr) source = py.code.Source(expr) - frame = space.getexecutioncontext().framestack.top() + frame = space.getexecutioncontext().gettopframe() w_locals = frame.getdictscope() w_locals = space.call_method(w_locals, 'copy') for key, w_value in kwds_w.items(): Modified: pypy/branch/avm/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/inline.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/inline.py Thu Nov 5 20:41:26 2009 @@ -381,6 +381,8 @@ excdata = self.translator.rtyper.getexceptiondata() exc_match = excdata.fn_exception_match for link in self.entrymap[self.graph_to_inline.exceptblock]: + if link.prevblock.exits[0] is not link: + continue copiedblock = self.copy_block(link.prevblock) VALUE, copiedlink = _find_exception_type(copiedblock) #print copiedblock.operations @@ -483,10 +485,14 @@ INPUT_SELF = inputv.concretetype if LINK_SELF != INPUT_SELF: # need to insert an upcast - assert ootype.isSubclass(LINK_SELF, INPUT_SELF) + if ootype.isSubclass(LINK_SELF, INPUT_SELF): + opname = 'ooupcast' + else: + assert ootype.isSubclass(INPUT_SELF, LINK_SELF) + opname = 'oodowncast' v = Variable() v.concretetype = INPUT_SELF - upcast = SpaceOperation('ooupcast', [linkv], v) + upcast = SpaceOperation(opname, [linkv], v) block.operations.append(upcast) passon_args[0] = v @@ -624,7 +630,6 @@ return (0.9999 * measure_median_execution_cost(graph) + count), True - def inlinable_static_callers(graphs): ok_to_call = dict.fromkeys(graphs) result = [] Modified: pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py Thu Nov 5 20:41:26 2009 @@ -434,6 +434,7 @@ # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', + 'bswap', 'bt', # zero-extending moves should not produce GC pointers 'movz', ]) @@ -1019,7 +1020,6 @@ break tracker = GcRootTracker(verbose=verbose, shuffle=shuffle) for fn in sys.argv[1:]: - tmpfn = fn + '.TMP' f = open(fn, 'r') firstline = f.readline() f.seek(0) @@ -1027,12 +1027,12 @@ tracker.reload_raw_table(f) f.close() else: - g = open(tmpfn, 'w') + assert fn.endswith('.s') + lblfn = fn[:-2] + '.lbl.s' + g = open(lblfn, 'w') tracker.process(f, g, filename=fn) f.close() g.close() - os.unlink(fn) - os.rename(tmpfn, fn) if output_raw_table: tracker.dump_raw_table(sys.stdout) tracker.clear() Modified: pypy/branch/avm/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/avm/pypy/translator/c/genc.py (original) +++ pypy/branch/avm/pypy/translator/c/genc.py Thu Nov 5 20:41:26 2009 @@ -486,14 +486,14 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - ofiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] + lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] - mk.definition('ASMFILES', ofiles) + mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) - mk.definition('OBJECTS', '$(ASMFILES) gcmaptable.s') + mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') - mk.rule('%.gcmap', '%.s', '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $@ || (rm -f $@ && exit 1)') - mk.rule('gcmaptable.s', '$(GCMAPFILES)', '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@ || (rm -f $@ && exit 1)') + mk.rule('%.lbl.s %.gcmap', '%.s', '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + mk.rule('gcmaptable.s', '$(GCMAPFILES)', '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') mk.write() #self.translator.platform, Modified: pypy/branch/avm/pypy/translator/c/test/test_symbolic.py ============================================================================== --- pypy/branch/avm/pypy/translator/c/test/test_symbolic.py (original) +++ pypy/branch/avm/pypy/translator/c/test/test_symbolic.py Thu Nov 5 20:41:26 2009 @@ -109,16 +109,3 @@ fn = t.compile_c() res = fn() assert res == 42 - -def test_is_early_constant(): - from pypy.rlib import jit - def f(x): - if jit._is_early_constant(x): - return 42 - return 0 - - - fn, t = getcompiled(f, [int]) - res = fn(5) - assert res == 0 - Modified: pypy/branch/avm/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/avm/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/avm/pypy/translator/cli/opcodes.py Thu Nov 5 20:41:26 2009 @@ -1,8 +1,7 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField,\ - FieldInfoForConst + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, DebugPrint from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -19,14 +18,22 @@ def _abs(type_): return [PushAllArgs, 'call %s class [mscorlib]System.Math::Abs(%s)' % (type_, type_), StoreResult] -def _check_ovf(op): +def _check_ovf(op, catch_arithmexic_exception=False): mapping = [('[mscorlib]System.OverflowException', 'exceptions.OverflowError')] + if catch_arithmexic_exception: + mapping.append(('[mscorlib]System.ArithmeticException', 'exceptions.OverflowError')) return [MapException(op, mapping)] def _check_zer(op): mapping = [('[mscorlib]System.DivideByZeroException', 'exceptions.ZeroDivisionError')] return [MapException(op, mapping)] +def _check_ovf_zer(op): + mapping = [('[mscorlib]System.OverflowException', 'exceptions.OverflowError'), + ('[mscorlib]System.DivideByZeroException', 'exceptions.ZeroDivisionError'), + ('[mscorlib]System.ArithmeticException', 'exceptions.OverflowError')] + return [MapException(op, mapping)] + # __________ object oriented & misc operations __________ misc_ops = { 'new': [New], @@ -36,6 +43,8 @@ 'oosend': [CallMethod], 'ooupcast': DoNothing, 'oodowncast': [DownCast], + 'cast_to_object': DoNothing, + 'cast_from_object': [DownCast], 'clibox': [Box], 'cliunbox': [Unbox], 'cli_newarray': [NewArray], @@ -46,9 +55,6 @@ 'cli_eventhandler': [EventHandler], 'cli_getstaticfield': [GetStaticField], 'cli_setstaticfield': [SetStaticField], - 'cli_fieldinfo_for_const': [FieldInfoForConst], - 'oois': 'ceq', - 'oononnull': [PushAllArgs, 'ldnull', 'ceq']+Not, 'classof': [PushAllArgs, 'callvirt instance class [mscorlib]System.Type object::GetType()'], 'instanceof': [CastTo, 'ldnull', 'cgt.un'], 'subclassof': [PushAllArgs, 'call bool [pypylib]pypy.runtime.Utils::SubclassOf(class [mscorlib]System.Type, class[mscorlib]System.Type)'], @@ -70,8 +76,11 @@ 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'debug_assert': Ignore, + 'debug_print': [DebugPrint], + 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, - 'is_early_constant': [PushPrimitive(ootype.Bool, False)], + 'jit_marker': Ignore, + 'promote_virtualizable': Ignore, } # __________ numeric operations __________ @@ -105,6 +114,10 @@ 'ullong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], 'ullong_invert': 'not', + # XXX: why nop nop nop? + 'ooisnull': [PushAllArgs, 'nop', 'nop', 'nop', 'ldnull', 'ceq'], + 'oononnull': [PushAllArgs, 'nop', 'nop', 'nop', 'nop', 'ldnull', 'ceq']+Not, + # when casting from bool we want that every truth value is casted # to 1: we can't simply DoNothing, because the CLI stack could # contains a truth value not equal to 1, so we should use the !=0 @@ -161,8 +174,8 @@ 'int_add_nonneg_ovf': _check_ovf('add.ovf'), 'int_sub_ovf': _check_ovf('sub.ovf'), 'int_mul_ovf': _check_ovf('mul.ovf'), - 'int_floordiv_ovf': 'div', # these can't overflow! - 'int_mod_ovf': 'rem', + 'int_floordiv_ovf': _check_ovf('div', catch_arithmexic_exception=True), + 'int_mod_ovf': _check_ovf('rem', catch_arithmexic_exception=True), 'int_lt_ovf': 'clt', 'int_le_ovf': _not('cgt'), 'int_eq_ovf': 'ceq', @@ -174,13 +187,11 @@ 'int_lshift_ovf': _check_ovf([PushArg(0),'conv.i8',PushArg(1), 'shl', 'conv.ovf.i4', StoreResult]), - 'int_lshift_ovf_val': _check_ovf([PushArg(0),'conv.i8',PushArg(1), 'shl', - 'conv.ovf.i4', StoreResult]), 'int_rshift_ovf': 'shr', # these can't overflow! 'int_xor_ovf': 'xor', - 'int_floordiv_ovf_zer': _check_zer('div'), - 'int_mod_ovf_zer': _check_zer('rem'), + 'int_floordiv_ovf_zer': _check_ovf_zer('div'), + 'int_mod_ovf_zer': _check_ovf_zer('rem'), 'int_mod_zer': _check_zer('rem'), 'uint_add': 'add', @@ -246,6 +257,9 @@ 'ullong_ge': _not('clt.un'), 'ullong_lshift': [PushAllArgs, 'conv.u4', 'shl'], 'ullong_rshift': [PushAllArgs, 'conv.i4', 'shr'], + + 'oois': 'ceq', + 'ooisnot': _not('ceq'), } opcodes = misc_ops.copy() Modified: pypy/branch/avm/pypy/translator/driver.py ============================================================================== --- pypy/branch/avm/pypy/translator/driver.py (original) +++ pypy/branch/avm/pypy/translator/driver.py Thu Nov 5 20:41:26 2009 @@ -120,7 +120,7 @@ else: task, postfix = parts if task in ('rtype', 'backendopt', 'llinterpret', - 'prejitbackendopt', 'pyjitpl'): + 'pyjitpl'): if ts: if ts == postfix: expose_task(task, explicit_task) @@ -355,31 +355,34 @@ task_rtype_ootype = taskdef(task_rtype_ootype, ['annotate'], "ootyping") OOTYPE = 'rtype_ootype' - def task_prejitbackendopt_lltype(self): - from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(self.translator, - inline_threshold=0, - merge_if_blocks=True, - constfold=True, - raisingop2direct_call=False, - remove_asserts=False) - # - task_prejitbackendopt_lltype = taskdef( - task_prejitbackendopt_lltype, - [RTYPE], - "Backendopt before jitting") - def task_pyjitpl_lltype(self): get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) # from pypy.jit.metainterp.warmspot import apply_jit - apply_jit(self.translator, policy=self.jitpolicy) + apply_jit(self.translator, policy=self.jitpolicy, + debug_level=self.config.translation.jit_debug, + backend_name=self.config.translation.jit_backend) # self.log.info("the JIT compiler was generated") # task_pyjitpl_lltype = taskdef(task_pyjitpl_lltype, - [RTYPE, '?prejitbackendopt_lltype'], + [RTYPE], + "JIT compiler generation") + + def task_pyjitpl_ootype(self): + get_policy = self.extra['jitpolicy'] + self.jitpolicy = get_policy(self) + # + from pypy.jit.metainterp.warmspot import apply_jit + apply_jit(self.translator, policy=self.jitpolicy, + debug_level=self.config.translation.jit_debug, + backend_name='cli') #XXX + # + self.log.info("the JIT compiler was generated") + # + task_pyjitpl_ootype = taskdef(task_pyjitpl_ootype, + [OOTYPE], "JIT compiler generation") def task_backendopt_lltype(self): Modified: pypy/branch/avm/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/avm/pypy/translator/goal/translate.py (original) +++ pypy/branch/avm/pypy/translator/goal/translate.py Thu Nov 5 20:41:26 2009 @@ -20,8 +20,6 @@ GOALS= [ ("annotate", "do type inference", "-a --annotate", ""), ("rtype", "do rtyping", "-t --rtype", ""), - ("prejitbackendopt", "backend optimize before jitting", - "--prejitbackendopt", ""), ("pyjitpl", "JIT generation step", "--pyjitpl", ""), ("backendopt", "do backend optimizations", "--backendopt", ""), ("source", "create source", "-s --source", ""), Modified: pypy/branch/avm/pypy/translator/interactive.py ============================================================================== --- pypy/branch/avm/pypy/translator/interactive.py (original) +++ pypy/branch/avm/pypy/translator/interactive.py Thu Nov 5 20:41:26 2009 @@ -73,6 +73,8 @@ kwds.get('standalone')) kwds.pop('policy', None) kwds.pop('standalone', None) + gc = kwds.pop('gc', None) + if gc: self.config.translation.gc = gc self.config.translation.set(**kwds) def ensure_opt(self, name, value=None, fallback=None): Modified: pypy/branch/avm/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/avm/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/avm/pypy/translator/jvm/opcodes.py Thu Nov 5 20:41:26 2009 @@ -218,6 +218,4 @@ 'truncate_longlong_to_int': jvm.L2I, 'cast_longlong_to_float': jvm.L2D, 'cast_primitive': [PushAllArgs, CastPrimitive, StoreResult], - 'is_early_constant': [PushPrimitive(ootype.Bool, False), StoreResult] - }) Modified: pypy/branch/avm/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/oosupport/metavm.py (original) +++ pypy/branch/avm/pypy/translator/oosupport/metavm.py Thu Nov 5 20:41:26 2009 @@ -456,6 +456,7 @@ is_primitive = self._get_primitive_name(callee) if is_primitive: + import pdb; pdb.set_trace() module, name = is_primitive generator.prepare_call_primitive(op, module, name) Modified: pypy/branch/avm/pypy/translator/oosupport/test_template/operations.py ============================================================================== --- pypy/branch/avm/pypy/translator/oosupport/test_template/operations.py (original) +++ pypy/branch/avm/pypy/translator/oosupport/test_template/operations.py Thu Nov 5 20:41:26 2009 @@ -191,14 +191,6 @@ return bool(x) self._check_all(fn) - def test_is_early_constant(self): - from pypy.rlib import jit - def f(x): - if jit._is_early_constant(x): - return 42 - return 0 - assert self.interpret(f, [5]) == 0 - def test_ullong_rshift(self): def f(x): return x >> 1 Modified: pypy/branch/avm/pypy/translator/test/test_driver.py ============================================================================== --- pypy/branch/avm/pypy/translator/test/test_driver.py (original) +++ pypy/branch/avm/pypy/translator/test/test_driver.py Thu Nov 5 20:41:26 2009 @@ -6,7 +6,7 @@ def test_ctr(): td = TranslationDriver() expected = ['annotate', 'backendopt', 'llinterpret', 'rtype', 'source', - 'compile', 'run', 'prejitbackendopt', 'pyjitpl'] + 'compile', 'run', 'pyjitpl'] assert set(td.exposed) == set(expected) assert td.backend_select_goals(['compile_c']) == ['compile_c'] @@ -35,7 +35,8 @@ 'compile_cli', 'compile_c', 'run_c', 'run_cli', 'compile_jvm', 'source_jvm', 'run_jvm', - 'prejitbackendopt_lltype', 'pyjitpl_lltype'] + 'pyjitpl_lltype', + 'pyjitpl_ootype'] assert set(td.exposed) == set(expected) td = TranslationDriver({'backend': None, 'type_system': 'lltype'}) @@ -49,6 +50,6 @@ 'backendopt_lltype'] expected = ['annotate', 'backendopt', 'llinterpret', 'rtype', 'source_c', - 'compile_c', 'run_c', 'prejitbackendopt', 'pyjitpl'] + 'compile_c', 'run_c', 'pyjitpl'] assert set(td.exposed) == set(expected) From fijal at codespeak.net Thu Nov 5 20:41:46 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 5 Nov 2009 20:41:46 +0100 (CET) Subject: [pypy-svn] r69005 - in pypy/extradoc/talk/rupy2009: . examples Message-ID: <20091105194146.D7090318137@codespeak.net> Author: fijal Date: Thu Nov 5 20:41:46 2009 New Revision: 69005 Added: pypy/extradoc/talk/rupy2009/beamerouterthememy.sty pypy/extradoc/talk/rupy2009/beamerthemeWarsaw.sty pypy/extradoc/talk/rupy2009/examples/ pypy/extradoc/talk/rupy2009/img1.jpg (contents, props changed) pypy/extradoc/talk/rupy2009/peasant_and_birdnester-400.jpg (contents, props changed) pypy/extradoc/talk/rupy2009/talk.pdf Log: Rudimentary first version Added: pypy/extradoc/talk/rupy2009/beamerouterthememy.sty ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/beamerouterthememy.sty Thu Nov 5 20:41:46 2009 @@ -0,0 +1,39 @@ +\ProvidesPackageRCS $Header: /cvsroot/latex-beamer/latex-beamer/themes/outer/beamerouterthemesplit.sty,v 1.4 2004/10/07 22:21:16 tantau Exp $ + +% Copyright 2003 by Till Tantau +% +% This program can be redistributed and/or modified under the terms +% of the GNU Public License, version 2. + +\mode + +\setbeamercolor{section in head/foot}{parent=palette quaternary} +\setbeamercolor{subsection in head/foot}{parent=palette primary} + +\setbeamercolor{author in head/foot}{parent=section in head/foot} +\setbeamercolor{title in head/foot}{parent=subsection in head/foot} + + + +\usesectionheadtemplate + {\hfill\insertsectionhead} + {\hfill\color{fg!50!bg}\insertsectionhead} + + + + +\defbeamertemplate*{footline}{split theme} +{% + \leavevmode% + \hbox{\begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm plus1fill,rightskip=.3cm]{author in head/foot}% + \usebeamerfont{author in head/foot}\insertshortauthor + \end{beamercolorbox}% + \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}% + \usebeamerfont{title in head/foot}\insertshorttitle + \end{beamercolorbox}}% + \vskip0pt% +} + + +\mode + Added: pypy/extradoc/talk/rupy2009/beamerthemeWarsaw.sty ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/beamerthemeWarsaw.sty Thu Nov 5 20:41:46 2009 @@ -0,0 +1,18 @@ +\ProvidesPackageRCS $Header: /cvsroot/latex-beamer/latex-beamer/themes/theme/beamerthemeWarsaw.sty,v 1.8 2004/10/07 20:53:10 tantau Exp $ + +% Copyright 2003 by Till Tantau +% +% This program can be redistributed and/or modified under the terms +% of the GNU Public License, version 2. + +\mode + +\useinnertheme[shadow=true]{rounded} +\useoutertheme{my} +\usecolortheme{orchid} +\usecolortheme{whale} + +\setbeamerfont{block title}{size={}} + +\mode + Added: pypy/extradoc/talk/rupy2009/img1.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/rupy2009/peasant_and_birdnester-400.jpg ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/rupy2009/talk.pdf ============================================================================== Files (empty file) and pypy/extradoc/talk/rupy2009/talk.pdf Thu Nov 5 20:41:46 2009 differ From arigo at codespeak.net Thu Nov 5 21:15:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 5 Nov 2009 21:15:32 +0100 (CET) Subject: [pypy-svn] r69006 - pypy/branch/gc-dump-malloc/pypy/rpython/memory Message-ID: <20091105201532.8DF6716842D@codespeak.net> Author: arigo Date: Thu Nov 5 21:15:29 2009 New Revision: 69006 Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Log: Gr gr gr gr gr. Xxx temporary and all that. Modified: pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/memory/gctypelayout.py Thu Nov 5 21:15:29 2009 @@ -26,7 +26,6 @@ ("finalizer", FINALIZERTYPE), ("fixedsize", lltype.Signed), ("ofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - hints={'immutable': True}, ) VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info", ("header", TYPE_INFO), @@ -34,7 +33,6 @@ ("ofstovar", lltype.Signed), ("ofstolength", lltype.Signed), ("varofstoptrs", lltype.Ptr(OFFSETS_TO_GC_PTR)), - hints={'immutable': True}, ) TYPE_INFO_PTR = lltype.Ptr(TYPE_INFO) VARSIZE_TYPE_INFO_PTR = lltype.Ptr(VARSIZE_TYPE_INFO) From magcius at codespeak.net Thu Nov 5 21:50:21 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 5 Nov 2009 21:50:21 +0100 (CET) Subject: [pypy-svn] r69007 - in pypy/branch/avm: . lib-python pypy pypy/doc/config pypy/interpreter pypy/module/pypyjit pypy/rlib pypy/translator pypy/translator/backendopt pypy/translator/backendopt/test pypy/translator/c/gcc pypy/translator/c/gcc/test pypy/translator/c/gcc/test/darwin pypy/translator/c/gcc/test/elf pypy/translator/c/src pypy/translator/goal Message-ID: <20091105205021.3B979168433@codespeak.net> Author: magcius Date: Thu Nov 5 21:50:19 2009 New Revision: 69007 Added: pypy/branch/avm/pypy/doc/config/translation.jit_debug.txt - copied unchanged from r67353, pypy/branch/pyjitpl5/pypy/doc/config/translation.jit_debug.txt pypy/branch/avm/pypy/translator/c/gcc/test/darwin/ - copied from r67353, pypy/branch/pyjitpl5/pypy/translator/c/gcc/test/darwin/ pypy/branch/avm/pypy/translator/c/gcc/test/elf/ (props changed) - copied from r67353, pypy/branch/pyjitpl5/pypy/translator/c/gcc/test/elf/ Removed: pypy/branch/avm/pypy/translator/c/gcc/test/track0.s pypy/branch/avm/pypy/translator/c/gcc/test/track1.s pypy/branch/avm/pypy/translator/c/gcc/test/track2.s pypy/branch/avm/pypy/translator/c/gcc/test/track3.s pypy/branch/avm/pypy/translator/c/gcc/test/track4.s pypy/branch/avm/pypy/translator/c/gcc/test/track5.s pypy/branch/avm/pypy/translator/c/gcc/test/track6.s pypy/branch/avm/pypy/translator/c/gcc/test/track7.s Modified: pypy/branch/avm/ (props changed) pypy/branch/avm/lib-python/conftest.py pypy/branch/avm/pypy/ (props changed) pypy/branch/avm/pypy/conftest.py pypy/branch/avm/pypy/interpreter/pycode.py pypy/branch/avm/pypy/module/pypyjit/__init__.py pypy/branch/avm/pypy/rlib/jit.py pypy/branch/avm/pypy/translator/backendopt/all.py pypy/branch/avm/pypy/translator/backendopt/removenoops.py pypy/branch/avm/pypy/translator/backendopt/test/test_all.py pypy/branch/avm/pypy/translator/backendopt/test/test_merge_if_blocks.py pypy/branch/avm/pypy/translator/backendopt/test/test_removeassert.py pypy/branch/avm/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py pypy/branch/avm/pypy/translator/c/src/asm_gcc_x86.h pypy/branch/avm/pypy/translator/c/src/mem.h pypy/branch/avm/pypy/translator/goal/app_main.py pypy/branch/avm/pypy/translator/simplify.py Log: Merged in from trunk. Last time, I promise Modified: pypy/branch/avm/lib-python/conftest.py ============================================================================== --- pypy/branch/avm/lib-python/conftest.py (original) +++ pypy/branch/avm/lib-python/conftest.py Thu Nov 5 21:50:19 2009 @@ -633,9 +633,9 @@ if exit_status: raise self.ExternalFailure(test_stdout, test_stderr) - def repr_failure(self, excinfo, outerr): + def repr_failure(self, excinfo): if not excinfo.errisinstance(self.ExternalFailure): - return super(ReallyRunFileExternal, self).repr_failure(excinfo, outerr) + return super(ReallyRunFileExternal, self).repr_failure(excinfo) out, err = excinfo.value.args return out + err Modified: pypy/branch/avm/pypy/conftest.py ============================================================================== --- pypy/branch/avm/pypy/conftest.py (original) +++ pypy/branch/avm/pypy/conftest.py Thu Nov 5 21:50:19 2009 @@ -371,10 +371,8 @@ def runtest(self): target = self.obj - args = self._args - assert not args if option.runappdirect: - return target(*args) + return target() space = gettestobjspace() func = app2interp_temp(target) print "executing", func @@ -400,10 +398,8 @@ def runtest(self): target = self.obj - args = self._args - assert not args if option.runappdirect: - return target(*args) + return target() space = target.im_self.space func = app2interp_temp(target.im_func) w_instance = self.parent.w_instance Modified: pypy/branch/avm/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/avm/pypy/interpreter/pycode.py (original) +++ pypy/branch/avm/pypy/interpreter/pycode.py Thu Nov 5 21:50:19 2009 @@ -380,7 +380,10 @@ ] return space.newtuple([new_inst, space.newtuple(tup)]) + def get_repr(self): + return "" % ( + self.co_name, self.co_filename, self.co_firstlineno) + def repr(self, space): - return space.wrap("" % ( - self.co_name, self.co_filename, self.co_firstlineno)) + return space.wrap(self.get_repr()) repr.unwrap_spec = ['self', ObjSpace] Modified: pypy/branch/avm/pypy/module/pypyjit/__init__.py ============================================================================== --- pypy/branch/avm/pypy/module/pypyjit/__init__.py (original) +++ pypy/branch/avm/pypy/module/pypyjit/__init__.py Thu Nov 5 21:50:19 2009 @@ -11,3 +11,8 @@ def setup_after_space_initialization(self): # force the __extend__ hacks to occur early import pypy.module.pypyjit.interp_jit + # add the 'defaults' attribute + from pypy.rlib.jit import PARAMETERS + space = self.space + w_obj = space.wrap(PARAMETERS) + space.setattr(space.wrap(self), space.wrap('defaults'), w_obj) Modified: pypy/branch/avm/pypy/rlib/jit.py ============================================================================== --- pypy/branch/avm/pypy/rlib/jit.py (original) +++ pypy/branch/avm/pypy/rlib/jit.py Thu Nov 5 21:50:19 2009 @@ -9,6 +9,10 @@ def hint(x, **kwds): return x +def dont_look_inside(func): + func._look_inside_me_ = False + return func + class Entry(ExtRegistryEntry): _about_ = hint @@ -75,12 +79,14 @@ class JitHintError(Exception): """Inconsistency in the JIT hints.""" -PARAMETERS = {'threshold': 40, - 'trace_eagerness': 10, +PARAMETERS = {'threshold': 1000, + 'trace_eagerness': 200, 'hash_bits': 14, } unroll_parameters = unrolling_iterable(PARAMETERS.keys()) +# ____________________________________________________________ + class JitDriver: """Base class to declare fine-grained user control on the JIT. So far, there must be a singleton instance of JitDriver. This style @@ -90,7 +96,8 @@ virtualizables = [] - def __init__(self, greens=None, reds=None, virtualizables=None): + def __init__(self, greens=None, reds=None, virtualizables=None, + can_inline=None, get_printable_location=None): if greens is not None: self.greens = greens if reds is not None: @@ -102,27 +109,25 @@ for v in self.virtualizables: assert v in self.reds self._alllivevars = dict.fromkeys(self.greens + self.reds) - self._params = PARAMETERS.copy() - if hasattr(self, 'on_enter_jit'): - self._make_on_enter_jit_wrappers() self._make_extregistryentries() + self.get_printable_location = get_printable_location + self.can_inline = can_inline def _freeze_(self): return True - def jit_merge_point(self, **livevars): + def jit_merge_point(_self, **livevars): # special-cased by ExtRegistryEntry - assert dict.fromkeys(livevars) == self._alllivevars + assert dict.fromkeys(livevars) == _self._alllivevars - def can_enter_jit(self, **livevars): + def can_enter_jit(_self, **livevars): # special-cased by ExtRegistryEntry - assert dict.fromkeys(livevars) == self._alllivevars + assert dict.fromkeys(livevars) == _self._alllivevars def _set_param(self, name, value): # special-cased by ExtRegistryEntry # (internal, must receive a constant 'name') assert name in PARAMETERS - self._params[name] = value def set_param(self, name, value): """Set one of the tunable JIT parameter.""" @@ -151,77 +156,6 @@ self.set_param(name, value) set_user_param._annspecialcase_ = 'specialize:arg(0)' - def compute_invariants(self, reds, *greens): - """This can compute a value or tuple that is passed as a green - argument 'invariants' to on_enter_jit(). It should in theory - only depend on the 'greens', but in practice it can peek at the - reds currently stored in 'self'. This allows the extraction in - an interpreter-specific way of whatever red information that - ultimately depends on the greens only. - """ - compute_invariants._annspecialcase_ = 'specialize:arg(0)' - - def _emulate_method_calls(self, bk, livevars_s): - # annotate "self.on_enter_jit()" if it is defined. - # self.on_enter_jit(invariants, reds, *greenvars) is called with a - # copy of the value of the red variables in 'reds'. The red variables - # can be modified in order to give hints to the JIT about the - # redboxes. - from pypy.annotation import model as annmodel - if hasattr(self, 'on_enter_jit'): - args_s = [] - for name in self.greens + self.reds: - args_s.append(livevars_s['s_' + name]) - - key = "rlib.jit.JitDriver._on_enter_jit" - s_func = bk.immutablevalue(self._on_enter_jit_wrapper) - s_result = bk.emulate_pbc_call(key, s_func, args_s) - assert annmodel.s_None.contains(s_result) - - key = "rlib.jit.JitDriver._compute_invariants" - s_func = bk.immutablevalue(self._compute_invariants_wrapper) - bk.emulate_pbc_call(key, s_func, args_s) - - def _make_on_enter_jit_wrappers(self): - # build some unrolling wrappers around on_enter_jit() and - # compute_invariants() which takes all green and red arguments - # and puts the red ones in a fresh instance of the - # RedVarsHolder. This logic is here in jit.py because it needs - # to be annotated and rtyped as a high-level function. - - num_green_args = len(self.greens) - unroll_reds = unrolling_iterable(self.reds) - - class RedVarsHolder: - def __init__(self, *redargs): - i = 0 - for name in unroll_reds: - setattr(self, name, redargs[i]) - i += 1 - - self._RedVarsHolder = RedVarsHolder - - def _on_enter_jit_wrapper(*allargs): - # This is what theoretically occurs when we are entering the - # JIT. In truth, compute_invariants() is called only once - # per set of greens and its result is cached. On the other - # hand, on_enter_jit() is compiled into machine code and so - # it runs every time the execution jumps from the regular - # interpreter to the machine code. Also note that changes - # to the attribute of RedVarsHolder are reflected back in - # the caller. - reds = RedVarsHolder(*allargs[num_green_args:]) - greens = allargs[:num_green_args] - invariants = self.compute_invariants(reds, *greens) - return self.on_enter_jit(invariants, reds, *greens) - self._on_enter_jit_wrapper = _on_enter_jit_wrapper - - def _compute_invariants_wrapper(*allargs): - reds = RedVarsHolder(*allargs[num_green_args:]) - greens = allargs[:num_green_args] - return self.compute_invariants(reds, *greens) - self._compute_invariants_wrapper = _compute_invariants_wrapper - def _make_extregistryentries(self): # workaround: we cannot declare ExtRegistryEntries for functions # used as methods of a frozen object, but we can attach the @@ -257,7 +191,9 @@ raise JitHintError("%s expects the following keyword " "arguments: %s" % (self.instance, expected)) - driver._emulate_method_calls(self.bookkeeper, kwds_s) + for name in driver.greens: + s_green_key = kwds_s['s_' + name] + s_green_key.hash() # force the hash cache to appear return annmodel.s_None def specialize_call(self, hop, **kwds_i): @@ -296,6 +232,7 @@ def specialize_call(self, hop): from pypy.rpython.lltypesystem import lltype + hop.exception_cannot_occur() driver = self.instance.im_self name = hop.args_s[0].const v_value = hop.inputarg(lltype.Signed, arg=1) Modified: pypy/branch/avm/pypy/translator/backendopt/all.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/all.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/all.py Thu Nov 5 21:50:19 2009 @@ -55,6 +55,15 @@ if translator.rtyper.type_system.name == 'ootypesystem': check_virtual_methods() + if config.remove_asserts: + constfold(config, graphs) + remove_asserts(translator, graphs) + + if config.really_remove_asserts: + for graph in graphs: + removenoops.remove_debug_assert(graph) + # the dead operations will be killed by the remove_obvious_noops below + # remove obvious no-ops def remove_obvious_noops(): for graph in graphs: @@ -114,9 +123,6 @@ call_count_pred=call_count_pred) constfold(config, graphs) - if config.remove_asserts: - remove_asserts(translator, graphs) - if config.heap2stack: assert graphs is translator.graphs # XXX for now malloc_to_stack(translator) Modified: pypy/branch/avm/pypy/translator/backendopt/removenoops.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/removenoops.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/removenoops.py Thu Nov 5 21:50:19 2009 @@ -101,6 +101,12 @@ "removed %s cast_pointers in %s" % (num_removed, graph.name)) return num_removed +def remove_debug_assert(graph): + for block in graph.iterblocks(): + for i, op in list(enumerate(block.operations))[::-1]: + if op.opname == "debug_assert": + del block.operations[i] + def remove_superfluous_keep_alive(graph): for block in graph.iterblocks(): used = {} Modified: pypy/branch/avm/pypy/translator/backendopt/test/test_all.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/test/test_all.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/test/test_all.py Thu Nov 5 21:50:19 2009 @@ -5,7 +5,7 @@ from pypy.translator.backendopt.test.test_malloc import TestLLTypeMallocRemoval as LLTypeMallocRemovalTest from pypy.translator.backendopt.test.test_malloc import TestOOTypeMallocRemoval as OOTypeMallocRemovalTest from pypy.translator.translator import TranslationContext, graphof -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Constant, summary from pypy.annotation import model as annmodel from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.rarithmetic import intmask @@ -211,6 +211,27 @@ assert Constant(7) not in link.args assert Constant(11) not in link.args + def test_isinstance(self): + class A: + pass + class B(A): + pass + def g(n): + if n > 10: + return A() + else: + b = B() + b.value = 321 + return b + def fn(n): + x = g(n) + assert isinstance(x, B) + return x.value + t = self.translateopt(fn, [int], really_remove_asserts=True, + remove_asserts=True) + graph = graphof(t, fn) + assert "direct_call" not in summary(graph) + class TestLLType(BaseTester): type_system = 'lltype' check_malloc_removed = LLTypeMallocRemovalTest.check_malloc_removed Modified: pypy/branch/avm/pypy/translator/backendopt/test/test_merge_if_blocks.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/test/test_merge_if_blocks.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/test/test_merge_if_blocks.py Thu Nov 5 21:50:19 2009 @@ -193,3 +193,33 @@ actual = interp.eval_graph(graph, [i]) assert actual == expected +def test_replace_exitswitch_by_constant_bug(): + class X: + pass + def constant9(): + x = X() + x.n = 3 + x.n = 9 + return x.n + def fn(): + n = constant9() + if n == 1: return 5 + elif n == 2: return 6 + elif n == 3: return 8 + elif n == 4: return -123 + elif n == 5: return 12973 + else: return n + + t = TranslationContext() + a = t.buildannotator() + a.build_types(fn, []) + rtyper = t.buildrtyper() + rtyper.specialize() + graph = t.graphs[0] + remove_same_as(graph) + merge_if_blocks_once(graph) + from pypy.translator.backendopt import malloc, inline + inline.auto_inlining(t, 20) + malloc.remove_mallocs(t, t.graphs) + from pypy.translator import simplify + simplify.join_blocks(graph) Modified: pypy/branch/avm/pypy/translator/backendopt/test/test_removeassert.py ============================================================================== --- pypy/branch/avm/pypy/translator/backendopt/test/test_removeassert.py (original) +++ pypy/branch/avm/pypy/translator/backendopt/test/test_removeassert.py Thu Nov 5 21:50:19 2009 @@ -23,6 +23,7 @@ remove_asserts(t, [graph]) assert contains_raise(graph) == remaining_raise check_graph(graph, args, expected_result, t) + return t, graph def test_simple(): @@ -40,6 +41,12 @@ remove_asserts(t, [graph]) assert summary(graph) == {'int_ge': 1, 'debug_assert': 1, 'int_sub': 1} check_graph(graph, [1], 0, t) + from pypy.translator.backendopt.removenoops import remove_debug_assert + remove_debug_assert(graph) + assert summary(graph) == {'int_ge': 1, 'int_sub': 1} + from pypy.translator.simplify import transform_dead_op_vars + transform_dead_op_vars(graph) + assert summary(graph) == {'int_sub': 1} def test_and(): def fn(n): @@ -69,7 +76,14 @@ x = g(n) assert isinstance(x, B) return x.value - check(fn, [5], 321) + t, graph = check(fn, [5], 321) + assert summary(graph)['debug_assert'] == 1 + from pypy.translator.backendopt.removenoops import remove_debug_assert + remove_debug_assert(graph) + assert "debug_assert" not in summary(graph) + from pypy.translator.simplify import transform_dead_op_vars + transform_dead_op_vars(graph, t) + assert summary(graph)["direct_call"] == 1 def test_with_exception(): def g(n): @@ -82,3 +96,5 @@ except ValueError: return 42 check(fn, [-8], 42, remaining_raise=True) + + Modified: pypy/branch/avm/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/avm/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/avm/pypy/translator/c/gcc/test/test_trackgcroot.py Thu Nov 5 21:50:19 2009 @@ -46,7 +46,7 @@ assert len(bytes) == 1+1+2+3+4+4+5+5+1 assert decompress_callshape(bytes) == list(shape) -def test_find_functions(): +def test_find_functions_elf(): source = """\ \t.p2align 4,,15 .globl pypy_g_make_tree @@ -70,29 +70,64 @@ assert parts[3] == (True, lines[8:11]) assert parts[4] == (False, lines[11:]) - +def test_find_functions_darwin(): + source = """\ +\t.text +\t.align 4,0x90 +.globl _pypy_g_ll_str__StringR_Ptr_GcStruct_rpy_strin_rpy_strin +_pypy_g_ll_str__StringR_Ptr_GcStruct_rpy_strin_rpy_strin: +L0: +\tFOO +\t.align 4,0x90 +_static: +\tSTATIC +\t.align 4,0x90 +.globl _pypy_g_ll_issubclass__object_vtablePtr_object_vtablePtr +_pypy_g_ll_issubclass__object_vtablePtr_object_vtablePtr: +\tBAR +\t.cstring +\t.ascii "foo" +\t.text +\t.align 4,0x90 +.globl _pypy_g_RPyRaiseException +_pypy_g_RPyRaiseException: +\tBAZ +\t.section stuff +""" + lines = source.splitlines(True) + parts = list(GcRootTracker(format='darwin').find_functions(iter(lines))) + assert len(parts) == 7 + assert parts[0] == (False, lines[:3]) + assert parts[1] == (True, lines[3:7]) + assert parts[2] == (True, lines[7:11]) + assert parts[3] == (True, lines[11:13]) + assert parts[4] == (False, lines[13:18]) + assert parts[5] == (True, lines[18:20]) + assert parts[6] == (False, lines[20:]) + def test_computegcmaptable(): tests = [] - for path in this_dir.listdir("track*.s"): - n = path.purebasename[5:] - try: - n = int(n) - except ValueError: - pass - tests.append((n, path)) + for format in ('elf', 'darwin'): + for path in this_dir.join(format).listdir("track*.s"): + n = path.purebasename[5:] + try: + n = int(n) + except ValueError: + pass + tests.append((format, n, path)) tests.sort() - for _, path in tests: - yield check_computegcmaptable, path + for format, _, path in tests: + yield check_computegcmaptable, format, path r_globallabel = re.compile(r"([\w]+)[:]") r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") -def check_computegcmaptable(path): +def check_computegcmaptable(format, path): print print path.basename lines = path.readlines() expectedlines = lines[:] - tracker = FunctionGcRootTracker(lines) + tracker = FunctionGcRootTracker(lines, format=format) table = tracker.computegcmaptable(verbose=sys.maxint) tabledict = {} seen = {} Modified: pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/avm/pypy/translator/c/gcc/trackgcroot.py Thu Nov 5 21:50:19 2009 @@ -2,13 +2,32 @@ import re, sys, os, random -r_functionstart = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") -r_functionend = re.compile(r"\t.size\s+(\w+),\s*[.]-(\w+)\s*$") +r_functionstart_elf = re.compile(r"\t.type\s+(\w+),\s*[@]function\s*$") +r_functionend_elf = re.compile(r"\t.size\s+(\w+),\s*[.]-(\w+)\s*$") + +# darwin +r_textstart = re.compile(r"\t.text\s*$") +# see +# http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html +OTHERSECTIONS = ['section', 'zerofill', + 'const', 'static_const', 'cstring', + 'literal4', 'literal8', 'literal16', + 'constructor', 'desctructor', + 'symbol_stub', + 'data', 'static_data', + 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', + 'dyld', 'mod_init_func', 'mod_term_func', + 'const_data' + ] +r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") +r_functionstart_darwin = re.compile(r"_(\w+):\s*$") + +# inside functions r_label = re.compile(r"([.]?\w+)[:]\s*$") r_globl = re.compile(r"\t[.]globl\t(\w+)\s*$") r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jump = re.compile(r"\tj\w+\s+([.]?\w+)\s*$") -OPERAND = r"(?:[-\w$%+.:@]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])" +r_jump = re.compile(r"\tj\w+\s+([.]?[\w$]+)\s*$") +OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") r_jmp_switch = re.compile(r"\tjmp\t[*]([.]?\w+)[(]") @@ -26,9 +45,10 @@ class GcRootTracker(object): - def __init__(self, verbose=0, shuffle=False): + def __init__(self, verbose=0, shuffle=False, format='elf'): self.verbose = verbose self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py + self.format = format self.clear() def clear(self): @@ -54,10 +74,26 @@ shapes = {} shapelines = [] shapeofs = 0 - print >> output, """\t.text - .globl pypy_asm_stackwalk - .type pypy_asm_stackwalk, @function - pypy_asm_stackwalk: + def _globalname(name): + if self.format == 'darwin': + return '_' + name + return name + def _globl(name): + print >> output, "\t.globl %s" % _globalname(name) + def _label(name): + print >> output, "%s:" % _globalname(name) + def _variant(elf, darwin): + if self.format == 'darwin': + txt = darwin + else: + txt = elf + print >> output, "\t%s" % txt + + print >> output, "\t.text" + _globl('pypy_asm_stackwalk') + _variant('.type pypy_asm_stackwalk, @function', '') + _label('pypy_asm_stackwalk') + print >> output, """\ /* See description in asmgcroot.py */ movl 4(%esp), %edx /* my argument, which is the callback */ movl %esp, %eax /* my frame top address */ @@ -67,21 +103,22 @@ pushl %esi /* ASM_FRAMEDATA[1] */ pushl %ebx /* ASM_FRAMEDATA[0] */ movl %esp, %eax /* address of ASM_FRAMEDATA */ - pushl %eax + pushl %eax /* respect Mac OS X 16 bytes aligment */ + pushl %eax /* the one argument to the callback */ call *%edx /* invoke the callback */ - popl %eax + addl $8, %esp popl %ebx /* restore from ASM_FRAMEDATA[0] */ popl %esi /* restore from ASM_FRAMEDATA[1] */ popl %edi /* restore from ASM_FRAMEDATA[2] */ popl %ebp /* restore from ASM_FRAMEDATA[3] */ popl %eax ret - .size pypy_asm_stackwalk_init, .-pypy_asm_stackwalk_init - """ +""" + _variant('.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', '') print >> output, '\t.data' print >> output, '\t.align\t4' - print >> output, '\t.globl\t__gcmapstart' - print >> output, '__gcmapstart:' + _globl('__gcmapstart') + _label('__gcmapstart') for label, state, is_range in self.gcmaptable: try: n = shapes[state] @@ -96,25 +133,29 @@ n = ~ n print >> output, '\t.long\t%s' % (label,) print >> output, '\t.long\t%d' % (n,) - print >> output, '\t.globl\t__gcmapend' - print >> output, '__gcmapend:' - print >> output, '\t.section\t.rodata' - print >> output, '\t.globl\t__gccallshapes' - print >> output, '__gccallshapes:' + _globl('__gcmapend') + _label('__gcmapend') + _variant('.section\t.rodata', '.const') + _globl('__gccallshapes') + _label('__gccallshapes') output.writelines(shapelines) def find_functions(self, iterlines): + _find_functions = getattr(self, '_find_functions_' + self.format) + return _find_functions(iterlines) + + def _find_functions_elf(self, iterlines): functionlines = [] in_function = False for line in iterlines: - if r_functionstart.match(line): + if r_functionstart_elf.match(line): assert not in_function, ( "missed the end of the previous function") yield False, functionlines in_function = True functionlines = [] functionlines.append(line) - if r_functionend.match(line): + if r_functionend_elf.match(line): assert in_function, ( "missed the start of the current function") yield True, functionlines @@ -124,16 +165,46 @@ "missed the end of the previous function") yield False, functionlines + def _find_functions_darwin(self, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if r_textstart.match(line): + assert not in_text, "unexpected repeated .text start: %d" % n + in_text = True + elif r_sectionstart.match(line): + if in_function: + yield in_function, functionlines + functionlines = [] + in_text = False + in_function = False + elif in_text and r_functionstart_darwin.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + + if functionlines: + yield in_function, functionlines + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + if self.format == 'darwin': + entrypoint = '_' + entrypoint for in_function, lines in self.find_functions(iterlines): if in_function: lines = self.process_function(lines, entrypoint, filename) newfile.writelines(lines) + if self.verbose == 1: + sys.stderr.write('\n') def process_function(self, lines, entrypoint, filename): - tracker = FunctionGcRootTracker(lines, filetag=getidentifier(filename)) + tracker = FunctionGcRootTracker(lines, filetag=getidentifier(filename), + format=self.format) tracker.is_main = tracker.funcname == entrypoint - if self.verbose: + if self.verbose == 1: + sys.stderr.write('.') + elif self.verbose > 1: print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, tracker.funcname) table = tracker.computegcmaptable(self.verbose) @@ -151,12 +222,20 @@ class FunctionGcRootTracker(object): - def __init__(self, lines, filetag=0): - match = r_functionstart.match(lines[0]) - self.funcname = match.group(1) - match = r_functionend.match(lines[-1]) - assert self.funcname == match.group(1) - assert self.funcname == match.group(2) + def __init__(self, lines, filetag=0, format='elf'): + if format == 'elf': + match = r_functionstart_elf.match(lines[0]) + funcname = match.group(1) + match = r_functionend_elf.match(lines[-1]) + assert funcname == match.group(1) + assert funcname == match.group(2) + elif format == 'darwin': + match = r_functionstart_darwin.match(lines[0]) + funcname = '_'+match.group(1) + else: + assert False, "unknown format: %s" % format + + self.funcname = funcname self.lines = lines self.uses_frame_pointer = False self.r_localvar = r_localvarnofp @@ -204,7 +283,8 @@ loc = localvar.getlocation(insn.framesize, self.uses_frame_pointer) else: - assert localvar in REG2LOC + assert localvar in REG2LOC, "%s: %s" % (self.funcname, + localvar) loc = REG2LOC[localvar] assert isinstance(loc, int) if tag is None: @@ -232,13 +312,13 @@ def parse_instructions(self): self.insns = [InsnFunctionStart()] - in_APP = False + ignore_insns = False for lineno, line in enumerate(self.lines): self.currentlineno = lineno insn = [] match = r_insn.match(line) if match: - if not in_APP: + if not ignore_insns: opname = match.group(1) try: meth = getattr(self, 'visit_' + opname) @@ -247,10 +327,10 @@ insn = meth(line) elif r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) - elif line == '#APP\n': - in_APP = True - elif line == '#NO_APP\n': - in_APP = False + elif line == '\t/* ignore_in_trackgcroot */\n': + ignore_insns = True + elif line == '\t/* end_ignore_in_trackgcroot */\n': + ignore_insns = False else: match = r_label.match(line) if match: @@ -430,7 +510,8 @@ 'rep', 'movs', 'lods', 'stos', 'scas', 'cwtl', 'prefetch', # floating-point operations cannot produce GC pointers 'f', - 'cvt', # sse2 + 'cvt', 'ucomi', 'subs', 'subp' , 'adds', 'addp', 'xorp', 'movap', + 'mins', 'minp', 'maxs', 'maxp', # sse2 # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', @@ -652,6 +733,11 @@ target = match.group(1) if target in FUNCTIONS_NOT_RETURNING: return InsnStop() + if target in self.labels: + lineoffset = self.labels[target].lineno - self.currentlineno + if lineoffset >= 0: + assert lineoffset in (1,2) + return [InsnStackAdjust(-4)] return [InsnCall(self.currentlineno), InsnSetLocal('%eax')] # the result is there @@ -847,6 +933,8 @@ 'abort': None, '_exit': None, '__assert_fail': None, + '___assert_rtn': None, + 'L___assert_rtn$stub': None } CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] @@ -1018,7 +1106,11 @@ output_raw_table = True else: break - tracker = GcRootTracker(verbose=verbose, shuffle=shuffle) + if sys.platform == 'darwin': + format = 'darwin' + else: + format = 'elf' + tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) for fn in sys.argv[1:]: f = open(fn, 'r') firstline = f.readline() Modified: pypy/branch/avm/pypy/translator/c/src/asm_gcc_x86.h ============================================================================== --- pypy/branch/avm/pypy/translator/c/src/asm_gcc_x86.h (original) +++ pypy/branch/avm/pypy/translator/c/src/asm_gcc_x86.h Thu Nov 5 21:50:19 2009 @@ -4,12 +4,15 @@ #undef OP_INT_ADD_OVF #define OP_INT_ADD_OVF(x,y,r) \ - asm volatile("addl %2,%0\n\t" \ + asm volatile( \ + "/* ignore_in_trackgcroot */\n\t" \ + "addl %2,%0\n\t" \ "jno 0f\n\t" \ "pusha\n\t" \ - "call op_int_overflowed\n\t" \ + "call _op_int_overflowed\n\t" \ "popa\n\t" \ - "0:" \ + "0:\n\t" \ + "/* end_ignore_in_trackgcroot */" \ : "=r"(r) /* outputs */ \ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ @@ -19,24 +22,30 @@ #undef OP_INT_SUB_OVF #define OP_INT_SUB_OVF(x,y,r) \ - asm volatile("subl %2,%0\n\t" \ + asm volatile( \ + "/* ignore_in_trackgcroot */\n\t" \ + "subl %2,%0\n\t" \ "jno 0f\n\t" \ "pusha\n\t" \ - "call op_int_overflowed\n\t" \ + "call _op_int_overflowed\n\t" \ "popa\n\t" \ - "0:" \ + "0:\n\t" \ + "/* end_ignore_in_trackgcroot */" \ : "=r"(r) /* outputs */ \ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ #undef OP_INT_MUL_OVF #define OP_INT_MUL_OVF(x,y,r) \ - asm volatile("imull %2,%0\n\t" \ + asm volatile( \ + "/* ignore_in_trackgcroot */\n\t" \ + "imull %2,%0\n\t" \ "jno 0f\n\t" \ "pusha\n\t" \ - "call op_int_overflowed\n\t" \ + "call _op_int_overflowed\n\t" \ "popa\n\t" \ - "0:" \ + "0:\n\t" \ + "/* end_ignore_in_trackgcroot */" \ : "=r"(r) /* outputs */ \ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ @@ -45,7 +54,7 @@ /* prototypes */ extern void op_int_overflowed(void) - asm ("op_int_overflowed") + asm ("_op_int_overflowed") __attribute__((used)); /* implementations */ Modified: pypy/branch/avm/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/avm/pypy/translator/c/src/mem.h (original) +++ pypy/branch/avm/pypy/translator/c/src/mem.h Thu Nov 5 21:50:19 2009 @@ -20,7 +20,7 @@ extern char __gcmapstart; extern char __gcmapend; extern char __gccallshapes; -extern char __gcnoreorderhack; +#define __gcnoreorderhack __gcmapend /* The following pseudo-instruction is used by --gcrootfinder=asmgcc just after a call to tell gcc to put a GCROOT mark on each gc-pointer Modified: pypy/branch/avm/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/avm/pypy/translator/goal/app_main.py (original) +++ pypy/branch/avm/pypy/translator/goal/app_main.py Thu Nov 5 21:50:19 2009 @@ -122,7 +122,18 @@ def print_help(): print 'usage: %s [options]' % (sys.executable,) - print __doc__ + print __doc__.rstrip() + if 'pypyjit' in sys.builtin_module_names: + print_jit_help() + print + +def print_jit_help(): + import pypyjit + items = pypyjit.defaults.items() + items.sort() + for key, value in items: + print ' --jit %s=N %slow-level JIT parameter (default %s)' % ( + key, ' '*(18-len(key)), value) class CommandLineError(Exception): pass @@ -161,8 +172,9 @@ def get_argument(option, argv, i): arg = argv[i] - if len(arg) > 2: - return arg[2:], i + n = len(option) + if len(arg) > n: + return arg[n:], i else: i += 1 if i >= len(argv): @@ -269,6 +281,14 @@ break elif arg.startswith('-W'): warnoptions, i = get_argument('-W', argv, i) + elif arg.startswith('--jit'): + jitparam, i = get_argument('--jit', argv, i) + if 'pypyjit' not in sys.builtin_module_names: + print >> sys.stderr, ("Warning: No jit support in %s" % + (sys.executable,)) + else: + import pypyjit + pypyjit.set_param(jitparam) elif arg == '--': i += 1 break # terminates option list Modified: pypy/branch/avm/pypy/translator/simplify.py ============================================================================== --- pypy/branch/avm/pypy/translator/simplify.py (original) +++ pypy/branch/avm/pypy/translator/simplify.py Thu Nov 5 21:50:19 2009 @@ -472,12 +472,7 @@ def transform_dead_op_vars(graph, translator=None): """Remove dead operations and variables that are passed over a link but not used in the target block. Input is a graph.""" - blocks = {} - def visit(block): - if isinstance(block, Block): - blocks[block] = True - traverse(visit, graph) - return transform_dead_op_vars_in_blocks(blocks, translator) + return transform_dead_op_vars_in_blocks(list(graph.iterblocks()), translator) # the set of operations that can safely be removed # (they have no side effects, at least in R-Python) From fijal at codespeak.net Thu Nov 5 21:53:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 5 Nov 2009 21:53:05 +0100 (CET) Subject: [pypy-svn] r69008 - pypy/extradoc/talk/rupy2009 Message-ID: <20091105205305.075A4168433@codespeak.net> Author: fijal Date: Thu Nov 5 21:53:05 2009 New Revision: 69008 Added: pypy/extradoc/talk/rupy2009/talk.tex Modified: pypy/extradoc/talk/rupy2009/talk.pdf Log: progress.. Modified: pypy/extradoc/talk/rupy2009/talk.pdf ============================================================================== Files pypy/extradoc/talk/rupy2009/talk.pdf (original) and pypy/extradoc/talk/rupy2009/talk.pdf Thu Nov 5 21:53:05 2009 differ Added: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/talk.tex Thu Nov 5 21:53:05 2009 @@ -0,0 +1,112 @@ +\documentclass[utf8x]{beamer} + +\mode +{ + \usetheme{Warsaw} + + %\setbeamercovered{transparent} +} + +\usepackage[english]{babel} + +\usepackage[utf8x]{inputenc} + +\usepackage{times} +\usepackage[T1]{fontenc} + +\title{The speed of PyPy} + +\author{Maciej Fija?kowski} + +\institute[merlinux GmbH] +{ merlinux GmbH } + +\date{RuPy, November 7th 2009, Pozna?} + +\begin{document} + +\begin{frame} + \titlepage +\end{frame} + +\begin{frame} + + \frametitle{Story about Python's speed} + \begin{figure} + \includegraphics[width=.8\textwidth]{img1.jpg} + \end{figure} + +\end{frame} + +\begin{frame} + \frametitle{Speed of Python} + \begin{itemize} + \item Python is slow + \end{itemize} + \pause + \begin{figure} + \includegraphics[width=.6\textwidth]{peasant_and_birdnester-400.jpg} + \end{figure} + +\end{frame} + +\begin{frame} + \frametitle{Speed of Python} + \begin{itemize} + \item Is Python really slow? + \pause + \item Sometimes + \pause + \item Let's have a look at some examples + \end{itemize} + +\end{frame} + +\begin{frame} + \frametitle{Nomenclature} + \begin{itemize} + \item Python - a programming language + \item CPython - main implementation of Python + xxx + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Example run} + \begin{itemize} + \item Float example, stolen from factor blog + \end{itemize} + \vspace{.5cm} + \begin{tabular}{| l | c | r |} + \hline + & CPython & JVM (client mode) \\ + Average of 10 runs: & 7.6s & 0.77s \\ + \hline + \end{tabular} + \vspace{.5cm} + \pause + \begin{itemize} + \item Python is 10x slower than Java + \pause + \item Python is 10x slower than Java on this particular benchmark + \pause + \item CPython is 10x slower than Java on this particular benchmark + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{More about this example} + \begin{tabular}{| l | c | c | c | c |} + \hline + & CPython & JVM & Psyco & PyPy \\ + Average of 10 runs & 7.6s & 0.77s & 4.4s & 1.3s \\ + \hline + \end{tabular} + \vspace{.5cm} + \pause + \begin{itemize} + \item So, it's CPython that is slow on this particular benchmark + \end{itemize} +\end{frame} + +\end{document} From afa at codespeak.net Thu Nov 5 22:17:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 22:17:25 +0100 (CET) Subject: [pypy-svn] r69009 - in pypy/branch/msvc-asmgcroot/pypy: . config config/test doc doc/config doc/jit interpreter interpreter/astcompiler interpreter/astcompiler/test interpreter/pyparser interpreter/test jit/backend jit/backend/cli jit/backend/llgraph jit/backend/llsupport jit/backend/llsupport/test jit/backend/llvm/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test jit/tl jit/tool jit/tool/test lang/prolog/interpreter lang/smalltalk module/__builtin__ module/__builtin__/test module/marshal/test module/operator module/pypyjit module/pypyjit/test module/pypyjit/test/loops module/sys module/sys/test objspace objspace/flow objspace/std objspace/std/test rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test rpython/ootypesystem rpython/test rpython/tool tool tool/test translator/backendopt translator/backendopt/test translator/benchmark translator/c translator/c/gcc translator/c/gcc/test translator/c/src translator/c/test translator/cli translator/jvm translator/platform Message-ID: <20091105211725.7F60D168439@codespeak.net> Author: afa Date: Thu Nov 5 22:17:18 2009 New Revision: 69009 Added: pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withinlineddict.txt - copied unchanged from r68993, pypy/trunk/pypy/doc/config/objspace.std.withinlineddict.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.gcremovetypeptr.txt - copied unchanged from r68993, pypy/trunk/pypy/doc/config/translation.gcremovetypeptr.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.log.txt - copied unchanged from r68993, pypy/trunk/pypy/doc/config/translation.log.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.taggedpointers.txt - copied unchanged from r68993, pypy/trunk/pypy/doc/config/translation.taggedpointers.txt pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/detect_sse2.py - copied unchanged from r68993, pypy/trunk/pypy/jit/backend/x86/detect_sse2.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/support.py - copied unchanged from r68993, pypy/trunk/pypy/jit/backend/x86/support.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_support.py - copied unchanged from r68993, pypy/trunk/pypy/jit/backend/x86/test/test_support.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_warmstate.py - copied unchanged from r68993, pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/warmstate.py - copied unchanged from r68993, pypy/trunk/pypy/jit/metainterp/warmstate.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/ (props changed) - copied from r68993, pypy/trunk/pypy/jit/tool/ pypy/branch/msvc-asmgcroot/pypy/jit/tool/__init__.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/__init__.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/autopath.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/autopath.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/jitoutput.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/jitoutput.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/loopviewer.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/loopviewer.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/showstats.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/showstats.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/test/ (props changed) - copied from r68993, pypy/trunk/pypy/jit/tool/test/ pypy/branch/msvc-asmgcroot/pypy/jit/tool/test/__init__.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/test/__init__.py pypy/branch/msvc-asmgcroot/pypy/jit/tool/test/test_jitoutput.py - copied unchanged from r68993, pypy/trunk/pypy/jit/tool/test/test_jitoutput.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/interp_inspect.py - copied unchanged from r68993, pypy/trunk/pypy/module/__builtin__/interp_inspect.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/inlinedict.py - copied unchanged from r68993, pypy/trunk/pypy/objspace/std/inlinedict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_inlinedict.py - copied unchanged from r68993, pypy/trunk/pypy/objspace/std/test/test_inlinedict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_setobject.py - copied unchanged from r68993, pypy/trunk/pypy/objspace/std/test/test_setobject.py pypy/branch/msvc-asmgcroot/pypy/tool/logparser.py - copied unchanged from r68993, pypy/trunk/pypy/tool/logparser.py pypy/branch/msvc-asmgcroot/pypy/tool/test/test_logparser.py - copied unchanged from r68993, pypy/trunk/pypy/tool/test/test_logparser.py pypy/branch/msvc-asmgcroot/pypy/translator/benchmark/jitbench.py - copied unchanged from r68993, pypy/trunk/pypy/translator/benchmark/jitbench.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/debug.h - copied unchanged from r68993, pypy/trunk/pypy/translator/c/src/debug.h Removed: pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withblist.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withbucketdict.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withchunklist.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withfastslice.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withmultidict.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withmultilist.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withsmartresizablelist.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.backendopt.heap2stack.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.gcconfig.debugprint.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.gcconfig.removetypeptr.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/translation.gcconfig.txt pypy/branch/msvc-asmgcroot/pypy/jit/backend/loopviewer.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/showstats.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/loops/ pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictbucket.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/listmultiobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/smartresizablelist.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_blistimplementation.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictbucket.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_listmultiobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_resizable.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_set.py Modified: pypy/branch/msvc-asmgcroot/pypy/ (props changed) pypy/branch/msvc-asmgcroot/pypy/config/pypyoption.py pypy/branch/msvc-asmgcroot/pypy/config/test/test_pypyoption.py pypy/branch/msvc-asmgcroot/pypy/config/translationoption.py pypy/branch/msvc-asmgcroot/pypy/doc/config/ (props changed) pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withshadowtracking.txt pypy/branch/msvc-asmgcroot/pypy/doc/config/opt.txt pypy/branch/msvc-asmgcroot/pypy/doc/cpython_differences.txt pypy/branch/msvc-asmgcroot/pypy/doc/faq.txt pypy/branch/msvc-asmgcroot/pypy/doc/getting-started-python.txt pypy/branch/msvc-asmgcroot/pypy/doc/interpreter-optimizations.txt pypy/branch/msvc-asmgcroot/pypy/doc/jit/pyjitpl5.txt pypy/branch/msvc-asmgcroot/pypy/doc/translation.txt pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/assemble.py pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/codegen.py pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/consts.py pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/msvc-asmgcroot/pypy/interpreter/baseobjspace.py pypy/branch/msvc-asmgcroot/pypy/interpreter/executioncontext.py pypy/branch/msvc-asmgcroot/pypy/interpreter/function.py pypy/branch/msvc-asmgcroot/pypy/interpreter/gateway.py pypy/branch/msvc-asmgcroot/pypy/interpreter/main.py pypy/branch/msvc-asmgcroot/pypy/interpreter/mixedmodule.py pypy/branch/msvc-asmgcroot/pypy/interpreter/nestedscope.py pypy/branch/msvc-asmgcroot/pypy/interpreter/pycode.py pypy/branch/msvc-asmgcroot/pypy/interpreter/pyopcode.py pypy/branch/msvc-asmgcroot/pypy/interpreter/pyparser/future.py pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_code.py pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_executioncontext.py pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_function.py pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_typedef.py pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/branch/msvc-asmgcroot/pypy/interpreter/typedef.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/cli/runner.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/detect_cpu.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/llimpl.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/runner.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/gc.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/llmodel.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/branch/msvc-asmgcroot/pypy/jit/backend/test/support.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/assembler.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/ri386setup.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/runner.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_zrpy_gc.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/compile.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/history.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/jitprof.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/logger.py (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimize.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimizeopt.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/policy.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/pyjitpl.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/resume.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/support.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/oparser.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_compile.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_jitprof.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_logger.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_loop.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_oparser.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_recursive.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_resume.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_virtual.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_vlist.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/virtualizable.py pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/warmspot.py pypy/branch/msvc-asmgcroot/pypy/jit/tl/pypyjit.py pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/arithmetic.py pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/term.py pypy/branch/msvc-asmgcroot/pypy/lang/smalltalk/model.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/__init__.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/app_inspect.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/functional.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/interp_classobj.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_builtin.py pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_classobj.py pypy/branch/msvc-asmgcroot/pypy/module/marshal/test/test_marshal.py pypy/branch/msvc-asmgcroot/pypy/module/operator/__init__.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/interp_jit.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/conftest.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_can_inline.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_jit_setup.py pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/msvc-asmgcroot/pypy/module/sys/__init__.py pypy/branch/msvc-asmgcroot/pypy/module/sys/test/test_sysmodule.py pypy/branch/msvc-asmgcroot/pypy/objspace/descroperation.py pypy/branch/msvc-asmgcroot/pypy/objspace/flow/objspace.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/callmethod.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/celldict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictmultiobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/dicttype.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/listtype.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/marshal_impl.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/model.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/objectobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/objecttype.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/objspace.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/proxyobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/setobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/sharingdict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/stdtypedef.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_builtinshortcut.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_celldict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_obj.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_proxy.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_shadowtracking.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_sharingdict.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_typeobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/typeobject.py pypy/branch/msvc-asmgcroot/pypy/objspace/std/typetype.py pypy/branch/msvc-asmgcroot/pypy/objspace/taint.py pypy/branch/msvc-asmgcroot/pypy/rlib/debug.py pypy/branch/msvc-asmgcroot/pypy/rlib/jit.py pypy/branch/msvc-asmgcroot/pypy/rlib/objectmodel.py pypy/branch/msvc-asmgcroot/pypy/rlib/rgc.py pypy/branch/msvc-asmgcroot/pypy/rlib/rmmap.py pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_debug.py pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_objectmodel.py pypy/branch/msvc-asmgcroot/pypy/rpython/llinterp.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llarena.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llmemory.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lloperation.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lltype.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/opimpl.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/rtagged.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_lloperation.py pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_rtagged.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/base.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/generation.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/hybrid.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/markcompact.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/marksweep.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/semispace.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/framework.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/transform.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctypelayout.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gcwrapper.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gc.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gctypelayout.py pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/msvc-asmgcroot/pypy/rpython/ootypesystem/ootype.py pypy/branch/msvc-asmgcroot/pypy/rpython/rclass.py pypy/branch/msvc-asmgcroot/pypy/rpython/rint.py pypy/branch/msvc-asmgcroot/pypy/rpython/rvirtualizable2.py pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rdict.py pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rptr.py pypy/branch/msvc-asmgcroot/pypy/rpython/tool/rffi_platform.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/all.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/escape.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/inline.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocprediction.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocv.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/stat.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_constfold.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_escape.py pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_mallocprediction.py pypy/branch/msvc-asmgcroot/pypy/translator/c/database.py pypy/branch/msvc-asmgcroot/pypy/translator/c/funcgen.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gc.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/c/node.py pypy/branch/msvc-asmgcroot/pypy/translator/c/src/asm_gcc_x86.h pypy/branch/msvc-asmgcroot/pypy/translator/c/src/g_include.h pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_refcount.py (props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_rtagged.py pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_standalone.py pypy/branch/msvc-asmgcroot/pypy/translator/cli/ilgenerator.py pypy/branch/msvc-asmgcroot/pypy/translator/cli/metavm.py pypy/branch/msvc-asmgcroot/pypy/translator/cli/opcodes.py pypy/branch/msvc-asmgcroot/pypy/translator/cli/sdk.py pypy/branch/msvc-asmgcroot/pypy/translator/jvm/genjvm.py pypy/branch/msvc-asmgcroot/pypy/translator/platform/linux.py Log: More tweaks in trackgcroot for the mingw32 compiler: - the table-based switches table may appear in a .rdata section, inside the function code - the jump address is stored in a registry, set by the instruction just above. If the jump pattern is more complex, no instruction will appear to jump to the missed labels, and trackgcroot will fail. Modified: pypy/branch/msvc-asmgcroot/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/config/pypyoption.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/config/pypyoption.py Thu Nov 5 22:17:18 2009 @@ -117,8 +117,7 @@ OptionDescription("opcodes", "opcodes to enable in the interpreter", [ BoolOption("CALL_LIKELY_BUILTIN", "emit a special bytecode for likely calls to builtin functions", default=False, - requires=[("objspace.std.withmultidict", True), - ("translation.stackless", False)]), + requires=[("translation.stackless", False)]), BoolOption("CALL_METHOD", "emit a special bytecode for expr.name()", default=False), ]), @@ -184,8 +183,8 @@ BoolOption("withsmallint", "use tagged integers", default=False, - requires=[("translation.gc", "boehm"), - ("objspace.std.withprebuiltint", False)]), + requires=[("objspace.std.withprebuiltint", False), + ("translation.taggedpointers", True)]), BoolOption("withprebuiltint", "prebuild commonly used int objects", default=False), @@ -222,33 +221,25 @@ default=False, requires=[("objspace.std.withrope", True)]), - BoolOption("withmultidict", - "use dictionaries optimized for flexibility", - default=False), - BoolOption("withcelldict", "use dictionaries that are optimized for being used as module dicts", default=False, - requires=[("objspace.std.withmultidict", True), - ("objspace.opcodes.CALL_LIKELY_BUILTIN", False), + requires=[("objspace.opcodes.CALL_LIKELY_BUILTIN", False), ("objspace.honor__builtins__", False)]), BoolOption("withsharingdict", "use dictionaries that share the keys part", - default=False, - requires=[("objspace.std.withmultidict", True)]), + default=False), BoolOption("withdictmeasurement", "create huge files with masses of information " "about dictionaries", - default=False, - requires=[("objspace.std.withmultidict", True)]), + default=False), - BoolOption("withbucketdict", - "use dictionaries with chained hash tables " - "(default is open addressing)", + BoolOption("withinlineddict", + "make instances more compact by revoming a level of indirection", default=False, - requires=[("objspace.std.withmultidict", True)]), + requires=[("objspace.std.withshadowtracking", False)]), BoolOption("withrangelist", "enable special range list implementation that does not " @@ -267,8 +258,7 @@ "track whether an instance attribute shadows a type" " attribute", default=False, - requires=[("objspace.std.withmultidict", True), - ("objspace.std.withtypeversion", True), + requires=[("objspace.std.withtypeversion", True), ("translation.rweakref", True)]), BoolOption("withmethodcache", "try to cache method lookups", @@ -283,28 +273,6 @@ IntOption("methodcachesizeexp", " 2 ** methodcachesizeexp is the size of the of the method cache ", default=11), - BoolOption("withmultilist", - "use lists optimized for flexibility", - default=False, - requires=[("objspace.std.withrangelist", False), - ("objspace.name", "std"), - ("objspace.std.withtproxy", False)]), - BoolOption("withfastslice", - "make list slicing lazy", - default=False, - requires=[("objspace.std.withmultilist", True)]), - BoolOption("withchunklist", - "introducing a new nesting level to slow down list operations", - default=False, - requires=[("objspace.std.withmultilist", True)]), - BoolOption("withsmartresizablelist", - "only overallocate O(sqrt(n)) elements for lists", - default=False, - requires=[("objspace.std.withmultilist", True)]), - BoolOption("withblist", - "good asymptotic performance for very large lists", - default=False, - requires=[("objspace.std.withmultilist", True)]), BoolOption("optimized_int_add", "special case the addition of two integers in BINARY_ADD", default=False), @@ -356,7 +324,6 @@ config.objspace.opcodes.suggest(CALL_LIKELY_BUILTIN=True) if level in ['2', '3', 'jit']: config.objspace.opcodes.suggest(CALL_METHOD=True) - config.objspace.std.suggest(withmultidict=True) config.objspace.std.suggest(withshadowtracking=True) config.objspace.std.suggest(withrangelist=True) config.objspace.std.suggest(withmethodcache=True) @@ -395,6 +362,7 @@ if type_system != 'ootype': config.objspace.std.suggest(withsharingdict=True) config.objspace.std.suggest(withcelldict=True) + config.objspace.std.suggest(withinlineddict=True) def enable_allworkingmodules(config): Modified: pypy/branch/msvc-asmgcroot/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/config/test/test_pypyoption.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/config/test/test_pypyoption.py Thu Nov 5 22:17:18 2009 @@ -47,10 +47,10 @@ def test_set_pypy_opt_level(): conf = get_pypy_config() set_pypy_opt_level(conf, '2') - assert conf.objspace.std.withmultidict + assert conf.objspace.std.withshadowtracking conf = get_pypy_config() set_pypy_opt_level(conf, '0') - assert not conf.objspace.std.withmultidict + assert not conf.objspace.std.newshortcut def test_rweakref_required(): conf = get_pypy_config() Modified: pypy/branch/msvc-asmgcroot/pypy/config/translationoption.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/config/translationoption.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/config/translationoption.py Thu Nov 5 22:17:18 2009 @@ -27,7 +27,6 @@ requires={ "ootype": [ ("translation.backendopt.constfold", False), - ("translation.backendopt.heap2stack", False), ("translation.backendopt.clever_malloc_removal", False), ("translation.gc", "boehm"), # it's not really used, but some jit code expects a value here ] @@ -41,6 +40,9 @@ }, cmdline="-b --backend"), + BoolOption("log", "Include debug prints in the translation (PYPYLOG=...)", + default=True, cmdline="--log"), + # gc ChoiceOption("gc", "Garbage Collection Strategy", ["boehm", "ref", "marksweep", "semispace", "statistics", @@ -63,16 +65,15 @@ ["boehm", "ref", "framework", "none"], default="ref", cmdline=None, requires={ - "boehm": [("translation.gcrootfinder", "n/a")], - "ref": [("translation.gcrootfinder", "n/a")], - "none": [("translation.gcrootfinder", "n/a")], + "boehm": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "ref": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], + "none": [("translation.gcrootfinder", "n/a"), + ("translation.gcremovetypeptr", False)], }), - OptionDescription("gcconfig", "Configure garbage collectors", [ - BoolOption("debugprint", "Turn on debug printing for the GC", - default=False), - BoolOption("removetypeptr", "Remove the typeptr from every object", - default=False, cmdline="--gcremovetypeptr"), - ]), + BoolOption("gcremovetypeptr", "Remove the typeptr from every object", + default=False, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", ["n/a", "shadowstack", "asmgcc"], @@ -97,13 +98,12 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False), - ("translation.gcconfig.removetypeptr", False)], + requires=[("translation.thread", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), ChoiceOption("jit_backend", "choose the backend for the JIT", - ["auto", "x86", "llvm"], + ["auto", "x86", "x86-without-sse2", "llvm"], default="auto", cmdline="--jit-backend"), ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", ["off", "profile", "steps", "detailed"], @@ -169,6 +169,10 @@ IntOption("withsmallfuncsets", "Represent groups of less funtions than this as indices into an array", default=0), + BoolOption("taggedpointers", + "When true, enable the use of tagged pointers. " + "If false, use normal boxing", + default=False), # options for ootype OptionDescription("ootype", "Object Oriented Typesystem options", [ @@ -198,9 +202,6 @@ BoolOption("mallocs", "Remove mallocs", default=True), BoolOption("constfold", "Constant propagation", default=True), - BoolOption("heap2stack", "Escape analysis and stack allocation", - default=False, - requires=[("translation.stackless", False)]), # control profile based inlining StrOption("profile_based_inline", "Use call count profiling to drive inlining" @@ -359,7 +360,7 @@ elif word == 'jit': config.translation.suggest(jit=True) elif word == 'removetypeptr': - config.translation.gcconfig.suggest(removetypeptr=True) + config.translation.suggest(gcremovetypeptr=True) else: raise ValueError(word) Modified: pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withshadowtracking.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withshadowtracking.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/config/objspace.std.withshadowtracking.txt Thu Nov 5 22:17:18 2009 @@ -1,5 +1,5 @@ Enable "shadow tracking". This means a special dict representation is used -together with `multidicts`_. This dict representation is used only for instance +-- but only for instance dictionaries. The instance dictionary tracks whether an instance attribute shadows an attribute of its class. This makes method calls slightly faster in the following way: When calling a method the first thing that is checked is the @@ -7,5 +7,3 @@ instance dictionary is then checked for instance attributes shadowing the class attribute. If we know that there is no shadowing (since our instance dict tells us that) we can save this lookup on the instance dictionary. - -.. _`multidicts`: objspace.std.withmultidict.html Modified: pypy/branch/msvc-asmgcroot/pypy/doc/config/opt.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/config/opt.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/config/opt.txt Thu Nov 5 22:17:18 2009 @@ -18,6 +18,7 @@ `--opt=mem` minimize the run-time RAM consumption (in-progress) `--opt=2` all optimizations on; good run-time performance `--opt=3` same as `--opt=2`; remove asserts; gcc profiling `(**)`_ + `--opt=jit` includes the JIT and tweak other optimizations for it ============= ======================================================== .. _`(*)`: Modified: pypy/branch/msvc-asmgcroot/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/cpython_differences.txt Thu Nov 5 22:17:18 2009 @@ -120,11 +120,22 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html +Note that the time at which __del__ is called is not well-defined in any +implementation apart from CPython. A __del__ method may appear to be +called sometimes later in PyPy; for example, a file may stay open for a +bit longer, which can have visible effects (e.g. a file opened for +writing might still contain data not flushed yet). This also makes +"weak proxies" less useful (see ``weakref.proxy()``). They will appear +to stay alive a bit longer in PyPy, and suddenly they will really be +dead, raising a ``ReferenceError`` on the next access. Any code that +uses weak proxies must carefully catch such ``ReferenceError`` at any +place that uses them. + The built-in function ``id()`` returns numbers that are not addresses for most of PyPy's garbage collectors. This is most visible in the default repr: a typical PyPy object can pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because the physical address can change for some GCs). Calling +its real address (because an object can move around in some GCs). Calling ``id`` a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to Modified: pypy/branch/msvc-asmgcroot/pypy/doc/faq.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/faq.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/faq.txt Thu Nov 5 22:17:18 2009 @@ -205,6 +205,20 @@ .. _`py.cleanup`: http://codespeak.net/py/current/doc/bin.html +------------------------------------------------------------- +OSError: ... cannot restore segment prot after reloc... Help? +------------------------------------------------------------- + +On Linux, if SELinux is enabled, you may get errors along the lines of +"OSError: externmod.so: cannot restore segment prot after reloc: Permission +denied." This is caused by a slight abuse of the C compiler during +configuration, and can be disabled by running the following command with root +privileges:: + + # setenforce 0 + +This will disable SELinux's protection and allow PyPy to configure correctly. +Be sure to enable it again if you need it! PyPy translation tool chain ======================================================================== Modified: pypy/branch/msvc-asmgcroot/pypy/doc/getting-started-python.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/getting-started-python.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/getting-started-python.txt Thu Nov 5 22:17:18 2009 @@ -55,17 +55,26 @@ `optimization level`_ `1` in the next step. A level of `2` or `3` gives much better results, though. + Let me stress this another time: at ``--opt=1`` you get the Boehm + GC, which is here mostly for historical and for testing reasons. + You really do not want to pick it. The resulting ``pypy-c`` is + slow. + + Alternatively, if you want to enable the experimental JIT, choose + the optimization level ``jit``. + 3. Run:: cd pypy/translator/goal python translate.py --opt=3 targetpypystandalone.py - possibly replacing ``--opt=3`` with ``--opt=1`` or another + possibly replacing ``--opt=3`` with ``--opt=jit`` or another `optimization level`_ of your choice. - On Linux 32-bit Intel machines, if you don't need threads, you - can get some extra speed (and extra translation time) by adding - ``--gcrootfinder=asmgcc`` just after the ``--opt`` option. + On Linux 32-bit Intel machines, you can get some extra speed + (and extra translation time) by adding ``--gcrootfinder=asmgcc`` + just after the ``--opt`` option. (This option is implied by + ``--opt=jit``.) .. _`optimization level`: config/opt.html Modified: pypy/branch/msvc-asmgcroot/pypy/doc/interpreter-optimizations.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/interpreter-optimizations.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/interpreter-optimizations.txt Thu Nov 5 22:17:18 2009 @@ -103,7 +103,7 @@ integer object is created it is checked whether the integer is small enough to be retrieved from the cache. -You can enable this feature with the :config:`objspace.std.withsmallint` option. +This option is enabled by default. Integers as Tagged Pointers +++++++++++++++++++++++++++ @@ -115,6 +115,7 @@ them from normal pointers. This completely avoids the boxing step, saving time and memory. +You can enable this feature with the :config:`objspace.std.withsmallint` option. Dictionary Optimizations ------------------------ @@ -132,11 +133,9 @@ for string-keyed dictionaries. In addition there are more specialized dictionary implementations for various purposes (see below). -You can enable this feature with the :config:`objspace.std.withmultidict` +This is now the default implementation of dictionaries in the Python interpreter. option. -.. XXX chunked dicts, small dicts - Sharing Dicts +++++++++++++ @@ -202,36 +201,6 @@ You can enable this feature with the :config:`objspace.std.withrangelist` option. -Multi-Lists -+++++++++++ - -As with dictionaries it became clear that it is generally useful to allow lists -to change their internal representation over their lifetime. Therefore -multi-lists were implemented, mostly equivalently to multi-dicts. The special -representations you get by default are for empty lists, for lists containing -only strings and ranges again (the reason why range lists and multilists both -implement the same optimization is that range lists came earlier and that -multi-lists are not tested that much so far). - -You can enable this feature with the :config:`objspace.std.withmultilist` -option. - - -Fast List Slicing -+++++++++++++++++ - -A rather experimental special list representation used with multilists is the -slice list (the original idea is from `Neal Norwitz on pypy-dev`_). The -observation is that slices are often created for iterating over them, so it -seems wasteful to create a full copy of that portion of the list. Instead the -list slice is only created lazily, that is when either the original list or -the sliced list is mutated. - -You can enable this feature with the :config:`objspace.std.withfastslice` -option. - -.. _`Neal Norwitz on pypy-dev`: http://codespeak.net/pipermail/pypy-dev/2005q4/002538.html - User Class Optimizations ------------------------ Modified: pypy/branch/msvc-asmgcroot/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/jit/pyjitpl5.txt Thu Nov 5 22:17:18 2009 @@ -10,8 +10,8 @@ The JIT's `theory`_ is great in principle, but actual code is a different story. This section tries to give a high level overview of how PyPy's JIT is -implemented. It's helpful to have a basic understanding of the PyPy -`translation tool chain`_ before digging into the sources. +implemented. It's helpful to have a basic understanding of how the PyPy +`translation tool chain`_ works before digging into the sources. Almost all JIT specific code is found in the two pypy/jit subdirectories, metainterp, and backend. The metainterp directory holds platform independent @@ -26,12 +26,12 @@ --------- To add a JIT to an interpreter, PyPy only requires that two hints be added to -the interpreter source. These are jit_merge_point and can_enter_jit. +the target interpreter. These are jit_merge_point and can_enter_jit. jit_merge_point is supposed to go at the start of opcode dispatch. It allows -the JIT to bail back to the interpreter in case assembler code fails at some -point. can_enter_jit goes at the close of a application level loop. In the -Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python interpreter -defines its hints in pypy/module/pypyjit/interp_jit.py. +the JIT to bail back to the interpreter in case running machine code is no +longer suitable. can_enter_jit goes at the end of a application level loop. In +the Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python +interpreter defines its hints in pypy/module/pypyjit/interp_jit.py. The interpreter wishing to use the PyPy's JIT must define a list of *green* variables and a list of *red* variables. The *green* variables are loop @@ -46,15 +46,16 @@ -------------- After the RTyping phase of translation, where high level Python operations are -turned into low-level ones for the backend, the translator calls apply_jit() in -metainterp/warmspot.py to add a JIT compiler to the currently translating -interpreter. apply_jit() decides what assembler backend to use then delegates -the rest of the work to the WarmRunnerDesc class. WarmRunnerDesc finds the two -JIT hints in the function graphs. It rewrites the graph containing the -jit_merge_point hint, called the portal graph, to be able to handle special JIT -exceptions, which indicate special operations to the interpreter. The location -of the can_enter_jit hint is changed to check if the current loop is "hot" and -should be compiled. +turned into low-level ones for the backend, the translation driver calls +apply_jit() in metainterp/warmspot.py to add a JIT compiler to the currently +translating interpreter. apply_jit() decides what assembler backend to use then +delegates the rest of the work to the WarmRunnerDesc class. WarmRunnerDesc +finds the two JIT hints in the function graphs. It rewrites the graph +containing the jit_merge_point hint, called the portal graph, to be able to +handle special JIT exceptions, which indicate special coditions to the +interpreter upon exiting from the JIT. The location of the can_enter_jit hint +is replaced with a call to a function, maybe_compile_and_run in warmstate.py, +that checks if current loop is "hot" and should be compiled. Next, starting with the portal graph, metainterp/codewriter.py converts the graphs of the interpreter into JIT bytecode. Since this bytecode is stored in @@ -63,7 +64,7 @@ part of the interpreter. In these cases, it simply inserts an opaque call. Finally, translation finishes, including the bytecode of the interpreter in the -final binary, and interpreter is ready to use the runtime component of the JIT! +final binary, and interpreter is ready to use the runtime component of the JIT. Tracing @@ -81,7 +82,7 @@ of the interpreter interpreting the application level code. This allows it to see the exact operations that make up the application level loop. Tracing is preformed by MetaInterp and MIFrame classes in metainterp/pyjitpl.py. -maybe_compile_and_run() creates a MetaInterp and calls the +maybe_compile_and_run() creates a MetaInterp and calls its compile_and_run_once() method. This initializes the MIFrame for the input arguments of the loop, the red and green variables passed from the jit_merge_point hint, and sets it to start interpreting the bytecode of the @@ -89,8 +90,8 @@ Before starting the interpretation, the loop input arguments are wrapped in a *box*. Boxes (defined in metainterp/history.py) wrap the value and type of a -variable in the program the JIT is interpreting. There are two main varieties -of boxes: constant boxes and normal boxes. Constant boxes are used for values +value in the program the JIT is interpreting. There are two main varieties of +boxes: constant boxes and normal boxes. Constant boxes are used for values assumed to be known during tracing. These are not necessarily compile time constants. All values which are "promoted", assumed to be constant by the JIT for optimization purposes, are also stored in constant boxes. Normal boxes @@ -99,14 +100,17 @@ boxes: ConstInt, ConstPtr, ConstFloat, and ConstAddr. The meta-interpreter starts interpreting the JIT bytecode. Each operation is -executed and then recorded in a list of operations and arguments called the -trace. All possible operations generated by tracing are listed in -metainterp/resoperation.py. When a (interpreter-level) call to a function the -JIT has bytecode for occurs in the bytecode, another frame is added to the stack -and the tracing continues with the same history. This flattens the list of -operations over calls. Most importantly, it unrolls the opcode dispatch loop. -Interpretation continues until the can_enter_jit hint is seen. At this point, a -whole interation of the application level loop has been seen and recorded. +executed and then recorded in a list of operations, called the trace. +Operations can have a list of boxes that operate on, arguments. Some operations +(like GETFIELD and GETARRAYITEM) also have special objects that describe how +their arguments are layed out in memory. All possible operations generated by +tracing are listed in metainterp/resoperation.py. When a (interpreter-level) +call to a function the JIT has bytecode for occurs during tracing, another +MIFrame is added to the stack and the tracing continues with the same history. +This flattens the list of operations over calls. Most importantly, it unrolls +the opcode dispatch loop. Interpretation continues until the can_enter_jit hint +is seen. At this point, a whole interation of the application level loop has +been seen and recorded. Because only one iteration has been recorded the JIT only knows about one codepath in the loop. For example, if there's a if statement construct like @@ -119,11 +123,43 @@ and ``x`` is true when the JIT does tracing, only the codepath ``do_something_exciting`` will be added to the trace. In future runs, to ensure -that it is picking the right path, a special operation called a *guard -operation* is added to the list. A guard is a small test that checks if -assumptions the JIT makes during tracing are still true. In the example above, -a GUARD_TRUE guard will be generated for ``x`` before running -``do_something_exciting``. +that this path is still valid, a special operation called a *guard operation* is +added to the trace. A guard is a small test that checks if assumptions the JIT +makes during tracing are still true. In the example above, a GUARD_TRUE guard +will be generated for ``x`` before running ``do_something_exciting``. + +Once the meta-interpreter has verified that it has traced a loop, it decides how +to compile what it has. There is an optional optimization phase between these +actions which is covered future down this page. The backend converts the trace +operations into assembly for the particular machine. It then hands the compiled +loop back to the frontend. The next time the loop is seen in application code, +the optimized assembly can be run instead of the normal intepreter. + + +Optimizations +------------- + +The JIT employs several techniques, old and new, to make machine code run +faster. + +Virtuals and Virtualizables +*************************** + +A *virtual* value is an array, struct, or RPython level instance that is created +during the loop and does not escape from it via calls or longevity past the +loop. Since it is only used by the JIT, it be "optimized out"; the value +doesn't have to be allocated at all and its fields can be stored as first class +values instead of deferencing them in memory. Virtuals allow temporary objects +in the interpreter to be unwrapped. For example, a W_IntObject in the PyPy can +be unwrapped to just be its integer value as long as the object is known not to +escape the machine code. + +A *virtualizable* is similar to a virtual in that its structure is optimized out +in the machine code. Virtualizables, howerver, can escape from JIT controlled +code. + +Most of the JIT's optimizer is contained 2 files optimizefindnodes.py and +optimizeopt.py. More resources @@ -134,7 +170,7 @@ * `Tracing the Meta-Level: PyPy's Tracing JIT Compiler`__ -.. __: http://codespeak.net/svn/pypy/extradoc/talk/icooolps2009/bolz-tracing-jit.pdf +.. __: http://codespeak.net/svn/pypy/extradoc/talk/ipcoolps2009/bolz-tracing-jit.pdf as well as the `blog posts with the JIT tag.`__ Modified: pypy/branch/msvc-asmgcroot/pypy/doc/translation.txt ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/doc/translation.txt (original) +++ pypy/branch/msvc-asmgcroot/pypy/doc/translation.txt Thu Nov 5 22:17:18 2009 @@ -548,42 +548,8 @@ Another technique to reduce the memory allocation penalty is to use stack allocation for objects that can be proved not to life longer than the stack -frame they have been allocated in. If this is the case it is possible to -allocate the object on the stack. This makes allocation faster, since stack -allocation is just the increase of a pointer, and makes deallocation basically -free since deallocation happens automatically when the function returns. -Therefore we wrote an analysis, which analyses which malloc positions lead to -mallocs which "escape" the current function, e.g. have references to them -stored into a place where they can be accessed by something outside of the -stack of frames starting with the frame where the malloc occured. - -For this we choose a naive, pessimistic approach. The analysis -assumes that an object escapes if one of the following situation occurs: - - * the object is returned - - * the object is raised as an exception - - * the object is stored into a field of any another object - -The algorithm uses abstract interpretation together with a fix point search to -find a solution. - -After using the escape analysis to find malloc sites that don't escape, we -replace the mallocs by stack allocations. This cannot be done in all cases, -namely if the allocated object is variable-sized or if the allocation occurs in -a loop. Both cases should be avoided because they make stack overflows more -likely. Also objects that have a finalizer cannot be allocated on the stack, -since the finalizer might resurrect the object. - -The resulting performance improvements by this optimization were not as big -we hoped. We think that this is due to the fact that the Boehm garbage -collector becomes slower when the stack is bigger, thus compensating -any speed improvement achieved by having faster allocation. We did not -implement stack allocation with any of the other GCs that PyPy can -use. - -Enable this optimization with :config:`translation.backendopt.heap2stack`. +frame they have been allocated in. This proved not to really gain us any +speed, so over time it was removed again. The Stackless Transform Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/assemble.py Thu Nov 5 22:17:18 2009 @@ -363,7 +363,7 @@ def assemble(self): """Build a PyCode object.""" - # Unless it's interactive, every code object must in an a return. + # Unless it's interactive, every code object must end in a return. if not self.current_block.have_return: self.use_next_block() if self.add_none_to_final_return: Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/codegen.py Thu Nov 5 22:17:18 2009 @@ -615,6 +615,8 @@ for alias in imp.names: assert isinstance(alias, ast.alias) if alias.name not in future.futureFlags_2_5.compiler_features: + if alias.name == "braces": + self.error("not a chance", imp) self.error("future feature %s is not defined" % (alias.name,), imp) if imp.level == 0 and \ @@ -1264,9 +1266,11 @@ assert isinstance(func, ast.FunctionDef) # If there's a docstring, store it as the first constant. if self.is_docstring(func.body[0]): - doc_string = func.body[0] - assert isinstance(doc_string, ast.Expr) - doc_string.value.walkabout(self) + doc_expr = func.body[0] + assert isinstance(doc_expr, ast.Expr) + doc_str = doc_expr.value + assert isinstance(doc_str, ast.Str) + self.add_const(doc_str.s) start = 1 else: self.add_const(self.space.w_None) Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/consts.py Thu Nov 5 22:17:18 2009 @@ -1,15 +1,6 @@ -# operation flags -OP_ASSIGN = 0 # 'OP_ASSIGN' -OP_DELETE = 1 # 'OP_DELETE' -OP_APPLY = 2 # 'OP_APPLY' -OP_NONE = 3 - -SC_LOCAL = 1 -SC_GLOBAL = 2 -SC_FREE = 3 -SC_CELL = 4 -SC_UNKNOWN = 5 -SC_DEFAULT = 6 +""" +Various flags used during the compilation process. +""" CO_OPTIMIZED = 0x0001 CO_NEWLOCALS = 0x0002 @@ -18,6 +9,8 @@ CO_NESTED = 0x0010 CO_GENERATOR = 0x0020 CO_NOFREE = 0x0040 +CO_CONTAINSLOOP = 0x0080 +CO_CONTAINSGLOBALS = 0x0800 CO_GENERATOR_ALLOWED = 0x1000 CO_FUTURE_DIVISION = 0x2000 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/astcompiler/test/test_compiler.py Thu Nov 5 22:17:18 2009 @@ -725,7 +725,20 @@ yield self.st, test, "X.__name__", "X" -class AppTestPrint: +class AppTestCompiler: + + def test_docstring_not_loaded(self): + import StringIO, dis, sys + ns = {} + exec "def f():\n 'hi'" in ns + f = ns["f"] + save = sys.stdout + sys.stdout = output = StringIO.StringIO() + try: + dis.dis(f) + finally: + sys.stdout = save + assert "0 ('hi')" not in output.getvalue() def test_print_to(self): exec """from StringIO import StringIO Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/baseobjspace.py Thu Nov 5 22:17:18 2009 @@ -20,26 +20,24 @@ in a 'normal' object space like StdObjSpace.""" __slots__ = () _settled_ = True + user_overridden_class = False def getdict(self): return None - def getdictvalue_w(self, space, attr): - return self.getdictvalue(space, space.wrap(attr)) - - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): w_dict = self.getdict() if w_dict is not None: - return space.finditem(w_dict, w_attr) + return space.finditem_str(w_dict, attr) return None - def getdictvalue_attr_is_in_class(self, space, w_attr): - return self.getdictvalue(space, w_attr) + def getdictvalue_attr_is_in_class(self, space, attr): + return self.getdictvalue(space, attr) - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): w_dict = self.getdict() if w_dict is not None: - space.set_str_keyed_item(w_dict, w_attr, w_value, shadows_type) + space.set_str_keyed_item(w_dict, attr, w_value, shadows_type) return True return False @@ -281,7 +279,7 @@ self.timer.stop("startup " + modname) def finish(self): - w_exitfunc = self.sys.getdictvalue_w(self, 'exitfunc') + w_exitfunc = self.sys.getdictvalue(self, 'exitfunc') if w_exitfunc is not None: self.call_function(w_exitfunc) from pypy.interpreter.module import Module @@ -550,12 +548,6 @@ # that can be defined in term of more primitive ones. Subclasses # may also override specific functions for performance. - #def is_(self, w_x, w_y): -- not really useful. Must be subclassed - # "'x is y'." - # w_id_x = self.id(w_x) - # w_id_y = self.id(w_y) - # return self.eq(w_id_x, w_id_y) - def not_(self, w_obj): return self.wrap(not self.is_true(w_obj)) @@ -571,8 +563,11 @@ """shortcut for space.int_w(space.hash(w_obj))""" return self.int_w(self.hash(w_obj)) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): - return self.setitem(w_obj, w_key, w_value) + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): + return self.setitem(w_obj, self.wrap(key), w_value) + + def finditem_str(self, w_obj, key): + return self.finditem(w_obj, self.wrap(key)) def finditem(self, w_obj, w_key): try: @@ -777,7 +772,7 @@ w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) for w_supertype in self.unpackiterable(w_mro): - w_value = w_supertype.getdictvalue_w(self, name) + w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value return None Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/executioncontext.py Thu Nov 5 22:17:18 2009 @@ -178,6 +178,9 @@ while frame is not None and not frame.f_back_forced: frame.f_back_some = f_back = ExecutionContext._extract_back_from_frame(frame) frame.f_back_forced = True + # now that we force the whole chain, we also have to set the + # forward links to None + frame.f_forward = None frame = f_back return orig_frame.f_back_some Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/function.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/function.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/function.py Thu Nov 5 22:17:18 2009 @@ -16,12 +16,20 @@ funccallunrolling = unrolling_iterable(range(4)) + at jit.purefunction +def _get_immutable_code(func): + assert not func.can_change_code + return func.code + class Function(Wrappable): """A function is a code object captured with some environment: an object space, a dictionary of globals, default arguments, and an arbitrary 'closure' passed to the code object.""" - def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): + can_change_code = True + + def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, + forcename=None): self.space = space self.name = forcename or code.co_name self.w_doc = None # lazily read from code.getdocstring() @@ -48,7 +56,12 @@ return self.getcode().funcrun_obj(self, w_obj, args) def getcode(self): - return jit.hint(self.code, promote=True) + if jit.we_are_jitted(): + if not self.can_change_code: + self = jit.hint(self, promote=True) + return _get_immutable_code(self) + return jit.hint(self.code, promote=True) + return self.code def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway @@ -309,7 +322,7 @@ def fget_func_defaults(space, self): values_w = self.defs_w - if not values_w: + if not values_w or None in values_w: return space.w_None return space.newtuple(values_w) @@ -368,6 +381,9 @@ def fset_func_code(space, self, w_code): from pypy.interpreter.pycode import PyCode + if not self.can_change_code: + raise OperationError(space.w_TypeError, + space.wrap("Cannot change code attribute of builtin functions")) code = space.interp_w(Code, w_code) closure_len = 0 if self.closure: @@ -478,9 +494,8 @@ return space.wrap(s) else: objrepr = space.str_w(space.repr(self.w_instance)) - info = 'bound method %s.%s of %s' % (typename, name, objrepr) - # info = "method %s of %s object" % (name, typename) - return self.w_instance.getrepr(self.space, info) + s = '' % (typename, name, objrepr) + return space.wrap(s) def descr_method_getattribute(self, w_attr): space = self.space @@ -568,7 +583,11 @@ "'%s' object is not callable" % typename)) return space.wrap(ClassMethod(w_function)) +class FunctionWithFixedCode(Function): + can_change_code = False + class BuiltinFunction(Function): + can_change_code = False def __init__(self, func): assert isinstance(func, Function) Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/gateway.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/gateway.py Thu Nov 5 22:17:18 2009 @@ -16,6 +16,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import eval from pypy.interpreter.function import Function, Method, ClassMethod +from pypy.interpreter.function import FunctionWithFixedCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace, Wrappable from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, Signature @@ -788,7 +789,7 @@ space = cache.space defs = gateway._getdefaults(space) # needs to be implemented by subclass code = gateway._code - fn = Function(space, code, None, defs, forcename = gateway.name) + fn = FunctionWithFixedCode(space, code, None, defs, forcename = gateway.name) if not space.config.translating: # for tests and py.py fn._freeze_() if gateway.as_classmethod: Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/main.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/main.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/main.py Thu Nov 5 22:17:18 2009 @@ -149,11 +149,11 @@ w_traceback) # call sys.excepthook if present - w_hook = space.sys.getdictvalue_w(space, 'excepthook') + w_hook = space.sys.getdictvalue(space, 'excepthook') if w_hook is not None: # hack: skip it if it wasn't modified by the user, # to do instead the faster verbose/nonverbose thing below - w_original = space.sys.getdictvalue_w(space, '__excepthook__') + w_original = space.sys.getdictvalue(space, '__excepthook__') if w_original is None or not space.is_w(w_hook, w_original): space.call_function(w_hook, w_type, w_value, w_traceback) return False # done Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/mixedmodule.py Thu Nov 5 22:17:18 2009 @@ -32,7 +32,7 @@ def get(self, name): space = self.space - w_value = self.getdictvalue_w(space, name) + w_value = self.getdictvalue(space, name) if w_value is None: raise OperationError(space.w_AttributeError, space.wrap(name)) return w_value @@ -41,33 +41,41 @@ w_builtin = self.get(name) return self.space.call_function(w_builtin, *args_w) - def getdictvalue(self, space, w_name): - w_value = space.finditem(self.w_dict, w_name) + def getdictvalue(self, space, name): + w_value = space.finditem_str(self.w_dict, name) if self.lazy and w_value is None: - name = space.str_w(w_name) - w_name = space.new_interned_w_str(w_name) - try: - loader = self.loaders[name] - except KeyError: - return None - else: - #print "trying to load", name - w_value = loader(space) - #print "loaded", w_value - # obscure - func = space.interpclass_w(w_value) - if type(func) is Function: - try: - bltin = func._builtinversion_ - except AttributeError: - bltin = BuiltinFunction(func) - bltin.w_module = self.w_name - func._builtinversion_ = bltin - bltin.name = name - w_value = space.wrap(bltin) - space.setitem(self.w_dict, w_name, w_value) + return self._load_lazily(space, name) return w_value + def _load_lazily(self, space, name): + w_name = space.new_interned_str(name) + try: + loader = self.loaders[name] + except KeyError: + return None + else: + w_value = loader(space) + func = space.interpclass_w(w_value) + # the idea of the following code is that all functions that are + # directly in a mixed-module are "builtin", e.g. they get a + # special type without a __get__ + # note that this is not just all functions that contain a + # builtin code object, as e.g. methods of builtin types have to + # be normal Functions to get the correct binding behaviour + if (isinstance(func, Function) and + type(func) is not BuiltinFunction): + try: + bltin = func._builtinversion_ + except AttributeError: + bltin = BuiltinFunction(func) + bltin.w_module = self.w_name + func._builtinversion_ = bltin + bltin.name = name + w_value = space.wrap(bltin) + space.setitem(self.w_dict, w_name, w_value) + return w_value + + def getdict(self): if self.lazy: space = self.space Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/nestedscope.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/nestedscope.py Thu Nov 5 22:17:18 2009 @@ -200,6 +200,7 @@ cell = f.cells[varindex] cell.set(w_newvalue) + @jit.unroll_safe def MAKE_CLOSURE(f, numdefaults, *ignored): w_codeobj = f.popvalue() codeobj = f.space.interp_w(pycode.PyCode, w_codeobj) Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/pycode.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/pycode.py Thu Nov 5 22:17:18 2009 @@ -11,23 +11,20 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.baseobjspace import ObjSpace, W_Root +from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED, + CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, + CO_GENERATOR, CO_CONTAINSLOOP, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash +from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT # helper def unpack_str_tuple(space,w_str_tuple): return [space.str_w(w_el) for w_el in space.unpackiterable(w_str_tuple)] -# code object contants, for co_flags below -CO_OPTIMIZED = 0x0001 -CO_NEWLOCALS = 0x0002 -CO_VARARGS = 0x0004 -CO_VARKEYWORDS = 0x0008 -CO_NESTED = 0x0010 -CO_GENERATOR = 0x0020 # Magic numbers for the bytecode version in code objects. # See comments in pypy/module/__builtin__/importing. @@ -84,6 +81,10 @@ self.hidden_applevel = hidden_applevel self.magic = magic self._signature = cpython_code_signature(self) + self._initialize() + + def _initialize(self): + self._init_flags() # Precompute what arguments need to be copied into cellvars self._args_as_cellvars = [] @@ -116,10 +117,28 @@ self._compute_flatcall() - if space.config.objspace.std.withcelldict: + if self.space.config.objspace.std.withcelldict: from pypy.objspace.std.celldict import init_code init_code(self) + def _init_flags(self): + co_code = self.co_code + next_instr = 0 + while next_instr < len(co_code): + opcode = ord(co_code[next_instr]) + next_instr += 1 + if opcode >= HAVE_ARGUMENT: + next_instr += 2 + while opcode == opcodedesc.EXTENDED_ARG.index: + opcode = ord(co_code[next_instr]) + next_instr += 3 + if opcode == opcodedesc.JUMP_ABSOLUTE.index: + self.co_flags |= CO_CONTAINSLOOP + elif opcode == opcodedesc.LOAD_GLOBAL.index: + self.co_flags |= CO_CONTAINSGLOBALS + elif opcode == opcodedesc.LOAD_NAME.index: + self.co_flags |= CO_CONTAINSGLOBALS + co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace def signature(self): @@ -160,18 +179,6 @@ list(code.co_cellvars), hidden_applevel, cpython_magic) - def _code_new_w(space, argcount, nlocals, stacksize, flags, - code, consts, names, varnames, filename, - name, firstlineno, lnotab, freevars, cellvars, - hidden_applevel=False): - """Initialize a new code objects from parameters given by - the pypy compiler""" - return PyCode(space, argcount, nlocals, stacksize, flags, code, - consts[:], names, varnames, filename, name, firstlineno, - lnotab, freevars, cellvars, hidden_applevel) - - _code_new_w = staticmethod(_code_new_w) - def _compute_flatcall(self): # Speed hack! Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/pyopcode.py Thu Nov 5 22:17:18 2009 @@ -619,9 +619,9 @@ f.pushvalue(w_newclass) def STORE_NAME(f, varindex, *ignored): - w_varname = f.getname_w(varindex) + varname = f.getname_u(varindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_locals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_locals, varname, w_newvalue) def DELETE_NAME(f, varindex, *ignored): w_varname = f.getname_w(varindex) @@ -656,9 +656,9 @@ f.space.delattr(w_obj, w_attributename) def STORE_GLOBAL(f, nameindex, *ignored): - w_varname = f.getname_w(nameindex) + varname = f.getname_u(nameindex) w_newvalue = f.popvalue() - f.space.set_str_keyed_item(f.w_globals, w_varname, w_newvalue) + f.space.set_str_keyed_item(f.w_globals, varname, w_newvalue) def DELETE_GLOBAL(f, nameindex, *ignored): w_varname = f.getname_w(nameindex) @@ -673,25 +673,24 @@ return f.LOAD_GLOBAL(nameindex) # fall-back - def _load_global(f, w_varname): - w_value = f.space.finditem(f.w_globals, w_varname) + def _load_global(f, varname): + w_value = f.space.finditem_str(f.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins - w_value = f.get_builtin().getdictvalue(f.space, w_varname) + w_value = f.get_builtin().getdictvalue(f.space, varname) if w_value is None: - f._load_global_failed(w_varname) + f._load_global_failed(varname) return w_value _load_global._always_inline_ = True - def _load_global_failed(f, w_varname): - varname = f.space.str_w(w_varname) + def _load_global_failed(f, varname): message = "global name '%s' is not defined" % varname raise OperationError(f.space.w_NameError, f.space.wrap(message)) _load_global_failed._dont_inline_ = True def LOAD_GLOBAL(f, nameindex, *ignored): - f.pushvalue(f._load_global(f.getname_w(nameindex))) + f.pushvalue(f._load_global(f.getname_u(nameindex))) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(f, varindex, *ignored): @@ -782,7 +781,7 @@ else: w_flag = None - w_import = f.get_builtin().getdictvalue_w(f.space, '__import__') + w_import = f.get_builtin().getdictvalue(f.space, '__import__') if w_import is None: raise OperationError(space.w_ImportError, space.wrap("__import__ not found")) @@ -989,8 +988,8 @@ def CALL_LIKELY_BUILTIN(f, oparg, *ignored): # overridden by faster version in the standard object space. from pypy.module.__builtin__ import OPTIMIZED_BUILTINS - w_varname = f.space.wrap(OPTIMIZED_BUILTINS[oparg >> 8]) - w_function = f._load_global(w_varname) + varname = OPTIMIZED_BUILTINS[oparg >> 8] + w_function = f._load_global(varname) nargs = oparg&0xFF try: w_result = f.space.call_valuestack(w_function, nargs, f) Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/pyparser/future.py Thu Nov 5 22:17:18 2009 @@ -207,9 +207,7 @@ self.col_offset = col_offset self.lineno = line self.consume_empty_line() - else: - return - + def consume_mandatory_whitespace(self): if self.getc() not in whitespace + '\\': raise DoneException @@ -242,13 +240,19 @@ if self.getc() not in letters: raise DoneException p = self.pos - while 1: - self.pos += 1 - if self.getc() not in alphanumerics: - break - name = self.s[p:self.pos] - self.consume_whitespace() - return name + try: + while self.getc() in alphanumerics: + self.pos += 1 + except DoneException: + # If there's any name at all, we want to call self.set_flag(). + # Something else while get the DoneException again. + if self.pos == p: + raise + end = self.pos + else: + end = self.pos + self.consume_whitespace() + return self.s[p:end] def get_more(self, paren_list=False): if paren_list and self.getc() == ')': Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_code.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_code.py Thu Nov 5 22:17:18 2009 @@ -188,3 +188,31 @@ # CO_NESTED assert f(4).func_code.co_flags & 0x10 assert f.func_code.co_flags & 0x10 == 0 + # check for CO_CONTAINSLOOP + assert not f.func_code.co_flags & 0x0080 + # check for CO_CONTAINSGLOBALS + assert not f.func_code.co_flags & 0x0800 + + + exec """if 1: + def f(): + return [l for l in range(100)] + def g(): + return [l for l in [1, 2, 3, 4]] +""" + + # check for CO_CONTAINSLOOP + assert f.func_code.co_flags & 0x0080 + assert g.func_code.co_flags & 0x0080 + # check for CO_CONTAINSGLOBALS + assert f.func_code.co_flags & 0x0800 + assert not g.func_code.co_flags & 0x0800 + + exec """if 1: + b = 2 + def f(x): + exec "a = 1"; + return a + b + x +""" + # check for CO_CONTAINSGLOBALS + assert f.func_code.co_flags & 0x0800 Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_executioncontext.py Thu Nov 5 22:17:18 2009 @@ -713,7 +713,7 @@ self.leave_two_jitted_levels(ec, frame, frame2) - def test_check_escaping_jitted_with_two_differen_virtualizables(self): + def test_check_escaping_jitted_with_two_different_virtualizables(self): ec, frame, frame2 = self.enter_two_jitted_levels() frame3 = self.Frame(ec, frame) @@ -741,3 +741,19 @@ ec._unchain(frame3) assert not frame2.escaped + def test_frame_top_is_virtualizable(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + frame3 = self.Frame(ec, frame2) + ec.jitted = False + ec._chain(frame3) + ec.gettopframe() + frame3.force_f_back() + ec._unchain(frame3) + assert not frame2.f_forward + assert ec.gettopframe() is frame2 + ec.jitted = True + ec._unchain(frame2) + assert not frame.f_forward + assert ec.gettopframe() is frame + ec._unchain(frame) + assert ec.gettopframe() is None Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_function.py Thu Nov 5 22:17:18 2009 @@ -16,7 +16,7 @@ assert f.func_defaults == None assert f.func_dict == {} assert type(f.func_globals) == dict - #self.assertEquals(f.func_closure, None) XXX + assert f.func_closure is None assert f.func_doc == None assert f.func_name == 'f' assert f.__module__ == 'mymodulename' @@ -70,6 +70,27 @@ f = g(42) raises(TypeError, FuncType, f.func_code, f.func_globals, 'f2', None, None) + def test_write_code(self): + def f(): + return 42 + def g(): + return 41 + assert f() == 42 + assert g() == 41 + raises(TypeError, "f.func_code = 1") + f.func_code = g.func_code + assert f() == 41 + def h(): + return f() # a closure + raises(ValueError, "f.func_code = h.func_code") + + def test_write_code_builtin_forbidden(self): + def f(*args): + return 42 + raises(TypeError, "dir.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") + + class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): @@ -205,6 +226,21 @@ meth = func.__get__(obj, object) assert meth() == obj + def test_no_get_builtin(self): + assert not hasattr(dir, '__get__') + class A(object): + ord = ord + a = A() + assert a.ord('a') == 97 + + def test_builtin_as_special_method_is_not_bound(self): + class A(object): + __getattr__ = len + a = A() + assert a.a == 1 + assert a.ab == 2 + assert a.abcdefghij == 10 + def test_call_builtin(self): s = 'hello' raises(TypeError, len) @@ -346,11 +382,13 @@ pass assert repr(A.f) == "" assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") def test_method_call(self): @@ -644,3 +682,15 @@ assert space.eq_w(w_res, space.wrap(44)) + +class TestFunction: + + def test_func_defaults(self): + from pypy.interpreter import gateway + def g(w_a=gateway.NoneNotWrapped): + pass + app_g = gateway.interp2app_temp(g) + space = self.space + w_g = space.wrap(app_g) + w_defs = space.getattr(w_g, space.wrap("func_defaults")) + assert space.is_w(w_defs, space.w_None) Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_typedef.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_typedef.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_typedef.py Thu Nov 5 22:17:18 2009 @@ -125,7 +125,7 @@ checks[2], checks[3])) subclasses = {} for key, subcls in typedef._subclass_cache.items(): - cls = key[0] + cls = key[1] subclasses.setdefault(cls, {}) subclasses[cls][subcls] = True for cls, set in subclasses.items(): Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/test/test_zzpickle_and_slow.py Thu Nov 5 22:17:18 2009 @@ -363,6 +363,7 @@ raises(TypeError, len, liter) assert list(liter) == list(result) + @py.test.mark.xfail def test_pickle_dictiter(self): import pickle tdict = {'2':2, '3':3, '5':5} Modified: pypy/branch/msvc-asmgcroot/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/interpreter/typedef.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/interpreter/typedef.py Thu Nov 5 22:17:18 2009 @@ -97,23 +97,24 @@ # / \ # 5 6 -def get_unique_interplevel_subclass(cls, hasdict, wants_slots, needsdel=False, - weakrefable=False): +def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots, + needsdel=False, weakrefable=False): "NOT_RPYTHON: initialization-time only" if hasattr(cls, '__del__') and getattr(cls, "handle_del_manually", False): needsdel = False assert cls.typedef.acceptable_as_base_class - key = cls, hasdict, wants_slots, needsdel, weakrefable + key = config, cls, hasdict, wants_slots, needsdel, weakrefable try: return _subclass_cache[key] except KeyError: - subcls = _getusercls(cls, hasdict, wants_slots, needsdel, weakrefable) + subcls = _getusercls(config, cls, hasdict, wants_slots, needsdel, + weakrefable) _subclass_cache[key] = subcls return subcls get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _subclass_cache = {} -def enum_interplevel_subclasses(cls): +def enum_interplevel_subclasses(config, cls): """Return a list of all the extra interp-level subclasses of 'cls' that can be built by get_unique_interplevel_subclass().""" result = [] @@ -121,14 +122,13 @@ for flag2 in (False, True): for flag3 in (False, True): for flag4 in (False, True): - result.append(get_unique_interplevel_subclass(cls, flag1, - flag2, flag3, - flag4)) + result.append(get_unique_interplevel_subclass( + config, cls, flag1, flag2, flag3, flag4)) result = dict.fromkeys(result) assert len(result) <= 6 return result.keys() -def _getusercls(cls, wants_dict, wants_slots, wants_del, weakrefable): +def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable): typedef = cls.typedef if wants_dict and typedef.hasdict: wants_dict = False @@ -136,47 +136,47 @@ if wants_del: if wants_dict: # case 5. Parent class is 3. - parentcls = get_unique_interplevel_subclass(cls, True, True, + parentcls = get_unique_interplevel_subclass(config, cls, True, True, False, True) else: # case 6. Parent class is 4. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, True) - return _usersubclswithfeature(parentcls, "del") + return _usersubclswithfeature(config, parentcls, "del") elif wants_dict: if wants_slots: # case 3. Parent class is 1. - parentcls = get_unique_interplevel_subclass(cls, True, False, + parentcls = get_unique_interplevel_subclass(config, cls, True, False, False, True) - return _usersubclswithfeature(parentcls, "slots") + return _usersubclswithfeature(config, parentcls, "slots") else: # case 1 (we need to add weakrefable unless it's already in 'cls') if not typedef.weakrefable: - return _usersubclswithfeature(cls, "user", "dict", "weakref") + return _usersubclswithfeature(config, cls, "user", "dict", "weakref") else: - return _usersubclswithfeature(cls, "user", "dict") + return _usersubclswithfeature(config, cls, "user", "dict") else: if weakrefable and not typedef.weakrefable: # case 4. Parent class is 2. - parentcls = get_unique_interplevel_subclass(cls, False, True, + parentcls = get_unique_interplevel_subclass(config, cls, False, True, False, False) - return _usersubclswithfeature(parentcls, "weakref") + return _usersubclswithfeature(config, parentcls, "weakref") else: # case 2 (if the base is already weakrefable, case 2 == case 4) - return _usersubclswithfeature(cls, "user", "slots") + return _usersubclswithfeature(config, cls, "user", "slots") -def _usersubclswithfeature(parentcls, *features): - key = parentcls, features +def _usersubclswithfeature(config, parentcls, *features): + key = config, parentcls, features try: return _usersubclswithfeature_cache[key] except KeyError: - subcls = _builduserclswithfeature(parentcls, *features) + subcls = _builduserclswithfeature(config, parentcls, *features) _usersubclswithfeature_cache[key] = subcls return subcls _usersubclswithfeature_cache = {} _allusersubcls_cache = {} -def _builduserclswithfeature(supercls, *features): +def _builduserclswithfeature(config, supercls, *features): "NOT_RPYTHON: initialization-time only" name = supercls.__name__ name += ''.join([name.capitalize() for name in features]) @@ -190,6 +190,8 @@ if "user" in features: # generic feature needed by all subcls class Proto(object): + user_overridden_class = True + def getclass(self, space): return hint(self.w__class__, promote=True) @@ -242,7 +244,16 @@ return self.slots_w[index] add(Proto) - if "dict" in features: + wantdict = "dict" in features + if wantdict and config.objspace.std.withinlineddict: + from pypy.objspace.std.objectobject import W_ObjectObject + from pypy.objspace.std.inlinedict import make_mixin + if supercls is W_ObjectObject: + Mixin = make_mixin(config) + add(Mixin) + wantdict = False + + if wantdict: class Proto(object): def getdict(self): return self.w__dict__ @@ -253,32 +264,22 @@ def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space, - sharing=True) - elif space.config.objspace.std.withshadowtracking: - from pypy.objspace.std import dictmultiobject - self.w__dict__ = dictmultiobject.W_DictMultiObject(space) - self.w__dict__.implementation = \ - dictmultiobject.ShadowDetectingDictImplementation( - space, w_subtype) - else: - self.w__dict__ = space.newdict() + self.w__dict__ = space.newdict( + instance=True, classofinstance=w_subtype) self.user_setup_slots(w_subtype.nslots) def setclass(self, space, w_subtype): # only used by descr_set___class__ self.w__class__ = w_subtype if space.config.objspace.std.withshadowtracking: - self.w__dict__.implementation.set_shadows_anything() + self.w__dict__.set_shadows_anything() - def getdictvalue_attr_is_in_class(self, space, w_name): + def getdictvalue_attr_is_in_class(self, space, name): w_dict = self.w__dict__ if space.config.objspace.std.withshadowtracking: - if not w_dict.implementation.shadows_anything(): + if not w_dict.shadows_anything(): return None - return space.finditem(w_dict, w_name) + return space.finditem_str(w_dict, name) add(Proto) @@ -292,9 +293,8 @@ if not space.is_true(space.isinstance(w_dict, space.w_dict)): raise OperationError(space.w_TypeError, space.wrap("setting dictionary to a non-dict")) - if space.config.objspace.std.withmultidict: - from pypy.objspace.std import dictmultiobject - assert isinstance(w_dict, dictmultiobject.W_DictMultiObject) + from pypy.objspace.std import dictmultiobject + assert isinstance(w_dict, dictmultiobject.W_DictMultiObject) return w_dict check_new_dictionary._dont_inline_ = True Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/cli/runner.py Thu Nov 5 22:17:18 2009 @@ -319,7 +319,7 @@ def __init__(self, TYPE): DescrWithKey.__init__(self, TYPE) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap ARRAY = ootype.Array(TYPE) def create(): if isinstance(TYPE, ootype.OOType): @@ -482,7 +482,7 @@ def __init__(self, TYPE, fieldname): DescrWithKey.__init__(self, (TYPE, fieldname)) from pypy.jit.backend.llgraph.runner import boxresult - from pypy.jit.metainterp.warmspot import unwrap + from pypy.jit.metainterp.warmstate import unwrap _, T = TYPE._lookup_field(fieldname) def getfield(objbox): obj = objbox.getref(TYPE) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/detect_cpu.py Thu Nov 5 22:17:18 2009 @@ -1,13 +1,14 @@ """ Processor auto-detection """ +import autopath import sys, os class ProcessorAutodetectError(Exception): pass -def autodetect(): +def autodetect_main_model(): mach = None try: import platform @@ -40,11 +41,21 @@ except KeyError: raise ProcessorAutodetectError, "unsupported processor '%s'" % mach +def autodetect(): + model = autodetect_main_model() + if model in ('i386', 'x86'): + from pypy.jit.backend.x86.detect_sse2 import detect_sse2 + if not detect_sse2(): + model = 'x86-without-sse2' + return model + def getcpuclass(backend_name="auto"): if backend_name == "auto": backend_name = autodetect() if backend_name in ('i386', 'x86'): from pypy.jit.backend.x86.runner import CPU + elif backend_name == 'x86-without-sse2': + from pypy.jit.backend.x86.runner import CPU386_NO_SSE2 as CPU elif backend_name == 'cli': from pypy.jit.backend.cli.runner import CliCPU as CPU elif backend_name == 'llvm': Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/llimpl.py Thu Nov 5 22:17:18 2009 @@ -807,7 +807,6 @@ op_getarrayitem_gc_pure = op_getarrayitem_gc def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - from pypy.jit.metainterp.warmspot import unwrap array = ootype.cast_from_object(typedescr.ARRAY, obj) if ootype.typeOf(objnewvalue) == ootype.Object: newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/llgraph/runner.py Thu Nov 5 22:17:18 2009 @@ -9,7 +9,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.jit.metainterp import history from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp.warmspot import unwrap +from pypy.jit.metainterp.warmstate import unwrap from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.backend import model from pypy.jit.backend.llgraph import llimpl, symbolic Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/gc.py Thu Nov 5 22:17:18 2009 @@ -325,8 +325,7 @@ # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer - self.layoutbuilder = framework.JITTransformerLayoutBuilder( - gcdescr.config) + self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() self.translator._jit2gc = { 'layoutbuilder': self.layoutbuilder, Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/llmodel.py Thu Nov 5 22:17:18 2009 @@ -28,8 +28,11 @@ else: translator = None self.gc_ll_descr = get_ll_description(gcdescr, translator) - self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, - 'typeptr', + if translator and translator.config.translation.gcremovetypeptr: + self.vtable_offset = None + else: + self.vtable_offset, _ = symbolic.get_field_token(rclass.OBJECT, + 'typeptr', translate_support_code) self._setup_prebuilt_error('ovf', OverflowError) self._setup_prebuilt_error('zer', ZeroDivisionError) @@ -424,8 +427,9 @@ classint = classbox.getint() descrsize = self.class_sizes[classint] res = self.gc_ll_descr.gc_malloc(descrsize) - as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) - as_array[self.vtable_offset/WORD] = classint + if self.vtable_offset is not None: + as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) + as_array[self.vtable_offset/WORD] = classint return BoxPtr(res) def do_new_array(self, countbox, arraydescr): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/llsupport/test/test_gc.py Thu Nov 5 22:17:18 2009 @@ -147,19 +147,20 @@ class TestFramework: def setup_method(self, meth): - class FakeTranslator: - pass - class config: + class config_: class translation: gc = 'hybrid' gcrootfinder = 'asmgcc' gctransformer = 'framework' + gcremovetypeptr = False + class FakeTranslator: + config = config_ class FakeCPU: def cast_adr_to_int(self, adr): ptr = llmemory.cast_adr_to_ptr(adr, gc_ll_descr.WB_FUNCPTR) assert ptr._obj._callable == llop1._write_barrier_failing_case return 42 - gcdescr = get_description(config) + gcdescr = get_description(config_) translator = FakeTranslator() llop1 = FakeLLOp() gc_ll_descr = GcLLDescr_framework(gcdescr, FakeTranslator(), llop1) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/test/support.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/test/support.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/test/support.py Thu Nov 5 22:17:18 2009 @@ -1,6 +1,6 @@ import py import sys -from pypy.jit.metainterp.history import log +from pypy.rlib.debug import debug_print from pypy.translator.translator import TranslationContext class BaseCompiledMixin(object): @@ -25,6 +25,7 @@ for arg in args: assert isinstance(arg, int) + self.pre_translation_hook() t = self._get_TranslationContext() t.config.translation.type_system = self.type_system # force typesystem-specific options if listcomp: @@ -63,8 +64,15 @@ entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings], annmodel.SomeInteger()) warmrunnerdesc.finish() + self.post_translation_hook() return self._compile_and_run(t, entry_point, entry_point_graph, args) + def pre_translation_hook(self): + pass + + def post_translation_hook(self): + pass + def check_loops(self, *args, **kwds): pass @@ -111,15 +119,23 @@ cbuilder = CBuilder(t, entry_point, config=t.config) cbuilder.generate_source() exe_name = cbuilder.compile() - log('---------- Test starting ----------') + debug_print('---------- Test starting ----------') stdout = cbuilder.cmdexec(" ".join([str(arg) for arg in args])) res = int(stdout) - log('---------- Test done (%d) ----------' % (res,)) + debug_print('---------- Test done (%d) ----------' % (res,)) return res class CliCompiledMixin(BaseCompiledMixin): type_system = 'ootype' + def pre_translation_hook(self): + from pypy.translator.oosupport.support import patch_os + self.olddefs = patch_os() + + def post_translation_hook(self): + from pypy.translator.oosupport.support import unpatch_os + unpatch_os(self.olddefs) # restore original values + def _compile_and_run(self, t, entry_point, entry_point_graph, args): from pypy.translator.cli.test.runtest import compile_graph func = compile_graph(entry_point_graph, t, nowrap=True, standalone=True) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/assembler.py Thu Nov 5 22:17:18 2009 @@ -13,15 +13,15 @@ from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * from pypy.jit.metainterp.resoperation import rop - - +from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ + NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ + CHUNK_SIZE # our calling convention - we pass first 6 args in registers # and the rest stays on the stack RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words -MAX_FAIL_BOXES = 1000 if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -76,30 +76,23 @@ self.malloc_array_func_addr = 0 self.malloc_str_func_addr = 0 self.malloc_unicode_func_addr = 0 - self.fail_boxes_int = lltype.malloc(lltype.GcArray(lltype.Signed), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_ptr = lltype.malloc(lltype.GcArray(llmemory.GCREF), - MAX_FAIL_BOXES, zero=True) - self.fail_boxes_float = lltype.malloc(lltype.GcArray(lltype.Float), - MAX_FAIL_BOXES, zero=True) + self.fail_boxes_int = NonmovableGrowableArraySigned() + self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() + self.fail_boxes_float = NonmovableGrowableArrayFloat() def leave_jitted_hook(self): - fail_boxes_ptr = self.fail_boxes_ptr - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(fail_boxes_ptr)) + # XXX BIG FAT WARNING XXX + # At this point, we should not call anyone here, because + # RPython-level exception might be set. Here be dragons + i = 0 + while i < self.fail_boxes_ptr.lgt: + chunk = self.fail_boxes_ptr.chunks[i] + llop.gc_assume_young_pointers(lltype.Void, + llmemory.cast_ptr_to_adr(chunk)) + i += 1 def make_sure_mc_exists(self): if self.mc is None: - rffi.cast(lltype.Signed, self.fail_boxes_int) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_ptr) # workaround - rffi.cast(lltype.Signed, self.fail_boxes_float) # workaround - self.fail_box_int_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_int)) - self.fail_box_ptr_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_ptr)) - self.fail_box_float_addr = rffi.cast(lltype.Signed, - lltype.direct_arrayitems(self.fail_boxes_float)) - # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() @@ -213,23 +206,22 @@ # This uses XCHG to put zeroes in fail_boxes_ptr after # reading them self.mc.XOR(target, target) - self.mc.XCHG(target, addr_add(imm(self.fail_box_ptr_addr), - imm(i*WORD))) + adr = self.fail_boxes_ptr.get_addr_for_num(i) + self.mc.XCHG(target, heap(adr)) else: - self.mc.MOV(target, addr_add(imm(self.fail_box_int_addr), - imm(i*WORD))) + adr = self.fail_boxes_int.get_addr_for_num(i) + self.mc.MOV(target, heap(adr)) if target is not loc: self.mc.MOV(loc, target) for i in range(len(floatlocs)): loc = floatlocs[i] if loc is None: continue + adr = self.fail_boxes_float.get_addr_for_num(i) if isinstance(loc, REG): - self.mc.MOVSD(loc, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(loc, heap64(adr)) else: - self.mc.MOVSD(xmmtmp, addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2))) + self.mc.MOVSD(xmmtmp, heap64(adr)) self.mc.MOVSD(loc, xmmtmp) return adr_stackadjust @@ -526,7 +518,8 @@ self.set_vtable(eax, loc_vtable) def set_vtable(self, loc, loc_vtable): - self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) + if self.cpu.vtable_offset is not None: + self.mc.MOV(mem(loc, self.cpu.vtable_offset), loc_vtable) # XXX genop_new is abused for all varsized mallocs with Boehm, for now # (instead of genop_new_array, genop_newstr, genop_newunicode) @@ -721,7 +714,24 @@ def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): offset = self.cpu.vtable_offset - self.mc.CMP(mem(locs[0], offset), locs[1]) + if offset is not None: + self.mc.CMP(mem(locs[0], offset), locs[1]) + else: + # XXX hard-coded assumption: to go from an object to its class + # we use the following algorithm: + # - read the typeid from mem(locs[0]), i.e. at offset 0 + # - keep the lower 16 bits read there + # - multiply by 4 and use it as an offset in type_info_group. + loc = locs[1] + assert isinstance(loc, IMM32) + classptr = loc.value + # here, we have to go back from 'classptr' to the value expected + # from reading the 16 bits in the object header + type_info_group = llop.gc_get_type_info_group(llmemory.Address) + type_info_group = rffi.cast(lltype.Signed, type_info_group) + expected_typeid = (classptr - type_info_group) >> 2 + self.mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + # return self.implement_guard(addr, self.mc.JNE) def _no_const_locs(self, args): @@ -741,41 +751,38 @@ return addr def generate_failure(self, mc, faildescr, failargs, locs, exc): - assert len(failargs) < MAX_FAIL_BOXES pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), loc) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), loc) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr - mc.MOV(addr_add(imm(base), imm(i*WORD)), loc) + adr = self.fail_boxes_int.get_addr_for_num(i) + mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: mc.MOVSD(xmm0, loc) - mc.MOVSD(addr64_add(imm(self.fail_box_float_addr), - imm(i*WORD*2)), xmm0) + adr = self.fail_boxes_float.get_addr_for_num(i) + mc.MOVSD(heap64(adr), xmm0) else: if arg.type == REF: - base = self.fail_box_ptr_addr + adr = self.fail_boxes_ptr.get_addr_for_num(i) else: - base = self.fail_box_int_addr + adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) - mc.MOV(addr_add(imm(base), imm(i*WORD)), eax) + mc.MOV(heap(adr), eax) if self.debug_markers: mc.MOV(eax, imm(pos)) - mc.MOV(addr_add(imm(self.fail_box_int_addr), - imm(len(locs) * WORD)), - eax) + mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/ri386setup.py Thu Nov 5 22:17:18 2009 @@ -310,6 +310,11 @@ CMP = Instruction() CMP.common_modes(7) +# special mode for comparing a 16-bit operand with an immediate +CMP16 = Instruction() +CMP16.mode2(MODRM, IMM32, ['\x66', '\x81', orbyte(7<<3), modrm(1), + immediate(2,'h')]) + NOP = Instruction() NOP.mode0(['\x90']) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/runner.py Thu Nov 5 22:17:18 2009 @@ -5,7 +5,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history -from pypy.jit.backend.x86.assembler import Assembler386, MAX_FAIL_BOXES +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU @@ -38,28 +38,25 @@ self.assembler.assemble_bridge(faildescr, inputargs, operations) def set_future_value_int(self, index, intvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_int[index] = intvalue + self.assembler.fail_boxes_int.setitem(index, intvalue) def set_future_value_float(self, index, floatvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_float[index] = floatvalue + self.assembler.fail_boxes_float.setitem(index, floatvalue) def set_future_value_ref(self, index, ptrvalue): - assert index < MAX_FAIL_BOXES, "overflow!" - self.assembler.fail_boxes_ptr[index] = ptrvalue + self.assembler.fail_boxes_ptr.setitem(index, ptrvalue) def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int[index] + return self.assembler.fail_boxes_int.getitem(index) def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float[index] + return self.assembler.fail_boxes_float.getitem(index) def get_latest_value_ref(self, index): - ptrvalue = self.assembler.fail_boxes_ptr[index] + ptrvalue = self.assembler.fail_boxes_ptr.getitem(index) # clear after reading - self.assembler.fail_boxes_ptr[index] = lltype.nullptr( - llmemory.GCREF.TO) + self.assembler.fail_boxes_ptr.setitem(index, lltype.nullptr( + llmemory.GCREF.TO)) return ptrvalue def execute_token(self, executable_token): @@ -91,6 +88,10 @@ return CPU386.cast_adr_to_int(adr) +class CPU386_NO_SSE2(CPU386): + supports_floats = False + + CPU = CPU386 import pypy.jit.metainterp.executor Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_regalloc.py Thu Nov 5 22:17:18 2009 @@ -198,8 +198,8 @@ ptr = lltype.malloc(S) self.interpret(ops, [0, ptr]) assert self.getptr(0, lltype.Ptr(S)) == ptr - assert not self.cpu.assembler.fail_boxes_ptr[0] - assert not self.cpu.assembler.fail_boxes_ptr[1] + assert not self.cpu.assembler.fail_boxes_ptr.getitem(0) + assert not self.cpu.assembler.fail_boxes_ptr.getitem(1) def test_exception_bridge_no_exception(self): ops = ''' Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Thu Nov 5 22:17:18 2009 @@ -218,6 +218,8 @@ return [] # MOV [constant-address], accum if instrname == "MOV16": return [] # skipped + if instrname == "CMP16": + return [] # skipped if instrname == "LEA": if (args[1][1].__class__ != i386.MODRM or args[1][1].is_register()): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/backend/x86/test/test_zrpy_gc.py Thu Nov 5 22:17:18 2009 @@ -15,6 +15,7 @@ from pypy.jit.backend.x86.runner import CPU386 from pypy.jit.backend.llsupport.gc import GcRefList, GcRootMap_asmgcc from pypy.jit.backend.x86.regalloc import X86StackManager +from pypy.tool.udir import udir stack_pos = X86StackManager.stack_pos @@ -33,7 +34,7 @@ def get_g(main): main._dont_inline_ = True - def g(num, n): + def g(name, n): x = X() x.foo = 2 main(n, x) @@ -46,12 +47,16 @@ def get_entry(g): def entrypoint(args): - num = 0 - if len(args) == 2: - num = int(args[1]) + name = '' + n = 2000 + argc = len(args) + if argc > 1: + name = args[1] + if argc > 2: + n = int(args[2]) r_list = [] for i in range(20): - r = g(num, 2000) + r = g(name, n) r_list.append(r) rgc.collect() rgc.collect(); rgc.collect() @@ -73,7 +78,8 @@ # t = TranslationContext() t.config.translation.gc = gc - t.config.translation.gcconfig.debugprint = True + if gc != 'boehm': + t.config.translation.gcremovetypeptr = True for name, value in kwds.items(): setattr(t.config.translation, name, value) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) @@ -88,7 +94,8 @@ def run(cbuilder, args=''): # - data = cbuilder.cmdexec(args) + pypylog = udir.join('test_zrpy_gc.log') + data = cbuilder.cmdexec(args, env={'PYPYLOG': str(pypylog)}) return data.strip() def compile_and_run(f, gc, **kwds): @@ -134,13 +141,14 @@ assert name not in name_to_func name_to_func[name] = len(name_to_func) print name_to_func - def allfuncs(num, n): + def allfuncs(name, n): x = X() x.foo = 2 - main_allfuncs(num, n, x) + main_allfuncs(name, n, x) x.foo = 5 return weakref.ref(x) - def main_allfuncs(num, n, x): + def main_allfuncs(name, n, x): + num = name_to_func[name] n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s = funcs[num][0](n, x) while n > 0: myjitdriver.can_enter_jit(num=num, n=n, x=x, x0=x0, x1=x1, @@ -158,14 +166,14 @@ cls.name_to_func = name_to_func cls.cbuilder = compile(get_entry(allfuncs), "hybrid", gcrootfinder="asmgcc", jit=True) - def run(self, name): - num = self.name_to_func[name] - res = self.cbuilder.cmdexec(str(num)) + def run(self, name, n=2000): + pypylog = udir.join('TestCompileHybrid.log') + res = self.cbuilder.cmdexec("%s %d" %(name, n), + env={'PYPYLOG': str(pypylog)}) assert int(res) == 20 def run_orig(self, name, n, x): - num = self.name_to_func[name] - self.main_allfuncs(num, n, x) + self.main_allfuncs(name, n, x) def define_compile_hybrid_1(cls): # a moving GC. Supports malloc_varsize_nonmovable. Simple test, works Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/compile.py Thu Nov 5 22:17:18 2009 @@ -2,10 +2,11 @@ from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import debug_start, debug_stop from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.history import TreeLoop, log, Box, History, LoopToken +from pypy.jit.metainterp.history import TreeLoop, Box, History, LoopToken from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt from pypy.jit.metainterp.history import BoxPtr, BoxObj, BoxFloat, Const from pypy.jit.metainterp import history @@ -13,6 +14,9 @@ from pypy.jit.metainterp.typesystem import llhelper, oohelper from pypy.jit.metainterp.optimizeutil import InvalidLoop +class GiveUp(Exception): + pass + def show_loop(metainterp_sd, loop=None, error=None): # debugging if option.view or option.viewloops: @@ -89,11 +93,16 @@ globaldata.loopnumbering += 1 metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd, loop) loop.check_consistency() - metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, loop.token) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_loop(loop.inputargs, loop.operations, + loop.token) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): @@ -106,12 +115,16 @@ def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): n = faildescr.get_index() metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) - metainterp_sd.profiler.start_backend() if not we_are_translated(): show_loop(metainterp_sd) TreeLoop.check_consistency_of(inputargs, operations) pass - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.profiler.start_backend() + debug_start("jit-backend") + try: + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + finally: + debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if not we_are_translated(): metainterp_sd.stats.compiled() @@ -259,8 +272,8 @@ new_loop_token) # store the new loop in compiled_merge_points too glob = metainterp_sd.globaldata - greenargs = glob.unpack_greenkey(self.original_greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points( + self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) @@ -284,7 +297,6 @@ old_loop_tokens, new_loop) except InvalidLoop: - assert 0, "InvalidLoop in optimize_bridge?" return None # Did it work? if target_loop_token is not None: Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/history.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/history.py Thu Nov 5 22:17:18 2009 @@ -3,24 +3,21 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic -from pypy.rlib.objectmodel import compute_hash +from pypy.rlib.objectmodel import compute_hash, compute_unique_id from pypy.rlib.rarithmetic import intmask from pypy.tool.uid import uid from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('compiler') -py.log.setconsumer('compiler', ansi_log) - # ____________________________________________________________ INT = 'i' REF = 'r' FLOAT = 'f' +FAILARGS_LIMIT = 1000 + def getkind(TYPE, supports_floats=True): if TYPE is lltype.Void: return "void" @@ -67,14 +64,9 @@ except AttributeError: return box.value -class ReprRPython: - def __init__(self): - self.seen = {} - def repr_rpython(self, box, typechars): - n = self.seen.setdefault(box, len(self.seen)) - return '%s/%s%d' % (box._get_hash_(), typechars, n) - -repr_rpython = ReprRPython().repr_rpython +def repr_rpython(box, typechars): + return '%s/%s%d' % (box._get_hash_(), typechars, + compute_unique_id(box)) class AbstractValue(object): @@ -117,6 +109,9 @@ def repr_rpython(self): return '%s' % self + def _get_str(self): + raise NotImplementedError + class AbstractDescr(AbstractValue): __slots__ = () @@ -930,8 +925,9 @@ # ---------------------------------------------------------------- class Options: - def __init__(self, listops=False): + def __init__(self, listops=False, failargs_limit=FAILARGS_LIMIT): self.listops = listops + self.failargs_limit = failargs_limit def _freeze_(self): return True Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/jitprof.py Thu Nov 5 22:17:18 2009 @@ -30,6 +30,8 @@ ncounters = len(names) _setup() +JITPROF_LINES = ncounters + 1 + 5 # one for TOTAL, 5 for calls, update if needed + class BaseProfiler(object): pass @@ -89,7 +91,7 @@ def start(self): self.starttime = self.timer() self.t1 = self.starttime - self.times = [0, 0, 0, 0] + self.times = [0, 0] self.counters = [0] * ncounters self.calls = [[0, 0], [0, 0], [0, 0]] self.current = [] @@ -128,11 +130,15 @@ def start_backend(self): self._start(BACKEND) def end_backend(self): self._end (BACKEND) - def start_running(self): self._start(RUNNING) - def end_running(self): self._end (RUNNING) + # Don't record times for 'running' and 'blackhole' because there are + # too many of them: calling time.time() is a major blocker. + # If you are interested in these numbers, use 'PYPYLOG=file' and + # look at the resulting file with pypy/tool/logparser.py. + def start_running(self): self.count(RUNNING) + def end_running(self): pass - def start_blackhole(self): self._start(BLACKHOLE) - def end_blackhole(self): self._end (BLACKHOLE) + def start_blackhole(self): self.count(BLACKHOLE) + def end_blackhole(self): pass def count(self, kind, inc=1): self.counters[kind] += inc @@ -151,8 +157,8 @@ calls = self.calls self._print_line_time("Tracing", cnt[TRACING], tim[TRACING]) self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) - self._print_line_time("Running asm", cnt[RUNNING], tim[RUNNING]) - self._print_line_time("Blackhole", cnt[BLACKHOLE], tim[BLACKHOLE]) + self._print_intline("Running asm", cnt[RUNNING]) + self._print_intline("Blackhole", cnt[BLACKHOLE]) line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) os.write(2, line) self._print_intline("ops", cnt[OPS]) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/logger.py Thu Nov 5 22:17:18 2009 @@ -1,48 +1,43 @@ import os -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ BoxInt, ConstAddr, ConstFloat, BoxFloat, AbstractFailDescr -from pypy.rlib.streamio import open_file_as_stream class Logger(object): def __init__(self, ts, guard_number=False): - self.log_stream = None self.ts = ts self.guard_number=guard_number - def create_log(self, extension='.ops'): - if self.log_stream is not None: - return self.log_stream - s = os.environ.get('PYPYJITLOG') - if not s: - return None - s += extension - try: - self.log_stream = open_file_as_stream(s, 'w') - except OSError: - os.write(2, "could not create log file\n") - return None - return self.log_stream - def log_loop(self, inputargs, operations, number=0, type=None): - if self.log_stream is None: + if not have_debug_prints(): return - if type is not None: - self.log_stream.write("# Loop%d (%s), %d ops\n" % (number, - type, - len(operations))) - self._log_operations(inputargs, operations, {}) + if type is None: + debug_start("jit-log-noopt-loop") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-loop") + else: + debug_start("jit-log-opt-loop") + debug_print("# Loop", number, ":", type, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-loop") def log_bridge(self, inputargs, operations, number=-1): - if self.log_stream is None: + if not have_debug_prints(): return - if number != -1: - self.log_stream.write("# bridge out of Guard%d, %d ops\n" % (number, - len(operations))) - self._log_operations(inputargs, operations, {}) - + if number == -1: + debug_start("jit-log-noopt-bridge") + self._log_operations(inputargs, operations) + debug_stop("jit-log-noopt-bridge") + else: + debug_start("jit-log-opt-bridge") + debug_print("# bridge out of Guard", number, + "with", len(operations), "ops") + self._log_operations(inputargs, operations) + debug_stop("jit-log-opt-bridge") def repr_of_descr(self, descr): return descr.repr_of_descr() @@ -70,15 +65,18 @@ else: return '?' - def _log_operations(self, inputargs, operations, memo): + def _log_operations(self, inputargs, operations): + if not have_debug_prints(): + return + memo = {} if inputargs is not None: args = ", ".join([self.repr_of_arg(memo, arg) for arg in inputargs]) - self.log_stream.write('[' + args + ']\n') + debug_print('[' + args + ']') for i in range(len(operations)): op = operations[i] if op.opnum == rop.DEBUG_MERGE_POINT: loc = op.args[0]._get_str() - self.log_stream.write("debug_merge_point('%s')\n" % (loc,)) + debug_print("debug_merge_point('%s')" % (loc,)) continue args = ", ".join([self.repr_of_arg(memo, arg) for arg in op.args]) if op.result is not None: @@ -99,6 +97,5 @@ for arg in op.fail_args]) + ']' else: fail_args = '' - self.log_stream.write(res + op.getopname() + - '(' + args + ')' + fail_args + '\n') - self.log_stream.flush() + debug_print(res + op.getopname() + + '(' + args + ')' + fail_args) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimize.py Thu Nov 5 22:17:18 2009 @@ -1,3 +1,5 @@ +from pypy.rlib.debug import debug_start, debug_stop + # ____________________________________________________________ from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder @@ -5,6 +7,13 @@ from pypy.jit.metainterp.specnode import equals_specnodes def optimize_loop(metainterp_sd, old_loop_tokens, loop): + debug_start("jit-optimize") + try: + return _optimize_loop(metainterp_sd, old_loop_tokens, loop) + finally: + debug_stop("jit-optimize") + +def _optimize_loop(metainterp_sd, old_loop_tokens, loop): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) finder = PerfectSpecializationFinder(cpu) @@ -21,6 +30,13 @@ from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 def optimize_bridge(metainterp_sd, old_loop_tokens, bridge): + debug_start("jit-optimize") + try: + return _optimize_bridge(metainterp_sd, old_loop_tokens, bridge) + finally: + debug_stop("jit-optimize") + +def _optimize_bridge(metainterp_sd, old_loop_tokens, bridge): cpu = metainterp_sd.cpu metainterp_sd.logger_noopt.log_loop(bridge.inputargs, bridge.operations) finder = BridgeSpecializationFinder(cpu) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/optimizeopt.py Thu Nov 5 22:17:18 2009 @@ -520,6 +520,8 @@ assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) + if len(newboxes) > self.metainterp_sd.options.failargs_limit: + raise compile.GiveUp descr.store_final_boxes(op, newboxes) def optimize_default(self, op): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/policy.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/policy.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/policy.py Thu Nov 5 22:17:18 2009 @@ -77,8 +77,9 @@ getkind(v.concretetype, supports_floats) getkind(op.result.concretetype, supports_floats) except NotImplementedError, e: - history.log.WARNING('%s, ignoring graph' % (e,)) - history.log.WARNING(' %s' % (graph,)) + from pypy.jit.metainterp.codewriter import log + log.WARNING('%s, ignoring graph' % (e,)) + log.WARNING(' %s' % (graph,)) return True return False Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/pyjitpl.py Thu Nov 5 22:17:18 2009 @@ -1,9 +1,9 @@ -import py +import py, os from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype -from pypy.rlib.objectmodel import we_are_translated, r_dict +from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box @@ -16,6 +16,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED +from pypy.jit.metainterp.compile import GiveUp # ____________________________________________________________ @@ -874,12 +875,10 @@ self.pc = pc self.exception_target = exception_target self.env = env - if self.metainterp.staticdata.state.debug_level >= DEBUG_DETAILED: - values = ' '.join([box.repr_rpython() for box in self.env]) - log = self.metainterp.staticdata.log - log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, - self.pc, values, - self.exception_target)) + ## values = ' '.join([box.repr_rpython() for box in self.env]) + ## log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, + ## self.pc, values, + ## self.exception_target)) def run_one_step(self): # Execute the frame forward. This method contains a loop that leaves @@ -1019,15 +1018,13 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: + debug_print(self.jit_starting_line) self._setup_class_sizes() self.cpu.setup_once() - self.log(self.jit_starting_line) if not self.profiler.initialized: self.profiler.start() self.profiler.initialized = True self.globaldata.initialized = True - self.logger_noopt.create_log('.noopt') - self.logger_ops.create_log('.ops') def _setup_class_sizes(self): class_sizes = {} @@ -1079,12 +1076,8 @@ # ---------------- logging ------------------------ - def log(self, msg, event_kind='info'): - if self.state.debug_level > DEBUG_PROFILE: - if not we_are_translated(): - getattr(history.log, event_kind)(msg) - else: - debug_print(msg) + def log(self, msg): + debug_print(msg) # ____________________________________________________________ @@ -1097,15 +1090,25 @@ # state = staticdata.state if state is not None: - self.unpack_greenkey = state.unwrap_greenkey - self.compiled_merge_points = r_dict(state.comparekey,state.hashkey) - # { (greenargs): [MergePoints] } + self.jit_cell_at_key = state.jit_cell_at_key else: - self.compiled_merge_points = {} # for tests only; not RPython - self.unpack_greenkey = tuple + # for tests only; not RPython + class JitCell: + compiled_merge_points = None + _jitcell_dict = {} + def jit_cell_at_key(greenkey): + greenkey = tuple(greenkey) + return _jitcell_dict.setdefault(greenkey, JitCell()) + self.jit_cell_at_key = jit_cell_at_key if staticdata.virtualizable_info: self.blackhole_virtualizable = staticdata.virtualizable_info.null_vable + def get_compiled_merge_points(self, greenkey): + cell = self.jit_cell_at_key(greenkey) + if cell.compiled_merge_points is None: + cell.compiled_merge_points = [] + return cell.compiled_merge_points + def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) lst = self.fail_descr_list @@ -1127,12 +1130,6 @@ def is_blackholing(self): return self.history is None - def blackholing_text(self): - if self.history is None: - return " (BlackHole)" - else: - return "" - def newframe(self, jitcode): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 @@ -1154,7 +1151,11 @@ return True else: if not self.is_blackholing(): - self.compile_done_with_this_frame(resultbox) + try: + self.compile_done_with_this_frame(resultbox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() sd = self.staticdata if sd.result_type == 'void': assert resultbox is None @@ -1188,7 +1189,11 @@ return True self.popframe() if not self.is_blackholing(): - self.compile_exit_frame_with_exception(excvaluebox) + try: + self.compile_exit_frame_with_exception(excvaluebox) + except GiveUp: + self.staticdata.profiler.count(ABORT_BRIDGE) + self.switch_to_blackhole() raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base()) def check_recursion_invariant(self): @@ -1316,9 +1321,11 @@ op.name = self.framestack[-1].jitcode.name def switch_to_blackhole(self): + debug_print('~~~ ABORTING TRACING') + debug_stop('jit-tracing') + debug_start('jit-blackhole') self.history = None # start blackholing self.staticdata.stats.aborted() - self.staticdata.log('~~~ ABORTING TRACING', event_kind='event') self.staticdata.profiler.end_tracing() self.staticdata.profiler.start_blackhole() @@ -1333,8 +1340,6 @@ # Execute the frames forward until we raise a DoneWithThisFrame, # a ContinueRunningNormally, or a GenerateMergePoint exception. self.staticdata.stats.entered() - self.staticdata.log('~~~ ENTER' + self.blackholing_text(), - event_kind='event') try: while True: self.framestack[-1].run_one_step() @@ -1346,8 +1351,6 @@ self.staticdata.profiler.end_blackhole() else: self.staticdata.profiler.end_tracing() - self.staticdata.log('~~~ LEAVE' + self.blackholing_text(), - event_kind='event') def interpret(self): if we_are_translated(): @@ -1358,11 +1361,22 @@ except: import sys if sys.exc_info()[0] is not None: - history.log.info(sys.exc_info()[0].__name__) + codewriter.log.info(sys.exc_info()[0].__name__) raise def compile_and_run_once(self, *args): - self.staticdata.log('Switching from interpreter to compiler') + debug_start('jit-tracing') + self.staticdata._setup_once() + self.create_empty_history() + try: + return self._compile_and_run_once(*args) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _compile_and_run_once(self, *args): original_boxes = self.initialize_state_from_start(*args) self.current_merge_points = [(original_boxes, 0)] num_green_args = self.staticdata.num_green_args @@ -1378,9 +1392,24 @@ return self.designate_target_loop(gmp) def handle_guard_failure(self, key): - from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase assert isinstance(key, compile.ResumeGuardDescr) - resumedescr = self.initialize_state_from_guard_failure(key) + warmrunnerstate = self.staticdata.state + must_compile = warmrunnerstate.must_compile_from_failure(key) + if must_compile: + debug_start('jit-tracing') + else: + debug_start('jit-blackhole') + self.initialize_state_from_guard_failure(key, must_compile) + try: + return self._handle_guard_failure(key) + finally: + if self.history is None: + debug_stop('jit-blackhole') + else: + debug_stop('jit-tracing') + + def _handle_guard_failure(self, key): + from pypy.jit.metainterp.warmspot import ContinueRunningNormallyBase original_greenkey = key.original_greenkey # notice that here we just put the greenkey # use -1 to mark that we will have to give up @@ -1398,7 +1427,7 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(resumedescr) + warmrunnerstate.reset_counter_from_failure(key) raise def forget_consts(self, boxes, startindex=0): @@ -1486,8 +1515,7 @@ self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - old_loop_tokens = glob.compiled_merge_points.setdefault(greenargs, []) + old_loop_tokens = glob.get_compiled_merge_points(greenkey) self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) @@ -1498,10 +1526,8 @@ num_green_args = self.staticdata.num_green_args greenkey = live_arg_boxes[:num_green_args] glob = self.staticdata.globaldata - greenargs = glob.unpack_greenkey(greenkey) - try: - old_loop_tokens = glob.compiled_merge_points[greenargs] - except KeyError: + old_loop_tokens = glob.get_compiled_merge_points(greenkey) + if len(old_loop_tokens) == 0: return self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) target_loop_token = compile.compile_new_bridge(self, old_loop_tokens, @@ -1555,7 +1581,7 @@ def _initialize_from_start(self, original_boxes, num_green_args, *args): if args: - from pypy.jit.metainterp.warmspot import wrap + from pypy.jit.metainterp.warmstate import wrap box = wrap(self.cpu, args[0], num_green_args > 0) original_boxes.append(box) self._initialize_from_start(original_boxes, num_green_args-1, @@ -1563,9 +1589,7 @@ def initialize_state_from_start(self, *args): self.in_recursion = -1 # always one portal around - self.staticdata._setup_once() self.staticdata.profiler.start_tracing() - self.create_empty_history() num_green_args = self.staticdata.num_green_args original_boxes = [] self._initialize_from_start(original_boxes, num_green_args, *args) @@ -1577,12 +1601,11 @@ self.initialize_virtualizable(original_boxes) return original_boxes - def initialize_state_from_guard_failure(self, resumedescr): + def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around inputargs = self.load_values_from_failure(resumedescr) warmrunnerstate = self.staticdata.state - must_compile = warmrunnerstate.must_compile_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) self.history.inputargs = inputargs @@ -1591,7 +1614,6 @@ self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true self.rebuild_state_after_failure(resumedescr, inputargs) - return resumedescr def load_values_from_failure(self, resumedescr): cpu = self.cpu @@ -1801,6 +1823,3 @@ assert target_loop_token is not None self.argboxes = args self.target_loop_token = target_loop_token - -class GiveUp(Exception): - pass Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/resume.py Thu Nov 5 22:17:18 2009 @@ -1,17 +1,17 @@ -import sys +import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import have_debug_prints +from pypy.rlib.debug import debug_start, debug_stop, debug_print # Logic to encode the chain of frames and the state of the boxes at a # guard operation, and to decode it again. This is a bit advanced, # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles and also to compress the information -debug = False - class Snapshot(object): __slots__ = ('prev', 'boxes') @@ -92,8 +92,6 @@ UNASSIGNED = tag(-1, TAGBOX) NULLREF = tag(-1, TAGCONST) -VIRTUAL_FLAG = int((sys.maxint+1) // 2) -assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two class ResumeDataLoopMemo(object): @@ -259,7 +257,7 @@ self._number_virtuals(liveboxes) storage.rd_consts = self.memo.consts - if debug: + if have_debug_prints(): dump_storage(storage, liveboxes) return liveboxes[:] @@ -310,6 +308,13 @@ self.fielddescrs[i], box, fieldbox) + def debug_prints(self): + assert len(self.fielddescrs) == len(self.fieldnums) + for i in range(len(self.fielddescrs)): + debug_print("\t\t", + str(self.fielddescrs[i]), + str(untag(self.fieldnums[i]))) + class VirtualInfo(AbstractVirtualStructInfo): def __init__(self, known_class, fielddescrs): AbstractVirtualStructInfo.__init__(self, fielddescrs) @@ -319,11 +324,9 @@ return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, None, self.known_class) - def repr_rpython(self): - return 'VirtualInfo("%s", %s, %s)' % ( - self.known_class, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvirtualinfo", self.known_class.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VStructInfo(AbstractVirtualStructInfo): def __init__(self, typedescr, fielddescrs): @@ -333,11 +336,9 @@ def allocate(self, metainterp): return metainterp.execute_and_record(rop.NEW, self.typedescr) - def repr_rpython(self): - return 'VStructInfo("%s", %s, %s)' % ( - self.typedescr, - ['"%s"' % (fd,) for fd in self.fielddescrs], - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvstructinfo", self.typedescr.repr_rpython()) + AbstractVirtualStructInfo.debug_prints(self) class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr): @@ -357,10 +358,10 @@ self.arraydescr, box, ConstInt(i), itembox) - def repr_rpython(self): - return 'VArrayInfo("%s", %s)' % ( - self.arraydescr, - [untag(i) for i in self.fieldnums]) + def debug_prints(self): + debug_print("\tvarrayinfo", self.arraydescr) + for i in self.fieldnums: + debug_print("\t\t", str(untag(i))) def rebuild_from_resumedata(metainterp, newboxes, storage, expects_virtualizables): @@ -428,33 +429,29 @@ def dump_storage(storage, liveboxes): "For profiling only." - import os - from pypy.rlib import objectmodel - fd = os.open('log.storage', os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0666) - os.write(fd, 'Log(%d, [\n' % objectmodel.compute_unique_id(storage)) + from pypy.rlib.objectmodel import compute_unique_id + debug_start("jit-resume") + debug_print('Log storage', compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list - while True: - os.write(fd, '\t("%s", %d, %d),\n' % ( - frameinfo.jitcode, frameinfo.pc, frameinfo.exception_target)) + while frameinfo is not None: + try: + jitcodename = frameinfo.jitcode.name + except AttributeError: + jitcodename = str(compute_unique_id(frameinfo.jitcode)) + debug_print('\tjitcode/pc', jitcodename, + frameinfo.pc, frameinfo.exception_target, + 'at', compute_unique_id(frameinfo)) frameinfo = frameinfo.prev - if frameinfo is None: - break - os.write(fd, '\t],\n\t[\n') numb = storage.rd_numb - while True: - os.write(fd, '\t\t%s,\n' % ([untag(i) for i in numb.nums],)) + while numb is not None: + debug_print('\tnumb', str([untag(i) for i in numb.nums]), + 'at', compute_unique_id(numb)) numb = numb.prev - if numb is None: - break - os.write(fd, '\t], [\n') for const in storage.rd_consts: - os.write(fd, '\t"%s",\n' % (const.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - os.write(fd, '\t"%s",\n' % (box.repr_rpython(),)) - os.write(fd, '\t], [\n') + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - os.write(fd, '\t%s,\n' % (virtual.repr_rpython(),)) - os.write(fd, '\t])\n') - os.close(fd) + virtual.debug_prints() + debug_stop("jit-resume") Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/support.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/support.py Thu Nov 5 22:17:18 2009 @@ -74,16 +74,15 @@ def maybe_on_top_of_llinterp(rtyper, fnptr): # Run a generated graph on top of the llinterp for testing. # When translated, this just returns the fnptr. - llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) funcobj = get_funcobj(fnptr) if hasattr(funcobj, 'graph'): + llinterp = LLInterpreter(rtyper) #, exc_data_ptr=exc_data_ptr) def on_top_of_llinterp(*args): return llinterp.eval_graph(funcobj.graph, list(args)) else: - assert isinstance(fnptr, ootype._meth) - assert hasattr(fnptr, '_callable') + assert hasattr(funcobj, '_callable') def on_top_of_llinterp(*args): - return fnptr._callable(*args) + return funcobj._callable(*args) return on_top_of_llinterp class Entry(ExtRegistryEntry): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/oparser.py Thu Nov 5 22:17:18 2009 @@ -50,18 +50,45 @@ class OpParser(object): - def __init__(self, descr, cpu, namespace, type_system, boxkinds, + def __init__(self, input, cpu, namespace, type_system, boxkinds, invent_fail_descr=default_fail_descr): - self.descr = descr + self.input = input self.vars = {} self.cpu = cpu - self.consts = namespace + self._consts = namespace self.type_system = type_system self.boxkinds = boxkinds or {} - self._cache = namespace.setdefault('_CACHE_', {}) + if namespace is not None: + self._cache = namespace.setdefault('_CACHE_', {}) + else: + self._cache = {} self.invent_fail_descr = invent_fail_descr self.looptoken = LoopToken() + def get_const(self, name, typ): + if self._consts is None: + return name + obj = self._consts[name] + if self.type_system == 'lltype': + if typ == 'ptr': + return ConstPtr(obj) + else: + assert typ == 'class' + return ConstAddr(llmemory.cast_ptr_to_adr(obj), + self.cpu) + else: + if typ == 'ptr': + return ConstObj(obj) + else: + assert typ == 'class' + return ConstObj(ootype.cast_to_object(obj)) + + def get_descr(self, poss_descr): + if poss_descr.startswith('<'): + return None + else: + return self._consts[poss_descr] + def box_for_var(self, elem): try: return self._cache[self.type_system, elem] @@ -122,11 +149,7 @@ llstr(info))) if arg.startswith('ConstClass('): name = arg[len('ConstClass('):-1] - if self.type_system == 'lltype': - return ConstAddr(llmemory.cast_ptr_to_adr(self.consts[name]), - self.cpu) - else: - return ConstObj(ootype.cast_to_object(self.consts[name])) + return self.get_const(name, 'class') elif arg == 'None': return None elif arg == 'NULL': @@ -136,10 +159,7 @@ return ConstObj(ConstObj.value) elif arg.startswith('ConstPtr('): name = arg[len('ConstPtr('):-1] - if self.type_system == 'lltype': - return ConstPtr(self.consts[name]) - else: - return ConstObj(self.consts[name]) + return self.get_const(name, 'ptr') return self.vars[arg] def parse_op(self, line): @@ -168,10 +188,7 @@ poss_descr = allargs[-1].strip() if poss_descr.startswith('descr='): - if poss_descr.startswith('descr=<'): - descr = None - else: - descr = self.consts[poss_descr[len('descr='):]] + descr = self.get_descr(poss_descr[len('descr='):]) allargs = allargs[:-1] for arg in allargs: arg = arg.strip() @@ -232,7 +249,7 @@ return self.parse_op_no_result(line) def parse(self): - lines = self.descr.splitlines() + lines = self.input.splitlines() ops = [] newlines = [] for line in lines: @@ -284,17 +301,19 @@ inpargs = self.parse_header_line(line[1:-1]) return base_indent, inpargs -def parse(descr, cpu=None, namespace=None, type_system='lltype', - boxkinds=None, invent_fail_descr=default_fail_descr): - if namespace is None: +def parse(input, cpu=None, namespace=None, type_system='lltype', + boxkinds=None, invent_fail_descr=default_fail_descr, + no_namespace=False): + if namespace is None and not no_namespace: namespace = {} - return OpParser(descr, cpu, namespace, type_system, boxkinds, + return OpParser(input, cpu, namespace, type_system, boxkinds, invent_fail_descr).parse() def pure_parse(*args, **kwds): kwds['invent_fail_descr'] = None return parse(*args, **kwds) + def _box_counter_more_than(s): if s.isdigit(): Box._counter = max(Box._counter, int(s)+1) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_compile.py Thu Nov 5 22:17:18 2009 @@ -41,7 +41,7 @@ optimize_loop = staticmethod(optimize.optimize_loop) debug_level = 0 -class FakeGlobalData(): +class FakeGlobalData: loopnumbering = 0 class FakeMetaInterpStaticData: Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_jitprof.py Thu Nov 5 22:17:18 2009 @@ -6,9 +6,11 @@ from pypy.jit.metainterp.jitprof import * class FakeProfiler(Profiler): - def __init__(self): + def start(self): self.counter = 123456 + Profiler.start(self) self.events = [] + self.times = [0, 0, 0, 0] def timer(self): self.counter += 1 @@ -22,6 +24,12 @@ Profiler._end(self, event) self.events.append(~event) + def start_running(self): self._start(RUNNING) + def end_running(self): self._end(RUNNING) + + def start_blackhole(self): self._start(BLACKHOLE) + def end_blackhole(self): self._end(BLACKHOLE) + class ProfilerMixin(LLJitMixin): def meta_interp(self, *args, **kwds): kwds = kwds.copy() Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_logger.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,5 @@ - +import sys +from pypy.rlib import debug from pypy.jit.metainterp.test.oparser import pure_parse from pypy.jit.metainterp import logger from pypy.jit.metainterp.typesystem import llhelper @@ -9,12 +10,20 @@ class Descr(AbstractDescr): pass +def capturing(func, *args, **kwds): + log_stream = StringIO() + debug._stderr = log_stream + try: + func(*args, **kwds) + finally: + debug._stderr = sys.stderr + return log_stream.getvalue() + class Logger(logger.Logger): def log_loop(self, loop, namespace={}): - self.log_stream = StringIO() self.namespace = namespace - logger.Logger.log_loop(self, loop.inputargs, loop.operations) - return self.log_stream.getvalue() + return capturing(logger.Logger.log_loop, self, + loop.inputargs, loop.operations) def repr_of_descr(self, descr): for k, v in self.namespace.items(): @@ -116,16 +125,12 @@ def test_intro_loop(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_loop([], [], 1, "foo") - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# Loop1 (foo), 0 ops" + output = capturing(bare_logger.log_loop, [], [], 1, "foo") + assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): bare_logger = logger.Logger(self.ts) - bare_logger.log_stream = StringIO() - bare_logger.log_bridge([], [], 3) - output = bare_logger.log_stream.getvalue() - assert output.splitlines()[0] == "# bridge out of Guard3, 0 ops" + output = capturing(bare_logger.log_bridge, [], [], 3) + assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_loop.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_loop.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_loop.py Thu Nov 5 22:17:18 2009 @@ -594,6 +594,7 @@ res = self.meta_interp(f, [200]) + class TestOOtype(LoopTest, OOJitMixin): pass Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_oparser.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_oparser.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_oparser.py Thu Nov 5 22:17:18 2009 @@ -160,3 +160,17 @@ ''' loop = parse(x) # assert did not explode + +example_loop_log = '''\ +# bridge out of Guard12, 6 ops +[i0, i1, i2] +i4 = int_add(i0, 2) +i6 = int_sub(i1, 1) +i8 = int_gt(i6, 3) +guard_true(i8, descr=) [i4, i6] +debug_merge_point('(no jitdriver.get_printable_location!)') +jump(i6, i4, descr=) +''' + +def test_parse_no_namespace(): + loop = parse(example_loop_log, no_namespace=True) Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_optimizeopt.py Thu Nov 5 22:17:18 2009 @@ -23,11 +23,17 @@ self.pc = pc self.exception_target = exc_target +class Fake(object): + failargs_limit = 1000 + storedebug = None + class FakeMetaInterpStaticData(object): def __init__(self, cpu): self.cpu = cpu self.profiler = EmptyProfiler() + self.options = Fake() + self.globaldata = Fake() def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_recursive.py Thu Nov 5 22:17:18 2009 @@ -393,6 +393,80 @@ self.check_aborted_count(8) self.check_enter_count_at_most(30) + def test_max_failure_args(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + return o + + res = self.meta_interp(loop, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + self.check_aborted_count(5) + + def test_max_failure_args_exc(self): + FAILARGS_LIMIT = 10 + jitdriver = JitDriver(greens = [], reds = ['o', 'i', 'n']) + + class A(object): + def __init__(self, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9): + self.i0 = i0 + self.i1 = i1 + self.i2 = i2 + self.i3 = i3 + self.i4 = i4 + self.i5 = i5 + self.i6 = i6 + self.i7 = i7 + self.i8 = i8 + self.i9 = i9 + + + def loop(n): + i = 0 + o = A(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + while i < n: + jitdriver.can_enter_jit(o=o, i=i, n=n) + jitdriver.jit_merge_point(o=o, i=i, n=n) + o = A(i, i + 1, i + 2, i + 3, i + 4, i + 5, + i + 6, i + 7, i + 8, i + 9) + i += 1 + raise ValueError + + def main(n): + try: + loop(n) + return 1 + except ValueError: + return 0 + + res = self.meta_interp(main, [20], failargs_limit=FAILARGS_LIMIT, + listops=True) + assert not res + self.check_aborted_count(5) + def test_set_param_inlining(self): myjitdriver = JitDriver(greens=[], reds=['n', 'recurse']) def loop(n, recurse=False): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_resume.py Thu Nov 5 22:17:18 2009 @@ -8,7 +8,9 @@ from pypy.jit.metainterp import executor class Storage: - pass + rd_frame_info_list = None + rd_numb = None + rd_consts = [] def test_tag(): assert tag(3, 1) == rffi.r_short(3<<2|1) @@ -591,6 +593,7 @@ snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) snapshot = Snapshot(snapshot, [b1, b2, b3]) storage.rd_snapshot = snapshot + storage.rd_frame_info_list = None return storage def test_virtual_adder_int_constants(): @@ -761,6 +764,7 @@ [b4s, c1s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -807,6 +811,7 @@ [c1s, b4s]) # new fields liveboxes = [] modifier._number_virtuals(liveboxes) + dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None b4t = BoxPtr() Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_virtual.py Thu Nov 5 22:17:18 2009 @@ -1,5 +1,6 @@ import py from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.objectmodel import compute_unique_id from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.lltypesystem import lltype, rclass @@ -123,7 +124,8 @@ def test_two_loops_with_escaping_virtual(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'node']) def externfn(node): - llop.debug_print(lltype.Void, node) + llop.debug_print(lltype.Void, compute_unique_id(node), + node.value, node.extra) return node.value * 2 def f(n): node = self._new() Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_vlist.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_vlist.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_vlist.py Thu Nov 5 22:17:18 2009 @@ -97,9 +97,9 @@ assert res == f(10) self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0) - @py.test.mark.xfail def test_vlist_alloc_and_set(self): - # this fails, because [non-null] * n is not supported yet + # the check_loops fails, because [non-null] * n is not supported yet + # (it is implemented as a residual call) jitdriver = JitDriver(greens = [], reds = ['n']) def f(n): l = [1] * 20 @@ -116,6 +116,7 @@ res = self.meta_interp(f, [10], listops=True) assert res == f(10) + py.test.skip("'[non-null] * n' gives a residual call so far") self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0) def test_append_pop(self): Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_warmspot.py Thu Nov 5 22:17:18 2009 @@ -1,5 +1,5 @@ import py -from pypy.jit.metainterp.warmspot import ll_meta_interp, hash_whatever +from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE from pypy.rlib.jit import unroll_safe @@ -8,15 +8,6 @@ from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -def test_translate_hash_whatever(): - from pypy.rpython.test.test_llinterp import interpret - from pypy.rpython.lltypesystem import lltype - def fn(x): - return hash_whatever(lltype.typeOf(x), x) - for type_system in ('lltype', 'ootype'): - res = interpret(fn, [42], type_system=type_system) - assert res == 42 - class Exit(Exception): def __init__(self, result): self.result = result @@ -73,20 +64,6 @@ res = self.meta_interp(f, [60]) assert res == f(30) - def test_hash_collision(self): - mydriver = JitDriver(reds = ['n'], greens = ['m']) - def f(n): - m = 0 - while n > 0: - mydriver.can_enter_jit(n=n, m=m) - mydriver.jit_merge_point(n=n, m=m) - n -= 1 - if not (n % 11): - m = (m+n) & 3 - return m - res = self.meta_interp(f, [110], hash_bits=1) - assert res == f(110) - def test_location(self): def get_printable_location(n): return 'GREEN IS %d.' % n @@ -154,6 +131,7 @@ assert warmrunnerdescr.state.optimize_bridge is optimize.optimize_bridge def test_static_debug_level(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler @@ -200,6 +178,7 @@ assert not "Running asm" in err def test_set_param_debug(self): + py.test.skip("debug_level is being deprecated") from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/test/test_ztranslation.py Thu Nov 5 22:17:18 2009 @@ -4,6 +4,8 @@ from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, unroll_parameters from pypy.rlib.jit import PARAMETERS, dont_look_inside from pypy.jit.metainterp.jitprof import Profiler +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.ootypesystem import ootype class TranslationTest: @@ -18,6 +20,7 @@ # - set_param interface # - profiler # - full optimizer + # - jitdriver hooks class Frame(object): _virtualizable2_ = ['i'] @@ -25,8 +28,24 @@ def __init__(self, i): self.i = i + class JitCellCache: + entry = None + jitcellcache = JitCellCache() + def set_jitcell_at(entry): + jitcellcache.entry = entry + def get_jitcell_at(): + return jitcellcache.entry + def get_printable_location(): + return '(hello world)' + def can_inline(): + return False + jitdriver = JitDriver(greens = [], reds = ['frame', 'total'], - virtualizables = ['frame']) + virtualizables = ['frame'], + get_jitcell_at=get_jitcell_at, + set_jitcell_at=set_jitcell_at, + get_printable_location=get_printable_location, + can_inline=can_inline) def f(i): for param in unroll_parameters: defl = PARAMETERS[param] Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/virtualizable.py Thu Nov 5 22:17:18 2009 @@ -6,7 +6,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history -from pypy.jit.metainterp.warmspot import wrap, unwrap +from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: Modified: pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/metainterp/warmspot.py Thu Nov 5 22:17:18 2009 @@ -10,7 +10,6 @@ from pypy.objspace.flow.model import checkgraph, Link, copygraph from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print from pypy.rpython.lltypesystem.lloperation import llop @@ -24,7 +23,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE -from pypy.rlib.nonconst import NonConstant # ____________________________________________________________ # Bootstrapping @@ -63,7 +61,7 @@ clear_tcache() return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds) -def jittify_and_run(interp, graph, args, repeat=1, hash_bits=None, +def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator @@ -74,9 +72,6 @@ warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) warmrunnerdesc.state.set_param_debug(debug_level) - warmrunnerdesc.state.create_tables_now() # for tests - if hash_bits: - warmrunnerdesc.state.set_param_hash_bits(hash_bits) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -167,7 +162,7 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() self.rewrite_jit_merge_point(policy) - self.make_driverhook_graph() + self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -269,8 +264,9 @@ warmrunnerdesc=self) def make_enter_function(self): - WarmEnterState = make_state_class(self) - state = WarmEnterState() + from pypy.jit.metainterp.warmstate import WarmEnterState + state = WarmEnterState(self) + maybe_compile_and_run = state.make_entry_point() self.state = state def crash_in_jit(e): @@ -288,7 +284,7 @@ if self.translator.rtyper.type_system.name == 'lltypesystem': def maybe_enter_jit(*args): try: - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) except JitException: raise # go through except Exception, e: @@ -296,7 +292,7 @@ maybe_enter_jit._always_inline_ = True else: def maybe_enter_jit(*args): - state.maybe_compile_and_run(*args) + maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit @@ -312,24 +308,38 @@ args_s, s_result) annhelper.finish() - def make_driverhook_graph(self): + def make_driverhook_graphs(self): + from pypy.rlib.jit import BaseJitCell + bk = self.rtyper.annotator.bookkeeper + classdef = bk.getuniqueclassdef(BaseJitCell) + s_BaseJitCell_or_None = annmodel.SomeInstance(classdef, + can_be_None=True) + s_BaseJitCell_not_None = annmodel.SomeInstance(classdef) + s_Str = annmodel.SomeString() + # + annhelper = MixLevelHelperAnnotator(self.translator.rtyper) + self.set_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.set_jitcell_at, annmodel.s_None, + s_BaseJitCell_not_None) + self.get_jitcell_at_ptr = self._make_hook_graph( + annhelper, self.jitdriver.get_jitcell_at, s_BaseJitCell_or_None) self.can_inline_ptr = self._make_hook_graph( - self.jitdriver.can_inline, bool) + annhelper, self.jitdriver.can_inline, annmodel.s_Bool) self.get_printable_location_ptr = self._make_hook_graph( - self.jitdriver.get_printable_location, str) + annhelper, self.jitdriver.get_printable_location, s_Str) + annhelper.finish() - def _make_hook_graph(self, func, rettype): - from pypy.annotation.signature import annotationoftype + def _make_hook_graph(self, annhelper, func, s_result, s_first_arg=None): if func is None: return None - annhelper = MixLevelHelperAnnotator(self.translator.rtyper) - s_result = annotationoftype(rettype) - RETTYPE = annhelper.rtyper.getrepr(s_result).lowleveltype - FUNC, PTR = self.cpu.ts.get_FuncType(self.green_args_spec, RETTYPE) + # + extra_args_s = [] + if s_first_arg is not None: + extra_args_s.append(s_first_arg) + # args_s = self.portal_args_s[:len(self.green_args_spec)] - graph = annhelper.getgraph(func, args_s, s_result) - funcptr = annhelper.graph2delayed(graph, FUNC) - annhelper.finish() + graph = annhelper.getgraph(func, extra_args_s + args_s, s_result) + funcptr = annhelper.graph2delayed(graph) return funcptr def make_args_specification(self): @@ -346,6 +356,7 @@ self.green_args_spec.append(TYPE) else: self.red_args_types.append(history.getkind(TYPE)) + self.num_green_args = len(self.green_args_spec) RESTYPE = graph.getreturnvar().concretetype (self.JIT_ENTER_FUNCTYPE, self.PTR_JIT_ENTER_FUNCTYPE) = self.cpu.ts.get_FuncType(ALLARGS, lltype.Void) @@ -465,6 +476,7 @@ # accepts boxes as argument, but unpacks them immediately # before we raise the exception -- the boxes' values will # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap for i, name, ARG in portalfunc_ARGS: v = unwrap(ARG, argboxes[i]) setattr(self, name, v) @@ -585,390 +597,3 @@ assert len(reds_v) == numreds return ([v for v in greens_v if v.concretetype is not lltype.Void], [v for v in reds_v if v.concretetype is not lltype.Void]) - -def unwrap(TYPE, box): - if TYPE is lltype.Void: - return None - if isinstance(TYPE, lltype.Ptr): - return box.getref(TYPE) - if isinstance(TYPE, ootype.OOType): - return box.getref(TYPE) - if TYPE == lltype.Float: - return box.getfloat() - else: - return lltype.cast_primitive(TYPE, box.getint()) -unwrap._annspecialcase_ = 'specialize:arg(0)' - -def wrap(cpu, value, in_const_box=False): - if isinstance(lltype.typeOf(value), lltype.Ptr): - if lltype.typeOf(value).TO._gckind == 'gc': - value = lltype.cast_opaque_ptr(llmemory.GCREF, value) - if in_const_box: - return history.ConstPtr(value) - else: - return history.BoxPtr(value) - else: - adr = llmemory.cast_ptr_to_adr(value) - value = cpu.cast_adr_to_int(adr) - # fall through to the end of the function - elif isinstance(lltype.typeOf(value), ootype.OOType): - value = ootype.cast_to_object(value) - if in_const_box: - return history.ConstObj(value) - else: - return history.BoxObj(value) - elif isinstance(value, float): - if in_const_box: - return history.ConstFloat(value) - else: - return history.BoxFloat(value) - else: - value = intmask(value) - if in_const_box: - return history.ConstInt(value) - else: - return history.BoxInt(value) -wrap._annspecialcase_ = 'specialize:ll' - -def equal_whatever(TYPE, x, y): - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_streq(x, y) - return x == y -equal_whatever._annspecialcase_ = 'specialize:arg(0)' - -def hash_whatever(TYPE, x): - # Hash of lltype or ootype object. - # Only supports strings, unicodes and regular instances, - # as well as primitives that can meaningfully be cast to Signed. - if isinstance(TYPE, lltype.Ptr): - if TYPE.TO is rstr.STR or TYPE.TO is rstr.UNICODE: - return rstr.LLHelpers.ll_strhash(x) # assumed not null - else: - if x: - return lltype.identityhash(x) - else: - return 0 - elif TYPE is ootype.String or TYPE is ootype.Unicode: - return x.ll_hash() - elif isinstance(TYPE, ootype.OOType): - if x: - return ootype.identityhash(x) - else: - return 0 - else: - return lltype.cast_primitive(lltype.Signed, x) -hash_whatever._annspecialcase_ = 'specialize:arg(0)' - -# ____________________________________________________________ - -def make_state_class(warmrunnerdesc): - jitdriver = warmrunnerdesc.jitdriver - num_green_args = len(jitdriver.greens) - warmrunnerdesc.num_green_args = num_green_args - green_args_spec = unrolling_iterable(warmrunnerdesc.green_args_spec) - green_args_names = unrolling_iterable(jitdriver.greens) - green_args_spec_names = unrolling_iterable(zip( - warmrunnerdesc.green_args_spec, jitdriver.greens)) - red_args_types = unrolling_iterable(warmrunnerdesc.red_args_types) - # - metainterp_sd = warmrunnerdesc.metainterp_sd - vinfo = metainterp_sd.virtualizable_info - if vinfo is None: - vable_static_fields = [] - vable_array_fields = [] - else: - vable_static_fields = unrolling_iterable( - zip(vinfo.static_extra_types, vinfo.static_fields)) - vable_array_fields = unrolling_iterable( - zip(vinfo.arrayitem_extra_types, vinfo.array_fields)) - # - if num_green_args: - MAX_HASH_TABLE_BITS = 28 - else: - MAX_HASH_TABLE_BITS = 1 - THRESHOLD_LIMIT = sys.maxint // 2 - # - getlength = warmrunnerdesc.cpu.ts.getlength - getarrayitem = warmrunnerdesc.cpu.ts.getarrayitem - setarrayitem = warmrunnerdesc.cpu.ts.setarrayitem - # - rtyper = warmrunnerdesc.translator.rtyper - can_inline_ptr = warmrunnerdesc.can_inline_ptr - get_printable_location_ptr = warmrunnerdesc.get_printable_location_ptr - # - class MachineCodeEntryPoint(object): - next = None # linked list - def __init__(self, entry_loop_token, *greenargs): - self.entry_loop_token = entry_loop_token - i = 0 - for name in green_args_names: - setattr(self, 'green_' + name, greenargs[i]) - i = i + 1 - def equalkey(self, *greenargs): - i = 0 - for TYPE, name in green_args_spec_names: - myvalue = getattr(self, 'green_' + name) - if not equal_whatever(TYPE, myvalue, greenargs[i]): - return False - i = i + 1 - return True - def set_future_values(self, *redargs): - i = 0 - for typecode in red_args_types: - set_future_value(i, redargs[i], typecode) - i = i + 1 - if vinfo is not None: - virtualizable = redargs[vinfo.index_of_virtualizable - - num_green_args] - virtualizable = vinfo.cast_to_vtype(virtualizable) - for typecode, fieldname in vable_static_fields: - x = getattr(virtualizable, fieldname) - set_future_value(i, x, typecode) - i = i + 1 - for typecode, fieldname in vable_array_fields: - lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)): - x = getarrayitem(lst, j) - set_future_value(i, x, typecode) - i = i + 1 - - def set_future_value(j, value, typecode): - cpu = metainterp_sd.cpu - if typecode == 'ref': - refvalue = cpu.ts.cast_to_ref(value) - cpu.set_future_value_ref(j, refvalue) - elif typecode == 'int': - intvalue = lltype.cast_primitive(lltype.Signed, value) - cpu.set_future_value_int(j, intvalue) - elif typecode == 'float': - assert isinstance(value, float) - cpu.set_future_value_float(j, value) - else: - assert False - set_future_value._annspecialcase_ = 'specialize:ll_and_arg(2)' - - class WarmEnterState: - def __init__(self): - # initialize the state with the default values of the - # parameters specified in rlib/jit.py - for name, default_value in PARAMETERS.items(): - meth = getattr(self, 'set_param_' + name) - meth(default_value) - - def set_param_threshold(self, threshold): - if threshold < 2: - threshold = 2 - self.increment_threshold = (THRESHOLD_LIMIT // threshold) + 1 - # the number is at least 1, and at most about half THRESHOLD_LIMIT - - def set_param_trace_eagerness(self, value): - self.trace_eagerness = value - - def set_param_trace_limit(self, value): - self.trace_limit = value - - def set_param_inlining(self, value): - self.inlining = value - - def set_param_hash_bits(self, value): - if value < 1: - value = 1 - elif value > MAX_HASH_TABLE_BITS: - value = MAX_HASH_TABLE_BITS - # the tables are initialized with the correct size only in - # create_tables_now() - self.hashbits = value - self.hashtablemask = 0 - self.mccounters = [0] - self.mcentrypoints = [None] - # invariant: (self.mccounters[j] < 0) if and only if - # (self.mcentrypoints[j] is not None) - - def set_param_optimizer(self, optimizer): - if optimizer == OPTIMIZER_SIMPLE: - from pypy.jit.metainterp import simple_optimize - self.optimize_loop = simple_optimize.optimize_loop - self.optimize_bridge = simple_optimize.optimize_bridge - elif optimizer == OPTIMIZER_FULL: - from pypy.jit.metainterp import optimize - self.optimize_loop = optimize.optimize_loop - self.optimize_bridge = optimize.optimize_bridge - else: - raise ValueError("unknown optimizer") - - def set_param_debug(self, value): - self.debug_level = value - metainterp_sd.profiler.set_printing(value >= DEBUG_PROFILE) - - def create_tables_now(self): - count = 1 << self.hashbits - self.hashtablemask = count - 1 - self.mccounters = [0] * count - self.mcentrypoints = [None] * count - - # Only use the hash of the arguments as the profiling key. - # Indeed, this is all a heuristic, so if things are designed - # correctly, the occasional mistake due to hash collision is - # not too bad. - - def maybe_compile_and_run(self, *args): - globaldata = metainterp_sd.globaldata - if NonConstant(False): - # make sure we always see the saner optimizer from an annotation - # point of view, otherwise we get lots of blocked ops - self.set_param_optimizer(OPTIMIZER_FULL) - - # get the greenargs and look for the cell corresponding to the hash - greenargs = args[:num_green_args] - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - counter = self.mccounters[argshash] - if vinfo: - virtualizable = args[vinfo.index_of_virtualizable] - virtualizable = vinfo.cast_to_vtype(virtualizable) - assert virtualizable != globaldata.blackhole_virtualizable, "reentering same frame via blackhole" - if counter >= 0: - # update the profiling counter - n = counter + self.increment_threshold - if n <= THRESHOLD_LIMIT: # bound not reached - self.mccounters[argshash] = n - return - if self.hashtablemask == 0: # must really create the tables now - self.create_tables_now() - return - metainterp = MetaInterp(metainterp_sd) - try: - loop_token = metainterp.compile_and_run_once(*args) - except warmrunnerdesc.ContinueRunningNormally: - # the trace got too long, reset the counter - self.mccounters[argshash] = 0 - raise - - else: - # machine code was already compiled for these greenargs - # (or we have a hash collision) - cell = self.mcentrypoints[argshash] - if not cell.equalkey(*greenargs): - # hash collision - loop_token = self.handle_hash_collision(cell, argshash, - *args) - if loop_token is None: - return - else: - # get the assembler and fill in the boxes - cell.set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token - # ---------- execute assembler ---------- - while True: # until interrupted by an exception - metainterp_sd.profiler.start_running() - fail_index = metainterp_sd.cpu.execute_token(loop_token) - metainterp_sd.profiler.end_running() - fail_descr = globaldata.get_fail_descr_from_number(fail_index) - loop_token = fail_descr.handle_fail(metainterp_sd) - - maybe_compile_and_run._dont_inline_ = True - - def handle_hash_collision(self, firstcell, argshash, *args): - greenargs = args[:num_green_args] - # search the linked list for the correct cell - cell = firstcell - while cell.next is not None: - nextcell = cell.next - if nextcell.equalkey(*greenargs): - # found, move to the front of the linked list - cell.next = nextcell.next - nextcell.next = firstcell - self.mcentrypoints[argshash] = nextcell - nextcell.set_future_values(*args[num_green_args:]) - return nextcell.entry_loop_token - cell = nextcell - # not found at all, do profiling - counter = self.mccounters[argshash] - assert counter < 0 # by invariant - n = counter + self.increment_threshold - if n < 0: # bound not reached - self.mccounters[argshash] = n - return None - metainterp = MetaInterp(metainterp_sd) - # XXX ContinueRunningNormally => "reset" counters - return metainterp.compile_and_run_once(*args) - handle_hash_collision._dont_inline_ = True - - def unwrap_greenkey(self, greenkey): - greenargs = () - i = 0 - for TYPE in green_args_spec: - value = unwrap(TYPE, greenkey[i]) - greenargs += (value,) - i = i + 1 - return greenargs - unwrap_greenkey._always_inline_ = True - - def comparekey(greenargs1, greenargs2): - i = 0 - for TYPE in green_args_spec: - if not equal_whatever(TYPE, greenargs1[i], greenargs2[i]): - return False - i = i + 1 - return True - comparekey = staticmethod(comparekey) - - def hashkey(greenargs): - return intmask(WarmEnterState.getkeyhash(*greenargs)) - hashkey = staticmethod(hashkey) - - def getkeyhash(*greenargs): - result = r_uint(0x345678) - i = 0 - mult = r_uint(1000003) - for TYPE in green_args_spec: - if i > 0: - result = result * mult - mult = mult + 82520 + 2*len(greenargs) - item = greenargs[i] - result = result ^ hash_whatever(TYPE, item) - i = i + 1 - return result # returns a r_uint - getkeyhash._always_inline_ = True - getkeyhash = staticmethod(getkeyhash) - - def must_compile_from_failure(self, key): - key.counter += 1 - return key.counter >= self.trace_eagerness - - def reset_counter_from_failure(self, key): - key.counter = 0 - - def attach_unoptimized_bridge_from_interp(self, greenkey, - entry_loop_token): - greenargs = self.unwrap_greenkey(greenkey) - newcell = MachineCodeEntryPoint(entry_loop_token, *greenargs) - argshash = self.getkeyhash(*greenargs) & self.hashtablemask - oldcell = self.mcentrypoints[argshash] - newcell.next = oldcell # link - self.mcentrypoints[argshash] = newcell - self.mccounters[argshash] = -THRESHOLD_LIMIT-1 - - if can_inline_ptr is None: - def can_inline_callable(self, greenkey): - return True - else: - def can_inline_callable(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) - return fn(*args) - - if get_printable_location_ptr is None: - def get_location_str(self, greenkey): - return '(no jitdriver.get_printable_location!)' - else: - def get_location_str(self, greenkey): - args = self.unwrap_greenkey(greenkey) - fn = support.maybe_on_top_of_llinterp(rtyper, - get_printable_location_ptr) - res = fn(*args) - if not we_are_translated() and not isinstance(res, str): - res = hlstr(res) - return res - - return WarmEnterState Modified: pypy/branch/msvc-asmgcroot/pypy/jit/tl/pypyjit.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/jit/tl/pypyjit.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/jit/tl/pypyjit.py Thu Nov 5 22:17:18 2009 @@ -41,6 +41,7 @@ config.objspace.usemodules._weakref = False config.objspace.usemodules._sre = False set_pypy_opt_level(config, level='jit') +config.objspace.std.withinlineddict = True if BACKEND == 'c': config.objspace.std.multimethods = 'mrd' Modified: pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/arithmetic.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/arithmetic.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/arithmetic.py Thu Nov 5 22:17:18 2009 @@ -4,7 +4,6 @@ from pypy.lang.prolog.interpreter import engine, helper, term, error from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound from pypy.rlib.rarithmetic import intmask -from pypy.rlib.jit import we_are_jitted, hint from pypy.rlib.unroll import unrolling_iterable arithmetic_functions = {} Modified: pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/lang/prolog/interpreter/term.py Thu Nov 5 22:17:18 2009 @@ -26,10 +26,6 @@ __slots__ = () _immutable_ = True - def __init__(self): - raise NotImplementedError("abstract base class") - return self - def getvalue(self, heap): return self @@ -270,12 +266,11 @@ error.throw_type_error("evaluable", self.get_prolog_signature()) -class Number(NonVar): +class Number(NonVar, UnboxedValue): TAG = tag() STANDARD_ORDER = 2 _immutable_ = True - def __init__(self, num): - self.num = num + __slots__ = ("num", ) @specialize.arg(3) def basic_unify(self, other, heap, occurs_check=False): Modified: pypy/branch/msvc-asmgcroot/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/lang/smalltalk/model.py Thu Nov 5 22:17:18 2009 @@ -19,7 +19,7 @@ from pypy.rlib.rarithmetic import intmask from pypy.lang.smalltalk import constants, error from pypy.tool.pairtype import extendabletype -from pypy.rlib.objectmodel import instantiate +from pypy.rlib.objectmodel import instantiate, UnboxedValue from pypy.rlib.bitmanipulation import splitter class W_Object(object): @@ -96,14 +96,15 @@ False means swapping failed""" return False -class W_SmallInteger(W_Object): + +# the UnboxedValue mixin means the object can potentially be stored in unboxed +# form + +class W_SmallInteger(W_Object, UnboxedValue): """Boxed integer value""" # TODO can we tell pypy that its never larger then 31-bit? __slots__ = ('value',) # the only allowed slot here - def __init__(self, value): - self.value = value - def getclass(self, space): """Return SmallInteger from special objects array.""" return space.w_SmallInteger Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/__init__.py Thu Nov 5 22:17:18 2009 @@ -32,8 +32,6 @@ # redirected to functional.py, applevel version # is still needed and should stay where it is. 'sorted' : 'app_functional.sorted', - 'globals' : 'app_inspect.globals', - 'locals' : 'app_inspect.locals', 'vars' : 'app_inspect.vars', 'dir' : 'app_inspect.dir', @@ -110,6 +108,9 @@ 'classmethod' : 'descriptor.ClassMethod', 'property' : 'descriptor.W_Property', + 'globals' : 'interp_inspect.globals', + 'locals' : 'interp_inspect.locals', + } def pick_builtin(self, w_globals): Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/app_inspect.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/app_inspect.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/app_inspect.py Thu Nov 5 22:17:18 2009 @@ -5,16 +5,9 @@ import sys -def globals(): - "Return the dictionary containing the current scope's global variables." - return sys._getframe(0).f_globals - -def locals(): - """Return a dictionary containing the current scope's local variables. -Note that this may be the real dictionary of local variables, or a copy.""" - return sys._getframe(0).f_locals - def _caller_locals(): + # note: the reason why this is working is because the functions in here are + # compiled by geninterp, so they don't have a frame return sys._getframe(0).f_locals def vars(*obj): Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/functional.py Thu Nov 5 22:17:18 2009 @@ -74,8 +74,7 @@ # save duplication by redirecting every error to applevel return range_fallback(space, w_x, w_y, w_step) - if (space.config.objspace.std.withmultilist or - space.config.objspace.std.withrangelist): + if space.config.objspace.std.withrangelist: return range_withspecialized_implementation(space, start, step, howmany) res_w = [None] * howmany @@ -92,14 +91,9 @@ ).interphook('range') def range_withspecialized_implementation(space, start, step, howmany): - if space.config.objspace.std.withrangelist: - from pypy.objspace.std.rangeobject import W_RangeListObject - return W_RangeListObject(start, step, howmany) - if space.config.objspace.std.withmultilist: - from pypy.objspace.std.listmultiobject import W_ListMultiObject - from pypy.objspace.std.listmultiobject import RangeImplementation - impl = RangeImplementation(space, start, step, howmany) - return W_ListMultiObject(space, impl) + assert space.config.objspace.std.withrangelist + from pypy.objspace.std.rangeobject import W_RangeListObject + return W_RangeListObject(start, step, howmany) @specialize.arg(2) Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/interp_classobj.py Thu Nov 5 22:17:18 2009 @@ -304,12 +304,7 @@ class W_InstanceObject(Wrappable): def __init__(self, space, w_class, w_dict=None): if w_dict is None: - if space.config.objspace.std.withsharingdict: - from pypy.objspace.std import dictmultiobject - w_dict = dictmultiobject.W_DictMultiObject(space, - sharing=True) - else: - w_dict = space.newdict() + w_dict = space.newdict(instance=True) assert isinstance(w_class, W_ClassObject) self.w_class = w_class self.w_dict = w_dict @@ -388,7 +383,7 @@ if w_meth is not None: space.call_function(w_meth, w_name, w_value) else: - self.setdictvalue(space, w_name, w_value) + self.setdictvalue(space, name, w_value) def descr_delattr(self, space, w_name): name = unwrap_attr(space, w_name) Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_builtin.py Thu Nov 5 22:17:18 2009 @@ -1,5 +1,5 @@ import autopath - +import sys class AppTestBuiltinApp: def setup_class(cls): @@ -397,6 +397,7 @@ def test_compile(self): co = compile('1+2', '?', 'eval') assert eval(co) == 3 + compile("from __future__ import with_statement", "", "exec") raises(SyntaxError, compile, '-', '?', 'eval') raises(ValueError, compile, '"\\xt"', '?', 'eval') raises(ValueError, compile, '1+2', '?', 'maybenot') Modified: pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/__builtin__/test/test_classobj.py Thu Nov 5 22:17:18 2009 @@ -772,8 +772,7 @@ def is_sharing(space, w_inst): from pypy.objspace.std.sharingdict import SharedDictImplementation, W_DictMultiObject w_d = w_inst.getdict() - return space.wrap(isinstance(w_d, W_DictMultiObject) and - isinstance(w_d.implementation, SharedDictImplementation)) + return space.wrap(isinstance(w_d, SharedDictImplementation)) cls.w_is_sharing = cls.space.wrap(gateway.interp2app(is_sharing)) Modified: pypy/branch/msvc-asmgcroot/pypy/module/marshal/test/test_marshal.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/marshal/test/test_marshal.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/marshal/test/test_marshal.py Thu Nov 5 22:17:18 2009 @@ -172,15 +172,6 @@ self.marshal_check(unichr(sys.maxunicode)) -class AppTestMultiDict(object): - def setup_class(cls): - from pypy.conftest import gettestobjspace - cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) - AppTestMarshal.setup_class.im_func(cls) - - test__dict__tcid_ = AppTestMarshal.test__dict__tcid_.im_func - test__dict_5_colon__6_comma__7_colon__8_tcid_ = AppTestMarshal.test__dict_5_colon__6_comma__7_colon__8_tcid_.im_func - class AppTestRope(AppTestMarshal): def setup_class(cls): from pypy.conftest import gettestobjspace Modified: pypy/branch/msvc-asmgcroot/pypy/module/operator/__init__.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/operator/__init__.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/operator/__init__.py Thu Nov 5 22:17:18 2009 @@ -9,7 +9,7 @@ def __init__(self, space, w_name): def create_lambda(name, alsoname): - return lambda space : self.getdictvalue(space, space.wrap(alsoname)) + return lambda space : self.getdictvalue(space, alsoname) MixedModule.__init__(self, space, w_name) for name, alsoname in self.mapping.iteritems(): Modified: pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/interp_jit.py Thu Nov 5 22:17:18 2009 @@ -11,12 +11,11 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, Arguments from pypy.interpreter.eval import Frame -from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS +from pypy.interpreter.pycode import PyCode, CO_CONTAINSLOOP from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.function import Function from pypy.interpreter.pyopcode import ExitFrame from pypy.rpython.annlowlevel import cast_base_ptr_to_instance -from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT from opcode import opmap from pypy.rlib.objectmodel import we_are_translated @@ -28,19 +27,7 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] def can_inline(next_instr, bytecode): - co_code = bytecode.co_code - next_instr = 0 - while next_instr < len(co_code): - opcode = ord(co_code[next_instr]) - next_instr += 1 - if opcode >= HAVE_ARGUMENT: - next_instr += 2 - while opcode == opcodedesc.EXTENDED_ARG.index: - opcode = ord(co_code[next_instr]) - next_instr += 3 - if opcode == JUMP_ABSOLUTE: - return False - return True + return not bool(bytecode.co_flags & CO_CONTAINSLOOP) def get_printable_location(next_instr, bytecode): from pypy.tool.stdlib_opcode import opcode_method_names @@ -50,9 +37,16 @@ def leave(next_instr, pycode, frame, ec): from pypy.interpreter.executioncontext import ExecutionContext # can't use a method here, since this function is seen later than the main - # annotation + # annotation XXX no longer true, could be fixed ExecutionContext._jit_rechain_frame(ec, frame) +def get_jitcell_at(next_instr, bytecode): + return bytecode.jit_cells.get(next_instr, None) + +def set_jitcell_at(newcell, next_instr, bytecode): + bytecode.jit_cells[next_instr] = newcell + + class PyPyJitDriver(JitDriver): reds = ['frame', 'ec'] greens = ['next_instr', 'pycode'] @@ -68,7 +62,9 @@ pypyjitdriver = PyPyJitDriver(can_inline = can_inline, get_printable_location = get_printable_location, - leave = leave) + leave = leave, + get_jitcell_at = get_jitcell_at, + set_jitcell_at = set_jitcell_at) class __extend__(PyFrame): @@ -94,6 +90,20 @@ pycode=f.getcode()) return jumpto + +PyCode__initialize = PyCode._initialize + +class __extend__(PyCode): + __metaclass__ = extendabletype + + def _initialize(self): + PyCode__initialize(self) + self.jit_cells = {} + + def _freeze_(self): + self.jit_cells = {} + return False + # ____________________________________________________________ # # Public interface Modified: pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/conftest.py Thu Nov 5 22:17:18 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): group = parser.addgroup("pypyjit options") - group.addoption("--pypy-c", action="store", default=None, dest="pypy_c", + group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_can_inline.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_can_inline.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_can_inline.py Thu Nov 5 22:17:18 2009 @@ -1,14 +1,28 @@ - +from pypy.interpreter import pycode from pypy.module.pypyjit.interp_jit import can_inline +class FakeSpace(object): + class config: + class objspace: + class std: + withcelldict = True + wrap = new_interned_str = unwrap = lambda _, x: x + def fromcache(self, X): + return X(self) + + def test_one(): + space = FakeSpace() def f(): pass - assert can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert can_inline(0, code) def f(): while i < 0: pass - assert not can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert not can_inline(0, code) def f(a, b): return a + b - assert can_inline(0, f.func_code) + code = pycode.PyCode._from_code(space, f.func_code) + assert can_inline(0, code) Modified: pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_jit_setup.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_jit_setup.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_jit_setup.py Thu Nov 5 22:17:18 2009 @@ -9,8 +9,8 @@ # this just checks that the module is setting up things correctly, and # the resulting code makes sense on top of CPython. import pypyjit - pypyjit.set_param(threshold=5, hash_bits=9) - pypyjit.set_param("trace_eagerness=3,hash_bits=7") + pypyjit.set_param(threshold=5, inlining=1) + pypyjit.set_param("trace_eagerness=3,inlining=0") def f(x, y): return x*y+1 Modified: pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/pypyjit/test/test_pypy_c.py Thu Nov 5 22:17:18 2009 @@ -1,14 +1,38 @@ from pypy.conftest import gettestobjspace, option from pypy.tool.udir import udir import py +from py.test import skip import sys, os +class BytecodeTrace(list): + def get_opnames(self, prefix=""): + return [op.getopname() for op in self + if op.getopname().startswith(prefix)] + + def __repr__(self): + return "%s%s" % (self.bytecode, list.__repr__(self)) + +ZERO_OP_BYTECODES = [ + 'POP_TOP', + 'ROT_TWO', + 'ROT_THREE', + 'DUP_TOP', + 'ROT_FOUR', + 'NOP', + 'DUP_TOPX', + 'LOAD_CONST', + 'JUMP_FORWARD', + #'JUMP_ABSOLUTE' in theory, but contains signals stuff + #'LOAD_FAST' should be here, but currently needs a guard for nonzeroness + 'STORE_FAST', + ] + class PyPyCJITTests(object): - def run_source(self, source, testcases): + def run_source(self, source, *testcases): source = py.code.Source(source) filepath = self.tmpdir.join('case%d.py' % self.counter) logfilepath = filepath.new(ext='.log') - self.counter += 1 + self.__class__.counter += 1 f = filepath.open('w') print >> f, source # some support code... @@ -17,12 +41,11 @@ pypyjit.set_param(threshold=3) def check(args, expected): - for i in range(3): - print >> sys.stderr, 'trying:', args - result = main(*args) - print >> sys.stderr, 'got:', repr(result) - assert result == expected - assert type(result) is type(expected) + print >> sys.stderr, 'trying:', args + result = main(*args) + print >> sys.stderr, 'got:', repr(result) + assert result == expected + assert type(result) is type(expected) """) for testcase in testcases * 2: print >> f, "check(%r, %r)" % testcase @@ -32,26 +55,43 @@ # we don't have os.popen() yet on pypy-c... if sys.platform.startswith('win'): py.test.skip("XXX this is not Windows-friendly") - child_stdin, child_stdout = os.popen2('PYPYJITLOG="%s" "%s" "%s"' % ( + child_stdin, child_stdout = os.popen2('PYPYLOG=":%s" "%s" "%s"' % ( logfilepath, self.pypy_c, filepath)) child_stdin.close() result = child_stdout.read() child_stdout.close() assert result assert result.splitlines()[-1].strip() == 'OK :-)' - assert logfilepath.check() + self.parse_loops(logfilepath) + def parse_loops(self, opslogfile): + from pypy.jit.metainterp.test.oparser import parse + from pypy.tool import logparser + assert opslogfile.check() + log = logparser.parse_log_file(str(opslogfile)) + parts = logparser.extract_category(log, 'jit-log-opt-') + # skip entry bridges, they can contain random things + self.loops = [parse(part, no_namespace=True) for part in parts + if "entry bridge" not in part] + self.sliced_loops = [] # contains all bytecodes of all loops + for loop in self.loops: + for op in loop.operations: + if op.getopname() == "debug_merge_point": + sliced_loop = BytecodeTrace() + sliced_loop.bytecode = op.args[0]._get_str().rsplit(" ", 1)[1] + self.sliced_loops.append(sliced_loop) + else: + sliced_loop.append(op) + self.check_0_op_bytecodes() - def test_f(self): - self.run_source(""" - def main(n): - return (n+5)+6 - """, - [([100], 111), - ([-5], 6), - ([sys.maxint], sys.maxint+11), - ([-sys.maxint-5], long(-sys.maxint+6)), - ]) + def check_0_op_bytecodes(self): + for bytecodetrace in self.sliced_loops: + if bytecodetrace.bytecode not in ZERO_OP_BYTECODES: + continue + assert not bytecodetrace + + def get_by_bytecode(self, name): + return [ops for ops in self.sliced_loops if ops.bytecode == name] def test_f1(self): self.run_source(''' @@ -67,7 +107,7 @@ i = i + 1 return x ''', - [([2117], 1083876708)]) + ([2117], 1083876708)) def test_factorial(self): self.run_source(''' @@ -78,10 +118,11 @@ n -= 1 return r ''', - [([5], 120), - ([20], 2432902008176640000L)]) + ([5], 120), + ([20], 2432902008176640000L)) def test_factorialrec(self): + skip("does not make sense yet") self.run_source(''' def main(n): if n > 1: @@ -89,8 +130,8 @@ else: return 1 ''', - [([5], 120), - ([20], 2432902008176640000L)]) + ([5], 120), + ([20], 2432902008176640000L)) def test_richards(self): self.run_source(''' @@ -100,24 +141,163 @@ def main(): return richards.main(iterations = 1) ''' % (sys.path,), - [([], 42)]) + ([], 42)) - def test_inplace_op(self): + def test_simple_call(self): self.run_source(''' - def main(x, y): - r = 5 - r += x - r += -y - return r - ''', [([17, 3], 19), - ([sys.maxint-3, 5], long(sys.maxint - 3)), - ([17, -sys.maxint - 1], sys.maxint + 23) - ]) + def f(i): + return i + 1 + def main(n): + i = 0 + while i < n: + i = f(f(i)) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("LOAD_GLOBAL") + assert len(ops) == 2 + assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + "getfield_gc", "ooisnull", + "guard_false"] + assert not ops[1] # second LOAD_GLOBAL folded away + ops = self.get_by_bytecode("CALL_FUNCTION") + assert len(ops) == 2 + for bytecode in ops: + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 10 + + def test_method_call(self): + self.run_source(''' + class A(object): + def __init__(self, a): + self.a = a + def f(self, i): + return self.a + i + def main(n): + i = 0 + a = A(1) + while i < n: + x = a.f(i) + i = a.f(x) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("LOOKUP_METHOD") + assert len(ops) == 2 + assert not ops[0].get_opnames("call") + assert not ops[0].get_opnames("new") + assert len(ops[0].get_opnames("guard")) <= 8 + assert not ops[1] # second LOOKUP_METHOD folded away + + ops = self.get_by_bytecode("CALL_METHOD") + assert len(ops) == 2 + for bytecode in ops: + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 9 + assert len(ops[1]) < len(ops[0]) + + ops = self.get_by_bytecode("LOAD_ATTR") + assert len(ops) == 2 + assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", + "ooisnull", "guard_false"] + assert not ops[1] # second LOAD_ATTR folded away + + def test_default_and_kw(self): + self.run_source(''' + def f(i, j=1): + return i + j + def main(n): + i = 0 + while i < n: + i = f(f(i), j=1) + return i + ''', + ([20], 20), + ([31], 32)) + ops = self.get_by_bytecode("CALL_FUNCTION") + assert len(ops) == 2 + for bytecode in ops: + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(ops[0].get_opnames("guard")) <= 14 + assert len(ops[1].get_opnames("guard")) <= 3 + + def test_virtual_instance(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 0 + while i < n: + a = A() + assert isinstance(a, A) + a.x = 2 + i = i + a.x + return i + ''', + ([20], 20), + ([31], 32)) + + callA, callisinstance = self.get_by_bytecode("CALL_FUNCTION") + assert not callA.get_opnames("call") + assert not callA.get_opnames("new") + assert len(callA.get_opnames("guard")) <= 9 + assert not callisinstance.get_opnames("call") + assert not callisinstance.get_opnames("new") + assert len(callisinstance.get_opnames("guard")) <= 2 + + bytecode, = self.get_by_bytecode("STORE_ATTR") + assert bytecode.get_opnames() == [] + + def test_mixed_type_loop(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 0.0 + j = 2 + while i < n: + i = j + i + return i, type(i) is float + ''', + ([20], (20, True)), + ([31], (32, True))) + + bytecode, = self.get_by_bytecode("BINARY_ADD") + assert not bytecode.get_opnames("call") + assert not bytecode.get_opnames("new") + assert len(bytecode.get_opnames("guard")) <= 2 + + def test_call_builtin_function(self): + self.run_source(''' + class A(object): + pass + def main(n): + i = 2 + l = [] + while i < n: + i += 1 + l.append(i) + return i, len(l) + ''', + ([20], (20, 18)), + ([31], (31, 29))) + + bytecode, = self.get_by_bytecode("CALL_METHOD") + assert len(bytecode.get_opnames("new_with_vtable")) == 1 # the forcing of the int + assert len(bytecode.get_opnames("call")) == 1 # the call to append + assert len(bytecode.get_opnames("guard")) == 1 # guard_no_exception after the call + class AppTestJIT(PyPyCJITTests): def setup_class(cls): if not option.runappdirect: py.test.skip("meant only for pypy-c") + # the next line skips stuff if the pypy-c is not a jit build cls.space = gettestobjspace(usemodules=['pypyjit']) cls.tmpdir = udir.join('pypy-jit') cls.tmpdir.ensure(dir=1) @@ -127,7 +307,7 @@ class TestJIT(PyPyCJITTests): def setup_class(cls): if option.pypy_c is None: - py.test.skip("pass --pypy-c!") + py.test.skip("pass --pypy!") cls.tmpdir = udir.join('pypy-jit') cls.tmpdir.ensure(dir=1) cls.counter = 0 Modified: pypy/branch/msvc-asmgcroot/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/sys/__init__.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/sys/__init__.py Thu Nov 5 22:17:18 2009 @@ -100,12 +100,11 @@ w_modules = self.get('modules') self.space.setitem(w_modules, w_name, w_module) - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): """ specialize access to dynamic exc_* attributes. """ - value = MixedModule.getdictvalue(self, space, w_attr) + value = MixedModule.getdictvalue(self, space, attr) if value is not None: return value - attr = space.str_w(w_attr) if attr == 'exc_type': operror = space.getexecutioncontext().sys_exc_info() if operror is None: Modified: pypy/branch/msvc-asmgcroot/pypy/module/sys/test/test_sysmodule.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/module/sys/test/test_sysmodule.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/module/sys/test/test_sysmodule.py Thu Nov 5 22:17:18 2009 @@ -65,6 +65,26 @@ assert exc_val2 ==e2 assert tb2.tb_lineno - tb.tb_lineno == 5 + def test_dynamic_attributes(self): + try: + raise Exception + except Exception,e: + import sys + exc_type = sys.exc_type + exc_val = sys.exc_value + tb = sys.exc_traceback + try: + raise Exception # 7 lines below the previous one + except Exception,e2: + exc_type2 = sys.exc_type + exc_val2 = sys.exc_value + tb2 = sys.exc_traceback + assert exc_type ==Exception + assert exc_val ==e + assert exc_type2 ==Exception + assert exc_val2 ==e2 + assert tb2.tb_lineno - tb.tb_lineno == 7 + def test_exc_info_normalization(self): import sys try: Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/descroperation.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/descroperation.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/descroperation.py Thu Nov 5 22:17:18 2009 @@ -1,7 +1,7 @@ import operator from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace -from pypy.interpreter.function import Function, Method +from pypy.interpreter.function import Function, Method, FunctionWithFixedCode from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import default_identity_hash from pypy.tool.sourcetools import compile2, func_with_new_name @@ -36,9 +36,9 @@ if w_descr is not None: if space.is_data_descr(w_descr): return space.get(w_descr, w_obj) - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) else: - w_value = w_obj.getdictvalue(space, w_name) + w_value = w_obj.getdictvalue(space, name) if w_value is not None: return w_value if w_descr is not None: @@ -54,7 +54,7 @@ space.set(w_descr, w_obj, w_value) return shadows_type = True - if w_obj.setdictvalue(space, w_name, w_value, shadows_type): + if w_obj.setdictvalue(space, name, w_value, shadows_type): return raiseattrerror(space, w_obj, name, w_descr) @@ -81,7 +81,7 @@ def get_and_call_args(space, w_descr, w_obj, args): descr = space.interpclass_w(w_descr) # a special case for performance and to avoid infinite recursion - if type(descr) is Function: + if isinstance(descr, Function): return descr.call_obj_args(w_obj, args) else: w_impl = space.get(w_descr, w_obj) @@ -89,8 +89,15 @@ def get_and_call_function(space, w_descr, w_obj, *args_w): descr = space.interpclass_w(w_descr) + typ = type(descr) # a special case for performance and to avoid infinite recursion - if type(descr) is Function: + if typ is Function or typ is FunctionWithFixedCode: + # isinstance(typ, Function) would not be correct here: + # for a BuiltinFunction we must not use that shortcut, because a + # builtin function binds differently than a normal function + # see test_builtin_as_special_method_is_not_bound + # in interpreter/test/test_function.py + # the fastcall paths are purely for performance, but the resulting # increase of speed is huge return descr.funccall(w_obj, *args_w) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/flow/objspace.py Thu Nov 5 22:17:18 2009 @@ -441,7 +441,7 @@ # XXX XXX Ha Ha # the reason to do this is: if you interrupt the flowing of a function # with the bytecode interpreter will raise an applevel - # KeyboardInterrup and you will get an AttributeError: space does not + # KeyboardInterrupt and you will get an AttributeError: space does not # have w_KeyboardInterrupt, which is not very helpful raise KeyboardInterrupt w_KeyboardInterrupt = property(w_KeyboardInterrupt) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/callmethod.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/callmethod.py Thu Nov 5 22:17:18 2009 @@ -54,15 +54,17 @@ if w_descr is None: # this handles directly the common case # module.function(args..) - w_value = w_obj.getdictvalue(space, w_name) - elif type(w_descr) is function.Function: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) - if w_value is None: - # fast method path: a function object in the class, - # nothing in the instance - f.pushvalue(w_descr) - f.pushvalue(w_obj) - return + w_value = w_obj.getdictvalue(space, name) + else: + typ = type(w_descr) + if typ is function.Function or typ is function.FunctionWithFixedCode: + w_value = w_obj.getdictvalue_attr_is_in_class(space, name) + if w_value is None: + # fast method path: a function object in the class, + # nothing in the instance + f.pushvalue(w_descr) + f.pushvalue(w_obj) + return if w_value is None: w_value = space.getattr(w_obj, w_name) f.pushvalue(w_value) @@ -85,15 +87,16 @@ """An optimized version of space.call_method() based on the same principle as above. """ - w_name = space.wrap(methname) w_getattribute = space.lookup(w_obj, '__getattribute__') if w_getattribute is object_getattribute(space): w_descr = space.lookup(w_obj, methname) - if type(w_descr) is function.Function: - w_value = w_obj.getdictvalue_attr_is_in_class(space, w_name) + typ = type(w_descr) + if typ is function.Function or typ is function.FunctionWithFixedCode: + w_value = w_obj.getdictvalue_attr_is_in_class(space, methname) if w_value is None: # fast method path: a function object in the class, # nothing in the instance return space.call_function(w_descr, w_obj, *arg_w) + w_name = space.wrap(methname) w_meth = space.getattr(w_obj, w_name) return space.call_function(w_meth, *arg_w) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/celldict.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/celldict.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,4 @@ -from pypy.objspace.std.dictmultiobject import DictImplementation +from pypy.interpreter.pycode import CO_CONTAINSGLOBALS from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib import jit @@ -15,7 +15,7 @@ def __repr__(self): return "" % (self.w_value, ) -class ModuleDictImplementation(DictImplementation): +class ModuleDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} @@ -44,24 +44,21 @@ w_value = cell.invalidate() cell = impl.content[name] = ModuleCell(w_value) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - name = self.space.str_w(w_key) + def impl_setitem_str(self, name, w_value, shadows_type=True): self.getcell(name).w_value = w_value if name in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(name) del self.unshadowed_builtins[name] - return self - - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -71,84 +68,71 @@ raise KeyError cell.invalidate() del self.content[key] - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - res = self.getcell(space.str_w(w_lookup), False) - if res is None: - return None - return res.w_value + return self.impl_getitem_str(space.str_w(w_lookup)) + elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_lookup) - def iteritems(self): - return ModuleDictItemIteratorImplementation(self.space, self) - - def iterkeys(self): - return ModuleDictKeyIteratorImplementation(self.space, self) + def impl_getitem_str(self, lookup): + res = self.getcell(lookup, False) + if res is None: + return None + return res.w_value - def itervalues(self): - return ModuleDictValueIteratorImplementation(self.space, self) + def impl_iter(self): + return ModuleDictIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return [cell.w_value for cell in self.content.itervalues()] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), cell.w_value]) for (key, cell) in self.content.iteritems()] - def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + def impl_clear(self): + # inefficient, but who cares for k, cell in self.content.iteritems(): - newimpl.setitem(self.space.wrap(k), cell.w_value) cell.invalidate() for k in self.unshadowed_builtins: self.invalidate_unshadowed_builtin(k) - return newimpl - -# grrrrr. just a copy-paste from StrKeyIteratorImplementation in dictmultiobject -class ModuleDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() + self.content.clear() + self.unshadowed_builtins.clear() - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None -class ModuleDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + for k, cell in self.content.iteritems(): + r_dict_content[self.space.wrap(k)] = cell.w_value + cell.invalidate() + for k in self.unshadowed_builtins: + self.invalidate_unshadowed_builtin(k) + self._clear_fields() + return self - def next_entry(self): - # note that this 'for' loop only runs once, at most - for cell in self.iterator: - return cell.w_value - else: - return None + def _clear_fields(self): + self.content = None + self.unshadowed_builtins = None -class ModuleDictItemIteratorImplementation(IteratorImplementation): +class ModuleDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() @@ -156,14 +140,9 @@ def next_entry(self): # note that this 'for' loop only runs once, at most for key, cell in self.iterator: - return self.space.newtuple([self.space.wrap(key), cell.w_value]) + return (self.space.wrap(key), cell.w_value) else: - return None - - - - - + return None, None class State(object): @@ -171,24 +150,27 @@ self.space = space self.invalidcell = ModuleCell() self.always_invalid_cache = [] - self.neverused_dictimpl = ModuleDictImplementation(space) + self.neverused_dictcontent = {} class GlobalCacheHolder(object): def __init__(self, space): self.cache = None state = space.fromcache(State) - self.dictimpl = state.neverused_dictimpl + self.dictcontent = state.neverused_dictcontent def getcache(self, space, code, w_globals): - implementation = getimplementation(w_globals) - if self.dictimpl is implementation: + if type(w_globals) is ModuleDictImplementation: + content = w_globals.content + else: + content = None + if self.dictcontent is content: return self.cache - return self.getcache_slow(space, code, w_globals, implementation) + return self.getcache_slow(space, code, w_globals, content) getcache._always_inline_ = True - def getcache_slow(self, space, code, w_globals, implementation): + def getcache_slow(self, space, code, w_globals, content): state = space.fromcache(State) - if not isinstance(implementation, ModuleDictImplementation): + if content is None: cache = state.always_invalid_cache if len(code.co_names_w) > len(cache): cache = [state.invalidcell] * len(code.co_names_w) @@ -196,23 +178,28 @@ else: cache = [state.invalidcell] * len(code.co_names_w) self.cache = cache - self.dictimpl = implementation + self.dictcontent = content return cache getcache_slow._dont_inline_ = True def init_code(code): - code.globalcacheholder = GlobalCacheHolder(code.space) + if code.co_flags & CO_CONTAINSGLOBALS: + code.globalcacheholder = GlobalCacheHolder(code.space) + else: + code.globalcacheholder = None def get_global_cache(space, code, w_globals): from pypy.interpreter.pycode import PyCode assert isinstance(code, PyCode) holder = code.globalcacheholder - return holder.getcache(space, code, w_globals) + if holder is not None: + return holder.getcache(space, code, w_globals) + return None def getimplementation(w_dict): - if type(w_dict) is W_DictMultiObject: - return w_dict.implementation + if type(w_dict) is ModuleDictImplementation and w_dict.r_dict_content is None: + return w_dict else: return None @@ -226,7 +213,7 @@ LOAD_GLOBAL._always_inline_ = True def find_cell_from_dict(implementation, name): - if isinstance(implementation, ModuleDictImplementation): + if implementation is not None: return implementation.getcell(name, False) return None @@ -234,8 +221,8 @@ def load_global_fill_cache(f, nameindex): name = f.space.str_w(f.getname_w(nameindex)) implementation = getimplementation(f.w_globals) - if isinstance(implementation, ModuleDictImplementation): - cell = find_cell_from_dict(implementation, name) + if implementation is not None: + cell = implementation.getcell(name, False) if cell is None: builtin_impl = getimplementation(f.get_builtin().getdict()) cell = find_cell_from_dict(builtin_impl, name) @@ -245,5 +232,5 @@ if cell is not None: f.cache_for_globals[nameindex] = cell return cell.w_value - return f._load_global(f.getname_w(nameindex)) + return f._load_global(f.getname_u(nameindex)) load_global_fill_cache._dont_inline_ = True Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/dictmultiobject.py Thu Nov 5 22:17:18 2009 @@ -22,92 +22,245 @@ space.is_w(w_lookup_type, space.w_float) ) +class W_DictMultiObject(W_Object): + from pypy.objspace.std.dicttype import dict_typedef as typedef -# DictImplementation lattice + r_dict_content = None + + @staticmethod + def allocate_and_init_instance(space, w_type=None, module=False, + instance=False, classofinstance=None, + from_strdict_shared=None): + if from_strdict_shared is not None: + assert w_type is None + assert not module and not instance and classofinstance is None + w_self = StrDictImplementation(space) + w_self.content = from_strdict_shared + return w_self + if space.config.objspace.std.withcelldict and module: + from pypy.objspace.std.celldict import ModuleDictImplementation + assert w_type is None + return ModuleDictImplementation(space) + elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: + assert w_type is None + return WaryDictImplementation(space) + elif space.config.objspace.std.withdictmeasurement: + assert w_type is None + return MeasuringDictImplementation(space) + elif space.config.objspace.std.withsharingdict and instance: + from pypy.objspace.std.sharingdict import SharedDictImplementation + assert w_type is None + return SharedDictImplementation(space) + elif (space.config.objspace.std.withshadowtracking and instance and + classofinstance is not None): + assert w_type is None + return ShadowDetectingDictImplementation(space, classofinstance) + elif instance: + assert w_type is None + return StrDictImplementation(space) + else: + if w_type is None: + w_type = space.w_dict + w_self = space.allocate_instance(W_DictMultiObject, w_type) + W_DictMultiObject.__init__(w_self, space) + w_self.initialize_as_rdict() + return w_self -# a dictionary starts with an EmptyDictImplementation, and moves down -# in this list: -# -# EmptyDictImplementation -# / \ -# | | -# StrDictImplementation | -# \ / -# RDictImplementation -# -# (in addition, any dictionary can go back to EmptyDictImplementation) + def __init__(self, space): + self.space = space -class DictImplementation(object): - - def get(self, w_lookup): + def initialize_as_rdict(self): + assert self.r_dict_content is None + self.r_dict_content = r_dict(self.space.eq_w, self.space.hash_w) + return self.r_dict_content + + + def initialize_content(w_self, list_pairs_w): + for w_k, w_v in list_pairs_w: + w_self.setitem(w_k, w_v) + + def __repr__(w_self): + """ representation for debugging purposes """ + return "%s()" % (w_self.__class__.__name__, ) + + def unwrap(w_dict, space): + result = {} + items = w_dict.items() + for w_pair in items: + key, val = space.unwrap(w_pair) + result[key] = val + return result + + def missing_method(w_dict, space, w_key): + if not space.is_w(space.type(w_dict), space.w_dict): + w_missing = space.lookup(w_dict, "__missing__") + if w_missing is None: + return None + return space.call_function(w_missing, w_dict, w_key) + else: + return None + + def set_str_keyed_item(w_dict, key, w_value, shadows_type=True): + w_dict.setitem_str(key, w_value, shadows_type) + + # _________________________________________________________________ + # implementation methods + def impl_getitem(self, w_key): #return w_value or None raise NotImplementedError("abstract base class") - def setitem_str(self, w_key, w_value, shadows_type=True): - #return implementation + def impl_getitem_str(self, w_key): + #return w_value or None raise NotImplementedError("abstract base class") - def setitem(self, w_key, w_value): - #return implementation + def impl_setitem_str(self, key, w_value, shadows_type=True): raise NotImplementedError("abstract base class") - def delitem(self, w_key): - #return implementation - raise NotImplementedError("abstract base class") - - def length(self): + def impl_setitem(self, w_key, w_value): raise NotImplementedError("abstract base class") - def iteritems(self): + def impl_delitem(self, w_key): raise NotImplementedError("abstract base class") - def iterkeys(self): + + def impl_length(self): raise NotImplementedError("abstract base class") - def itervalues(self): + + def impl_iter(self): raise NotImplementedError("abstract base class") + def impl_clear(self): + raise NotImplementedError("abstract base class") - def keys(self): - iterator = self.iterkeys() + def impl_keys(self): + iterator = self.impl_iter() result = [] while 1: - w_key = iterator.next() + w_key, w_value = iterator.next() if w_key is not None: result.append(w_key) else: return result - def values(self): - iterator = self.itervalues() + def impl_values(self): + iterator = self.impl_iter() result = [] while 1: - w_value = iterator.next() + w_key, w_value = iterator.next() if w_value is not None: result.append(w_value) else: return result - def items(self): - iterator = self.iteritems() + def impl_items(self): + iterator = self.impl_iter() result = [] while 1: - w_item = iterator.next() - if w_item is not None: - result.append(w_item) + w_key, w_value = iterator.next() + if w_key is not None: + result.append(self.space.newtuple([w_key, w_value])) else: return result -# the following method only makes sense when the option to use the -# CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen -# by the annotator - def get_builtin_indexed(self, i): - w_key = self.space.wrap(OPTIMIZED_BUILTINS[i]) - return self.get(w_key) + # the following method only makes sense when the option to use the + # CALL_LIKELY_BUILTIN opcode is set. Otherwise it won't even be seen + # by the annotator + def impl_get_builtin_indexed(self, i): + key = OPTIMIZED_BUILTINS[i] + return self.impl_getitem_str(key) -# this method will only be seen whan a certain config option is used - def shadows_anything(self): + # this method will only be seen whan a certain config option is used + def impl_shadows_anything(self): return True - def set_shadows_anything(self): + def impl_set_shadows_anything(self): pass + # _________________________________________________________________ + # fallback implementation methods + + def impl_fallback_setitem(self, w_key, w_value): + self.r_dict_content[w_key] = w_value + + def impl_fallback_setitem_str(self, key, w_value, shadows_type=True): + return self.impl_fallback_setitem(self.space.wrap(key), w_value) + + def impl_fallback_delitem(self, w_key): + del self.r_dict_content[w_key] + + def impl_fallback_length(self): + return len(self.r_dict_content) + + def impl_fallback_getitem(self, w_key): + return self.r_dict_content.get(w_key, None) + + def impl_fallback_getitem_str(self, key): + return self.r_dict_content.get(self.space.wrap(key), None) + + def impl_fallback_iter(self): + return RDictIteratorImplementation(self.space, self) + + def impl_fallback_keys(self): + return self.r_dict_content.keys() + def impl_fallback_values(self): + return self.r_dict_content.values() + def impl_fallback_items(self): + return [self.space.newtuple([w_key, w_val]) + for w_key, w_val in self.r_dict_content.iteritems()] + + def impl_fallback_clear(self): + self.r_dict_content.clear() + + def impl_fallback_get_builtin_indexed(self, i): + key = OPTIMIZED_BUILTINS[i] + return self.impl_fallback_getitem_str(key) + + def impl_fallback_shadows_anything(self): + return True + + def impl_fallback_set_shadows_anything(self): + pass + + +implementation_methods = [ + ("getitem", 1), + ("getitem_str", 1), + ("length", 0), + ("setitem_str", 3), + ("setitem", 2), + ("delitem", 1), + ("iter", 0), + ("items", 0), + ("values", 0), + ("keys", 0), + ("clear", 0), + ("get_builtin_indexed", 1), + ("shadows_anything", 0), + ("set_shadows_anything", 0), +] + + +def _make_method(name, implname, fallback, numargs): + args = ", ".join(["a" + str(i) for i in range(numargs)]) + code = """def %s(self, %s): + if self.r_dict_content is not None: + return self.%s(%s) + return self.%s(%s)""" % (name, args, fallback, args, implname, args) + d = {} + exec py.code.Source(code).compile() in d + implementation_method = d[name] + implementation_method.func_defaults = getattr(W_DictMultiObject, implname).func_defaults + return implementation_method + +def _install_methods(): + for name, numargs in implementation_methods: + implname = "impl_" + name + fallbackname = "impl_fallback_" + name + func = _make_method(name, implname, fallbackname, numargs) + setattr(W_DictMultiObject, name, func) +_install_methods() + +registerimplementation(W_DictMultiObject) + +# DictImplementation lattice +# XXX fix me # Iterator Implementation base classes @@ -120,19 +273,19 @@ def next(self): if self.dictimplementation is None: - return None + return None, None if self.len != self.dictimplementation.length(): self.len = -1 # Make this error state sticky raise OperationError(self.space.w_RuntimeError, self.space.wrap("dictionary changed size during iteration")) # look for the next entry - w_result = self.next_entry() - if w_result is not None: + if self.pos < self.len: + result = self.next_entry() self.pos += 1 - return w_result + return result # no more entries self.dictimplementation = None - return None + return None, None def next_entry(self): """ Purely abstract method @@ -148,170 +301,92 @@ # concrete subclasses of the above -class EmptyDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - - def get(self, w_lookup): - space = self.space - if not _is_str(space, w_lookup) and not _is_sane_hash(space, - space.type(w_lookup)): - # give hash a chance to raise an exception - space.hash(w_lookup) - return None - - def setitem(self, w_key, w_value): - space = self.space - if _is_str(space, w_key): - return StrDictImplementation(space).setitem_str(w_key, w_value) - else: - return space.DefaultDictImpl(space).setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - return StrDictImplementation(self.space).setitem_str(w_key, w_value) - #return SmallStrDictImplementation(self.space, w_key, w_value) - - def delitem(self, w_key): - space = self.space - if not _is_str(space, w_key) and not _is_sane_hash(space, - space.type(w_key)): - # count hash - space.hash(w_key) - raise KeyError - - def length(self): - return 0 - - def iteritems(self): - return EmptyIteratorImplementation(self.space, self) - def iterkeys(self): - return EmptyIteratorImplementation(self.space, self) - def itervalues(self): - return EmptyIteratorImplementation(self.space, self) - - def keys(self): - return [] - def values(self): - return [] - def items(self): - return [] - -class EmptyIteratorImplementation(IteratorImplementation): - def next_entry(self): - return None - - -class StrDictImplementation(DictImplementation): +class StrDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = {} - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def setitem_str(self, w_key, w_value, shadows_type=True): - self.content[self.space.str_w(w_key)] = w_value - return self + def impl_setitem_str(self, key, w_value, shadows_type=True): + self.content[key] = w_value - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): del self.content[space.str_w(w_key)] - if self.content: - return self - else: - return space.emptydictimpl + return elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return len(self.content) - def get(self, w_lookup): + def impl_getitem_str(self, key): + return self.content.get(key, None) + + def impl_getitem(self, w_key): space = self.space # -- This is called extremely often. Hack for performance -- - if type(w_lookup) is space.StringObjectCls: - return self.content.get(w_lookup.unwrap(space), None) + if type(w_key) is space.StringObjectCls: + return self.impl_getitem_str(w_key.unwrap(space)) # -- End of performance hack -- - w_lookup_type = space.type(w_lookup) + w_lookup_type = space.type(w_key) if space.is_w(w_lookup_type, space.w_str): - return self.content.get(space.str_w(w_lookup), None) + return self.impl_getitem_str(space.str_w(w_key)) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) - - def iteritems(self): - return StrItemIteratorImplementation(self.space, self) + return self._as_rdict().getitem(w_key) - def iterkeys(self): - return StrKeyIteratorImplementation(self.space, self) + def impl_iter(self): + return StrIteratorImplementation(self.space, self) - def itervalues(self): - return StrValueIteratorImplementation(self.space, self) - - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) for key in self.content.iterkeys()] - def values(self): + def impl_values(self): return self.content.values() - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), w_value]) for (key, w_value) in self.content.iteritems()] + def impl_clear(self): + self.content.clear() + def _as_rdict(self): - newimpl = self.space.DefaultDictImpl(self.space) + r_dict_content = self.initialize_as_rdict() for k, w_v in self.content.items(): - newimpl.setitem(self.space.wrap(k), w_v) - return newimpl - -# the following are very close copies of the base classes above + r_dict_content[self.space.wrap(k)] = w_v + self._clear_fields() + return self -class StrKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() + def _clear_fields(self): + self.content = None - def next_entry(self): - # note that this 'for' loop only runs once, at most - for key in self.iterator: - return self.space.wrap(key) - else: - return None - -class StrValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None - -class StrItemIteratorImplementation(IteratorImplementation): +class StrIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for key, w_value in self.iterator: - return self.space.newtuple([self.space.wrap(key), w_value]) + for str, w_value in self.iterator: + return self.space.wrap(str), w_value else: - return None + return None, None class ShadowDetectingDictImplementation(StrDictImplementation): @@ -324,29 +399,29 @@ else: self._shadows_anything = False - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): if shadows_type: self._shadows_anything = True - return StrDictImplementation.setitem_str( - self, w_key, w_value, shadows_type) + StrDictImplementation.impl_setitem_str( + self, key, w_value, shadows_type) - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): if not self._shadows_anything: w_obj = self.w_type.lookup(space.str_w(w_key)) if w_obj is not None: self._shadows_anything = True - return StrDictImplementation.setitem_str( - self, w_key, w_value, False) + StrDictImplementation.impl_setitem_str( + self, self.space.str_w(w_key), w_value, False) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) - def shadows_anything(self): + def impl_shadows_anything(self): return (self._shadows_anything or self.w_type.version_tag is not self.original_version_tag) - def set_shadows_anything(self): + def impl_set_shadows_anything(self): self._shadows_anything = True class WaryDictImplementation(StrDictImplementation): @@ -354,15 +429,13 @@ StrDictImplementation.__init__(self, space) self.shadowed = [None] * len(BUILTIN_TO_INDEX) - def setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = w_value self.content[key] = w_value - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): @@ -371,96 +444,30 @@ i = BUILTIN_TO_INDEX.get(key, -1) if i != -1: self.shadowed[i] = None - return self elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def get_builtin_indexed(self, i): + def impl_get_builtin_indexed(self, i): return self.shadowed[i] -class RDictImplementation(DictImplementation): - def __init__(self, space): - self.space = space - self.content = r_dict(space.eq_w, space.hash_w) - - def __repr__(self): - return "%s<%s>" % (self.__class__.__name__, self.content) - - def setitem(self, w_key, w_value): - self.content[w_key] = w_value - return self - - def setitem_str(self, w_key, w_value, shadows_type=True): - return self.setitem(w_key, w_value) - - def delitem(self, w_key): - del self.content[w_key] - if self.content: - return self - else: - return self.space.emptydictimpl - - def length(self): - return len(self.content) - def get(self, w_lookup): - return self.content.get(w_lookup, None) - - def iteritems(self): - return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): - return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): - return RDictValueIteratorImplementation(self.space, self) - - def keys(self): - return self.content.keys() - def values(self): - return self.content.values() - def items(self): - return [self.space.newtuple([w_key, w_val]) - for w_key, w_val in self.content.iteritems()] - - -class RDictKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iterkeys() - - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_key in self.iterator: - return w_key - else: - return None - -class RDictValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.itervalues() - def next_entry(self): - # note that this 'for' loop only runs once, at most - for w_value in self.iterator: - return w_value - else: - return None - -class RDictItemIteratorImplementation(IteratorImplementation): +class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.content.iteritems() + self.iterator = dictimplementation.r_dict_content.iteritems() def next_entry(self): # note that this 'for' loop only runs once, at most - for w_key, w_value in self.iterator: - return self.space.newtuple([w_key, w_value]) + for item in self.iterator: + return item else: - return None + return None, None +# XXX fix this thing import time, py class DictInfo(object): @@ -518,7 +525,7 @@ def __del__(self): self.info.lifetime = time.time() - self.info.createtime -class MeasuringDictImplementation(DictImplementation): +class MeasuringDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.content = r_dict(space.eq_w, space.hash_w) @@ -544,7 +551,7 @@ else: self.info.misses += 1 - def setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): if not self.info.seen_non_string_in_write and not self._is_str(w_key): self.info.seen_non_string_in_write = True self.info.size_on_non_string_seen_in_write = len(self.content) @@ -552,11 +559,10 @@ self.info.writes += 1 self.content[w_key] = w_value self.info.maxcontents = max(self.info.maxcontents, len(self.content)) - return self - def setitem_str(self, w_key, w_value, shadows_type=True): + def impl_setitem_str(self, key, w_value, shadows_type=True): self.info.setitem_strs += 1 - return self.setitem(w_key, w_value) - def delitem(self, w_key): + self.impl_setitem(self.space.wrap(key), w_value) + def impl_delitem(self, w_key): if not self.info.seen_non_string_in_write \ and not self.info.seen_non_string_in_read_first \ and not self._is_str(w_key): @@ -565,38 +571,39 @@ self.info.delitems += 1 self.info.writes += 1 del self.content[w_key] - return self - def length(self): + def impl_length(self): self.info.lengths += 1 return len(self.content) - def get(self, w_lookup): + def impl_getitem_str(self, key): + return self.impl_getitem(self.space.wrap(key)) + def impl_getitem(self, w_key): self.info.gets += 1 - self._read(w_lookup) - return self.content.get(w_lookup, None) + self._read(w_key) + return self.content.get(w_key, None) - def iteritems(self): + def impl_iteritems(self): self.info.iteritems += 1 self.info.iterations += 1 return RDictItemIteratorImplementation(self.space, self) - def iterkeys(self): + def impl_iterkeys(self): self.info.iterkeys += 1 self.info.iterations += 1 return RDictKeyIteratorImplementation(self.space, self) - def itervalues(self): + def impl_itervalues(self): self.info.itervalues += 1 self.info.iterations += 1 return RDictValueIteratorImplementation(self.space, self) - def keys(self): + def impl_keys(self): self.info.keys += 1 self.info.listings += 1 return self.content.keys() - def values(self): + def impl_values(self): self.info.values += 1 self.info.listings += 1 return self.content.values() - def items(self): + def impl_items(self): self.info.items += 1 self.info.listings += 1 return [self.space.newtuple([w_key, w_val]) @@ -630,71 +637,6 @@ os.close(fd) os.write(2, "Reporting done.\n") -class W_DictMultiObject(W_Object): - from pypy.objspace.std.dicttype import dict_typedef as typedef - - def __init__(w_self, space, module=False, sharing=False): - if space.config.objspace.std.withcelldict and module: - from pypy.objspace.std.celldict import ModuleDictImplementation - w_self.implementation = ModuleDictImplementation(space) - elif space.config.objspace.opcodes.CALL_LIKELY_BUILTIN and module: - w_self.implementation = WaryDictImplementation(space) - elif space.config.objspace.std.withdictmeasurement: - w_self.implementation = MeasuringDictImplementation(space) - elif space.config.objspace.std.withsharingdict and sharing: - from pypy.objspace.std.sharingdict import SharedDictImplementation - w_self.implementation = SharedDictImplementation(space) - else: - w_self.implementation = space.emptydictimpl - w_self.space = space - - def initialize_content(w_self, list_pairs_w): - impl = w_self.implementation - for w_k, w_v in list_pairs_w: - impl = impl.setitem(w_k, w_v) - w_self.implementation = impl - - def initialize_from_strdict_shared(w_self, strdict): - impl = StrDictImplementation(w_self.space) - impl.content = strdict - w_self.implementation = impl - - def __repr__(w_self): - """ representation for debugging purposes """ - return "%s(%s)" % (w_self.__class__.__name__, w_self.implementation) - - def unwrap(w_dict, space): - result = {} - items = w_dict.implementation.items() - for w_pair in items: - key, val = space.unwrap(w_pair) - result[key] = val - return result - - def missing_method(w_dict, space, w_key): - if not space.is_w(space.type(w_dict), space.w_dict): - w_missing = space.lookup(w_dict, "__missing__") - if w_missing is None: - return None - return space.call_function(w_missing, w_dict, w_key) - else: - return None - - def len(w_self): - return w_self.implementation.length() - - def get(w_dict, w_key, w_default): - w_value = w_dict.implementation.get(w_key) - if w_value is not None: - return w_value - else: - return w_default - - def set_str_keyed_item(w_dict, w_key, w_value, shadows_type=True): - w_dict.implementation = w_dict.implementation.setitem_str( - w_key, w_value, shadows_type) - -registerimplementation(W_DictMultiObject) init_signature = Signature(['seq_or_map'], None, 'kwargs') @@ -715,7 +657,7 @@ raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) w_k, w_v = pair - w_dict.implementation = w_dict.implementation.setitem(w_k, w_v) + w_dict.setitem(w_k, w_v) else: if space.is_true(w_src): from pypy.objspace.std.dicttype import update1 @@ -724,71 +666,67 @@ from pypy.objspace.std.dicttype import update1 update1(space, w_dict, w_kwds) -def getitem__DictMulti_ANY(space, w_dict, w_lookup): - w_value = w_dict.implementation.get(w_lookup) +def getitem__DictMulti_ANY(space, w_dict, w_key): + w_value = w_dict.getitem(w_key) if w_value is not None: return w_value - w_missing_item = w_dict.missing_method(space, w_lookup) + w_missing_item = w_dict.missing_method(space, w_key) if w_missing_item is not None: return w_missing_item - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def setitem__DictMulti_ANY_ANY(space, w_dict, w_newkey, w_newvalue): - w_dict.implementation = w_dict.implementation.setitem(w_newkey, w_newvalue) + w_dict.setitem(w_newkey, w_newvalue) -def delitem__DictMulti_ANY(space, w_dict, w_lookup): +def delitem__DictMulti_ANY(space, w_dict, w_key): try: - w_dict.implementation = w_dict.implementation.delitem(w_lookup) + w_dict.delitem(w_key) except KeyError: - space.raise_key_error(w_lookup) + space.raise_key_error(w_key) def len__DictMulti(space, w_dict): - return space.wrap(w_dict.implementation.length()) + return space.wrap(w_dict.length()) -def contains__DictMulti_ANY(space, w_dict, w_lookup): - return space.newbool(w_dict.implementation.get(w_lookup) is not None) +def contains__DictMulti_ANY(space, w_dict, w_key): + return space.newbool(w_dict.getitem(w_key) is not None) dict_has_key__DictMulti_ANY = contains__DictMulti_ANY def iter__DictMulti(space, w_dict): - return W_DictMultiIterObject(space, w_dict.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_dict.iter(), KEYSITER) def eq__DictMulti_DictMulti(space, w_left, w_right): if space.is_w(w_left, w_right): return space.w_True - if w_left.implementation.length() != w_right.implementation.length(): + if w_left.length() != w_right.length(): return space.w_False - iteratorimplementation = w_left.implementation.iteritems() + iteratorimplementation = w_left.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) - w_rightval = w_right.implementation.get(w_key) + w_rightval = w_right.getitem(w_key) if w_rightval is None: return space.w_False if not space.eq_w(w_val, w_rightval): return space.w_False return space.w_True -def characterize(space, aimpl, bimpl): +def characterize(space, w_a, w_b): """ (similar to CPython) returns the smallest key in acontent for which b's value is different or absent and this value """ w_smallest_diff_a_key = None w_its_value = None - iteratorimplementation = aimpl.iteritems() + iteratorimplementation = w_a.iter() while 1: - w_item = iteratorimplementation.next() - if w_item is None: + w_key, w_val = iteratorimplementation.next() + if w_key is None: break - w_key = space.getitem(w_item, space.wrap(0)) - w_val = space.getitem(w_item, space.wrap(1)) if w_smallest_diff_a_key is None or space.is_true(space.lt(w_key, w_smallest_diff_a_key)): - w_bvalue = bimpl.get(w_key) + w_bvalue = w_b.getitem(w_key) if w_bvalue is None: w_its_value = w_val w_smallest_diff_a_key = w_key @@ -800,18 +738,16 @@ def lt__DictMulti_DictMulti(space, w_left, w_right): # Different sizes, no problem - leftimpl = w_left.implementation - rightimpl = w_right.implementation - if leftimpl.length() < rightimpl.length(): + if w_left.length() < w_right.length(): return space.w_True - if leftimpl.length() > rightimpl.length(): + if w_left.length() > w_right.length(): return space.w_False # Same size - w_leftdiff, w_leftval = characterize(space, leftimpl, rightimpl) + w_leftdiff, w_leftval = characterize(space, w_left, w_right) if w_leftdiff is None: return space.w_False - w_rightdiff, w_rightval = characterize(space, rightimpl, leftimpl) + w_rightdiff, w_rightval = characterize(space, w_right, w_left) if w_rightdiff is None: # w_leftdiff is not None, w_rightdiff is None return space.w_True @@ -824,54 +760,82 @@ def dict_copy__DictMulti(space, w_self): from pypy.objspace.std.dicttype import update1 - w_new = W_DictMultiObject(space) + w_new = W_DictMultiObject.allocate_and_init_instance(space) update1(space, w_new, w_self) return w_new def dict_items__DictMulti(space, w_self): - return space.newlist(w_self.implementation.items()) + return space.newlist(w_self.items()) def dict_keys__DictMulti(space, w_self): - return space.newlist(w_self.implementation.keys()) + return space.newlist(w_self.keys()) def dict_values__DictMulti(space, w_self): - return space.newlist(w_self.implementation.values()) + return space.newlist(w_self.values()) def dict_iteritems__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iteritems()) + return W_DictMultiIterObject(space, w_self.iter(), ITEMSITER) def dict_iterkeys__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.iterkeys()) + return W_DictMultiIterObject(space, w_self.iter(), KEYSITER) def dict_itervalues__DictMulti(space, w_self): - return W_DictMultiIterObject(space, w_self.implementation.itervalues()) + return W_DictMultiIterObject(space, w_self.iter(), VALUESITER) def dict_clear__DictMulti(space, w_self): - w_self.implementation = space.emptydictimpl + w_self.clear() -def dict_get__DictMulti_ANY_ANY(space, w_dict, w_lookup, w_default): - return w_dict.get(w_lookup, w_default) +def dict_get__DictMulti_ANY_ANY(space, w_dict, w_key, w_default): + w_value = w_dict.getitem(w_key) + if w_value is not None: + return w_value + else: + return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): defaults = space.unpackiterable(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) - w_item = w_dict.implementation.get(w_key) + w_item = w_dict.getitem(w_key) if w_item is None: if len_defaults > 0: return defaults[0] else: space.raise_key_error(w_key) else: - w_dict.implementation.delitem(w_key) + w_dict.delitem(w_key) return w_item +app = gateway.applevel(''' + def dictrepr(currently_in_repr, d): + # Now we only handle one implementation of dicts, this one. + # The fix is to move this to dicttype.py, and do a + # multimethod lookup mapping str to StdObjSpace.str + # This cannot happen until multimethods are fixed. See dicttype.py + dict_id = id(d) + if dict_id in currently_in_repr: + return '{...}' + currently_in_repr[dict_id] = 1 + try: + items = [] + # XXX for now, we cannot use iteritems() at app-level because + # we want a reasonable result instead of a RuntimeError + # even if the dict is mutated by the repr() in the loop. + for k, v in dict.items(d): + items.append(repr(k) + ": " + repr(v)) + return "{" + ', '.join(items) + "}" + finally: + try: + del currently_in_repr[dict_id] + except: + pass +''', filename=__file__) -from pypy.objspace.std.dictobject import dictrepr +dictrepr = app.interphook("dictrepr") def repr__DictMulti(space, w_dict): - if w_dict.implementation.length() == 0: + if w_dict.length() == 0: return space.wrap('{}') ec = space.getexecutioncontext() w_currently_in_repr = ec._py_repr @@ -884,12 +848,17 @@ # Iteration +KEYSITER = 0 +ITEMSITER = 1 +VALUESITER = 2 + class W_DictMultiIterObject(W_Object): from pypy.objspace.std.dicttype import dictiter_typedef as typedef - def __init__(w_self, space, iteratorimplementation): + def __init__(w_self, space, iteratorimplementation, itertype): w_self.space = space w_self.iteratorimplementation = iteratorimplementation + w_self.itertype = itertype registerimplementation(W_DictMultiIterObject) @@ -898,16 +867,19 @@ def next__DictMultiIterObject(space, w_dictiter): iteratorimplementation = w_dictiter.iteratorimplementation - w_result = iteratorimplementation.next() - if w_result is not None: - return w_result + w_key, w_value = iteratorimplementation.next() + if w_key is not None: + itertype = w_dictiter.itertype + if itertype == KEYSITER: + return w_key + elif itertype == VALUESITER: + return w_value + elif itertype == ITEMSITER: + return space.newtuple([w_key, w_value]) + else: + assert 0, "should be unreachable" raise OperationError(space.w_StopIteration, space.w_None) -# XXX __length_hint__() -##def len__DictMultiIterObject(space, w_dictiter): -## iteratorimplementation = w_dictiter.iteratorimplementation -## return space.wrap(iteratorimplementation.length()) - # ____________________________________________________________ from pypy.objspace.std import dicttype Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/dicttype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/dicttype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/dicttype.py Thu Nov 5 22:17:18 2009 @@ -141,8 +141,8 @@ # ____________________________________________________________ def descr__new__(space, w_dicttype, __args__): - w_obj = space.allocate_instance(space.DictObjectCls, w_dicttype) - space.DictObjectCls.__init__(w_obj, space) + from pypy.objspace.std.dictmultiobject import W_DictMultiObject + w_obj = W_DictMultiObject.allocate_and_init_instance(space, w_dicttype) return w_obj # ____________________________________________________________ @@ -187,13 +187,10 @@ w_typeobj = space.gettypeobject(dictiter_typedef) from pypy.interpreter.mixedmodule import MixedModule - if space.config.objspace.std.withmultidict: - raise OperationError( - space.w_RuntimeError, - space.wrap("cannot pickle dictiters with multidicts")) - else: - from pypy.objspace.std.dictobject import \ - W_DictIter_Keys, W_DictIter_Values, W_DictIter_Items + raise OperationError( + space.w_RuntimeError, + space.wrap("cannot pickle dictiters with multidicts")) + # XXXXXX get that working again # we cannot call __init__ since we don't have the original dict if isinstance(w_self, W_DictIter_Keys): Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/listtype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/listtype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/listtype.py Thu Nov 5 22:17:18 2009 @@ -40,14 +40,9 @@ # ____________________________________________________________ def descr__new__(space, w_listtype, __args__): - if space.config.objspace.std.withmultilist: - from pypy.objspace.std.listmultiobject import W_ListMultiObject - w_obj = space.allocate_instance(W_ListMultiObject, w_listtype) - W_ListMultiObject.__init__(w_obj, space) - else: - from pypy.objspace.std.listobject import W_ListObject - w_obj = space.allocate_instance(W_ListObject, w_listtype) - W_ListObject.__init__(w_obj, []) + from pypy.objspace.std.listobject import W_ListObject + w_obj = space.allocate_instance(W_ListObject, w_listtype) + W_ListObject.__init__(w_obj, []) return w_obj # ____________________________________________________________ Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/marshal_impl.py Thu Nov 5 22:17:18 2009 @@ -11,8 +11,6 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all from pypy.rlib.rarithmetic import LONG_BIT -from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.floatobject import repr__Float as repr_float from pypy.objspace.std.longobject import SHIFT as long_bits from pypy.objspace.std.objspace import StdObjSpace from pypy.interpreter.special import Ellipsis @@ -26,7 +24,6 @@ from pypy.objspace.std.floatobject import W_FloatObject from pypy.objspace.std.tupleobject import W_TupleObject from pypy.objspace.std.listobject import W_ListObject -from pypy.objspace.std.dictobject import W_DictObject from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.ropeobject import W_RopeObject @@ -35,7 +32,7 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.unicodeobject import W_UnicodeObject -import longobject, dictobject +import longobject from pypy.module.marshal.interp_marshal import register @@ -145,14 +142,14 @@ m.atom_int(TYPE_INT, w_int.intval) def unmarshal_Int(space, u, tc): - return wrapint(space, u.get_int()) + return space.newint(u.get_int()) register(TYPE_INT, unmarshal_Int) def unmarshal_Int64(space, u, tc): if LONG_BIT >= 64: lo = u.get_int() & (2**32-1) hi = u.get_int() - return wrapint(space, (hi << 32) | lo) + return space.newint((hi << 32) | lo) else: # fall back to a long # XXX at some point, we need to extend longobject @@ -186,7 +183,7 @@ m.put(pack_float(w_float.floatval)) else: m.start(TYPE_FLOAT) - m.put_pascal(space.str_w(repr_float(space, w_float))) + m.put_pascal(space.str_w(space.repr(w_float))) def unmarshal_Float(space, u, tc): return space.call_function(space.builtin.get('float'), @@ -194,7 +191,7 @@ register(TYPE_FLOAT, unmarshal_Float) def unmarshal_Float_bin(space, u, tc): - return W_FloatObject(unpack_float(u.get(8))) + return space.newfloat(unpack_float(u.get(8))) register(TYPE_BINARY_FLOAT, unmarshal_Float_bin) def marshal_w__Complex(space, w_complex, m): @@ -207,8 +204,8 @@ w_real = space.wrap(w_complex.realval) w_imag = space.wrap(w_complex.imagval) m.start(TYPE_COMPLEX) - m.put_pascal(space.str_w(repr_float(space, w_real))) - m.put_pascal(space.str_w(repr_float(space, w_imag))) + m.put_pascal(space.str_w(space.repr(w_real))) + m.put_pascal(space.str_w(space.repr(w_imag))) def unmarshal_Complex(space, u, tc): w_real = space.call_function(space.builtin.get('float'), @@ -222,7 +219,7 @@ def unmarshal_Complex_bin(space, u, tc): real = unpack_float(u.get(8)) imag = unpack_float(u.get(8)) - return W_ComplexObject(real, imag) + return space.newcomplex(real, imag) register(TYPE_BINARY_COMPLEX, unmarshal_Complex_bin) def marshal_w__Long(space, w_long, m): @@ -356,22 +353,15 @@ return space.newlist(items_w) register(TYPE_LIST, unmarshal_List) -def marshal_w__Dict(space, w_dict, m): - m.start(TYPE_DICT) - for w_key, w_value in w_dict.content.iteritems(): - m.put_w_obj(w_key) - m.put_w_obj(w_value) - m.atom(TYPE_NULL) - def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) - for w_tuple in w_dict.implementation.items(): + for w_tuple in w_dict.items(): w_key, w_value = space.viewiterable(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) -def unmarshal_Dict(space, u, tc): +def unmarshal_DictMulti(space, u, tc): # since primitive lists are not optimized and we don't know # the dict size in advance, use the dict's setitem instead # of building a list of tuples. @@ -383,7 +373,7 @@ w_value = u.get_w_obj(False) space.setitem(w_dic, w_key, w_value) return w_dic -register(TYPE_DICT, unmarshal_Dict) +register(TYPE_DICT, unmarshal_DictMulti) def unmarshal_NULL(self, u, tc): return None @@ -452,9 +442,9 @@ name = unmarshal_str(u) firstlineno = u.get_int() lnotab = unmarshal_str(u) - code = PyCode._code_new_w(space, argcount, nlocals, stacksize, flags, - code, consts_w, names, varnames, filename, - name, firstlineno, lnotab, freevars, cellvars) + code = PyCode(space, argcount, nlocals, stacksize, flags, + code, consts_w[:], names, varnames, filename, + name, firstlineno, lnotab, freevars, cellvars) return space.wrap(code) register(TYPE_CODE, unmarshal_pycode) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/model.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/model.py Thu Nov 5 22:17:18 2009 @@ -12,9 +12,6 @@ "withsmallint" : ["smallintobject.W_SmallIntObject"], "withstrslice" : ["strsliceobject.W_StringSliceObject"], "withstrjoin" : ["strjoinobject.W_StringJoinObject"], - "withmultidict" : ["dictmultiobject.W_DictMultiObject", - "dictmultiobject.W_DictMultiIterObject"], - "withmultilist" : ["listmultiobject.W_ListMultiObject"], "withrope" : ["ropeobject.W_RopeObject", "ropeobject.W_RopeIterObject"], "withropeunicode": ["ropeunicodeobject.W_RopeUnicodeObject", @@ -29,6 +26,7 @@ def __init__(self, config): """NOT_RPYTHON: inititialization only""" + self.config = config # All the Python types that we want to provide in this StdObjSpace class result: from pypy.objspace.std.objecttype import object_typedef @@ -65,9 +63,7 @@ from pypy.objspace.std import smallintobject from pypy.objspace.std import tupleobject from pypy.objspace.std import listobject - from pypy.objspace.std import dictobject from pypy.objspace.std import dictmultiobject - from pypy.objspace.std import listmultiobject from pypy.objspace.std import stringobject from pypy.objspace.std import ropeobject from pypy.objspace.std import ropeunicodeobject @@ -95,13 +91,18 @@ floatobject.W_FloatObject: [], tupleobject.W_TupleObject: [], listobject.W_ListObject: [], - dictobject.W_DictObject: [], - dictobject.W_DictIterObject: [], + dictmultiobject.W_DictMultiObject: [], + dictmultiobject.W_DictMultiIterObject: [], stringobject.W_StringObject: [], typeobject.W_TypeObject: [], sliceobject.W_SliceObject: [], longobject.W_LongObject: [], noneobject.W_NoneObject: [], + complexobject.W_ComplexObject: [], + setobject.W_BaseSetObject: [], + setobject.W_SetObject: [], + setobject.W_FrozensetObject: [], + setobject.W_SetIterObject: [], iterobject.W_SeqIterObject: [], iterobject.W_FastListIterObject: [], iterobject.W_FastTupleIterObject: [], @@ -111,14 +112,10 @@ pypy.interpreter.pycode.PyCode: [], pypy.interpreter.special.Ellipsis: [], } - self.typeorder[complexobject.W_ComplexObject] = [] - self.typeorder[setobject.W_SetObject] = [] - self.typeorder[setobject.W_FrozensetObject] = [] - self.typeorder[setobject.W_SetIterObject] = [] self.imported_but_not_registered = { - dictobject.W_DictObject: True, - dictobject.W_DictIterObject: True, + dictmultiobject.W_DictMultiObject: True, # XXXXXX + dictmultiobject.W_DictMultiIterObject: True, listobject.W_ListObject: True, stringobject.W_StringObject: True, tupleobject.W_TupleObject: True, @@ -132,12 +129,6 @@ else: self.imported_but_not_registered[implcls] = True - if config.objspace.std.withmultidict: - del self.typeorder[dictobject.W_DictObject] - del self.typeorder[dictobject.W_DictIterObject] - - if config.objspace.std.withmultilist: - del self.typeorder[listobject.W_ListObject] if config.objspace.std.withrope: del self.typeorder[stringobject.W_StringObject] @@ -187,6 +178,12 @@ (complexobject.W_ComplexObject, complexobject.delegate_Float2Complex), ] + self.typeorder[setobject.W_SetObject] += [ + (setobject.W_BaseSetObject, None) + ] + self.typeorder[setobject.W_FrozensetObject] += [ + (setobject.W_BaseSetObject, None) + ] if not config.objspace.std.withrope: self.typeorder[stringobject.W_StringObject] += [ (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode), @@ -252,9 +249,9 @@ from pypy.objspace.std import stdtypedef result = self.typeorder.copy() for cls in self.typeorder: - if (hasattr(cls, 'typedef') and + if (hasattr(cls, 'typedef') and cls.typedef is not None and cls.typedef.acceptable_as_base_class): - subclslist = enum_interplevel_subclasses(cls) + subclslist = enum_interplevel_subclasses(self.config, cls) for subcls in subclslist: if cls in subcls.__bases__: # only direct subclasses # for user subclasses we only accept "generic" Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/objectobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/objectobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/objectobject.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,4 @@ -from pypy.objspace.std.objspace import * +from pypy.objspace.std.objspace import W_Object, register_all class W_ObjectObject(W_Object): Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/objecttype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/objecttype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/objecttype.py Thu Nov 5 22:17:18 2009 @@ -63,7 +63,6 @@ space.wrap("default __new__ takes " "no parameters")) w_obj = space.allocate_instance(W_ObjectObject, w_type) - #W_ObjectObject.__init__(w_obj) return w_obj def descr__init__(space, w_obj, __args__): Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/objspace.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/objspace.py Thu Nov 5 22:17:18 2009 @@ -18,7 +18,7 @@ from pypy.objspace.std import stdtypedef from pypy.rlib.rarithmetic import base_int from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.jit import hint, we_are_jitted +from pypy.rlib.jit import hint from pypy.rlib.unroll import unrolling_iterable import sys import os @@ -140,13 +140,13 @@ w_globals = f.w_globals num = oparg >> 8 assert isinstance(w_globals, W_DictMultiObject) - w_value = w_globals.implementation.get_builtin_indexed(num) + w_value = w_globals.get_builtin_indexed(num) if w_value is None: builtins = f.get_builtin() assert isinstance(builtins, Module) w_builtin_dict = builtins.w_dict assert isinstance(w_builtin_dict, W_DictMultiObject) - w_value = w_builtin_dict.implementation.get_builtin_indexed(num) + w_value = w_builtin_dict.get_builtin_indexed(num) ## if w_value is not None: ## print "CALL_LIKELY_BUILTIN fast" if w_value is None: @@ -240,20 +240,9 @@ self.FrameClass = StdObjSpaceFrame - # XXX store the dict class on the space to access it in various places - if self.config.objspace.std.withmultidict: - from pypy.objspace.std import dictmultiobject - self.DictObjectCls = dictmultiobject.W_DictMultiObject - self.emptydictimpl = dictmultiobject.EmptyDictImplementation(self) - if self.config.objspace.std.withbucketdict: - from pypy.objspace.std import dictbucket - self.DefaultDictImpl = dictbucket.BucketDictImplementation - else: - self.DefaultDictImpl = dictmultiobject.RDictImplementation - else: - from pypy.objspace.std import dictobject - self.DictObjectCls = dictobject.W_DictObject - assert self.DictObjectCls in self.model.typeorder + # store the dict class on the space to access it in various places + from pypy.objspace.std import dictmultiobject + self.DictObjectCls = dictmultiobject.W_DictMultiObject from pypy.objspace.std import tupleobject self.TupleObjectCls = tupleobject.W_TupleObject @@ -445,6 +434,7 @@ def gettypeobject(self, typedef): # stdtypedef.TypeCache maps each StdTypeDef instance to its # unique-for-this-space W_TypeObject instance + assert typedef is not None return self.fromcache(stdtypedef.TypeCache).getorbuild(typedef) def wrap(self, x): @@ -577,18 +567,16 @@ return wraptuple(self, list_w) def newlist(self, list_w): - if self.config.objspace.std.withmultilist: - from pypy.objspace.std.listmultiobject import convert_list_w - return convert_list_w(self, list_w) - else: - from pypy.objspace.std.listobject import W_ListObject - return W_ListObject(list_w) + from pypy.objspace.std.listobject import W_ListObject + return W_ListObject(list_w) - def newdict(self, module=False): - if self.config.objspace.std.withmultidict and module: - from pypy.objspace.std.dictmultiobject import W_DictMultiObject - return W_DictMultiObject(self, module=True) - return self.DictObjectCls(self) + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): + from pypy.objspace.std.dictmultiobject import W_DictMultiObject + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) def newslice(self, w_start, w_end, w_step): return W_SliceObject(w_start, w_end, w_step) @@ -622,12 +610,14 @@ user-defined type, without actually __init__ializing the instance.""" w_type = self.gettypeobject(cls.typedef) if self.is_w(w_type, w_subtype): - instance = instantiate(cls) + instance = instantiate(cls) elif cls.typedef.acceptable_as_base_class: # the purpose of the above check is to avoid the code below # to be annotated at all for 'cls' if it is not necessary w_subtype = w_type.check_user_subclass(w_subtype) - subcls = get_unique_interplevel_subclass(cls, w_subtype.hasdict, w_subtype.nslots != 0, w_subtype.needsdel, w_subtype.weakrefable) + subcls = get_unique_interplevel_subclass( + self.config, cls, w_subtype.hasdict, w_subtype.nslots != 0, + w_subtype.needsdel, w_subtype.weakrefable) instance = instantiate(subcls) instance.user_setup(self, w_subtype) else: @@ -679,7 +669,6 @@ return self.int_w(l_w[0]), self.int_w(l_w[1]), self.int_w(l_w[2]) def is_(self, w_one, w_two): - # XXX a bit of hacking to gain more speed if w_one is w_two: return self.w_True return self.w_False @@ -719,7 +708,7 @@ e = None if w_descr is not None: if not self.is_data_descr(w_descr): - w_value = w_obj.getdictvalue_attr_is_in_class(self, w_name) + w_value = w_obj.getdictvalue_attr_is_in_class(self, name) if w_value is not None: return w_value try: @@ -728,7 +717,7 @@ if not e.match(self, self.w_AttributeError): raise else: - w_value = w_obj.getdictvalue(self, w_name) + w_value = w_obj.getdictvalue(self, name) if w_value is not None: return w_value @@ -740,18 +729,27 @@ else: raiseattrerror(self, w_obj, name) + def finditem_str(self, w_obj, key): + # performance shortcut to avoid creating the OperationError(KeyError) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem_str(key) + return ObjSpace.finditem_str(self, w_obj, key) + def finditem(self, w_obj, w_key): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: - return w_obj.get(w_key, None) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + return w_obj.getitem(w_key) return ObjSpace.finditem(self, w_obj, w_key) - def set_str_keyed_item(self, w_obj, w_key, w_value, shadows_type=True): + def set_str_keyed_item(self, w_obj, key, w_value, shadows_type=True): # performance shortcut to avoid creating the OperationError(KeyError) - if type(w_obj) is self.DictObjectCls: - w_obj.set_str_keyed_item(w_key, w_value, shadows_type) + if (isinstance(w_obj, self.DictObjectCls) and + not w_obj.user_overridden_class): + w_obj.set_str_keyed_item(key, w_value, shadows_type) else: - self.setitem(w_obj, w_key, w_value) + self.setitem(w_obj, self.wrap(key), w_value) def getindex_w(self, w_obj, w_exception, objdescr=None): # Performance shortcut for the common case of w_obj being an int. Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/proxyobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/proxyobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/proxyobject.py Thu Nov 5 22:17:18 2009 @@ -34,19 +34,19 @@ raise OperationError(space.w_TypeError, space.wrap("You cannot override __class__ for transparent proxies")) - def getdictvalue(self, space, w_attr): + def getdictvalue(self, space, attr): try: return space.call_function(self.w_controller, space.wrap('__getattribute__'), - w_attr) + space.wrap(attr)) except OperationError, e: if not e.match(space, space.w_AttributeError): raise return None - def setdictvalue(self, space, w_attr, w_value, shadows_type=True): + def setdictvalue(self, space, attr, w_value, shadows_type=True): try: space.call_function(self.w_controller, space.wrap('__setattr__'), - w_attr, w_value) + space.wrap(attr), w_value) return True except OperationError, e: if not e.match(space, space.w_AttributeError): @@ -64,19 +64,12 @@ return False def getdict(self): - return self.getdictvalue(self.space, self.space.wrap('__dict__')) + return self.getdictvalue(self.space, '__dict__') def setdict(self, space, w_dict): - if not self.setdictvalue(space, space.wrap('__dict__'), w_dict): + if not self.setdictvalue(space, '__dict__', w_dict): baseobjspace.W_Root.setdict(self, space, w_dict) -## def __getattr__(self, attr): -## # NOT_RPYTHON -## try: -## return self.getdictvalue(self.space, self.space.wrap(attr)) -## except OperationError, e: -## raise AttributeError(attr) - W_Transparent.__name__ = name return W_Transparent @@ -109,7 +102,7 @@ from pypy.objspace.std.listtype import list_typedef as typedef class W_TransparentDict(W_TransparentObject): - from pypy.objspace.std.dictobject import W_DictObject as original + from pypy.objspace.std.dictmultiobject import W_DictMultiObject as original from pypy.objspace.std.dicttype import dict_typedef as typedef registerimplementation(W_TransparentList) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/setobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/setobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/setobject.py Thu Nov 5 22:17:18 2009 @@ -8,6 +8,17 @@ from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef class W_BaseSetObject(W_Object): + typedef = None + + # make sure that Base is used for Set and Frozenset in multimethod + # declarations + @classmethod + def is_implementation_for(cls, typedef): + if typedef is frozensettypedef or typedef is settypedef: + assert cls is W_BaseSetObject + return True + return False + def __init__(w_self, space, setdata=None): if setdata is None: @@ -48,6 +59,7 @@ W_BaseSetObject.__init__(w_self, space, setdata) w_self.hash = -1 +registerimplementation(W_BaseSetObject) registerimplementation(W_SetObject) registerimplementation(W_FrozensetObject) @@ -99,14 +111,7 @@ def make_setdata_from_w_iterable(space, w_iterable=None): data = r_dict(space.eq_w, space.hash_w) if w_iterable is not None: - w_iterator = space.iter(w_iterable) - while True: - try: - w_item = space.next(w_iterator) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - break + for w_item in space.viewiterable(w_iterable): data[w_item] = None return data @@ -211,14 +216,12 @@ #end helper functions -def set_update__Set_Set(space, w_left, w_other): +def set_update__Set_BaseSet(space, w_left, w_other): # optimization only (the general case works too) ld, rd = w_left.setdata, w_other.setdata new_ld, rd = _union_dict(ld, rd, True) return space.w_None -set_update__Set_Frozenset = set_update__Set_Set - def set_update__Set_ANY(space, w_left, w_other): """Update a set with the union of itself and another.""" ld, rd = w_left.setdata, make_setdata_from_w_iterable(space, w_other) @@ -226,7 +229,7 @@ return space.w_None def inplace_or__Set_Set(space, w_left, w_other): - set_update__Set_Set(space, w_left, w_other) + set_update__Set_BaseSet(space, w_left, w_other) return w_left inplace_or__Set_Frozenset = inplace_or__Set_Set @@ -316,6 +319,22 @@ eq__Frozenset_ANY = eq__Set_ANY +def ne__Set_Set(space, w_left, w_other): + return space.wrap(not _is_eq(w_left.setdata, w_other.setdata)) + +ne__Set_Frozenset = ne__Set_Set +ne__Frozenset_Frozenset = ne__Set_Set +ne__Frozenset_Set = ne__Set_Set + +def ne__Set_settypedef(space, w_left, w_other): + rd = make_setdata_from_w_iterable(space, w_other) + return space.wrap(_is_eq(w_left.setdata, rd)) + +ne__Set_frozensettypedef = ne__Set_settypedef +ne__Frozenset_settypedef = ne__Set_settypedef +ne__Frozenset_frozensettypedef = ne__Set_settypedef + + def ne__Set_ANY(space, w_left, w_other): # more workarounds return space.w_True Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/sharingdict.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/sharingdict.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,3 @@ -from pypy.objspace.std.dictmultiobject import DictImplementation, StrDictImplementation from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash from pypy.rlib.jit import purefunction, hint, we_are_jitted, unroll_safe @@ -75,41 +74,42 @@ self.emptylist = [] -class SharedDictImplementation(DictImplementation): +class SharedDictImplementation(W_DictMultiObject): def __init__(self, space): self.space = space self.structure = space.fromcache(State).empty_structure self.entries = space.fromcache(State).emptylist - def get(self, w_lookup): + def impl_getitem(self, w_lookup): space = self.space w_lookup_type = space.type(w_lookup) if space.is_w(w_lookup_type, space.w_str): - lookup = space.str_w(w_lookup) - i = self.structure.lookup_position(lookup) - if i == -1: - return None - return self.entries[i] + return self.impl_getitem_str(space.str_w(w_lookup)) elif _is_sane_hash(space, w_lookup_type): return None else: - return self._as_rdict().get(w_lookup) + return self._as_rdict().getitem(w_lookup) - def setitem(self, w_key, w_value): + def impl_getitem_str(self, lookup): + i = self.structure.lookup_position(lookup) + if i == -1: + return None + return self.entries[i] + + def impl_setitem(self, w_key, w_value): space = self.space if space.is_w(space.type(w_key), space.w_str): - return self.setitem_str(w_key, w_value) + self.impl_setitem_str(self.space.str_w(w_key), w_value) else: - return self._as_rdict().setitem(w_key, w_value) + self._as_rdict().setitem(w_key, w_value) @unroll_safe - def setitem_str(self, w_key, w_value, shadows_type=True): - key = self.space.str_w(w_key) + def impl_setitem_str(self, key, w_value, shadows_type=True): i = self.structure.lookup_position(key) if i != -1: self.entries[i] = w_value - return self + return new_structure = self.structure.get_next_structure(key) if new_structure.length > len(self.entries): new_entries = [None] * new_structure.size_estimate() @@ -120,75 +120,72 @@ self.entries[new_structure.length - 1] = w_value assert self.structure.length + 1 == new_structure.length self.structure = new_structure - return self - def delitem(self, w_key): + def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) if space.is_w(w_key_type, space.w_str): key = space.str_w(w_key) - if (self.structure.last_key is not None and - key == self.structure.last_key): - self.entries[self.structure.length - 1] = None - self.structure = self.structure.back_struct - return self - return self._as_rdict().delitem(w_key) + pos = self.structure.lookup_position(key) + if pos == -1: + raise KeyError + struct_len = self.structure.length + num_back = struct_len - pos - 1 + + if num_back > 0: + for i in range(pos, struct_len - 1): + self.entries[i] = self.entries[i + 1] + # don't make the entries list shorter, new keys might be added soon + self.entries[struct_len - 1] = None + structure = self.structure + keys = [None] * num_back + for i in range(num_back): + keys[i] = structure.last_key + structure = structure.back_struct + # go back the structure that contains the deleted key + structure = structure.back_struct + for i in range(num_back - 1, -1, -1): + structure = structure.get_next_structure(keys[i]) + self.structure = structure elif _is_sane_hash(space, w_key_type): raise KeyError else: - return self._as_rdict().delitem(w_key) + self._as_rdict().delitem(w_key) - def length(self): + def impl_length(self): return self.structure.length - def iteritems(self): - return SharedItemIteratorImplementation(self.space, self) - - def iterkeys(self): - return SharedKeyIteratorImplementation(self.space, self) + def impl_iter(self): + return SharedIteratorImplementation(self.space, self) - def itervalues(self): - return SharedValueIteratorImplementation(self.space, self) - - def keys(self): + def impl_keys(self): space = self.space return [space.wrap(key) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] + for (key, item) in self.structure.keys.iteritems()] - def values(self): + def impl_values(self): return self.entries[:self.structure.length] - def items(self): + def impl_items(self): space = self.space return [space.newtuple([space.wrap(key), self.entries[item]]) - for (key, item) in self.structure.keys.iteritems() - if item >= 0] - - def _as_rdict(self, as_strdict=False): - if as_strdict: - newimpl = StrDictImplementation(self.space) - else: - newimpl = self.space.DefaultDictImpl(self.space) + for (key, item) in self.structure.keys.iteritems()] + def impl_clear(self): + space = self.space + self.structure = space.fromcache(State).empty_structure + self.entries = space.fromcache(State).emptylist + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() for k, i in self.structure.keys.items(): - if i >= 0: - newimpl.setitem_str(self.space.wrap(k), self.entries[i]) - return newimpl + r_dict_content[self.space.wrap(k)] = self.entries[i] + self._clear_fields() + return self + def _clear_fields(self): + self.structure = None + self.entries = None -class SharedValueIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.values = dictimplementation.entries - - def next(self): - if self.pos < self.len: - return self.values[self.pos] - else: - self.values = None - return None - -class SharedItemIteratorImplementation(IteratorImplementation): +class SharedIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) self.iterator = dictimplementation.structure.keys.iteritems() @@ -198,19 +195,6 @@ assert isinstance(implementation, SharedDictImplementation) for key, index in self.iterator: w_value = implementation.entries[index] - return self.space.newtuple([self.space.wrap(key), w_value]) - else: - return None - -class SharedKeyIteratorImplementation(IteratorImplementation): - def __init__(self, space, dictimplementation): - IteratorImplementation.__init__(self, space, dictimplementation) - self.iterator = dictimplementation.structure.keys.iteritems() - - def next_entry(self): - implementation = self.dictimplementation - assert isinstance(implementation, SharedDictImplementation) - for key, index in self.iterator: - return self.space.wrap(key) + return self.space.wrap(key), w_value else: - return None + return None, None Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/stdtypedef.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/stdtypedef.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/stdtypedef.py Thu Nov 5 22:17:18 2009 @@ -112,6 +112,13 @@ result.append(value) return result +def is_relevant_for_slice(target_type, typedef): + targettypedef = getattr(target_type, 'typedef', None) + if targettypedef == typedef: + return True + method = getattr(target_type, "is_implementation_for", lambda t: False) + return method(typedef) + def sliced_typeorders(typeorder, multimethod, typedef, i, local=False): """NOT_RPYTHON""" list_of_typeorders = [typeorder] * multimethod.arity @@ -124,8 +131,7 @@ if issubtypedef(thistypedef, typedef): lst = [] for target_type, conversion in order: - targettypedef = getattr(target_type, 'typedef', None) - if targettypedef == typedef: + if is_relevant_for_slice(target_type, typedef): lst.append((target_type, conversion)) sliced_typeorder[type] = lst list_of_typeorders[i] = sliced_typeorder Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_builtinshortcut.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_builtinshortcut.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_builtinshortcut.py Thu Nov 5 22:17:18 2009 @@ -1,5 +1,5 @@ from pypy.objspace.std.test import test_userobject -from pypy.objspace.std.test import test_set +from pypy.objspace.std.test import test_setobject from pypy.objspace.std.test import test_stringobject WITH_BUILTINSHORTCUT = {'objspace.std.builtinshortcut': True} @@ -80,7 +80,7 @@ assert u != s -class AppTestSet(test_set.AppTestAppSetTest): +class AppTestSet(test_setobject.AppTestAppSetTest): # this tests tons of funny comparison combinations that can easily go wrong def setup_class(cls): from pypy import conftest Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_celldict.py Thu Nov 5 22:17:18 2009 @@ -26,7 +26,7 @@ def rescue_builtins(space): w_dict = space.builtin.getdict() content = {} - for key, cell in w_dict.implementation.content.iteritems(): + for key, cell in w_dict.content.iteritems(): newcell = ModuleCell() newcell.w_value = cell.w_value content[key] = newcell @@ -35,9 +35,9 @@ cls.w_rescue_builtins = cls.space.wrap(rescue_builtins) def restore_builtins(space): w_dict = space.builtin.getdict() - if not isinstance(w_dict.implementation, ModuleDictImplementation): - w_dict.implementation = ModuleDictImplementation(space) - w_dict.implementation.content = stored_builtins.pop() + assert isinstance(w_dict, ModuleDictImplementation) + w_dict.content = stored_builtins.pop() + w_dict.fallback = None restore_builtins = gateway.interp2app(restore_builtins) cls.w_restore_builtins = cls.space.wrap(restore_builtins) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_dictmultiobject.py Thu Nov 5 22:17:18 2009 @@ -1,31 +1,490 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.dictmultiobject import \ W_DictMultiObject, setitem__DictMulti_ANY_ANY, getitem__DictMulti_ANY, \ - EmptyDictImplementation, RDictImplementation, StrDictImplementation, \ - MeasuringDictImplementation + StrDictImplementation from pypy.objspace.std.celldict import ModuleDictImplementation +from pypy.objspace.std.sharingdict import SharedDictImplementation from pypy.conftest import gettestobjspace -from pypy.objspace.std.test import test_dictobject -class TestW_DictMultiObject(test_dictobject.TestW_DictObject): + +class TestW_DictObject: + def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + cls.space = gettestobjspace() + + def test_empty(self): + space = self.space + d = self.space.newdict() + assert not self.space.is_true(d) + + def test_nonempty(self): + space = self.space + wNone = space.w_None + d = self.space.newdict() + d.initialize_content([(wNone, wNone)]) + assert space.is_true(d) + i = space.getitem(d, wNone) + equal = space.eq(i, wNone) + assert space.is_true(equal) + + def test_setitem(self): + space = self.space + wk1 = space.wrap('key') + wone = space.wrap(1) + d = self.space.newdict() + d.initialize_content([(space.wrap('zero'),space.wrap(0))]) + space.setitem(d,wk1,wone) + wback = space.getitem(d,wk1) + assert self.space.eq_w(wback,wone) + + def test_delitem(self): + space = self.space + wk1 = space.wrap('key') + d = self.space.newdict() + d.initialize_content( [(space.wrap('zero'),space.wrap(0)), + (space.wrap('one'),space.wrap(1)), + (space.wrap('two'),space.wrap(2))]) + space.delitem(d,space.wrap('one')) + assert self.space.eq_w(space.getitem(d,space.wrap('zero')),space.wrap(0)) + assert self.space.eq_w(space.getitem(d,space.wrap('two')),space.wrap(2)) + self.space.raises_w(self.space.w_KeyError, + space.getitem,d,space.wrap('one')) + + def test_wrap_dict(self): + assert isinstance(self.space.wrap({}), W_DictMultiObject) + + + def test_dict_compare(self): + w = self.space.wrap + w0, w1, w2, w3 = map(w, range(4)) + def wd(items): + d = self.space.newdict() + d.initialize_content(items) + return d + wd1 = wd([(w0, w1), (w2, w3)]) + wd2 = wd([(w2, w3), (w0, w1)]) + assert self.space.eq_w(wd1, wd2) + wd3 = wd([(w2, w2), (w0, w1)]) + assert not self.space.eq_w(wd1, wd3) + wd4 = wd([(w3, w3), (w0, w1)]) + assert not self.space.eq_w(wd1, wd4) + wd5 = wd([(w3, w3)]) + assert not self.space.eq_w(wd1, wd4) + def test_dict_call(self): + space = self.space + w = space.wrap + def wd(items): + d = space.newdict() + d.initialize_content(items) + return d + def mydict(w_args=w(()), w_kwds=w({})): + return space.call(space.w_dict, w_args, w_kwds) + def deepwrap(lp): + return [[w(a),w(b)] for a,b in lp] + d = mydict() + assert self.space.eq_w(d, w({})) + args = w(([['a',2],[23,45]],)) + d = mydict(args) + assert self.space.eq_w(d, wd(deepwrap([['a',2],[23,45]]))) + d = mydict(args, w({'a':33, 'b':44})) + assert self.space.eq_w(d, wd(deepwrap([['a',33],['b',44],[23,45]]))) + d = mydict(w_kwds=w({'a':33, 'b':44})) + assert self.space.eq_w(d, wd(deepwrap([['a',33],['b',44]]))) + self.space.raises_w(space.w_TypeError, mydict, w((23,))) + self.space.raises_w(space.w_ValueError, mydict, w(([[1,2,3]],))) + + def test_dict_pop(self): + space = self.space + w = space.wrap + def mydict(w_args=w(()), w_kwds=w({})): + return space.call(space.w_dict, w_args, w_kwds) + d = mydict(w_kwds=w({"1":2, "3":4})) + dd = mydict(w_kwds=w({"1":2, "3":4})) # means d.copy() + pop = space.getattr(dd, w("pop")) + result = space.call_function(pop, w("1")) + assert self.space.eq_w(result, w(2)) + assert self.space.eq_w(space.len(dd), w(1)) + + dd = mydict(w_kwds=w({"1":2, "3":4})) # means d.copy() + pop = space.getattr(dd, w("pop")) + result = space.call_function(pop, w("1"), w(44)) + assert self.space.eq_w(result, w(2)) + assert self.space.eq_w(space.len(dd), w(1)) + result = space.call_function(pop, w("1"), w(44)) + assert self.space.eq_w(result, w(44)) + assert self.space.eq_w(space.len(dd), w(1)) + + self.space.raises_w(space.w_KeyError, space.call_function, pop, w(33)) + + def test_get(self): + space = self.space + w = space.wrap + def mydict(w_args=w(()), w_kwds=w({})): + return space.call(space.w_dict, w_args, w_kwds) + d = mydict(w_kwds=w({"1":2, "3":4})) + get = space.getattr(d, w("get")) + assert self.space.eq_w(space.call_function(get, w("1")), w(2)) + assert self.space.eq_w(space.call_function(get, w("1"), w(44)), w(2)) + assert self.space.eq_w(space.call_function(get, w("33")), w(None)) + assert self.space.eq_w(space.call_function(get, w("33"), w(44)), w(44)) + + def test_initialize_from_strdict_shared(self): + space = self.space + w = space.wrap + d = {"a": w(1), "b": w(2)} + w_d = space.newdict(from_strdict_shared=d) + assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) + assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) + def test_initialize_from_strdict_really_shared(self): space = self.space w = space.wrap d = {"a": w(1), "b": w(2)} - w_d = space.DictObjectCls(space) - w_d.initialize_from_strdict_shared(d) + w_d = space.newdict(from_strdict_shared=d) assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) d["c"] = w(41) assert self.space.eq_w(space.getitem(w_d, w("c")), w(41)) -class AppTest_DictMultiObject(test_dictobject.AppTest_DictObject): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + + +class AppTest_DictObject: + + def test_equality(self): + d = {1:2} + f = {1:2} + assert d == f + assert d != {1:3} + + def test_clear(self): + d = {1:2, 3:4} + d.clear() + assert len(d) == 0 + + def test_copy(self): + d = {1:2, 3:4} + dd = d.copy() + assert d == dd + assert not d is dd + + def test_get(self): + d = {1:2, 3:4} + assert d.get(1) == 2 + assert d.get(1,44) == 2 + assert d.get(33) == None + assert d.get(33,44) == 44 + + def test_pop(self): + d = {1:2, 3:4} + dd = d.copy() + result = dd.pop(1) + assert result == 2 + assert len(dd) == 1 + dd = d.copy() + result = dd.pop(1, 44) + assert result == 2 + assert len(dd) == 1 + result = dd.pop(1, 44) + assert result == 44 + assert len(dd) == 1 + raises(KeyError, dd.pop, 33) + + def test_has_key(self): + d = {1:2, 3:4} + assert d.has_key(1) + assert not d.has_key(33) + + def test_items(self): + d = {1:2, 3:4} + its = d.items() + its.sort() + assert its == [(1,2),(3,4)] + + def test_iteritems(self): + d = {1:2, 3:4} + dd = d.copy() + for k, v in d.iteritems(): + assert v == dd[k] + del dd[k] + assert not dd + + def test_iterkeys(self): + d = {1:2, 3:4} + dd = d.copy() + for k in d.iterkeys(): + del dd[k] + assert not dd + + def test_itervalues(self): + d = {1:2, 3:4} + values = [] + for k in d.itervalues(): + values.append(k) + assert values == d.values() + + def test_keys(self): + d = {1:2, 3:4} + kys = d.keys() + kys.sort() + assert kys == [1,3] + + def test_popitem(self): + d = {1:2, 3:4} + it = d.popitem() + assert len(d) == 1 + assert it==(1,2) or it==(3,4) + it1 = d.popitem() + assert len(d) == 0 + assert (it!=it1) and (it1==(1,2) or it1==(3,4)) + + def test_setdefault(self): + d = {1:2, 3:4} + dd = d.copy() + x = dd.setdefault(1, 99) + assert d == dd + assert x == 2 + x = dd.setdefault(33, 99) + d[33] = 99 + assert d == dd + assert x == 99 + + def test_update(self): + d = {1:2, 3:4} + dd = d.copy() + d.update({}) + assert d == dd + d.update({3:5, 6:7}) + assert d == {1:2, 3:5, 6:7} + + def test_update_iterable(self): + d = {} + d.update((('a',1),)) + assert d == {'a': 1} + d.update([('a',2), ('c',3)]) + assert d == {'a': 2, 'c': 3} + + def test_update_nop(self): + d = {} + d.update() + assert d == {} + + def test_update_kwargs(self): + d = {} + d.update(foo='bar', baz=1) + assert d == {'foo': 'bar', 'baz': 1} + + def test_update_dict_and_kwargs(self): + d = {} + d.update({'foo': 'bar'}, baz=1) + assert d == {'foo': 'bar', 'baz': 1} + + def test_values(self): + d = {1:2, 3:4} + vals = d.values() + vals.sort() + assert vals == [2,4] + + def test_eq(self): + d1 = {1:2, 3:4} + d2 = {1:2, 3:4} + d3 = {1:2} + bool = d1 == d2 + assert bool == True + bool = d1 == d3 + assert bool == False + bool = d1 != d2 + assert bool == False + bool = d1 != d3 + assert bool == True + + def test_lt(self): + d1 = {1:2, 3:4} + d2 = {1:2, 3:4} + d3 = {1:2, 3:5} + d4 = {1:2} + bool = d1 < d2 + assert bool == False + bool = d1 < d3 + assert bool == True + bool = d1 < d4 + assert bool == False + + def test_lt2(self): + assert {'a': 1 } < { 'a': 2 } + assert not {'a': 1 } > { 'a': 2 } + assert not {'a': 1, 'b': 0 } > { 'a': 2, 'b': 0 } + assert {'a': 1, 'b': 0 } < { 'a': 2, 'b': 0 } + assert {'a': 1, 'b': 0 } < { 'a': 1, 'b': 2 } + assert not {'a': 1, 'b': 0 } < { 'a': 1, 'b': -2 } + assert {'a': 1 } < { 'b': 1} + assert {'a': 1, 'x': 2 } < { 'b': 1, 'x': 2} + + def test_str_repr(self): + assert '{}' == str({}) + assert '{1: 2}' == str({1: 2}) + assert "{'ba': 'bo'}" == str({'ba': 'bo'}) + # NOTE: the string repr depends on hash values of 1 and 'ba'!!! + ok_reprs = ["{1: 2, 'ba': 'bo'}", "{'ba': 'bo', 1: 2}"] + assert str({1: 2, 'ba': 'bo'}) in ok_reprs + assert '{}' == repr({}) + assert '{1: 2}' == repr({1: 2}) + assert "{'ba': 'bo'}" == repr({'ba': 'bo'}) + assert str({1: 2, 'ba': 'bo'}) in ok_reprs + + # Now test self-containing dict + d = {} + d[0] = d + assert str(d) == '{0: {...}}' + + # Mutating while repr'ing + class Machiavelli(object): + def __repr__(self): + d.clear() + return "42" + d = {Machiavelli(): True} + str(d) + assert d == {} + + def test_new(self): + d = dict() + assert d == {} + args = [['a',2], [23,45]] + d = dict(args) + assert d == {'a':2, 23:45} + d = dict(args, a=33, b=44) + assert d == {'a':33, 'b':44, 23:45} + d = dict(a=33, b=44) + assert d == {'a':33, 'b':44} + d = dict({'a':33, 'b':44}) + assert d == {'a':33, 'b':44} + try: d = dict(23) + except (TypeError, ValueError): pass + else: self.fail("dict(23) should raise!") + try: d = dict([[1,2,3]]) + except (TypeError, ValueError): pass + else: self.fail("dict([[1,2,3]]) should raise!") + + def test_fromkeys(self): + assert {}.fromkeys([1, 2], 1) == {1: 1, 2: 1} + assert {}.fromkeys([1, 2]) == {1: None, 2: None} + assert {}.fromkeys([]) == {} + assert {1: 0, 2: 0, 3: 0}.fromkeys([1, '1'], 'j') == ( + {1: 'j', '1': 'j'}) + + def test_str_uses_repr(self): + class D(dict): + def __repr__(self): + return 'hi' + assert repr(D()) == 'hi' + assert str(D()) == 'hi' + + def test_overridden_setitem(self): + class D(dict): + def __setitem__(self, key, value): + dict.__setitem__(self, key, 42) + d = D([('x', 'foo')], y = 'bar') + assert d['x'] == 'foo' + assert d['y'] == 'bar' + + d.setdefault('z', 'baz') + assert d['z'] == 'baz' + + d['foo'] = 'bar' + assert d['foo'] == 42 + + d.update({'w': 'foobar'}) + assert d['w'] == 'foobar' + + d = d.copy() + assert d['x'] == 'foo' + + d3 = D.fromkeys(['x', 'y'], 'foo') + assert d3['x'] == 42 + assert d3['y'] == 42 + + def test_overridden_setitem_customkey(self): + class D(dict): + def __setitem__(self, key, value): + dict.__setitem__(self, key, 42) + class Foo(object): + pass + + d = D() + key = Foo() + d[key] = 'bar' + assert d[key] == 42 + + def test_repr_with_overridden_items(self): + class D(dict): + def items(self): + return [] + + d = D([("foo", "foobar")]) + assert repr(d) == "{'foo': 'foobar'}" + + def test_popitem_with_overridden_delitem(self): + class D(dict): + def __delitem__(self, key): + assert False + d = D() + d['a'] = 42 + item = d.popitem() + assert item == ('a', 42) + + def test_dict_update_overridden_getitem(self): + class D(dict): + def __getitem__(self, key): + return 42 + d1 = {} + d2 = D(a='foo') + d1.update(d2) + assert d1['a'] == 42 # fails on CPython, d1['a'] == 'foo' + + def test_index_keyerror_unpacking(self): + d = {} + for v1 in ['Q', (1,)]: + try: + d[v1] + except KeyError, e: + v2 = e.args[0] + assert v1 == v2 + else: + assert False, 'Expected KeyError' + + def test_del_keyerror_unpacking(self): + d = {} + for v1 in ['Q', (1,)]: + try: + del d[v1] + except KeyError, e: + v2 = e.args[0] + assert v1 == v2 + else: + assert False, 'Expected KeyError' + + def test_pop_keyerror_unpacking(self): + d = {} + for v1 in ['Q', (1,)]: + try: + d.pop(v1) + except KeyError, e: + v2 = e.args[0] + assert v1 == v2 + else: + assert False, 'Expected KeyError' + + def test_no_len_on_dict_iter(self): + iterable = {1: 2, 3: 4} + raises(TypeError, len, iter(iterable)) + iterable = {"1": 2, "3": 4} + raises(TypeError, len, iter(iterable)) + iterable = {} + raises(TypeError, len, iter(iterable)) + + +class AppTest_DictMultiObject(AppTest_DictObject): def test_emptydict_unhashable(self): raises(TypeError, "{}[['x']]") @@ -45,11 +504,11 @@ assert getattr(a, s) == 42 -class TestW_DictSharing(test_dictobject.TestW_DictObject): +class TestW_DictSharing(TestW_DictObject): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withsharingdict": True}) -class AppTest_DictSharing(test_dictobject.AppTest_DictObject): +class AppTest_DictSharing(AppTest_DictObject): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withsharingdict": True}) @@ -91,9 +550,32 @@ raises(KeyError, "d['def']") -class C: pass -class FakeSpace(test_dictobject.FakeSpace): +class FakeString(str): + def unwrap(self, space): + self.unwrapped = True + return str(self) + +# the minimal 'space' needed to use a W_DictMultiObject +class FakeSpace: + def hash_w(self, obj): + return hash(obj) + def unwrap(self, x): + return x + def is_true(self, x): + return x + def is_(self, x, y): + return x is y + is_w = is_ + def eq(self, x, y): + return x == y + eq_w = eq + def newlist(self, l): + return [] + DictObjectCls = W_DictMultiObject + def type(self, w_obj): + return type(w_obj) + w_str = str def str_w(self, string): assert isinstance(string, str) return string @@ -107,152 +589,185 @@ def newtuple(self, l): return tuple(l) + def newdict(self, module=False, instance=False, classofinstance=None, + from_strdict_shared=None): + return W_DictMultiObject.allocate_and_init_instance( + self, module=module, instance=instance, + classofinstance=classofinstance, + from_strdict_shared=from_strdict_shared) + + def allocate_instance(self, cls, type): + return object.__new__(cls) + + def fromcache(self, cls): + return cls(self) + w_StopIteration = StopIteration w_None = None - StringObjectCls = None # xxx untested: shortcut in StrDictImpl.getitem + StringObjectCls = FakeString + w_dict = None + iter = iter + viewiterable = list + + +class Config: + class objspace: + class std: + withdictmeasurement = False + withsharingdict = False + withsmalldicts = False + withcelldict = False + withshadowtracking = False + class opcodes: + CALL_LIKELY_BUILTIN = False + +FakeSpace.config = Config() + class TestDictImplementation: def setup_method(self,method): self.space = FakeSpace() - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DictObjectCls = W_DictMultiObject - self.space.DefaultDictImpl = RDictImplementation def test_stressdict(self): from random import randint - d = self.space.DictObjectCls(self.space) + d = self.space.newdict() N = 10000 pydict = {} for i in range(N): x = randint(-N, N) setitem__DictMulti_ANY_ANY(self.space, d, x, i) pydict[x] = i - for x in pydict: - assert pydict[x] == getitem__DictMulti_ANY(self.space, d, x) + for key, value in pydict.iteritems(): + assert value == getitem__DictMulti_ANY(self.space, d, key) -class TestRDictImplementation: - ImplementionClass = RDictImplementation - DevolvedClass = RDictImplementation - EmptyClass = EmptyDictImplementation - DefaultDictImpl = RDictImplementation +class BaseTestRDictImplementation: def setup_method(self,method): - self.space = FakeSpace() - self.space.DictObjectCls = W_DictMultiObject - self.space.emptydictimpl = EmptyDictImplementation(self.space) - self.space.DefaultDictImpl = self.DefaultDictImpl - self.string = self.space.wrap("fish") - self.string2 = self.space.wrap("fish2") + self.fakespace = FakeSpace() + self.string = self.fakespace.wrap("fish") + self.string2 = self.fakespace.wrap("fish2") self.impl = self.get_impl() def get_impl(self): - "Needs to be empty, or one entry with key self.string" - return self.ImplementionClass(self.space) + return self.ImplementionClass(self.fakespace) + + def fill_impl(self): + self.impl.setitem(self.string, 1000) + self.impl.setitem(self.string2, 2000) + + def check_not_devolved(self): + assert self.impl.r_dict_content is None def test_setitem(self): - assert self.impl.setitem(self.string, 1000) is self.impl + self.impl.setitem(self.string, 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 + self.check_not_devolved() def test_setitem_str(self): - assert self.impl.setitem_str(self.space.str_w(self.string), 1000) is self.impl + self.impl.setitem_str(self.fakespace.str_w(self.string), 1000) assert self.impl.length() == 1 - assert self.impl.get(self.string) == 1000 + assert self.impl.getitem(self.string) == 1000 + assert self.impl.getitem_str(self.string) == 1000 + self.check_not_devolved() def test_delitem(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() assert self.impl.length() == 2 - newimpl = self.impl.delitem(self.string) + self.impl.delitem(self.string2) assert self.impl.length() == 1 - assert newimpl is self.impl - newimpl = self.impl.delitem(self.string2) + self.impl.delitem(self.string) assert self.impl.length() == 0 - assert isinstance(newimpl, self.EmptyClass) + self.check_not_devolved() def test_keys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() keys = self.impl.keys() keys.sort() assert keys == [self.string, self.string2] + self.check_not_devolved() def test_values(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() values = self.impl.values() values.sort() assert values == [1000, 2000] + self.check_not_devolved() def test_items(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) + self.fill_impl() items = self.impl.items() items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() - def test_iterkeys(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iterkeys() - keys = [] - while 1: - key = iteratorimplementation.next() - if key is None: - break - keys.append(key) - keys.sort() - assert keys == [self.string, self.string2] - - def test_itervalues(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.itervalues() - values = [] - while 1: - value = iteratorimplementation.next() - if value is None: - break - values.append(value) - values.sort() - assert values == [1000, 2000] - - def test_iteritems(self): - self.impl.setitem(self.string, 1000) - self.impl.setitem(self.string2, 2000) - iteratorimplementation = self.impl.iteritems() + def test_iter(self): + self.fill_impl() + iteratorimplementation = self.impl.iter() items = [] while 1: item = iteratorimplementation.next() - if item is None: + if item == (None, None): break items.append(item) items.sort() assert items == zip([self.string, self.string2], [1000, 2000]) + self.check_not_devolved() def test_devolve(self): impl = self.impl for x in xrange(100): - impl = impl.setitem(self.space.str_w(str(x)), x) - impl = impl.setitem(x, x) - assert isinstance(impl, self.DevolvedClass) + impl.setitem(self.fakespace.str_w(str(x)), x) + impl.setitem(x, x) + assert impl.r_dict_content is not None -class TestStrDictImplementation(TestRDictImplementation): +class TestStrDictImplementation(BaseTestRDictImplementation): ImplementionClass = StrDictImplementation -class TestMeasuringDictImplementation(TestRDictImplementation): - ImplementionClass = MeasuringDictImplementation - DevolvedClass = MeasuringDictImplementation - EmptyClass = MeasuringDictImplementation + def test_str_shortcut(self): + self.fill_impl() + s = FakeString(self.string) + assert self.impl.getitem(s) == 1000 + assert s.unwrapped + +## class TestMeasuringDictImplementation(BaseTestRDictImplementation): +## ImplementionClass = MeasuringDictImplementation +## DevolvedClass = MeasuringDictImplementation + +class TestModuleDictImplementation(BaseTestRDictImplementation): + ImplementionClass = ModuleDictImplementation + +class TestModuleDictImplementationWithBuiltinNames(BaseTestRDictImplementation): + ImplementionClass = ModuleDictImplementation + + string = "int" + string2 = "isinstance" + +class TestSharedDictImplementation(BaseTestRDictImplementation): + ImplementionClass = SharedDictImplementation -class TestModuleDictImplementation(TestRDictImplementation): + +class BaseTestDevolvedDictImplementation(BaseTestRDictImplementation): + def fill_impl(self): + BaseTestRDictImplementation.fill_impl(self) + self.impl._as_rdict() + + def check_not_devolved(self): + pass + +class TestDevolvedStrDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = StrDictImplementation + +class TestDevolvedModuleDictImplementation(BaseTestDevolvedDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation -class TestModuleDictImplementationWithBuiltinNames(TestRDictImplementation): +class TestDevolvedModuleDictImplementationWithBuiltinNames(BaseTestDevolvedDictImplementation): ImplementionClass = ModuleDictImplementation - EmptyClass = ModuleDictImplementation string = "int" string2 = "isinstance" +class TestDevolvedSharedDictImplementation(BaseTestDevolvedDictImplementation): + ImplementionClass = SharedDictImplementation + Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_obj.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_obj.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_obj.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,3 @@ -# -*- coding: iso-8859-1 -*- from pypy.conftest import option class AppTestObject: Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_proxy.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_proxy.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_proxy.py Thu Nov 5 22:17:18 2009 @@ -117,8 +117,3 @@ d = self.proxy(dict, c.perform) d['z'] = 4 assert sorted(list(d.iterkeys())) == ['a', 'b', 'c', 'z'] - -class AppTestDictStrProxy(AppTestDictProxy): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmultidict": True, - "objspace.std.withtproxy": True}) Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_shadowtracking.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_shadowtracking.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_shadowtracking.py Thu Nov 5 22:17:18 2009 @@ -13,15 +13,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.g = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.f = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_shadowing_via__dict__(self): space = self.space @@ -32,15 +32,15 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["g"] = "foo" """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__["f"] = "foo" """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__dict__(self): space = self.space @@ -51,11 +51,11 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): a.__dict__ = {} """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing__class__(self): space = self.space @@ -66,14 +66,14 @@ a = A() return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() space.appexec([w_inst], """(a): class B(object): def g(self): return 42 a.__class__ = B """) - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() def test_changing_the_type(self): space = self.space @@ -84,13 +84,13 @@ a.x = 72 return a """) - assert not w_inst.w__dict__.implementation.shadows_anything() + assert not w_inst.getdict().shadows_anything() w_x = space.appexec([w_inst], """(a): a.__class__.x = 42 return a.x """) assert space.unwrap(w_x) == 72 - assert w_inst.w__dict__.implementation.shadows_anything() + assert w_inst.getdict().shadows_anything() class AppTestShadowTracking(object): def setup_class(cls): Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_sharingdict.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_sharingdict.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_sharingdict.py Thu Nov 5 22:17:18 2009 @@ -1,6 +1,8 @@ +import py from pypy.conftest import gettestobjspace -from pypy.objspace.std.sharingdict import SharedStructure, NUM_DIGITS +from pypy.objspace.std.sharingdict import SharedStructure, NUM_DIGITS, SharedDictImplementation from pypy.interpreter import gateway +from pypy.objspace.std.test.test_dictmultiobject import FakeSpace def instance_with_keys(structure, *keys): for key in keys: @@ -27,3 +29,40 @@ assert empty_structure.size_estimate() == 3 assert empty_structure.other_structs.get("a").size_estimate() == 6 assert empty_structure.other_structs.get("x").size_estimate() == 2 + +def test_delete(): + space = FakeSpace() + d = SharedDictImplementation(space) + d.setitem_str("a", 1) + d.setitem_str("b", 2) + d.setitem_str("c", 3) + d.delitem("b") + assert d.r_dict_content is None + assert d.entries == [1, 3, None] + assert d.structure.keys == {"a": 0, "c": 1} + assert d.getitem("a") == 1 + assert d.getitem("c") == 3 + assert d.getitem("b") is None + py.test.raises(KeyError, d.delitem, "b") + + d.delitem("c") + assert d.entries == [1, None, None] + assert d.structure.keys == {"a": 0} + + d.delitem("a") + assert d.entries == [None, None, None] + assert d.structure.keys == {} + + d = SharedDictImplementation(space) + d.setitem_str("a", 1) + d.setitem_str("b", 2) + d.setitem_str("c", 3) + d.setitem_str("d", 4) + d.setitem_str("e", 5) + d.setitem_str("f", 6) + d.setitem_str("g", 7) + d.setitem_str("h", 8) + d.setitem_str("i", 9) + d.delitem("d") + assert d.entries == [1, 2, 3, 5, 6, 7, 8, 9, None] + assert d.structure.keys == {"a": 0, "b": 1, "c": 2, "e": 3, "f": 4, "g": 5, "h": 6, "i": 7} Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_typeobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_typeobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/test/test_typeobject.py Thu Nov 5 22:17:18 2009 @@ -915,10 +915,6 @@ return 0 raises(TypeError, X) -class AppTestWithMultidictTypes: - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) - def test_dictproxy_is_updated(self): class A(object): x = 1 Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/typeobject.py Thu Nov 5 22:17:18 2009 @@ -131,10 +131,7 @@ def compute_default_mro(w_self): return compute_C3_mro(w_self.space, w_self) - def getdictvalue(w_self, space, w_attr): - return w_self.getdictvalue_w(space, space.str_w(w_attr)) - - def getdictvalue_w(w_self, space, attr): + def getdictvalue(w_self, space, attr): w_value = w_self.dict_w.get(attr, None) if w_self.lazyloaders and w_value is None: if attr in w_self.lazyloaders: @@ -172,7 +169,7 @@ if w_class is w_starttype: look = True elif look: - w_value = w_class.getdictvalue_w(space, name) + w_value = w_class.getdictvalue(space, name) if w_value is not None: return w_value return None @@ -182,7 +179,7 @@ def _lookup(w_self, key): space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_value return None @@ -193,7 +190,7 @@ # attribute was found space = w_self.space for w_class in w_self.mro_w: - w_value = w_class.getdictvalue_w(space, key) + w_value = w_class.getdictvalue(space, key) if w_value is not None: return w_class, w_value return None, None @@ -273,7 +270,7 @@ "NOT_RPYTHON. Forces the lazy attributes to be computed." if 'lazyloaders' in w_self.__dict__: for attr in w_self.lazyloaders.keys(): - w_self.getdictvalue_w(w_self.space, attr) + w_self.getdictvalue(w_self.space, attr) del w_self.lazyloaders return False @@ -281,8 +278,7 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - newdic = space.DictObjectCls(space) - newdic.initialize_from_strdict_shared(w_self.dict_w) + newdic = space.newdict(from_strdict_shared=w_self.dict_w) return W_DictProxyObject(newdic) def unwrap(w_self, space): Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/std/typetype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/std/typetype.py Thu Nov 5 22:17:18 2009 @@ -174,7 +174,7 @@ return space.wrap("""type(object) -> the object's type type(name, bases, dict) -> a new type""") w_type = _check(space, w_type) - w_result = w_type.getdictvalue_w(space, '__doc__') + w_result = w_type.getdictvalue(space, '__doc__') if w_result is None: return space.w_None else: Modified: pypy/branch/msvc-asmgcroot/pypy/objspace/taint.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/objspace/taint.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/objspace/taint.py Thu Nov 5 22:17:18 2009 @@ -20,14 +20,11 @@ ## def getdict(self): ## return taint(self.w_obj.getdict()) -## def getdictvalue_w(self, space, attr): -## return taint(self.w_obj.getdictvalue_w(space, attr)) +## def getdictvalue(self, space, attr): +## return taint(self.w_obj.getdictvalue(space, attr)) -## def getdictvalue(self, space, w_attr): -## return taint(self.w_obj.getdictvalue(space, w_attr)) - -## def setdictvalue(self, space, w_attr, w_value): -## return self.w_obj.setdictvalue(space, w_attr, w_value) +## def setdictvalue(self, space, attr, w_value): +## return self.w_obj.setdictvalue(space, attr, w_value) ## ... Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/debug.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/debug.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/debug.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,4 @@ - +import sys, time from pypy.rpython.extregistry import ExtRegistryEntry def ll_assert(x, msg): @@ -20,11 +20,40 @@ hop.genop('debug_assert', vlist) +class DebugLog(list): + def debug_print(self, *args): + self.append(('debug_print',) + args) + def debug_start(self, category, time=None): + self.append(('debug_start', category, time)) + def debug_stop(self, category, time=None): + for i in xrange(len(self)-1, -1, -1): + if self[i][0] == 'debug_start': + assert self[i][1] == category, ( + "nesting error: starts with %r but stops with %r" % + (self[i][1], category)) + starttime = self[i][2] + if starttime is not None or time is not None: + self[i:] = [(category, starttime, time, self[i+1:])] + else: + self[i:] = [(category, self[i+1:])] + return + assert False, ("nesting error: no start corresponding to stop %r" % + (category,)) + def __repr__(self): + import pprint + return pprint.pformat(list(self)) + +_log = None # patched from tests to be an object of class DebugLog + # or compatible +_stderr = sys.stderr # alternatively, this is patched from tests + # (redirects debug_print(), but not debug_start/stop) + def debug_print(*args): - import sys for arg in args: - print >> sys.stderr, arg, - print >> sys.stderr + print >> _stderr, arg, + print >> _stderr + if _log is not None: + _log.debug_print(*args) class Entry(ExtRegistryEntry): _about_ = debug_print @@ -35,7 +64,72 @@ def specialize_call(self, hop): vlist = hop.inputargs(*hop.args_r) hop.exception_cannot_occur() - hop.genop('debug_print', vlist) + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop('debug_print', vlist) + + +if sys.stderr.isatty(): + _start_colors_1 = "\033[1m\033[31m" + _start_colors_2 = "\033[31m" + _stop_colors = "\033[0m" +else: + _start_colors_1 = "" + _start_colors_2 = "" + _stop_colors = "" + +def debug_start(category): + print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + category, _stop_colors) + if _log is not None: + _log.debug_start(category) + +def debug_stop(category): + print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + category, _stop_colors) + if _log is not None: + _log.debug_stop(category) + +class Entry(ExtRegistryEntry): + _about_ = debug_start, debug_stop + + def compute_result_annotation(self, s_category): + return None + + def specialize_call(self, hop): + fn = self.instance + string_repr = hop.rtyper.type_system.rstr.string_repr + vlist = hop.inputargs(string_repr) + hop.exception_cannot_occur() + t = hop.rtyper.annotator.translator + if t.config.translation.log: + hop.genop(fn.__name__, vlist) + + +def have_debug_prints(): + # returns True if the next calls to debug_print show up, + # and False if they would not have any effect. + return True + +class Entry(ExtRegistryEntry): + _about_ = have_debug_prints + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + t = self.bookkeeper.annotator.translator + if t.config.translation.log: + return annmodel.s_Bool + else: + return self.bookkeeper.immutablevalue(False) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + t = hop.rtyper.annotator.translator + hop.exception_cannot_occur() + if t.config.translation.log: + return hop.genop('have_debug_prints', [], resulttype=lltype.Bool) + else: + return hop.inputconst(lltype.Bool, False) def llinterpcall(RESTYPE, pythonfunction, *args): Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/jit.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/jit.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/jit.py Thu Nov 5 22:17:18 2009 @@ -94,7 +94,6 @@ PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, - 'hash_bits': 14, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, @@ -114,6 +113,7 @@ virtualizables = [] def __init__(self, greens=None, reds=None, virtualizables=None, + get_jitcell_at=None, set_jitcell_at=None, can_inline=None, get_printable_location=None, leave=None): if greens is not None: @@ -128,6 +128,8 @@ assert v in self.reds self._alllivevars = dict.fromkeys(self.greens + self.reds) self._make_extregistryentries() + self.get_jitcell_at = get_jitcell_at + self.set_jitcell_at = set_jitcell_at self.get_printable_location = get_printable_location self.can_inline = can_inline self.leave = leave @@ -194,6 +196,10 @@ # # Annotation and rtyping of some of the JitDriver methods +class BaseJitCell(object): + __slots__ = () + + class ExtEnterLeaveMarker(ExtRegistryEntry): # Replace a call to myjitdriver.jit_merge_point(**livevars) # with an operation jit_marker('jit_merge_point', myjitdriver, livevars...) @@ -218,17 +224,21 @@ def annotate_hooks(self, **kwds_s): driver = self.instance.im_self + s_jitcell = self.bookkeeper.valueoftype(BaseJitCell) + self.annotate_hook(driver.get_jitcell_at, driver.greens, **kwds_s) + self.annotate_hook(driver.set_jitcell_at, driver.greens, [s_jitcell], + **kwds_s) self.annotate_hook(driver.can_inline, driver.greens, **kwds_s) self.annotate_hook(driver.get_printable_location, driver.greens, **kwds_s) self.annotate_hook(driver.leave, driver.greens + driver.reds, **kwds_s) - def annotate_hook(self, func, variables, **kwds_s): + def annotate_hook(self, func, variables, args_s=[], **kwds_s): if func is None: return bk = self.bookkeeper s_func = bk.immutablevalue(func) uniquekey = 'jitdriver.%s' % func.func_name - args_s = [] + args_s = args_s[:] for name in variables: s_arg = kwds_s['s_' + name] args_s.append(s_arg) Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/objectmodel.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/objectmodel.py Thu Nov 5 22:17:18 2009 @@ -359,7 +359,8 @@ class UnboxedValue(object): """A mixin class to use for classes that have exactly one field which - is an integer. They are represented as a tagged pointer.""" + is an integer. They are represented as a tagged pointer, if the + translation.taggedpointers config option is used.""" _mixin_ = True def __new__(cls, value): @@ -379,9 +380,9 @@ raise OverflowError("UnboxedValue: argument out of range") def __repr__(self): - return '' % (self.getvalue(),) + return '' % (self.get_untagged_value(),) - def getvalue(self): # helper, equivalent to reading the custom field + def get_untagged_value(self): # helper, equivalent to reading the custom field if isinstance(self.__class__.__slots__, str): return getattr(self, self.__class__.__slots__) else: Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/rgc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/rgc.py Thu Nov 5 22:17:18 2009 @@ -184,7 +184,7 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) -def malloc_nonmovable(TP, n=None): +def malloc_nonmovable(TP, n=None, zero=False): """ Allocate a non-moving buffer or return nullptr. When running directly, will pretend that gc is always moving (might be configurable in a future) @@ -195,21 +195,26 @@ class MallocNonMovingEntry(ExtRegistryEntry): _about_ = malloc_nonmovable - def compute_result_annotation(self, s_TP, s_n=None): + def compute_result_annotation(self, s_TP, s_n=None, s_zero=None): # basically return the same as malloc from pypy.annotation.builtin import malloc - return malloc(s_TP, s_n) + return malloc(s_TP, s_n, s_zero=s_zero) - def specialize_call(self, hop): + def specialize_call(self, hop, i_zero=None): from pypy.rpython.lltypesystem import lltype # XXX assume flavor and zero to be None by now assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc_nonmovable' flags = {'flavor': 'gc'} + if i_zero is not None: + flags['zero'] = hop.args_s[i_zero].const + nb_args = hop.nb_args - 1 + else: + nb_args = hop.nb_args vlist.append(hop.inputconst(lltype.Void, flags)) - if hop.nb_args == 2: + if nb_args == 2: vlist.append(hop.inputarg(lltype.Signed, arg=1)) opname += '_varsize' Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/rmmap.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/rmmap.py Thu Nov 5 22:17:18 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import rffi, lltype, llmemory from pypy.rlib import rposix from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib.nonconst import NonConstant import sys import os @@ -612,7 +613,10 @@ else: m.fd = os.dup(fd) - res = c_mmap(NULL, map_size, prot, flags, fd, 0) + # XXX if we use hintp below in alloc, the NonConstant + # is necessary since we want a general version of c_mmap + # to be annotated with a non-constant pointer. + res = c_mmap(NonConstant(NULL), map_size, prot, flags, fd, 0) if res == rffi.cast(PTR, -1): errno = _get_error_no() raise OSError(errno, os.strerror(errno)) Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_debug.py Thu Nov 5 22:17:18 2009 @@ -1,6 +1,9 @@ import py from pypy.rlib.debug import check_annotation, make_sure_not_resized +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints +from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret def test_check_annotation(): @@ -37,3 +40,46 @@ py.test.raises(TooLateForChange, interpret, f, [], list_comprehension_operations=True) + + +class DebugTests: + + def test_debug_print_start_stop(self): + def f(x): + debug_start("mycat") + debug_print("foo", 2, "bar", x) + debug_stop("mycat") + return have_debug_prints() + + try: + debug._log = dlog = debug.DebugLog() + res = f(3) + assert res == True + finally: + debug._log = None + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] + + try: + debug._log = dlog = debug.DebugLog() + res = self.interpret(f, [3]) + assert res == True + finally: + debug._log = None + assert dlog == [ + ("mycat", [ + ('debug_print', 'foo', 2, 'bar', 3), + ]), + ] + + +class TestLLType(DebugTests): + def interpret(self, f, args): + return interpret(f, args, type_system='lltype') + +class TestOOType(DebugTests): + def interpret(self, f, args): + return interpret(f, args, type_system='ootype') Modified: pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rlib/test/test_objectmodel.py Thu Nov 5 22:17:18 2009 @@ -124,13 +124,13 @@ __slots__ = 'smallint' assert C(17).smallint == 17 - assert C(17).getvalue() == 17 + assert C(17).get_untagged_value() == 17 class A(UnboxedValue): __slots__ = ['value'] assert A(12098).value == 12098 - assert A(12098).getvalue() == 12098 + assert A(12098).get_untagged_value() == 12098 def test_symbolic(): py.test.skip("xxx no test here") Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/llinterp.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/llinterp.py Thu Nov 5 22:17:18 2009 @@ -522,20 +522,6 @@ from pypy.translator.tool.lltracker import track track(*ll_objects) - def op_debug_print(self, *ll_args): - from pypy.rpython.lltypesystem.rstr import STR - line = [] - for arg in ll_args: - T = lltype.typeOf(arg) - if T == lltype.Ptr(STR): - arg = ''.join(arg.chars) - line.append(str(arg)) - line = ' '.join(line) - print line - tracer = self.llinterpreter.tracer - if tracer: - tracer.dump('\n[debug] %s\n' % (line,)) - def op_debug_pdb(self, *ll_args): if self.llinterpreter.tracer: self.llinterpreter.tracer.flush() @@ -904,6 +890,9 @@ def op_gc_stack_bottom(self): pass # marker for trackgcroot.py + def op_gc_get_type_info_group(self): + raise NotImplementedError("gc_get_type_info_group") + def op_do_malloc_fixedsize_clear(self): raise NotImplementedError("do_malloc_fixedsize_clear") Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llarena.py Thu Nov 5 22:17:18 2009 @@ -331,7 +331,49 @@ from pypy.rpython.extfunc import register_external from pypy.rlib.objectmodel import CDefinedIntSymbolic -if os.name == 'posix': +if sys.platform == 'linux2': + # This only works with linux's madvise(), which is really not a memory + # usage hint but a real command. It guarantees that after MADV_DONTNEED + # the pages are cleared again. + from pypy.rpython.tool import rffi_platform + MADV_DONTNEED = rffi_platform.getconstantinteger('MADV_DONTNEED', + '#include ') + linux_madvise = rffi.llexternal('madvise', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True) + linux_getpagesize = rffi.llexternal('getpagesize', [], rffi.INT, + sandboxsafe=True, _nowrapper=True) + + class LinuxPageSize: + def __init__(self): + self.pagesize = 0 + _freeze_ = __init__ + linuxpagesize = LinuxPageSize() + + def clear_large_memory_chunk(baseaddr, size): + pagesize = linuxpagesize.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, linux_getpagesize()) + linuxpagesize.pagesize = pagesize + if size > 2 * pagesize: + lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) + if lowbits: # clear the initial misaligned part, if any + partpage = pagesize - lowbits + llmemory.raw_memclear(baseaddr, partpage) + baseaddr += partpage + size -= partpage + length = size & -pagesize + madv_length = rffi.cast(rffi.SIZE_T, length) + madv_flags = rffi.cast(rffi.INT, MADV_DONTNEED) + err = linux_madvise(baseaddr, madv_length, madv_flags) + if rffi.cast(lltype.Signed, err) == 0: + baseaddr += length # madvise() worked + size -= length + if size > 0: # clear the final misaligned part, if any + llmemory.raw_memclear(baseaddr, size) + +elif os.name == 'posix': READ_MAX = (sys.maxint//4) + 1 # upper bound on reads to avoid surprises raw_os_open = rffi.llexternal('open', [rffi.CCHARP, rffi.INT, rffi.MODE_T], @@ -348,7 +390,7 @@ _dev_zero = rffi.str2charp('/dev/zero') # prebuilt def clear_large_memory_chunk(baseaddr, size): - # on Linux at least, reading from /dev/zero is the fastest way + # on some Unixy platforms, reading from /dev/zero is the fastest way # to clear arenas, because the kernel knows that it doesn't # need to even allocate the pages before they are used. @@ -373,6 +415,10 @@ else: # XXX any better implementation on Windows? + # Should use VirtualAlloc() to reserve the range of pages, + # and commit some pages gradually with support from the GC. + # Or it might be enough to decommit the pages and recommit + # them immediately. clear_large_memory_chunk = llmemory.raw_memclear Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/llmemory.py Thu Nov 5 22:17:18 2009 @@ -464,8 +464,10 @@ return lltype.nullptr(EXPECTED_TYPE.TO) def _cast_to_int(self): + # This is a bit annoying. We want this method to still work when the + # pointed-to object is dead if self: - return self.ptr._cast_to_int() + return self.ptr._cast_to_int(False) else: return 0 Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lloperation.py Thu Nov 5 22:17:18 2009 @@ -84,12 +84,17 @@ return op_impl fold = roproperty(get_fold_impl) - def is_pure(self, *ARGTYPES): + def is_pure(self, args_v): return (self.canfold or # canfold => pure operation self is llop.debug_assert or # debug_assert is pure enough # reading from immutable (self in (llop.getfield, llop.getarrayitem) and - ARGTYPES[0].TO._hints.get('immutable'))) + args_v[0].concretetype.TO._hints.get('immutable')) or + (self is llop.getfield and # reading from immutable_field + 'immutable_fields' in args_v[0].concretetype.TO._hints and + args_v[1].value in args_v[0].concretetype.TO + ._hints['immutable_fields'].fields)) + # XXX: what about ootype immutable arrays? def __repr__(self): return '' % (getattr(self, 'opname', '?'),) @@ -427,6 +432,7 @@ 'do_malloc_fixedsize_clear': LLOp(canunwindgc=True), 'do_malloc_varsize_clear': LLOp(canunwindgc=True), 'get_write_barrier_failing_case': LLOp(sideeffects=False), + 'gc_get_type_info_group': LLOp(sideeffects=False), # __________ GC operations __________ @@ -518,6 +524,9 @@ # __________ debugging __________ 'debug_view': LLOp(), 'debug_print': LLOp(canrun=True), + 'debug_start': LLOp(canrun=True), + 'debug_stop': LLOp(canrun=True), + 'have_debug_prints': LLOp(canrun=True), 'debug_pdb': LLOp(), 'debug_assert': LLOp(tryfold=True), 'debug_fatalerror': LLOp(), Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/lltype.py Thu Nov 5 22:17:18 2009 @@ -816,6 +816,7 @@ if not ptr: return nullptr(PTRTYPE.TO) return opaqueptr(PTRTYPE.TO, 'hidden', container = ptr._obj, + ORIGTYPE = CURTYPE, solid = ptr._solid) elif (isinstance(CURTYPE.TO, OpaqueType) and isinstance(PTRTYPE.TO, OpaqueType)): @@ -899,7 +900,7 @@ top_parent = parent return top_parent -def normalizeptr(p): +def normalizeptr(p, check=True): # If p is a pointer, returns the same pointer casted to the largest # containing structure (for the cast where p points to the header part). # Also un-hides pointers to opaque. Null pointers become None. @@ -907,12 +908,17 @@ T = typeOf(p) if not isinstance(T, Ptr): return p # primitive - if not p: + obj = p._getobj(check) + if not obj: return None # null pointer if type(p._obj0) is int: return p # a pointer obtained by cast_int_to_ptr - container = p._obj._normalizedcontainer() - if container is not p._obj: + container = obj._normalizedcontainer() + if type(container) is int: + # this must be an opaque ptr originating from an integer + assert isinstance(obj, _opaque) + return cast_int_to_ptr(obj.ORIGTYPE, container) + if container is not obj: p = _ptr(Ptr(typeOf(container)), container, p._solid) return p @@ -1190,13 +1196,15 @@ raise RuntimeError("widening %r inside %r instead of %r" % (CURTYPE, PARENTTYPE, PTRTYPE.TO)) return _ptr(PTRTYPE, struc, solid=self._solid) - def _cast_to_int(self): - if not self: + def _cast_to_int(self, check=True): + obj = self._getobj(check) + if not obj: return 0 # NULL pointer - obj = self._obj if isinstance(obj, int): return obj # special case for cast_int_to_ptr() results - obj = normalizeptr(self)._obj + obj = normalizeptr(self, check)._getobj(check) + if isinstance(obj, int): + return obj # special case for cast_int_to_ptr() results put into opaques result = intmask(obj._getid()) # assume that id() returns an addressish value which is # not zero and aligned to at least a multiple of 4 @@ -1739,6 +1747,9 @@ # if we are an opaque containing a normal Struct/GcStruct, # unwrap it if hasattr(self, 'container'): + # an integer, cast to a ptr, cast to an opaque + if type(self.container) is int: + return self.container return self.container._normalizedcontainer() else: return _parentable._normalizedcontainer(self) Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/opimpl.py Thu Nov 5 22:17:18 2009 @@ -3,6 +3,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import opimpls +from pypy.rlib import debug # ____________________________________________________________ # Implementation of the 'canfold' operations @@ -142,7 +143,12 @@ # we can constant-fold this if the innermost structure from which we # read the final field is immutable. T = lltype.typeOf(innermostcontainer).TO - if not T._hints.get('immutable'): + if T._hints.get('immutable'): + pass + elif ('immutable_fields' in T._hints and + offsets[-1] in T._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getinteriorfield on mutable struct") assert not isinstance(ob, lltype._interior_ptr) return ob @@ -390,7 +396,13 @@ def op_getfield(p, name): checkptr(p) - if not lltype.typeOf(p).TO._hints.get('immutable'): + TYPE = lltype.typeOf(p).TO + if TYPE._hints.get('immutable'): + pass + elif ('immutable_fields' in TYPE._hints and + name in TYPE._hints['immutable_fields'].fields): + pass + else: raise TypeError("cannot fold getfield on mutable struct") return getattr(p, name) @@ -400,10 +412,26 @@ raise TypeError("cannot fold getfield on mutable array") return p[index] +def _normalize(x): + if not isinstance(x, str): + TYPE = lltype.typeOf(x) + if (isinstance(TYPE, lltype.Ptr) and TYPE.TO._name == 'rpy_string' + or getattr(TYPE, '_name', '') == 'String'): # ootype + from pypy.rpython.annlowlevel import hlstr + return hlstr(x) + return x + def op_debug_print(*args): - for arg in args: - print arg, - print + debug.debug_print(*map(_normalize, args)) + +def op_debug_start(category): + debug.debug_start(_normalize(category)) + +def op_debug_stop(category): + debug.debug_stop(_normalize(category)) + +def op_have_debug_prints(): + return debug.have_debug_prints() def op_gc_stack_bottom(): pass # marker for trackgcroot.py Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/rtagged.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/rtagged.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/rtagged.py Thu Nov 5 22:17:18 2009 @@ -1,7 +1,7 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.rclass import getclassrepr, getinstancerepr, get_type_repr from pypy.rpython.lltypesystem import lltype -from pypy.rpython.lltypesystem.rclass import InstanceRepr, CLASSTYPE +from pypy.rpython.lltypesystem.rclass import InstanceRepr, CLASSTYPE, ll_inst_type from pypy.rpython.lltypesystem.rclass import MissingRTypeAttribute from pypy.rpython.lltypesystem.rclass import ll_issubclass_const from pypy.rpython.rmodel import TyperError, inputconst @@ -53,7 +53,7 @@ def convert_const_exact(self, value): self.setup() - number = value.getvalue() + number = value.get_untagged_value() return ll_int_to_unboxed(self.lowleveltype, number) def getvalue_from_unboxed(self, llops, vinst): @@ -62,7 +62,7 @@ c_one = inputconst(lltype.Signed, 1) return llops.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed) - def gettype_from_unboxed(self, llops, vinst): + def gettype_from_unboxed(self, llops, vinst, can_be_none=False): unboxedclass_repr = getclassrepr(self.rtyper, self.unboxedclassdef) cunboxedcls = inputconst(CLASSTYPE, unboxedclass_repr.getvtable()) if self.is_parent: @@ -81,8 +81,14 @@ vinst = llops.genop('cast_pointer', [vinst], resulttype=self.common_repr()) if can_be_tagged: - return llops.gendirectcall(ll_unboxed_getclass, vinst, + if can_be_none: + func = ll_unboxed_getclass_canbenone + else: + func = ll_unboxed_getclass + return llops.gendirectcall(func, vinst, cunboxedcls) + elif can_be_none: + return llops.gendirectcall(ll_inst_type, vinst) else: ctypeptr = inputconst(lltype.Void, 'typeptr') return llops.genop('getfield', [vinst, ctypeptr], @@ -100,7 +106,8 @@ def rtype_type(self, hop): [vinst] = hop.inputargs(self) - return self.gettype_from_unboxed(hop.llops, vinst) + return self.gettype_from_unboxed( + hop.llops, vinst, can_be_none=hop.args_s[0].can_be_none()) def rtype_setattr(self, hop): # only for UnboxedValue.__init__(), which is not actually called @@ -141,11 +148,15 @@ def ll_unboxed_to_int(p): return lltype.cast_ptr_to_int(p) >> 1 +def ll_unboxed_getclass_canbenone(instance, class_if_unboxed): + if instance: + return ll_unboxed_getclass(instance, class_if_unboxed) + return lltype.nullptr(lltype.typeOf(instance).TO.typeptr.TO) + def ll_unboxed_getclass(instance, class_if_unboxed): if lltype.cast_ptr_to_int(instance) & 1: return class_if_unboxed - else: - return instance.typeptr + return instance.typeptr def ll_unboxed_isinstance_const(obj, minid, maxid, answer_if_unboxed): if not obj: Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_lloperation.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_lloperation.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_lloperation.py Thu Nov 5 22:17:18 2009 @@ -4,6 +4,7 @@ from pypy.rpython.ootypesystem import ootype, ooopimpl from pypy.rpython.llinterp import LLFrame from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython import rclass LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys() if name.startswith('op_')] @@ -50,7 +51,72 @@ return s.x + s.y res = interpret(llf, [], policy=LowLevelAnnotatorPolicy()) assert res == 5 - + +def test_is_pure(): + from pypy.objspace.flow.model import Variable, Constant + assert llop.bool_not.is_pure([Variable()]) + assert llop.debug_assert.is_pure([Variable()]) + assert not llop.int_add_ovf.is_pure([Variable(), Variable()]) + # + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + v_s1 = Variable() + v_s1.concretetype = lltype.Ptr(S1) + assert not llop.setfield.is_pure([v_s1, Constant('x'), Variable()]) + assert not llop.getfield.is_pure([v_s1, Constant('y')]) + # + A1 = lltype.GcArray(lltype.Signed) + v_a1 = Variable() + v_a1.concretetype = lltype.Ptr(A1) + assert not llop.setarrayitem.is_pure([v_a1, Variable(), Variable()]) + assert not llop.getarrayitem.is_pure([v_a1, Variable()]) + assert llop.getarraysize.is_pure([v_a1]) + # + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + v_s2 = Variable() + v_s2.concretetype = lltype.Ptr(S2) + assert not llop.setfield.is_pure([v_s2, Constant('x'), Variable()]) + assert llop.getfield.is_pure([v_s2, Constant('y')]) + # + A2 = lltype.GcArray(lltype.Signed, hints={'immutable': True}) + v_a2 = Variable() + v_a2.concretetype = lltype.Ptr(A2) + assert not llop.setarrayitem.is_pure([v_a2, Variable(), Variable()]) + assert llop.getarrayitem.is_pure([v_a2, Variable()]) + assert llop.getarraysize.is_pure([v_a2]) + # + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + v_s3 = Variable() + v_s3.concretetype = lltype.Ptr(S3) + assert not llop.setfield.is_pure([v_s3, Constant('x'), Variable()]) + assert not llop.setfield.is_pure([v_s3, Constant('y'), Variable()]) + assert llop.getfield.is_pure([v_s3, Constant('x')]) + assert not llop.getfield.is_pure([v_s3, Constant('y')]) + +def test_getfield_pure(): + S1 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed)) + S2 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', lltype.Signed), ('y', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1); s1.x = 45 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s1, 'x') + s2 = lltype.malloc(S2); s2.x = 45 + assert llop.getfield(lltype.Signed, s2, 'x') == 45 + s3 = lltype.malloc(S3); s3.x = 46; s3.y = 47 + assert llop.getfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getfield, lltype.Signed, s3, 'y') + # + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s1, 'x') + assert llop.getinteriorfield(lltype.Signed, s2, 'x') == 45 + assert llop.getinteriorfield(lltype.Signed, s3, 'x') == 46 + py.test.raises(TypeError, llop.getinteriorfield, lltype.Signed, s3, 'y') # ___________________________________________________________________________ # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync. @@ -61,7 +127,7 @@ def test_llinterp_complete(): for opname, llop in LL_OPERATIONS.items(): - if llop.canfold: + if llop.canrun: continue if opname.startswith('gc_x_'): continue # ignore experimental stuff Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_rtagged.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_rtagged.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/lltypesystem/test/test_rtagged.py Thu Nov 5 22:17:18 2009 @@ -33,20 +33,20 @@ def test_instantiate(): def fn1(n): return C(n) - res = interpret(fn1, [42]) + res = interpret(fn1, [42], taggedpointers=True) value = lltype.cast_ptr_to_int(res) assert value == 42 * 2 + 1 # for now def test_attribute(): def fn1(n): return C(n).smallint - res = interpret(fn1, [42]) + res = interpret(fn1, [42], taggedpointers=True) assert res == 42 -def test_getvalue(): +def test_get_untagged_value(): def fn1(n): - return C(n).getvalue() - res = interpret(fn1, [42]) + return C(n).get_untagged_value() + res = interpret(fn1, [42], taggedpointers=True) assert res == 42 def test_overflowerror(): @@ -65,11 +65,11 @@ else: return 'A', 0 - res = interpret(fn2, [-117]) + res = interpret(fn2, [-117], taggedpointers=True) assert res.item0 == 'C' assert res.item1 == -117 - res = interpret(fn2, [sys.maxint]) + res = interpret(fn2, [sys.maxint], taggedpointers=True) assert res.item0 == 'B' assert res.item1 == sys.maxint @@ -95,13 +95,13 @@ else: return 'A', 0 - res = interpret(fn, [12]) + res = interpret(fn, [12], taggedpointers=True) assert res.item0 == 'C' assert res.item1 == 12 - res = interpret(fn, [-1]) + res = interpret(fn, [-1], taggedpointers=True) assert res.item0 == 'C' assert res.item1 == 111 - res = interpret(fn, [0]) + res = interpret(fn, [0], taggedpointers=True) assert res.item0 == 'B' assert res.item1 == 939393 @@ -118,9 +118,9 @@ x = C(n) return g(x) - res = interpret(fn, [-1]) + res = interpret(fn, [-1], taggedpointers=True) assert res == sys.maxint - res = interpret(fn, [56]) + res = interpret(fn, [56], taggedpointers=True) assert res == 56 def test_type(): @@ -131,11 +131,32 @@ x = C(n) return type(x) is B, type(x) is C - res = interpret(fn, [-212]) + res = interpret(fn, [-212], taggedpointers=True) assert res.item0 and not res.item1 - res = interpret(fn, [9874]) + res = interpret(fn, [9874], taggedpointers=True) assert res.item1 and not res.item0 +def test_type_of_None(): + # use extra function to prevent flow graph cleverness + def g(n): + if n < 0: + x = B(n) + elif n == 0: + x = None + else: + x = C(n) + return x + def fn(n): + x= g(n) + return type(x) is B, type(x) is C + + res = interpret(fn, [-212], taggedpointers=True) + assert res.item0 and not res.item1 + res = interpret(fn, [9874], taggedpointers=True) + assert res.item1 and not res.item0 + res = interpret(fn, [0], taggedpointers=True) + assert not res.item1 and not res.item0 + def test_str(): def fn(n): if n > 0: @@ -143,9 +164,9 @@ else: x = C(n) return str(x) - res = interpret(fn, [-832]) + res = interpret(fn, [-832], taggedpointers=True) assert ''.join(res.chars) == '' - res = interpret(fn, [1]) + res = interpret(fn, [1], taggedpointers=True) assert ''.join(res.chars).startswith('' - res = interpret(fn, [1]) + res = interpret(fn, [1], taggedpointers=True) assert ''.join(res.chars).startswith(' 0: - callback(item, arg) + if self.points_to_valid_gc_object(item): + callback(item, arg) item += llmemory.gcarrayofptr_singleitemoffset length -= 1 return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): - callback(obj + offsets[i], arg) + item = obj + offsets[i] + if self.points_to_valid_gc_object(item): + callback(item, arg) i += 1 if self.has_gcptr_in_varsize(typeid): item = obj + self.varsize_offset_to_variable_part(typeid) @@ -167,12 +175,22 @@ while length > 0: j = 0 while j < len(offsets): - callback(item + offsets[j], arg) + itemobj = item + offsets[j] + if self.points_to_valid_gc_object(itemobj): + callback(itemobj, arg) j += 1 item += itemlength length -= 1 trace._annspecialcase_ = 'specialize:arg(2)' + def points_to_valid_gc_object(self, addr): + return self.is_valid_gc_object(addr.address[0]) + + def is_valid_gc_object(self, addr): + return (addr != NULL and + (not self.config.taggedpointers or + llmemory.cast_adr_to_int(addr) & 1 == 0)) + def debug_check_consistency(self): """To use after a collection. If self.DEBUG is set, this enumerates all roots and traces all objects to check if we didn't @@ -207,8 +225,8 @@ self._debug_record(obj) def _debug_callback2(self, pointer, ignored): obj = pointer.address[0] - if obj: - self._debug_record(obj) + ll_assert(bool(obj), "NULL address from self.trace()") + self._debug_record(obj) def debug_check_object(self, obj): pass @@ -243,11 +261,17 @@ # Default implementation for id(), assuming that "external" objects # never move. Overriden in the HybridGC. obj = llmemory.cast_ptr_to_adr(ptr) - if self._is_external(obj): - result = obj - else: - result = self._compute_id(obj) - return llmemory.cast_adr_to_int(result) + + # is it a tagged pointer? or an external object? + if not self.is_valid_gc_object(obj) or self._is_external(obj): + return llmemory.cast_adr_to_int(obj) + + # tagged pointers have ids of the form 2n + 1 + # external objects have ids of the form 4n (due to word alignment) + # self._compute_id returns addresses of the form 2n + 1 + # if we multiply by 2, we get ids of the form 4n + 2, thus we get no + # clashes + return llmemory.cast_adr_to_int(self._compute_id(obj)) * 2 def _next_id(self): # return an id not currently in use (as an address instead of an int) Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/generation.py Thu Nov 5 22:17:18 2009 @@ -7,6 +7,7 @@ from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE from pypy.rlib.objectmodel import free_non_gc_object from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop # The following flag is never set on young objects, i.e. the ones living @@ -55,12 +56,8 @@ self.initial_nursery_size = nursery_size self.auto_nursery_size = auto_nursery_size self.min_nursery_size = min_nursery_size - self.old_objects_pointing_to_young = self.AddressStack() - # ^^^ a list of addresses inside the old objects space; it - # may contain static prebuilt objects as well. More precisely, - # it lists exactly the old and static objects whose - # GCFLAG_NO_YOUNG_PTRS bit is not set. - self.young_objects_with_weakrefs = self.AddressStack() + + # define nursery fields self.reset_nursery() self._setup_wb() @@ -74,6 +71,13 @@ self.lb_young_var_basesize = sz def setup(self): + self.old_objects_pointing_to_young = self.AddressStack() + # ^^^ a list of addresses inside the old objects space; it + # may contain static prebuilt objects as well. More precisely, + # it lists exactly the old and static objects whose + # GCFLAG_NO_YOUNG_PTRS bit is not set. + self.young_objects_with_weakrefs = self.AddressStack() + self.last_generation_root_objects = self.AddressStack() self.young_objects_with_id = self.AddressDict() SemiSpaceGC.setup(self) @@ -82,17 +86,23 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size( - self.config.gcconfig.debugprint) + newsize = estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) + self.reset_nursery() + + def _teardown(self): + self.collect() # should restore last gen objects flags + SemiSpaceGC._teardown(self) + def reset_nursery(self): self.nursery = NULL self.nursery_top = NULL self.nursery_free = NULL def set_nursery_size(self, newsize): + debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: @@ -107,13 +117,12 @@ while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) - llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", - self.largest_young_fixedsize) - llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", - self.largest_young_var_basesize) - llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) + debug_print("nursery_size =", newsize) + debug_print("largest_young_fixedsize =", + self.largest_young_fixedsize) + debug_print("largest_young_var_basesize =", + self.largest_young_var_basesize) + debug_print("nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) @@ -122,6 +131,7 @@ # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() + debug_stop("gc-set-nursery-size") @staticmethod def get_young_fixedsize(nursery_size): @@ -132,6 +142,8 @@ return nursery_size // 4 - 1 def is_in_nursery(self, addr): + ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0, + "odd-valued (i.e. tagged) pointer unexpected here") return self.nursery <= addr < self.nursery_top def malloc_fixedsize_clear(self, typeid, size, can_collect, @@ -238,11 +250,7 @@ self.weakrefs_grow_older() self.ids_grow_older() self.reset_nursery() - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) - if self.config.gcconfig.debugprint and not size_changing: - llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size) def make_a_copy(self, obj, objsize): tid = self.header(obj).tid @@ -305,10 +313,9 @@ def _trace_external_obj(self, pointer, obj): addr = pointer.address[0] - if addr != NULL: - newaddr = self.copy(addr) - pointer.address[0] = newaddr - self.write_into_last_generation_obj(obj, newaddr) + newaddr = self.copy(addr) + pointer.address[0] = newaddr + self.write_into_last_generation_obj(obj, newaddr) # ____________________________________________________________ # Implementation of nursery-only collections @@ -320,10 +327,9 @@ ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "--- minor collect ---") - llop.debug_print(lltype.Void, "nursery:", - self.nursery, "to", self.nursery_top) + debug_start("gc-minor") + debug_print("--- minor collect ---") + debug_print("nursery:", self.nursery, "to", self.nursery_top) # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() @@ -337,10 +343,9 @@ self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "survived (fraction of the size):", - float(scan - beginning) / self.nursery_size) + debug_print("survived (fraction of the size):", + float(scan - beginning) / self.nursery_size) + debug_stop("gc-minor") #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either @@ -366,8 +371,7 @@ hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count) + debug_print("collect_oldrefs_to_nursery", count) def collect_roots_in_nursery(self): # we don't need to trace prebuilt GcStructs during a minor collect: @@ -446,10 +450,17 @@ # addr_struct, "<-", addr) ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") + # if we have tagged pointers around, we first need to check whether + # we have valid pointer here, otherwise we can do it after the + # is_in_nursery check + if (self.config.taggedpointers and + not self.is_valid_gc_object(addr)): + return if self.is_in_nursery(addr): self.old_objects_pointing_to_young.append(addr_struct) self.header(addr_struct).tid &= ~GCFLAG_NO_YOUNG_PTRS - elif addr == NULL: + elif (not self.config.taggedpointers and + not self.is_valid_gc_object(addr)): return self.write_into_last_generation_obj(addr_struct, addr) remember_young_pointer._dont_inline_ = True @@ -571,19 +582,18 @@ pass return -1 -def best_nursery_size_for_L2cache(L2cache, debugprint=False): - if debugprint: - llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) +def best_nursery_size_for_L2cache(L2cache): # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2 if sys.platform == 'linux2': - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = sys.maxint try: fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) @@ -631,10 +641,13 @@ if number < L2cache: L2cache = number + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") + if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1 @@ -659,10 +672,11 @@ rffi.INT, sandboxsafe=True) - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ + debug_start("gc-L2cache") L2cache = 0 l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') try: @@ -686,14 +700,16 @@ lltype.free(len_p, flavor='raw') finally: lltype.free(l2cache_p, flavor='raw') + debug_print("L2cache =", L2cache) + debug_stop("gc-L2cache") if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache, debugprint) + return best_nursery_size_for_L2cache(L2cache) else: - # Print a warning even in non-debug builds + # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1 else: - def estimate_best_nursery_size(debugprint=False): + def estimate_best_nursery_size(): return -1 # XXX implement me for other platforms Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/hybrid.py Thu Nov 5 22:17:18 2009 @@ -8,7 +8,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.lltypesystem import rffi @@ -114,13 +115,13 @@ self.nonlarge_gcptrs_max = large_object_gcptrs - 1 assert self.nonlarge_gcptrs_max <= self.lb_young_var_basesize assert self.nonlarge_max <= self.nonlarge_gcptrs_max - self.large_objects_collect_trigger = self.space_size - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + + def setup(self): + self.large_objects_collect_trigger = self.param_space_size + self._initial_trigger = self.large_objects_collect_trigger self.rawmalloced_objects_to_trace = self.AddressStack() self.count_semispaceonly_collects = 0 - def setup(self): self.gen2_rawmalloced_objects = self.AddressStack() self.gen3_rawmalloced_objects = self.AddressStack() self.gen2_resizable_objects = self.AddressStack() @@ -270,12 +271,12 @@ def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, "allocated", - self._initial_trigger - - self.large_objects_collect_trigger, - "bytes, triggering full collection") + debug_start("gc-rawsize-collect") + debug_print("allocated", (self._initial_trigger - + self.large_objects_collect_trigger), + "bytes, triggering full collection") self.semispace_collect() + debug_stop("gc-rawsize-collect") def malloc_varsize_marknsweep(self, totalsize, resizable=False): # In order to free the large objects from time to time, we @@ -340,9 +341,8 @@ None) ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at start") - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count = 0 - self._nonmoving_copy_size = 0 + self._nonmoving_copy_count = 0 + self._nonmoving_copy_size = 0 def _set_gcflag_unvisited(self, obj, ignored): ll_assert(not (self.header(obj).tid & GCFLAG_UNVISITED), @@ -418,9 +418,8 @@ newaddr = self.allocate_external_object(totalsize_incl_hash) if not newaddr: return llmemory.NULL # can't raise MemoryError during a collect() - if self.config.gcconfig.debugprint: - self._nonmoving_copy_count += 1 - self._nonmoving_copy_size += raw_malloc_usage(totalsize) + self._nonmoving_copy_count += 1 + self._nonmoving_copy_size += raw_malloc_usage(totalsize) llmemory.raw_memcopy(obj - self.size_gc_header(), newaddr, totalsize) # check if we need to write a hash value at the end of the new obj @@ -463,11 +462,9 @@ def finished_full_collect(self): ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at end") - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hybrid] made nonmoving: ", - self._nonmoving_copy_size, "bytes in", - self._nonmoving_copy_count, "objs") + debug_print("| [hybrid] made nonmoving: ", + self._nonmoving_copy_size, "bytes in", + self._nonmoving_copy_count, "objs") # sweep the nonmarked rawmalloced objects if self.is_collecting_gen3(): self.sweep_rawmalloced_objects(generation=3) @@ -478,8 +475,7 @@ self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 - if self.config.gcconfig.debugprint: - self._initial_trigger = self.large_objects_collect_trigger + self._initial_trigger = self.large_objects_collect_trigger def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation @@ -512,17 +508,18 @@ surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 + debug = have_debug_prints() while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: - if self.config.gcconfig.debugprint: + if debug: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: - if self.config.gcconfig.debugprint: + if debug: alive_count+=1 alive_size+=raw_malloc_usage(self.get_size_incl_hash(obj)) if generation == 3: @@ -553,20 +550,22 @@ self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving now alive: ", - alive_size, "bytes in", - alive_count, "objs") - llop.debug_print(lltype.Void, - "| [hyb] gen", generation, - "nonmoving freed: ", - dead_size, "bytes in", - dead_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving now alive: ", + alive_size, "bytes in", + alive_count, "objs") + debug_print("| [hyb] gen", generation, + "nonmoving freed: ", + dead_size, "bytes in", + dead_count, "objs") def id(self, ptr): obj = llmemory.cast_ptr_to_adr(ptr) + + # is it a tagged pointer? + if not self.is_valid_gc_object(obj): + return llmemory.cast_adr_to_int(obj) + if self._is_external(obj): # a prebuilt or rawmalloced object if self.is_last_generation(obj): @@ -581,7 +580,7 @@ result = obj else: result = self._compute_id(obj) # common case - return llmemory.cast_adr_to_int(result) + return llmemory.cast_adr_to_int(result) * 2 # see comment in base.py # XXX a possible optimization would be to use three dicts, one # for each generation, instead of mixing gen2 and gen3 objects. Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/markcompact.py Thu Nov 5 22:17:18 2009 @@ -91,11 +91,13 @@ def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096): import py; py.test.skip("Disabled for now, sorry") + self.param_space_size = space_size MovingGCBase.__init__(self, config, chunk_size) - self.space_size = space_size - self.next_collect_after = space_size/2 # whatever... def setup(self): + self.space_size = self.param_space_size + self.next_collect_after = self.param_space_size/2 # whatever... + if self.config.gcconfig.debugprint: self.program_start_time = time.time() self.space = llarena.arena_malloc(self.space_size, True) @@ -370,11 +372,10 @@ def _mark_obj(self, pointer, ignored): obj = pointer.address[0] - if obj != NULL: - if self.marked(obj): - return - self.mark(obj) - self.to_see.append(obj) + if self.marked(obj): + return + self.mark(obj) + self.to_see.append(obj) def _mark_root_recursively(self, root): self.mark(root.address[0]) @@ -437,14 +438,17 @@ length = (obj + llmemory.gcarrayofptr_lengthoffset).signed[0] item = obj + llmemory.gcarrayofptr_itemsoffset while length > 0: - callback(item, arg) + if self.points_to_valid_gc_object(item): + callback(item, arg) item += llmemory.gcarrayofptr_singleitemoffset length -= 1 return offsets = self.offsets_to_gc_pointers(typeid) i = 0 while i < len(offsets): - callback(obj + offsets[i], arg) + item = obj + offsets[i] + if self.points_to_valid_gc_object(item): + callback(item, arg) i += 1 if self.has_gcptr_in_varsize(typeid): item = obj + self.varsize_offset_to_variable_part(typeid) @@ -454,7 +458,9 @@ while length > 0: j = 0 while j < len(offsets): - callback(item + offsets[j], arg) + itemobj = item + offsets[j] + if self.points_to_valid_gc_object(itemobj): + callback(itemobj, arg) j += 1 item += itemlength length -= 1 Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/marksweep.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/marksweep.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/marksweep.py Thu Nov 5 22:17:18 2009 @@ -8,6 +8,7 @@ from pypy.rlib.objectmodel import free_non_gc_object from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.gc.base import GCBase @@ -48,10 +49,14 @@ TRANSLATION_PARAMS = {'start_heap_size': 8*1024*1024} # XXX adjust def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, start_heap_size=4096): + self.param_start_heap_size = start_heap_size GCBase.__init__(self, config, chunk_size) + + def setup(self): + GCBase.setup(self) self.heap_usage = 0 # at the end of the latest collection self.bytes_malloced = 0 # since the latest collection - self.bytes_malloced_threshold = start_heap_size + self.bytes_malloced_threshold = self.param_start_heap_size self.total_collection_time = 0.0 self.malloced_objects = lltype.nullptr(self.HDR) self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) @@ -238,8 +243,7 @@ # call __del__, move the object to the list of object-without-del import time from pypy.rpython.lltypesystem.lloperation import llop - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, 'collecting...') + debug_start("gc-collect") start_time = time.time() self.collect_in_progress = True size_gc_header = self.gcheaderbuilder.size_gc_header @@ -402,31 +406,22 @@ 256 * 1024 * 1024) self.total_collection_time += collect_time self.prev_collect_end_time = end_time - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void, - " malloced since previous collection:", - old_malloced, "bytes") - llop.debug_print(lltype.Void, - " heap usage at start of collection: ", - self.heap_usage + old_malloced, "bytes") - llop.debug_print(lltype.Void, - " freed: ", - freed_size, "bytes") - llop.debug_print(lltype.Void, - " new heap usage: ", - curr_heap_size, "bytes") - llop.debug_print(lltype.Void, - " total time spent collecting: ", - self.total_collection_time, "seconds") - llop.debug_print(lltype.Void, - " collecting time: ", - collect_time) - llop.debug_print(lltype.Void, - " computing time: ", - collect_time) - llop.debug_print(lltype.Void, - " new threshold: ", - self.bytes_malloced_threshold) + debug_print(" malloced since previous collection:", + old_malloced, "bytes") + debug_print(" heap usage at start of collection: ", + self.heap_usage + old_malloced, "bytes") + debug_print(" freed: ", + freed_size, "bytes") + debug_print(" new heap usage: ", + curr_heap_size, "bytes") + debug_print(" total time spent collecting: ", + self.total_collection_time, "seconds") + debug_print(" collecting time: ", + collect_time) + debug_print(" computing time: ", + collect_time) + debug_print(" new threshold: ", + self.bytes_malloced_threshold) ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size @@ -459,8 +454,9 @@ #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) finalizer(obj) if not self.collect_in_progress: # another collection was caused? - llop.debug_print(lltype.Void, "outer collect interrupted " - "by recursive collect") + debug_print("outer collect interrupted " + "by recursive collect") + debug_stop("gc-collect") return if not last: if self.malloced_objects_with_finalizer == next: @@ -476,6 +472,7 @@ last.next = lltype.nullptr(self.HDR) hdr = next self.collect_in_progress = False + debug_stop("gc-collect") def _mark_root(self, root): # 'root' is the address of the GCPTR gcobjectaddr = root.address[0] @@ -504,8 +501,7 @@ def _add_reachable(pointer, objects): obj = pointer.address[0] - if obj: - objects.append(obj) + objects.append(obj) _add_reachable = staticmethod(_add_reachable) def statistics(self, index): Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/semispace.py Thu Nov 5 22:17:18 2009 @@ -6,12 +6,13 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rlib.objectmodel import free_non_gc_object -from pypy.rlib.debug import ll_assert +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.memory.gc.base import MovingGCBase -import sys, os, time +import sys, os first_gcflag = 1 << 16 GCFLAG_FORWARDED = first_gcflag @@ -31,8 +32,6 @@ inline_simple_malloc_varsize = True malloc_zero_filled = True first_unused_gcflag = first_gcflag << 5 - total_collection_time = 0.0 - total_collection_count = 0 HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? typeid_is_in_field = 'tid' @@ -51,14 +50,19 @@ def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, space_size=4096, max_space_size=sys.maxint//2+1): + self.param_space_size = space_size + self.param_max_space_size = max_space_size MovingGCBase.__init__(self, config, chunk_size) - self.space_size = space_size - self.max_space_size = max_space_size - self.red_zone = 0 def setup(self): - if self.config.gcconfig.debugprint: - self.program_start_time = time.time() + #self.total_collection_time = 0.0 + self.total_collection_count = 0 + + self.space_size = self.param_space_size + self.max_space_size = self.param_max_space_size + self.red_zone = 0 + + #self.program_start_time = time.time() self.tospace = llarena.arena_malloc(self.space_size, True) ll_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size @@ -69,6 +73,11 @@ self.objects_with_finalizers = self.AddressDeque() self.objects_with_weakrefs = self.AddressStack() + def _teardown(self): + debug_print("Teardown") + llarena.arena_free(self.fromspace) + llarena.arena_free(self.tospace) + # This class only defines the malloc_{fixed,var}size_clear() methods # because the spaces are filled with zeroes in advance. @@ -204,18 +213,13 @@ # (this is also a hook for the HybridGC) def semispace_collect(self, size_changing=False): - if self.config.gcconfig.debugprint: - llop.debug_print(lltype.Void) - llop.debug_print(lltype.Void, - ".----------- Full collection ------------------") - start_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used before collection: ", - start_usage, "bytes") - start_time = time.time() - else: - start_time = 0 # Help the flow space - start_usage = 0 # Help the flow space + debug_start("gc-collect") + debug_print() + debug_print(".----------- Full collection ------------------") + start_usage = self.free - self.tospace + debug_print("| used before collection: ", + start_usage, "bytes") + #start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space @@ -245,41 +249,33 @@ self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) - if self.config.gcconfig.debugprint: - end_time = time.time() - elapsed_time = end_time - start_time - self.total_collection_time += elapsed_time + if have_debug_prints(): + #end_time = time.time() + #elapsed_time = end_time - start_time + #self.total_collection_time += elapsed_time self.total_collection_count += 1 - total_program_time = end_time - self.program_start_time + #total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace - llop.debug_print(lltype.Void, - "| used after collection: ", - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| freed: ", - start_usage - end_usage, "bytes") - llop.debug_print(lltype.Void, - "| size of each semispace: ", - self.space_size, "bytes") - llop.debug_print(lltype.Void, - "| fraction of semispace now used: ", - end_usage * 100.0 / self.space_size, "%") - ct = self.total_collection_time + debug_print("| used after collection: ", + end_usage, "bytes") + debug_print("| freed: ", + start_usage - end_usage, "bytes") + debug_print("| size of each semispace: ", + self.space_size, "bytes") + debug_print("| fraction of semispace now used: ", + end_usage * 100.0 / self.space_size, "%") + #ct = self.total_collection_time cc = self.total_collection_count - llop.debug_print(lltype.Void, - "| number of semispace_collects: ", - cc) - llop.debug_print(lltype.Void, - "| i.e.: ", - cc / total_program_time, "per second") - llop.debug_print(lltype.Void, - "| total time in semispace_collect: ", - ct, "seconds") - llop.debug_print(lltype.Void, - "| i.e.: ", - ct * 100.0 / total_program_time, "%") - llop.debug_print(lltype.Void, - "`----------------------------------------------") + debug_print("| number of semispace_collects: ", + cc) + #debug_print("| i.e.: ", + # cc / total_program_time, "per second") + #debug_print("| total time in semispace_collect: ", + # ct, "seconds") + #debug_print("| i.e.: ", + # ct * 100.0 / total_program_time, "%") + debug_print("`----------------------------------------------") + debug_stop("gc-collect") def starting_full_collect(self): pass # hook for the HybridGC @@ -370,8 +366,7 @@ self.trace(obj, self._trace_copy, None) def _trace_copy(self, pointer, ignored): - if pointer.address[0] != NULL: - pointer.address[0] = self.copy(pointer.address[0]) + pointer.address[0] = self.copy(pointer.address[0]) def surviving(self, obj): # To use during a collection. Check if the object is currently @@ -494,8 +489,7 @@ return scan def _append_if_nonnull(pointer, stack): - if pointer.address[0] != NULL: - stack.append(pointer.address[0]) + stack.append(pointer.address[0]) _append_if_nonnull = staticmethod(_append_if_nonnull) def _finalization_state(self, obj): Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gc/test/test_direct.py Thu Nov 5 22:17:18 2009 @@ -68,7 +68,7 @@ self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) self.gc.set_root_walker(self.rootwalker) - self.layoutbuilder = TypeLayoutBuilder(self.GCClass, {}) + self.layoutbuilder = TypeLayoutBuilder(self.GCClass) self.get_type_id = self.layoutbuilder.get_type_id self.layoutbuilder.initialize_gc_query_function(self.gc) self.gc.setup() Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/asmgcroot.py Thu Nov 5 22:17:18 2009 @@ -255,7 +255,7 @@ if location == 0: break addr = self.getlocation(callee, location) - if addr.address[0] != llmemory.NULL: + if gc.points_to_valid_gc_object(addr): collect_stack_root(gc, addr) # # track where the caller_frame saved the registers from its own Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/framework.py Thu Nov 5 22:17:18 2009 @@ -130,12 +130,7 @@ if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] else: - if translator.config.translation.gcconfig.removetypeptr: - lltype2vtable = translator.rtyper.lltype2vtable - else: - lltype2vtable = {} - self.layoutbuilder = TransformerLayoutBuilder(GCClass, - lltype2vtable) + self.layoutbuilder = TransformerLayoutBuilder(translator, GCClass) self.layoutbuilder.transformer = self self.get_type_id = self.layoutbuilder.get_type_id @@ -168,6 +163,10 @@ root_walker.setup_root_walker() gcdata.gc.setup() + def frameworkgc__teardown(): + # run-time teardown code for tests! + gcdata.gc._teardown() + bk = self.translator.annotator.bookkeeper r_typeid16 = rffi.platform.numbertype_to_rclass[TYPE_ID] s_typeid16 = annmodel.SomeInteger(knowntype=r_typeid16) @@ -198,6 +197,10 @@ self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [], annmodel.s_None) + # for tests + self.frameworkgc__teardown_ptr = getfn(frameworkgc__teardown, [], + annmodel.s_None) + if root_walker.need_root_stack: self.incr_stack_ptr = getfn(root_walker.incr_stack, [annmodel.SomeInteger()], @@ -500,11 +503,16 @@ self.write_typeid_list() return newgcdependencies - def get_final_dependencies(self): - # returns an iterator enumerating the type_info_group's members, - # to make sure that they are all followed (only a part of them - # might have been followed by a previous enum_dependencies()). - return iter(self.layoutbuilder.type_info_group.members) + def get_finish_tables(self): + # We must first make sure that the type_info_group's members + # are all followed. Do it repeatedly while new members show up. + # Once it is really done, do finish_tables(). + seen = 0 + while seen < len(self.layoutbuilder.type_info_group.members): + curtotal = len(self.layoutbuilder.type_info_group.members) + yield self.layoutbuilder.type_info_group.members[seen:curtotal] + seen = curtotal + yield self.finish_tables() def write_typeid_list(self): """write out the list of type ids together with some info""" @@ -813,6 +821,9 @@ if hasattr(self.root_walker, 'thread_die_ptr'): hop.genop("direct_call", [self.root_walker.thread_die_ptr]) + def gct_gc_get_type_info_group(self, hop): + return hop.cast_result(self.c_type_info_group) + def gct_malloc_nonmovable_varsize(self, hop): TYPE = hop.spaceop.result.concretetype if self.gcdata.gc.can_malloc_nonmovable(): @@ -880,7 +891,7 @@ def gct_getfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_getfield_typeptr(hop) else: GCTransformer.gct_getfield(self, hop) @@ -888,7 +899,7 @@ def gct_setfield(self, hop): if (hop.spaceop.args[1].value == 'typeptr' and hop.spaceop.args[0].concretetype.TO._hints.get('typeptr') and - self.translator.config.translation.gcconfig.removetypeptr): + self.translator.config.translation.gcremovetypeptr): self.transform_setfield_typeptr(hop) else: GCTransformer.gct_setfield(self, hop) @@ -956,6 +967,16 @@ class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): + def __init__(self, translator, GCClass=None): + if GCClass is None: + from pypy.rpython.memory.gc.base import choose_gc_from_config + GCClass, _ = choose_gc_from_config(translator.config) + if translator.config.translation.gcremovetypeptr: + lltype2vtable = translator.rtyper.lltype2vtable + else: + lltype2vtable = None + super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable) + def has_finalizer(self, TYPE): rtti = get_rtti(TYPE) return rtti is not None and hasattr(rtti._obj, 'destructor_funcptr') @@ -982,18 +1003,6 @@ return fptr -class JITTransformerLayoutBuilder(TransformerLayoutBuilder): - # for the JIT: currently does not support removetypeptr - def __init__(self, config): - from pypy.rpython.memory.gc.base import choose_gc_from_config - try: - assert not config.translation.gcconfig.removetypeptr - except AttributeError: # for some tests - pass - GCClass, _ = choose_gc_from_config(config) - TransformerLayoutBuilder.__init__(self, GCClass, {}) - - def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): if previous_steps is None: previous_steps = [] @@ -1040,7 +1049,7 @@ end = gcdata.static_root_nongcend while addr != end: result = addr.address[0] - if result.address[0] != llmemory.NULL: + if gc.points_to_valid_gc_object(result): collect_static_in_prebuilt_nongc(gc, result) addr += sizeofaddr if collect_static_in_prebuilt_gc: @@ -1048,7 +1057,7 @@ end = gcdata.static_root_end while addr != end: result = addr.address[0] - if result.address[0] != llmemory.NULL: + if gc.points_to_valid_gc_object(result): collect_static_in_prebuilt_gc(gc, result) addr += sizeofaddr if collect_stack_root: @@ -1110,7 +1119,7 @@ addr = gcdata.root_stack_base end = gcdata.root_stack_top while addr != end: - if addr.address[0] != llmemory.NULL: + if gc.points_to_valid_gc_object(addr): collect_stack_root(gc, addr) addr += sizeofaddr if self.collect_stacks_from_other_threads is not None: @@ -1219,7 +1228,7 @@ end = stacktop - sizeofaddr addr = end.address[0] while addr != end: - if addr.address[0] != llmemory.NULL: + if gc.points_to_valid_gc_object(addr): callback(gc, addr) addr += sizeofaddr Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctransform/transform.py Thu Nov 5 22:17:18 2009 @@ -312,12 +312,12 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies - def get_final_dependencies(self): - pass - def finish_tables(self): pass + def get_finish_tables(self): + return self.finish_tables + def finish(self, backendopt=True): self.finish_helpers(backendopt=backendopt) self.finish_tables() Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gctypelayout.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup +from pypy.rpython.lltypesystem import rclass from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import ll_assert @@ -169,7 +170,7 @@ size_of_fixed_type_info = llmemory.sizeof(GCData.TYPE_INFO) - def __init__(self, GCClass, lltype2vtable): + def __init__(self, GCClass, lltype2vtable=None): self.GCClass = GCClass self.lltype2vtable = lltype2vtable self.make_type_info_group() @@ -218,17 +219,28 @@ # store it type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id - # store the vtable of the type (if any) immediately thereafter - # (note that if gcconfig.removetypeptr is False, lltype2vtable - # is empty) - vtable = self.lltype2vtable.get(TYPE, None) - if vtable is not None: - # check that if we have a vtable, we are not varsize - assert lltype.typeOf(fullinfo) == GCData.TYPE_INFO_PTR - vtable = lltype.normalizeptr(vtable) - self.type_info_group.add_member(vtable) + self.add_vtable_after_typeinfo(TYPE) return type_id + def add_vtable_after_typeinfo(self, TYPE): + # if gcremovetypeptr is False, then lltype2vtable is None and it + # means that we don't have to store the vtables in type_info_group. + if self.lltype2vtable is None: + return + # does the type have a vtable? + vtable = self.lltype2vtable.get(TYPE, None) + if vtable is not None: + # yes. check that in this case, we are not varsize + assert not TYPE._is_varsize() + vtable = lltype.normalizeptr(vtable) + self.type_info_group.add_member(vtable) + else: + # no vtable from lltype2vtable -- double-check to be sure + # that it's not a subclass of OBJECT. + while isinstance(TYPE, lltype.GcStruct): + assert TYPE is not rclass.OBJECT + _, TYPE = TYPE._first_struct() + def get_info(self, type_id): return llop.get_group_member(GCData.TYPE_INFO_PTR, self.type_info_group._as_ptr(), @@ -350,11 +362,16 @@ def gc_pointers_inside(v, adr, mutable_only=False): t = lltype.typeOf(v) if isinstance(t, lltype.Struct): - if mutable_only and t._hints.get('immutable'): - return + skip = () + if mutable_only: + if t._hints.get('immutable'): + return + if 'immutable_fields' in t._hints: + skip = t._hints['immutable_fields'].fields for n, t2 in t._flds.iteritems(): if isinstance(t2, lltype.Ptr) and t2.TO._gckind == 'gc': - yield adr + llmemory.offsetof(t, n) + if n not in skip: + yield adr + llmemory.offsetof(t, n) elif isinstance(t2, (lltype.Array, lltype.Struct)): for a in gc_pointers_inside(getattr(v, n), adr + llmemory.offsetof(t, n), Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gcwrapper.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gcwrapper.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/gcwrapper.py Thu Nov 5 22:17:18 2009 @@ -144,15 +144,15 @@ gc = gcheap.gc if collect_static_in_prebuilt_gc: for addrofaddr in gcheap.constantroots: - if addrofaddr.address[0]: + if self.gcheap.gc.points_to_valid_gc_object(addrofaddr): collect_static_in_prebuilt_gc(gc, addrofaddr) if collect_static_in_prebuilt_nongc: for addrofaddr in gcheap.constantrootsnongc: - if addrofaddr.address[0]: + if self.gcheap.gc.points_to_valid_gc_object(addrofaddr): collect_static_in_prebuilt_nongc(gc, addrofaddr) if collect_stack_root: for addrofaddr in gcheap.llinterp.find_roots(): - if addrofaddr.address[0]: + if self.gcheap.gc.points_to_valid_gc_object(addrofaddr): collect_stack_root(gc, addrofaddr) def _walk_prebuilt_gc(self, collect): # debugging only! not RPython @@ -208,7 +208,8 @@ def reccollect(constants, llvalue): if (isinstance(llvalue, lltype._abstract_ptr) - and llvalue._obj is not None and llvalue._obj not in constants): + and llvalue._obj is not None and llvalue._obj not in constants + and not isinstance(llvalue._obj, int)): TYPE = llvalue._T constants[llvalue._obj] = True if isinstance(TYPE, lltype.Struct): Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gc.py Thu Nov 5 22:17:18 2009 @@ -10,6 +10,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here +from pypy.rlib import rgc def stdout_ignore_ll_functions(msg): @@ -436,7 +437,6 @@ def test_can_move(self): TP = lltype.GcArray(lltype.Float) def func(): - from pypy.rlib import rgc return rgc.can_move(lltype.malloc(TP, 1)) assert self.interpret(func, []) == self.GC_CAN_MOVE @@ -444,7 +444,6 @@ def test_malloc_nonmovable(self): TP = lltype.GcArray(lltype.Char) def func(): - from pypy.rlib import rgc a = rgc.malloc_nonmovable(TP, 3) if a: assert not rgc.can_move(a) @@ -458,7 +457,6 @@ TP = lltype.GcStruct('T', ('s', lltype.Ptr(S))) def func(): try: - from pypy.rlib import rgc a = rgc.malloc_nonmovable(TP) rgc.collect() if a: @@ -473,7 +471,6 @@ def test_resizable_buffer(self): from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import hlstr - from pypy.rlib import rgc def f(): ptr = rgc.resizable_buffer_of_shape(STR, 1) @@ -484,6 +481,97 @@ assert self.interpret(f, []) == 2 + def test_tagged_simple(self): + from pypy.rlib.objectmodel import UnboxedValue + + class Unrelated(object): + pass + + u = Unrelated() + u.x = UnboxedObject(47) + def fn(n): + rgc.collect() # check that a prebuilt tagged pointer doesn't explode + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + u.x = x # invoke write barrier + rgc.collect() + return x.meth(100) + res = self.interpret(fn, [1000], taggedpointers=True) + assert res == 1102 + res = self.interpret(fn, [-1000], taggedpointers=True) + assert res == -897 + + def test_tagged_prebuilt(self): + + class F: + pass + + f = F() + f.l = [UnboxedObject(10)] + def fn(n): + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + f.l.append(x) + rgc.collect() + return f.l[-1].meth(100) + res = self.interpret(fn, [1000], taggedpointers=True) + assert res == 1102 + res = self.interpret(fn, [-1000], taggedpointers=True) + assert res == -897 + + def test_tagged_id(self): + from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id + + class Unrelated(object): + pass + + u = Unrelated() + u.x = UnboxedObject(0) + def fn(n): + id_prebuilt1 = compute_unique_id(u.x) + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + id_x1 = compute_unique_id(x) + rgc.collect() # check that a prebuilt tagged pointer doesn't explode + id_prebuilt2 = compute_unique_id(u.x) + id_x2 = compute_unique_id(x) + print u.x, id_prebuilt1, id_prebuilt2 + print x, id_x1, id_x2 + return ((id_x1 == id_x2) * 1 + + (id_prebuilt1 == id_prebuilt2) * 10 + + (id_x1 != id_prebuilt1) * 100) + res = self.interpret(fn, [1000], taggedpointers=True) + assert res == 111 + res = self.interpret(fn, [-1000], taggedpointers=True) + assert res == 111 + + +from pypy.rlib.objectmodel import UnboxedValue + +class TaggedBase(object): + __slots__ = () + def meth(self, x): + raise NotImplementedError + +class BoxedObject(TaggedBase): + attrvalue = 66 + def __init__(self, normalint): + self.normalint = normalint + def meth(self, x): + return self.normalint + x + 2 + +class UnboxedObject(TaggedBase, UnboxedValue): + __slots__ = 'smallint' + def meth(self, x): + return self.smallint + x + 3 + + class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gctypelayout.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gctypelayout.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_gctypelayout.py Thu Nov 5 22:17:18 2009 @@ -1,7 +1,8 @@ import py from pypy.rpython.memory.gctypelayout import TypeLayoutBuilder, GCData from pypy.rpython.memory.gctypelayout import offsets_to_gc_pointers -from pypy.rpython.lltypesystem import lltype, rclass +from pypy.rpython.memory.gctypelayout import gc_pointers_inside +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.test.test_llinterp import get_interpreter from pypy.objspace.flow.model import Constant @@ -36,7 +37,7 @@ for T, c in [(GC_S, 0), (GC_S2, 2), (GC_A, 0), (GC_A2, 0), (GC_S3, 2)]: assert len(offsets_to_gc_pointers(T)) == c -def test_layout_builder(lltype2vtable={}): +def test_layout_builder(lltype2vtable=None): # XXX a very minimal test layoutbuilder = TypeLayoutBuilder(FakeGC, lltype2vtable) for T1, T2 in [(GC_A, GC_S), (GC_A2, GC_S2), (GC_S3, GC_S2)]: @@ -66,7 +67,7 @@ layoutbuilder.size_of_fixed_type_info) def test_constfold(): - layoutbuilder = TypeLayoutBuilder(FakeGC, {}) + layoutbuilder = TypeLayoutBuilder(FakeGC) tid1 = layoutbuilder.get_type_id(GC_A) tid2 = layoutbuilder.get_type_id(GC_S3) class MockGC: @@ -90,3 +91,31 @@ interp, graph = get_interpreter(f, [], backendopt=True) assert interp.eval_graph(graph, []) == 11001 assert graph.startblock.exits[0].args == [Constant(11001, lltype.Signed)] + +def test_gc_pointers_inside(): + from pypy.rpython import rclass + PT = lltype.Ptr(lltype.GcStruct('T')) + S1 = lltype.GcStruct('S', ('x', PT), ('y', PT)) + S2 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable': True}) + accessor = rclass.FieldListAccessor() + S3 = lltype.GcStruct('S', ('x', PT), ('y', PT), + hints={'immutable_fields': accessor}) + accessor.initialize(S3, ['x']) + # + s1 = lltype.malloc(S1) + adr = llmemory.cast_ptr_to_adr(s1) + lst = list(gc_pointers_inside(s1._obj, adr, mutable_only=True)) + expected = [adr + llmemory.offsetof(S1, 'x'), + adr + llmemory.offsetof(S1, 'y')] + assert lst == expected or lst == expected[::-1] + # + s2 = lltype.malloc(S2) + adr = llmemory.cast_ptr_to_adr(s2) + lst = list(gc_pointers_inside(s2._obj, adr, mutable_only=True)) + assert lst == [] + # + s3 = lltype.malloc(S3) + adr = llmemory.cast_ptr_to_adr(s3) + lst = list(gc_pointers_inside(s3._obj, adr, mutable_only=True)) + assert lst == [adr + llmemory.offsetof(S3, 'y')] Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/memory/test/test_transformed_gc.py Thu Nov 5 22:17:18 2009 @@ -1,14 +1,16 @@ import py import sys -import struct +import struct, inspect from pypy.translator.c import gc from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.annotation import policy as annpolicy +from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rpython.memory.gctransform import framework from pypy.rpython.lltypesystem.lloperation import llop, void from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR from pypy.rlib.objectmodel import compute_unique_id, we_are_translated from pypy.rlib.debug import ll_assert +from pypy.rlib import rgc from pypy import conftest from pypy.rlib.rstring import StringBuilder @@ -21,11 +23,13 @@ t = TranslationContext() # XXX XXX XXX mess t.config.translation.gc = gcname - t.config.translation.gcconfig.removetypeptr = True + t.config.translation.gcremovetypeptr = True if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) - t.buildannotator().build_types(func, inputtypes) + ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) + ann.build_types(func, inputtypes) + if specialize: t.buildrtyper().specialize() if backendopt: @@ -35,64 +39,124 @@ t.viewcg() return t +ARGS = lltype.FixedSizeArray(lltype.Signed, 3) + class GCTest(object): gcpolicy = None stacklessgc = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False + taggedpointers = False + + def setup_class(cls): + funcs0 = [] + funcs2 = [] + cleanups = [] + name_to_func = {} + mixlevelstuff = [] + for fullname in dir(cls): + if not fullname.startswith('define'): + continue + definefunc = getattr(cls, fullname) + _, name = fullname.split('_', 1) + func_fixup = definefunc.im_func(cls) + cleanup = None + if isinstance(func_fixup, tuple): + func, cleanup, fixup = func_fixup + mixlevelstuff.append(fixup) + else: + func = func_fixup + func.func_name = "f_%s" % name + if cleanup: + cleanup.func_name = "clean_%s" % name + + nargs = len(inspect.getargspec(func)[0]) + name_to_func[name] = len(funcs0) + if nargs == 2: + funcs2.append(func) + funcs0.append(None) + elif nargs == 0: + funcs0.append(func) + funcs2.append(None) + else: + raise NotImplementedError( + "defined test functions should have 0/2 arguments") + # used to let test cleanup static root pointing to runtime + # allocated stuff + cleanups.append(cleanup) + + def entrypoint(args): + num = args[0] + func = funcs0[num] + if func: + res = func() + else: + func = funcs2[num] + res = func(args[1], args[2]) + cleanup = cleanups[num] + if cleanup: + cleanup() + return res - def runner(self, f, nbargs=0, statistics=False, transformer=False, - mixlevelstuff=None, **extraconfigopts): - if nbargs == 2: - def entrypoint(args): - x = args[0] - y = args[1] - r = f(x, y) - return r - elif nbargs == 0: - def entrypoint(args): - return f() - else: - raise NotImplementedError("pure laziness") - - from pypy.rpython.llinterp import LLInterpreter from pypy.translator.c.genc import CStandaloneBuilder - ARGS = lltype.FixedSizeArray(lltype.Signed, nbargs) s_args = annmodel.SomePtr(lltype.Ptr(ARGS)) - t = rtype(entrypoint, [s_args], gcname=self.gcname, - stacklessgc=self.stacklessgc, - **extraconfigopts) - if mixlevelstuff: - mixlevelstuff(t) + t = rtype(entrypoint, [s_args], gcname=cls.gcname, + stacklessgc=cls.stacklessgc, + taggedpointers=cls.taggedpointers) + + for fixup in mixlevelstuff: + if fixup: + fixup(t) + cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, - gcpolicy=self.gcpolicy) + gcpolicy=cls.gcpolicy) db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph if conftest.option.view: t.viewcg() - llinterp = LLInterpreter(t.rtyper) + cls.name_to_func = name_to_func + cls.entrygraph = entrygraph + cls.rtyper = t.rtyper + cls.db = db + + def runner(self, name, statistics=False, transformer=False): + db = self.db + name_to_func = self.name_to_func + entrygraph = self.entrygraph + from pypy.rpython.llinterp import LLInterpreter + + llinterp = LLInterpreter(self.rtyper) + + gct = db.gctransformer + + if self.__class__.__dict__.get('_used', False): + teardowngraph = gct.frameworkgc__teardown_ptr.value._obj.graph + llinterp.eval_graph(teardowngraph, []) + self.__class__._used = True # FIIIIISH - setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph + setupgraph = gct.frameworkgc_setup_ptr.value._obj.graph + # setup => resets the gc llinterp.eval_graph(setupgraph, []) def run(args): ll_args = lltype.malloc(ARGS, immortal=True) - for i in range(nbargs): - ll_args[i] = args[i] + ll_args[0] = name_to_func[name] + for i in range(len(args)): + ll_args[1+i] = args[i] res = llinterp.eval_graph(entrygraph, [ll_args]) return res if statistics: - statisticsgraph = db.gctransformer.statistics_ptr.value._obj.graph - ll_gc = db.gctransformer.c_const_gc.value + statisticsgraph = gct.statistics_ptr.value._obj.graph + ll_gc = gct.c_const_gc.value def statistics(index): return llinterp.eval_graph(statisticsgraph, [ll_gc, index]) return run, statistics elif transformer: - return run, db.gctransformer + return run, gct else: return run @@ -108,7 +172,7 @@ else: return -1 # xxx - def test_instances(self): + def define_instances(cls): class A(object): pass class B(A): @@ -128,12 +192,15 @@ b.last = first j += 1 return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_instances(self): + run, statistics = self.runner("instances", statistics=True) run([]) heap_size = self.heap_usage(statistics) - def test_llinterp_lists(self): + def define_llinterp_lists(cls): def malloc_a_lot(): i = 0 while i < 10: @@ -144,12 +211,15 @@ j += 1 a.append(j) return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_llinterp_lists(self): + run, statistics = self.runner("llinterp_lists", statistics=True) run([]) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_llinterp_tuples(self): + def define_llinterp_tuples(cls): def malloc_a_lot(): i = 0 while i < 10: @@ -161,41 +231,51 @@ j += 1 b.append((1, j, i)) return 0 - run, statistics = self.runner(malloc_a_lot, statistics=True) + return malloc_a_lot + + def test_llinterp_tuples(self): + run, statistics = self.runner("llinterp_tuples", statistics=True) run([]) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_global_list(self): + def skipdefine_global_list(cls): + gl = [] class Box: def __init__(self): - self.lst = [] + self.lst = gl box = Box() def append_to_list(i, j): box.lst.append([i] * 50) llop.gc__collect(lltype.Void) return box.lst[j][0] - run = self.runner(append_to_list, nbargs=2) + return append_to_list, None, None + + def test_global_list(self): + py.test.skip("doesn't fit in the model, tested elsewhere too") + run = self.runner("global_list") res = run([0, 0]) assert res == 0 for i in range(1, 5): res = run([i, i - 1]) assert res == i - 1 # crashes if constants are not considered roots - def test_string_concatenation(self): - + def define_string_concatenation(cls): def concat(j, dummy): lst = [] for i in range(j): lst.append(str(i)) return len("".join(lst)) - run, statistics = self.runner(concat, nbargs=2, statistics=True) + return concat + + def test_string_concatenation(self): + run, statistics = self.runner("string_concatenation", statistics=True) res = run([100, 0]) - assert res == concat(100, 0) + assert res == len(''.join([str(x) for x in range(100)])) heap_size = self.heap_usage(statistics) assert heap_size < 16000 * INT_SIZE / 4 # xxx - def test_nongc_static_root(self): + def define_nongc_static_root(cls): T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -205,11 +285,16 @@ static.p = t1 llop.gc__collect(lltype.Void) return static.p.x - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = lltype.nullptr(T1) + return f, cleanup, None + + def test_nongc_static_root(self): + run = self.runner("nongc_static_root") res = run([]) assert res == 42 - def test_finalizer(self): + def define_finalizer(cls): class B(object): pass b = B() @@ -230,11 +315,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_finalizer(self): + run = self.runner("finalizer") res = run([5, 42]) #XXX pure lazyness here too assert res == 6 - def test_finalizer_calls_malloc(self): + def define_finalizer_calls_malloc(cls): class B(object): pass b = B() @@ -259,11 +347,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_finalizer_calls_malloc(self): + run = self.runner("finalizer_calls_malloc") res = run([5, 42]) #XXX pure lazyness here too assert res == 12 - def test_finalizer_resurrects(self): + def define_finalizer_resurrects(cls): class B(object): pass b = B() @@ -290,11 +381,14 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) return b.num_deleted * 10 + aid + 100 * (b.a is None) - run = self.runner(f, nbargs=2) + return f + + def test_finalizer_resurrects(self): + run = self.runner("finalizer_resurrects") res = run([5, 42]) #XXX pure lazyness here too assert 160 <= res <= 165 - def test_weakref(self): + def define_weakref(cls): import weakref, gc class A(object): pass @@ -312,11 +406,14 @@ llop.gc__collect(lltype.Void) result = result and (ref() is None) return result - run = self.runner(f) + return f + + def test_weakref(self): + run = self.runner("weakref") res = run([]) assert res - def test_weakref_to_object_with_finalizer(self): + def define_weakref_to_object_with_finalizer(cls): import weakref, gc class A(object): count = 0 @@ -333,11 +430,14 @@ llop.gc__collect(lltype.Void) result = a.count == 1 and (ref() is None) return result - run = self.runner(f) + return f + + def test_weakref_to_object_with_finalizer(self): + run = self.runner("weakref_to_object_with_finalizer") res = run([]) assert res - def test_collect_during_collect(self): + def define_collect_during_collect(cls): class B(object): pass b = B() @@ -369,14 +469,18 @@ llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) b.bla = persistent_a1.id + persistent_a2.id + persistent_a3.id + persistent_a4.id - print b.num_deleted_c + # NB print would create a static root! + llop.debug_print(lltype.Void, b.num_deleted_c) return b.num_deleted - run = self.runner(f, nbargs=2) + return f + + def test_collect_during_collect(self): + run = self.runner("collect_during_collect") # runs collect recursively 4 times res = run([4, 42]) #XXX pure lazyness here too assert res == 12 - def test_collect_0(self): + def define_collect_0(cls): def concat(j, dummy): lst = [] for i in range(j): @@ -385,11 +489,14 @@ if we_are_translated(): llop.gc__collect(lltype.Void, 0) return result - run = self.runner(concat, nbargs=2) + return concat + + def test_collect_0(self): + run = self.runner("collect_0") res = run([100, 0]) - assert res == concat(100, 0) + assert res == len(''.join([str(x) for x in range(100)])) - def test_interior_ptrs(self): + def define_interior_ptrs(cls): from pypy.rpython.lltypesystem.lltype import Struct, GcStruct, GcArray from pypy.rpython.lltypesystem.lltype import Array, Signed, malloc @@ -443,11 +550,14 @@ f6()) assert func() == 111111 - run = self.runner(func) + return func + + def test_interior_ptrs(self): + run = self.runner("interior_ptrs") res = run([]) assert res == 111111 - def test_id(self): + def define_id(cls): class A(object): pass a1 = A() @@ -463,25 +573,29 @@ if id2 != compute_unique_id(a2): error += 2 if id3 != compute_unique_id(a3): error += 4 return error - run = self.runner(func) + return func + + def test_id(self): + run = self.runner("id") res = run([]) assert res == 0 - def test_can_move(self): + def define_can_move(cls): TP = lltype.GcArray(lltype.Float) def func(): - from pypy.rlib import rgc return rgc.can_move(lltype.malloc(TP, 1)) - run = self.runner(func) + return func + + def test_can_move(self): + run = self.runner("can_move") res = run([]) assert res == self.GC_CAN_MOVE - def test_malloc_nonmovable(self): + def define_malloc_nonmovable(cls): TP = lltype.GcArray(lltype.Char) def func(): #try: - from pypy.rlib import rgc - a = rgc.malloc_nonmovable(TP, 3) + a = rgc.malloc_nonmovable(TP, 3, zero=True) rgc.collect() if a: assert not rgc.can_move(a) @@ -490,15 +604,17 @@ #except Exception, e: # return 2 - run = self.runner(func) + return func + + def test_malloc_nonmovable(self): + run = self.runner("malloc_nonmovable") assert int(self.GC_CANNOT_MALLOC_NONMOVABLE) == run([]) - def test_malloc_nonmovable_fixsize(self): + def define_malloc_nonmovable_fixsize(cls): S = lltype.GcStruct('S', ('x', lltype.Float)) TP = lltype.GcStruct('T', ('s', lltype.Ptr(S))) def func(): try: - from pypy.rlib import rgc a = rgc.malloc_nonmovable(TP) rgc.collect() if a: @@ -508,13 +624,15 @@ except Exception, e: return 2 - run = self.runner(func) + return func + + def test_malloc_nonmovable_fixsize(self): + run = self.runner("malloc_nonmovable_fixsize") assert run([]) == int(self.GC_CANNOT_MALLOC_NONMOVABLE) - def test_resizable_buffer(self): + def define_resizable_buffer(cls): from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.annlowlevel import hlstr - from pypy.rlib import rgc def f(): ptr = rgc.resizable_buffer_of_shape(STR, 2) @@ -523,10 +641,13 @@ ptr.chars[1] = 'b' return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab" - run = self.runner(f) + return f + + def test_resizable_buffer(self): + run = self.runner("resizable_buffer") assert run([]) == 1 - def test_string_builder_over_allocation(self): + def define_string_builder_over_allocation(cls): import gc def fn(): s = StringBuilder(4) @@ -538,17 +659,19 @@ s.append_multiple_char('y', 1000) res = s.build()[1000] gc.collect() - return res - fn = self.runner(fn) + return ord(res) + return fn + + def test_string_builder_over_allocation(self): + fn = self.runner("string_builder_over_allocation") res = fn([]) - assert res == 'y' + assert res == ord('y') class GenericMovingGCTests(GenericGCTests): GC_CAN_MOVE = True GC_CANNOT_MALLOC_NONMOVABLE = True - def test_many_ids(self): - py.test.skip("fails for bad reasons in lltype.py :-(") + def define_many_ids(cls): class A(object): pass def f(): @@ -572,10 +695,28 @@ i += 1 j += 1 lltype.free(idarray, flavor='raw') - run = self.runner(f) + return 0 + return f + + def test_many_ids(self): + py.test.skip("fails for bad reasons in lltype.py :-(") + run = self.runner("many_ids") run([]) - def test_do_malloc_operations(self): + @classmethod + def ensure_layoutbuilder(cls, translator): + jit2gc = getattr(translator, '_jit2gc', None) + if jit2gc: + return jit2gc['layoutbuilder'] + GCClass = cls.gcpolicy.transformerclass.GCClass + layoutbuilder = framework.TransformerLayoutBuilder(translator, GCClass) + layoutbuilder.delay_encoding() + translator._jit2gc = { + 'layoutbuilder': layoutbuilder, + } + return layoutbuilder + + def define_do_malloc_operations(cls): P = lltype.GcStruct('P', ('x', lltype.Signed)) def g(): r = lltype.malloc(P) @@ -589,18 +730,13 @@ while i < 40: g() i += 1 + return 0 def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import rffi - GCClass = self.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) - layoutbuilder.delay_encoding() - translator._jit2gc = { - 'layoutbuilder': layoutbuilder, - } + layoutbuilder = cls.ensure_layoutbuilder(translator) + type_id = layoutbuilder.get_type_id(P) # # now fix the do_malloc_fixedsize_clear in the graph of g @@ -615,10 +751,13 @@ break else: assert 0, "oups, not found" - run = self.runner(f, mixlevelstuff=fix_graph_of_g) + return f, None, fix_graph_of_g + + def test_do_malloc_operations(self): + run = self.runner("do_malloc_operations") run([]) - def test_do_malloc_operations_in_call(self): + def define_do_malloc_operations_in_call(cls): P = lltype.GcStruct('P', ('x', lltype.Signed)) def g(): llop.do_malloc_fixedsize_clear(llmemory.GCREF) # placeholder @@ -629,18 +768,12 @@ while i < 40: g() i += q.x + return 0 def fix_graph_of_g(translator): from pypy.translator.translator import graphof from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import rffi - GCClass = self.gcpolicy.transformerclass.GCClass - lltype2vtable = translator.rtyper.lltype2vtable - layoutbuilder = framework.TransformerLayoutBuilder(GCClass, - lltype2vtable) - layoutbuilder.delay_encoding() - translator._jit2gc = { - 'layoutbuilder': layoutbuilder, - } + layoutbuilder = cls.ensure_layoutbuilder(translator) type_id = layoutbuilder.get_type_id(P) # # now fix the do_malloc_fixedsize_clear in the graph of g @@ -655,9 +788,14 @@ break else: assert 0, "oups, not found" - run = self.runner(f, mixlevelstuff=fix_graph_of_g) + return f, None, fix_graph_of_g + + def test_do_malloc_operations_in_call(self): + run = self.runner("do_malloc_operations_in_call") run([]) +# ________________________________________________________________ + class TestMarkSweepGC(GenericGCTests): gcname = "marksweep" class gcpolicy(gc.FrameworkGcPolicy): @@ -666,7 +804,7 @@ root_stack_depth = 200 - def test_cloning(self): + def define_cloning(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('b', lltype.Ptr(B)), ('unused', lltype.Ptr(B))) @@ -696,11 +834,14 @@ a2copy.b.x = 444 return a1.b.x * 1000000 + a2.b.x * 1000 + a3.b.x - run = self.runner(func) + return func + + def test_cloning(self): + run = self.runner("cloning") res = run([]) assert res == 111222333 - def test_cloning_varsize(self): + def define_cloning_varsize(cls): B = lltype.GcStruct('B', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('b', lltype.Ptr(B)), ('more', lltype.Array(lltype.Ptr(B)))) @@ -731,12 +872,14 @@ a2copy.more[1].x = 441 return a2.b.x * 1000000 + a2.more[0].x * 1000 + a2.more[1].x - run = self.runner(func) + return func + + def test_cloning_varsize(self): + run = self.runner("cloning_varsize") res = run([]) assert res == 22220221 - def test_cloning_highlevel(self): - from pypy.rlib import rgc + def define_cloning_highlevel(cls): class A: pass class B(A): @@ -763,14 +906,16 @@ assert n > 5 return 1 - run = self.runner(func, nbargs=2) + return func + + def test_cloning_highlevel(self): + run = self.runner("cloning_highlevel") res = run([3, 0]) assert res == 1 res = run([7, 0]) assert res == 1 - def test_cloning_highlevel_varsize(self): - from pypy.rlib import rgc + def define_cloning_highlevel_varsize(cls): class A: pass def func(n, dummy): @@ -792,11 +937,14 @@ n = n*10 + a.value return n - run = self.runner(func, nbargs=2) + return func + + def test_cloning_highlevel_varsize(self): + run = self.runner("cloning_highlevel_varsize") res = run([3, 0]) assert res == 456012789 - def test_tree_cloning(self): + def define_tree_cloning(cls): import os # this makes a tree of calls. Each leaf stores its path (a linked # list) in 'result'. Paths are mutated in-place but the leaves don't @@ -876,7 +1024,10 @@ check(result[i], i, 0, depth) os.write(2, 'ok\n') return 1 - run = self.runner(func, nbargs=2) + return func + + def test_tree_cloning(self): + run = self.runner("tree_cloning") res = run([3, 0]) assert res == 1 @@ -902,6 +1053,9 @@ class TestMarkCompactGC(GenericMovingGCTests): gcname = 'markcompact' + def setup_class(cls): + py.test.skip("Disabled for now, sorry") + class gcpolicy(gc.FrameworkGcPolicy): class transformerclass(framework.FrameworkGCTransformer): from pypy.rpython.memory.gc.markcompact import MarkCompactGC as GCClass @@ -919,7 +1073,7 @@ 'nursery_size': 128} root_stack_depth = 200 - def test_weakref_across_minor_collection(self): + def define_weakref_across_minor_collection(cls): import weakref class A: pass @@ -937,11 +1091,14 @@ llop.gc__collect(lltype.Void) assert ref() is a return a.foo + len(all) - run = self.runner(f) + return f + + def test_weakref_across_minor_collection(self): + run = self.runner("weakref_across_minor_collection") res = run([]) assert res == 20 + 20 - def test_nongc_static_root_minor_collect(self): + def define_nongc_static_root_minor_collect(cls): T1 = lltype.GcStruct("C", ('x', lltype.Signed)) T2 = lltype.Struct("C", ('p', lltype.Ptr(T1))) static = lltype.malloc(T2, immortal=True) @@ -958,12 +1115,17 @@ i = static.p.x llop.gc__collect(lltype.Void) return static.p.x + i - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = lltype.nullptr(T1) + return f, cleanup, None + + def test_nongc_static_root_minor_collect(self): + run = self.runner("nongc_static_root_minor_collect") res = run([]) assert res == 84 - def test_static_root_minor_collect(self): + def define_static_root_minor_collect(cls): class A: pass class B: @@ -983,12 +1145,17 @@ i = static.p.x llop.gc__collect(lltype.Void) return static.p.x + i - run = self.runner(f, nbargs=0) + def cleanup(): + static.p = None + return f, cleanup, None + + def test_static_root_minor_collect(self): + run = self.runner("static_root_minor_collect") res = run([]) assert res == 84 - def test_many_weakrefs(self): + def define_many_weakrefs(cls): # test for the case where allocating the weakref itself triggers # a collection import weakref @@ -1001,10 +1168,15 @@ ref = weakref.ref(a) assert ref() is a i += 1 - run = self.runner(f, nbargs=0) + return 0 + + return f + + def test_many_weakrefs(self): + run = self.runner("many_weakrefs") run([]) - def test_immutable_to_old_promotion(self): + def define_immutable_to_old_promotion(cls): T_CHILD = lltype.Ptr(lltype.GcStruct('Child', ('field', lltype.Signed))) T_PARENT = lltype.Ptr(lltype.GcStruct('Parent', ('sub', T_CHILD))) child = lltype.malloc(T_CHILD.TO) @@ -1026,7 +1198,10 @@ #all[x] = lltype.nullptr(T_PARENT.TO) return res.sub.field - run, transformer = self.runner(f, nbargs=2, transformer=True) + return f + + def test_immutable_to_old_promotion(self): + run, transformer = self.runner("immutable_to_old_promotion", transformer=True) run([1, 4]) if not transformer.GCClass.prebuilt_gc_objects_are_static_roots: assert len(transformer.layoutbuilder.addresses_of_static_ptrs) == 0 @@ -1043,7 +1218,7 @@ # * the GcArray pointer from gc.wr_to_objects_with_id # * the GcArray pointer from gc.object_id_dict. - def test_adr_of_nursery(self): + def define_adr_of_nursery(cls): class A(object): pass @@ -1061,7 +1236,12 @@ assert nf1 > nf0 assert nt1 > nf1 assert nt1 == nt0 - run = self.runner(f, nbargs=0) + return 0 + + return f + + def test_adr_of_nursery(self): + run = self.runner("adr_of_nursery") res = run([]) class TestGenerationalNoFullCollectGC(GCTest): @@ -1082,11 +1262,15 @@ def semispace_collect(self, size_changing=False): ll_assert(not self.__ready, "no full collect should occur in this test") + def _teardown(self): + self.__ready = False # collecting here is expected + GenerationGC._teardown(self) + GC_PARAMS = {'space_size': 2048, 'nursery_size': 512} root_stack_depth = 200 - def test_working_nursery(self): + def define_working_nursery(cls): def f(): total = 0 i = 0 @@ -1099,7 +1283,10 @@ total += len(lst) i += 1 return total - run = self.runner(f, nbargs=0) + return f + + def test_working_nursery(self): + run = self.runner("working_nursery") res = run([]) assert res == 40 * 5 @@ -1115,7 +1302,7 @@ 'large_object': 32} root_stack_depth = 200 - def test_ref_from_rawmalloced_to_regular(self): + def define_ref_from_rawmalloced_to_regular(cls): import gc S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcStruct('A', ('p', lltype.Ptr(S)), @@ -1133,11 +1320,14 @@ lst = setup(j) gc.collect() return lst.p.x - run = self.runner(f, nbargs=2) + return f + + def test_ref_from_rawmalloced_to_regular(self): + run = self.runner("ref_from_rawmalloced_to_regular") res = run([100, 100]) assert res == 200 - def test_assume_young_pointers(self): + def define_assume_young_pointers(cls): from pypy.rlib import rgc S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', @@ -1154,9 +1344,109 @@ rgc.collect(0) return s0.next.x - run = self.runner(f, nbargs=0) + def cleanup(): + s0.next = lltype.nullptr(S) + + return f, cleanup, None + + def test_assume_young_pointers(self): + run = self.runner("assume_young_pointers") res = run([]) assert res == 42 def test_malloc_nonmovable_fixsize(self): py.test.skip("not supported") + +# ________________________________________________________________ +# tagged pointers + +class TaggedPointerGCTests(GCTest): + taggedpointers = True + + def define_tagged_simple(cls): + class Unrelated(object): + pass + + u = Unrelated() + u.x = UnboxedObject(47) + def fn(n): + rgc.collect() # check that a prebuilt tagged pointer doesn't explode + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + u.x = x # invoke write barrier + rgc.collect() + return x.meth(100) + def func(): + return fn(1000) + fn(-1000) + assert func() == 205 + return func + + def test_tagged_simple(self): + func = self.runner("tagged_simple") + res = func([]) + assert res == 205 + + def define_tagged_prebuilt(cls): + + class F: + pass + + f = F() + f.l = [UnboxedObject(10)] + def fn(n): + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + f.l.append(x) + rgc.collect() + return f.l[-1].meth(100) + def func(): + return fn(1000) ^ fn(-1000) + assert func() == -1999 + return func + + def test_tagged_prebuilt(self): + func = self.runner("tagged_prebuilt") + res = func([]) + assert res == -1999 + +from pypy.rlib.objectmodel import UnboxedValue + +class TaggedBase(object): + __slots__ = () + def meth(self, x): + raise NotImplementedError + +class BoxedObject(TaggedBase): + attrvalue = 66 + def __init__(self, normalint): + self.normalint = normalint + def meth(self, x): + return self.normalint + x + 2 + +class UnboxedObject(TaggedBase, UnboxedValue): + __slots__ = 'smallint' + def meth(self, x): + return self.smallint + x + 3 + + +class TestMarkSweepTaggedPointerGC(TaggedPointerGCTests): + gcname = "marksweep" + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + GC_PARAMS = {'start_heap_size': 4096 } + root_stack_depth = 200 + +class TestHybridTaggedPointerGC(TaggedPointerGCTests): + gcname = "hybrid" + + class gcpolicy(gc.FrameworkGcPolicy): + class transformerclass(framework.FrameworkGCTransformer): + from pypy.rpython.memory.gc.generation import GenerationGC as \ + GCClass + GC_PARAMS = {'space_size': 2048, + 'nursery_size': 128} + root_stack_depth = 200 Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/ootypesystem/ootype.py Thu Nov 5 22:17:18 2009 @@ -1629,7 +1629,10 @@ self._array[index] = item def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null arrays class _null_array(_null_mixin(_array), _array): @@ -1776,7 +1779,10 @@ self.__dict__[name] = value def _identityhash(self): - return hash(self) + if self: + return intmask(id(self)) + else: + return 0 # for all null records def _items_in_order(self): return [self._items[name] for name in self._TYPE._fields_in_order] Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/rclass.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/rclass.py Thu Nov 5 22:17:18 2009 @@ -56,11 +56,14 @@ issubclass(subdef.classdesc.pyobj, UnboxedValue)] virtualizable2 = classdef.classdesc.read_attribute('_virtualizable2_', Constant(False)).value + config = rtyper.annotator.translator.config + usetagging = len(unboxed) != 0 and config.translation.taggedpointers + if virtualizable2: assert len(unboxed) == 0 assert gcflavor == 'gc' return rtyper.type_system.rvirtualizable2.Virtualizable2InstanceRepr(rtyper, classdef) - elif len(unboxed) > 0 and rtyper.type_system.name == 'lltypesystem': + elif usetagging and rtyper.type_system.name == 'lltypesystem': # the UnboxedValue class and its parent classes need a # special repr for their instances if len(unboxed) != 1: Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/rint.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/rint.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/rint.py Thu Nov 5 22:17:18 2009 @@ -408,10 +408,8 @@ fn = hop.rtyper.type_system.ll_str.ll_int2oct return hop.gendirectcall(fn, varg, true) -def ll_identity(n): - return n - -ll_hash_int = ll_identity +def ll_hash_int(n): + return intmask(n) def ll_check_chr(n): if 0 <= n <= 255: Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/rvirtualizable2.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,3 @@ -import py from pypy.rpython.rmodel import inputconst, log from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rdict.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rdict.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rdict.py Thu Nov 5 22:17:18 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rdict, rstr from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rlib.objectmodel import r_dict +from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong import py py.log.setconsumer("rtyper", py.log.STDOUT) @@ -564,6 +565,17 @@ res = self.interpret(fn, [3.0]) assert res == 42 + def test_dict_of_r_uint(self): + for r_t in [r_uint, r_longlong, r_ulonglong]: + d = {r_t(2): 3, r_t(4): 5} + def fn(x, y): + d[r_t(x)] = 123 + return d[r_t(y)] + res = self.interpret(fn, [4, 2]) + assert res == 3 + res = self.interpret(fn, [3, 3]) + assert res == 123 + class TestLLtype(BaseTestRdict, LLRtypeMixin): def test_dict_but_not_with_char_keys(self): Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rptr.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rptr.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/test/test_rptr.py Thu Nov 5 22:17:18 2009 @@ -112,6 +112,20 @@ interpret(fn, [11521]) +def test_odd_ints_opaque(): + T = GcStruct('T') + Q = GcOpaqueType('Q') + PT = Ptr(T) + PQ = Ptr(Q) + def fn(n): + t = cast_int_to_ptr(PT, n) + assert typeOf(t) == PT + assert cast_ptr_to_int(t) == n + o = cast_opaque_ptr(PQ, t) + assert cast_ptr_to_int(o) == n + + fn(13) + interpret(fn, [11521]) def test_Ptr(): S = GcStruct('s') Modified: pypy/branch/msvc-asmgcroot/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/rpython/tool/rffi_platform.py Thu Nov 5 22:17:18 2009 @@ -566,7 +566,7 @@ def get_python_include_dir(): from distutils import sysconfig gcv = sysconfig.get_config_vars() - return gcv['INCLUDEPY'] + return gcv.get('INCLUDEPY', '.') # this is for running on top of pypy-c def configure_external_library(name, eci, configurations, symbol=None, _cache={}): Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/all.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/all.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/all.py Thu Nov 5 22:17:18 2009 @@ -6,7 +6,6 @@ from pypy.translator.backendopt.stat import print_statistics from pypy.translator.backendopt.merge_if_blocks import merge_if_blocks from pypy.translator import simplify -from pypy.translator.backendopt.escape import malloc_to_stack from pypy.translator.backendopt import mallocprediction from pypy.translator.backendopt.removeassert import remove_asserts from pypy.translator.backendopt.support import log @@ -123,10 +122,6 @@ call_count_pred=call_count_pred) constfold(config, graphs) - if config.heap2stack: - assert graphs is translator.graphs # XXX for now - malloc_to_stack(translator) - if config.merge_if_blocks: log.mergeifblocks("starting to merge if blocks") for graph in graphs: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/escape.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/escape.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/escape.py Thu Nov 5 22:17:18 2009 @@ -1,4 +1,3 @@ -from pypy.annotation.model import setunion from pypy.objspace.flow.model import Variable, Constant from pypy.rpython.lltypesystem import lltype from pypy.translator.simplify import get_graph @@ -7,37 +6,31 @@ from pypy.tool.uid import uid class CreationPoint(object): - def __init__(self, creation_method, lltype): - self.changes = False + def __init__(self, creation_method, TYPE, op=None): self.escapes = False + self.returns = False self.creation_method = creation_method if creation_method == "constant": - self.changes = True self.escapes = True - self.malloced = False - self.lltype = lltype + self.TYPE = TYPE + self.op = op def __repr__(self): - return ("CreationPoint(<0x%x>, %r, %s, esc=%s, cha=%s)" % - (uid(self), self.lltype, self.creation_method, self.escapes, self.changes)) + return ("CreationPoint(<0x%x>, %r, %s, esc=%s)" % + (uid(self), self.TYPE, self.creation_method, self.escapes)) class VarState(object): - def __init__(self, crep=None): - self.creation_points = {} - if crep is not None: - self.creation_points[crep] = True + def __init__(self, *creps): + self.creation_points = set() + for crep in creps: + self.creation_points.add(crep) def contains(self, other): - for crep in other.creation_points: - if crep not in self.creation_points: - return False - return True + return other.creation_points.issubset(self.creation_points) def merge(self, other): - creation_points = setunion(self.creation_points, other.creation_points) - newstate = VarState() - newstate.creation_points = creation_points - return newstate + creation_points = self.creation_points.union(other.creation_points) + return VarState(*creation_points) def setescapes(self): changed = [] @@ -47,29 +40,28 @@ crep.escapes = True return changed - def setchanges(self): + def setreturns(self): changed = [] for crep in self.creation_points: - if not crep.changes: + if not crep.returns: changed.append(crep) - crep.changes = True + crep.returns = True return changed - + def does_escape(self): for crep in self.creation_points: if crep.escapes: return True return False - def does_change(self): + def does_return(self): for crep in self.creation_points: - if crep.changes: + if crep.returns: return True return False - + def __repr__(self): - crepsrepr = (", ".join([repr(crep) for crep in self.creation_points]), ) - return "VarState({%s})" % crepsrepr + return "" % (self.creation_points, ) class AbstractDataFlowInterpreter(object): def __init__(self, translation_context): @@ -108,15 +100,14 @@ def setstate(self, var, state): self.varstates[var] = state - def get_creationpoint(self, var, method="?"): + def get_creationpoint(self, var, method="?", op=None): if var in self.creationpoints: return self.creationpoints[var] - crep = CreationPoint(method, var.concretetype) + crep = CreationPoint(method, var.concretetype, op) self.creationpoints[var] = crep return crep def schedule_function(self, graph): - #print "scheduling function:", graph.name startblock = graph.startblock if graph in self.functionargs: args = self.functionargs[graph] @@ -136,54 +127,39 @@ return resultstate, args def flow_block(self, block, graph): - #print "flowing in block %s of function %s" % (block, graph.name) self.flown_blocks[block] = True if block is graph.returnblock: if isonheap(block.inputargs[0]): - changed = self.getstate(block.inputargs[0]).setescapes() - self.handle_changed(changed) + self.returns(self.getstate(block.inputargs[0])) return if block is graph.exceptblock: if isonheap(block.inputargs[0]): - changed = self.getstate(block.inputargs[0]).setescapes() - self.handle_changed(changed) + self.escapes(self.getstate(block.inputargs[0])) if isonheap(block.inputargs[1]): - changed = self.getstate(block.inputargs[1]).setescapes() - self.handle_changed(changed) + self.escapes(self.getstate(block.inputargs[1])) return self.curr_block = block self.curr_graph = graph - #print "inputargs", self.getstates(block.inputargs) for op in block.operations: self.flow_operation(op) - #print "checking exits..." for exit in block.exits: - #print "exit", exit args = self.getstates(exit.args) targetargs = self.getstates(exit.target.inputargs) - #print " newargs", args - #print " targetargs", targetargs - # flow every block at least once: + # flow every block at least once if (multicontains(targetargs, args) and exit.target in self.flown_blocks): - #print " not necessary" continue - #else: - #print " scheduling for flowin" for prevstate, origstate, var in zip(args, targetargs, exit.target.inputargs): if not isonheap(var): continue newstate = prevstate.merge(origstate) self.setstate(var, newstate) - #print " args", self.getstates(exit.target.inputargs) self.scheduled[exit.target] = graph def flow_operation(self, op): - #print "handling", op args = self.getstates(op.args) - #print "args:", args opimpl = getattr(self, 'op_'+op.opname, None) if opimpl is not None: res = opimpl(op, *args) @@ -194,18 +170,21 @@ if isonheap(op.result) or filter(None, args): for arg in args: if arg is not None: - changed = arg.setchanges() - self.handle_changed(changed) - changed = arg.setescapes() - self.handle_changed(changed) - #raise NotImplementedError("can't handle %s" % (op.opname, )) - #print "assuming that '%s' is irrelevant" % op + self.escapes(arg) def complete(self): while self.scheduled: block, graph = self.scheduled.popitem() self.flow_block(block, graph) + def escapes(self, arg): + changed = arg.setescapes() + self.handle_changed(changed) + + def returns(self, arg): + changed = arg.setreturns() + self.handle_changed(changed) + def handle_changed(self, changed): for crep in changed: if crep not in self.dependencies: @@ -222,13 +201,10 @@ def register_state_dependency(self, state1, state2): "state1 depends on state2: if state2 does escape/change, so does state1" # change state1 according to how state2 is now - #print "registering dependency of %s on %s" % (state1, state2) if state2.does_escape(): - changed = state1.setescapes() # mark all crep's as escaping - self.handle_changed(changed) - if state2.does_change(): - changed = state1.setchanges() # mark all crep's as changing - self.handle_changed(changed) + self.escapes(state1) + if state2.does_return(): + self.returns(state1) # register a dependency of the current block on state2: # that means that if state2 changes the current block will be reflown # triggering this function again and thus updating state1 @@ -242,14 +218,14 @@ flags = op.args[1].value if flags != {'flavor': 'gc'}: return NotImplemented - return VarState(self.get_creationpoint(op.result, "malloc")) + return VarState(self.get_creationpoint(op.result, "malloc", op)) def op_malloc_varsize(self, op, typestate, flagsstate, lengthstate): assert flagsstate is None flags = op.args[1].value if flags != {'flavor': 'gc'}: return NotImplemented - return VarState(self.get_creationpoint(op.result, "malloc_varsize")) + return VarState(self.get_creationpoint(op.result, "malloc_varsize", op)) def op_keepalive(self, op, state): return None @@ -258,49 +234,26 @@ return state def op_setfield(self, op, objstate, fieldname, valuestate): - changed = objstate.setchanges() - self.handle_changed(changed) if valuestate is not None: # be pessimistic for now: - # everything that gets stored into a structure escapes and changes - self.handle_changed(changed) - changed = valuestate.setchanges() - self.handle_changed(changed) - changed = valuestate.setescapes() - self.handle_changed(changed) + # everything that gets stored into a structure escapes + self.escapes(valuestate) return None def op_setarrayitem(self, op, objstate, indexstate, valuestate): - changed = objstate.setchanges() - self.handle_changed(changed) if valuestate is not None: - # everything that gets stored into a structure escapes and changes - self.handle_changed(changed) - changed = valuestate.setchanges() - self.handle_changed(changed) - changed = valuestate.setescapes() - self.handle_changed(changed) + # everything that gets stored into a structure escapes + self.escapes(valuestate) return None def op_getarrayitem(self, op, objstate, indexstate): if isonheap(op.result): - return VarState(self.get_creationpoint(op.result, "getarrayitem")) + return VarState(self.get_creationpoint(op.result, "getarrayitem", op)) def op_getfield(self, op, objstate, fieldname): if isonheap(op.result): # assume that getfield creates a new value - return VarState(self.get_creationpoint(op.result, "getfield")) - - def op_getsubstruct(self, op, objstate, fieldname): - # since this is really an embedded struct, it has the same - # state, the same creationpoints, etc. - return objstate - - def op_getarraysubstruct(self, op, arraystate, indexstate): - # since this is really a struct embedded somewhere in the array it has - # the same state, creationpoints, etc. in most cases the resulting - # pointer should not be used much anyway - return arraystate + return VarState(self.get_creationpoint(op.result, "getfield", op)) def op_getarraysize(self, op, arraystate): pass @@ -311,9 +264,8 @@ for arg in args: if arg is None: continue - # an external function can change every parameter: - changed = arg.setchanges() - self.handle_changed(changed) + # an external function can escape every parameter: + self.escapes(arg) funcargs = [None] * len(args) else: result, funcargs = self.schedule_function(graph) @@ -326,7 +278,7 @@ self.register_state_dependency(localarg, funcarg) if isonheap(op.result): # assume that a call creates a new value - return VarState(self.get_creationpoint(op.result, "direct_call")) + return VarState(self.get_creationpoint(op.result, "direct_call", op)) def op_indirect_call(self, op, function, *args): graphs = op.args[-1].value @@ -335,10 +287,7 @@ for localarg in args: if localarg is None: continue - changed = localarg.setescapes() - self.handle_changed(changed) - changed = localarg.setchanges() - self.handle_changed(changed) + self.escapes(localarg) else: for graph in graphs: result, funcargs = self.schedule_function(graph) @@ -350,7 +299,7 @@ self.register_state_dependency(localarg, funcarg) if isonheap(op.result): # assume that a call creates a new value - return VarState(self.get_creationpoint(op.result, "indirect_call")) + return VarState(self.get_creationpoint(op.result, "indirect_call", op)) def op_ptr_iszero(self, op, ptrstate): return None @@ -377,35 +326,38 @@ return False return True -def malloc_to_stack(t): - adi = AbstractDataFlowInterpreter(t) - for graph in t.graphs: - if graph.startblock not in adi.flown_blocks: - adi.schedule_function(graph) - adi.complete() - for graph in t.graphs: - loop_blocks = support.find_loop_blocks(graph) - for block, op in graph.iterblockops(): - if op.opname != 'malloc': - continue - STRUCT = op.args[0].value - # must not remove mallocs of structures that have a RTTI with a destructor - try: - destr_ptr = lltype.getRuntimeTypeInfo(STRUCT)._obj.destructor_funcptr - if destr_ptr: - continue - except (ValueError, AttributeError), e: - pass - varstate = adi.getstate(op.result) - assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: - if block not in loop_blocks: - print "moving object from heap to stack %s in %s" % (op, graph.name) - flags = op.args[1].value - assert flags == {'flavor': 'gc'} - op.args[1] = Constant({'flavor': 'stack'}, lltype.Void) - else: - print "%s in %s is a non-escaping malloc in a loop" % (op, graph.name) + +def is_malloc_like(adi, graph, seen): + if graph in seen: + return seen[graph] + return_state = adi.getstate(graph.getreturnvar()) + if return_state is None or len(return_state.creation_points) != 1: + seen[graph] = False + return False + crep, = return_state.creation_points + if crep.escapes: + seen[graph] = False + return False + if crep.creation_method in ["malloc", "malloc_varsize"]: + assert crep.returns + seen[graph] = True + return True + if crep.creation_method == "direct_call": + subgraph = get_graph(crep.op.args[0], adi.translation_context) + if subgraph is None: + seen[graph] = False + return False + res = is_malloc_like(adi, subgraph, seen) + seen[graph] = res + return res + seen[graph] = False + return False + + +def malloc_like_graphs(adi): + seen = {} + return [graph for graph in adi.seen_graphs() + if is_malloc_like(adi, graph, seen)] + Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/inline.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/inline.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/inline.py Thu Nov 5 22:17:18 2009 @@ -631,7 +631,7 @@ count), True def inlinable_static_callers(graphs): - ok_to_call = dict.fromkeys(graphs) + ok_to_call = set(graphs) result = [] for parentgraph in graphs: for block in parentgraph.iterblocks(): Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocprediction.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocprediction.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocprediction.py Thu Nov 5 22:17:18 2009 @@ -1,15 +1,16 @@ from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter +from pypy.translator.backendopt.escape import malloc_like_graphs from pypy.translator.backendopt.all import remove_mallocs from pypy.translator.backendopt import inline from pypy.rpython.lltypesystem import lltype -from pypy.translator import simplify +from pypy.translator.simplify import get_graph from pypy.translator.backendopt import removenoops from pypy.translator.backendopt.support import log SMALL_THRESHOLD = 15 BIG_THRESHOLD = 50 -def find_malloc_creps(graph, adi, translator): +def find_malloc_creps(graph, adi, translator, malloc_graphs): # mapping from malloc creation point to graphs that it flows into malloc_creps = {} # find all mallocs that don't escape @@ -29,8 +30,17 @@ pass varstate = adi.getstate(op.result) assert len(varstate.creation_points) == 1 - crep = varstate.creation_points.keys()[0] - if not crep.escapes: + crep, = varstate.creation_points + if not crep.escapes and not crep.returns: + malloc_creps[crep] = {} + if op.opname == 'direct_call': + called_graph = get_graph(op.args[0], translator) + if called_graph not in malloc_graphs: + continue + varstate = adi.getstate(op.result) + assert len(varstate.creation_points) == 1 + crep, = varstate.creation_points + if not crep.escapes and not crep.returns: malloc_creps[crep] = {} return malloc_creps @@ -64,16 +74,19 @@ del interesting_creps[crep] elif op.opname == "direct_call": #print op, interesting_creps - called_graph = simplify.get_graph(op.args[0], translator) + called_graph = get_graph(op.args[0], translator) interesting = {} - for i, var in enumerate(op.args[1:]): - #print i, var, + if called_graph is None: + graphvars = [None] * len(op.args) + else: + graphvars = called_graph.getargs() + [called_graph.getreturnvar()] + for var, graphvar in zip(op.args[1:] + [op.result], graphvars): varstate = adi.getstate(var) if varstate is None: #print "no varstate" continue if len(varstate.creation_points) == 1: - crep = varstate.creation_points.keys()[0] + crep, = varstate.creation_points if crep not in interesting_creps: #print "not interesting" continue @@ -81,16 +94,14 @@ del interesting_creps[crep] #print "graph not found" continue - if (called_graph, i) in seen: - seen[(called_graph, i)][graph] = True + if called_graph in seen: + seen[called_graph][graph] = True #print "seen already" else: #print "taking", crep - seen[(called_graph, i)] = {graph: True} - arg = called_graph.startblock.inputargs[i] - argstate = adi.getstate(arg) - argcrep = [c for c in argstate.creation_points - if c.creation_method == "arg"][0] + seen[called_graph] = {graph: True} + argstate = adi.getstate(graphvar) + argcrep, = argstate.creation_points interesting[argcrep] = True #print interesting if interesting: @@ -104,21 +115,21 @@ if graph.startblock not in adi.flown_blocks: adi.schedule_function(graph) adi.complete() + malloc_graphs = malloc_like_graphs(adi) targetset = dict.fromkeys(graphs) caller_candidates = {} seen = {} for graph in adi.seen_graphs(): - creps = find_malloc_creps(graph, adi, t) - #print "malloc creps", creps + creps = find_malloc_creps(graph, adi, t, malloc_graphs) if creps: find_calls_where_creps_go(creps, graph, adi, t, seen) if creps: if graph in targetset: caller_candidates[graph] = True callgraph = [] - for (called_graph, i), callers in seen.iteritems(): + for called_graph, callers in seen.iteritems(): for caller in callers: - if caller in targetset: + if caller in targetset and called_graph in targetset: callgraph.append((caller, called_graph)) else: log.inlineandremove.WARNING("would like to inline into" Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocv.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocv.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/mallocv.py Thu Nov 5 22:17:18 2009 @@ -1046,7 +1046,7 @@ op = getattr(llop, opname) except AttributeError: return - if not op.is_pure(*[v.concretetype for v in args_v]): + if not op.is_pure(args_v): return try: result = op(RESTYPE, *args) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/stat.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/stat.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/stat.py Thu Nov 5 22:17:18 2009 @@ -7,6 +7,7 @@ num_graphs = 0 num_blocks = 0 num_ops = 0 + num_mallocs = 0 per_graph = {} while stack: graph = stack.pop() @@ -16,6 +17,7 @@ num_graphs += 1 old_num_blocks = num_blocks old_num_ops = num_ops + old_num_mallocs = num_mallocs for block in graph.iterblocks(): num_blocks += 1 for op in block.operations: @@ -30,29 +32,34 @@ called_graphs = op.args[-1].value if called_graphs is not None: stack.extend(called_graphs) + elif op.opname.startswith("malloc"): + num_mallocs += 1 num_ops += 1 - per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops) + per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops, num_mallocs-old_num_mallocs) if save_per_graph_details: details = [] - for graph, (nblocks, nops) in per_graph.iteritems(): + for graph, (nblocks, nops, nmallocs) in per_graph.iteritems(): try: code = graph.func.func_code.co_code except AttributeError: code = "None" hash = md5(code).hexdigest() - details.append((hash, graph.name, nblocks, nops)) + details.append((hash, graph.name, nblocks, nops, nmallocs)) details.sort() f = open(save_per_graph_details, "w") try: - for hash, name, nblocks, nops in details: - print >>f, hash, name, nblocks, nops + for hash, name, nblocks, nops, nmallocs in details: + print >>f, hash, name, nblocks, nops, nmallocs finally: f.close() - return num_graphs, num_blocks, num_ops + return num_graphs, num_blocks, num_ops, num_mallocs def print_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False): - num_graphs, num_blocks, num_ops = get_statistics(graph, translator, save_per_graph_details, - ignore_stack_checks=ignore_stack_checks) + num_graphs, num_blocks, num_ops, num_mallocs = get_statistics( + graph, translator, save_per_graph_details, + ignore_stack_checks=ignore_stack_checks) print ("Statistics:\nnumber of graphs %s\n" "number of blocks %s\n" - "number of operations %s\n") % (num_graphs, num_blocks, num_ops) + "number of operations %s\n" + "number of mallocs %s\n" + ) % (num_graphs, num_blocks, num_ops, num_mallocs) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_constfold.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_constfold.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_constfold.py Thu Nov 5 22:17:18 2009 @@ -4,6 +4,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython import rclass from pypy.rlib import objectmodel from pypy.translator.backendopt.constfold import constant_fold_graph from pypy import conftest @@ -26,8 +27,10 @@ assert res == expected_result -def test_simple(): - S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) +def test_simple(S1=None): + if S1 is None: + S1 = lltype.GcStruct('S1', ('x', lltype.Signed), + hints={'immutable': True}) s1 = lltype.malloc(S1) s1.x = 123 def g(y): @@ -42,6 +45,14 @@ check_graph(graph, [], 124, t) +def test_immutable_fields(): + accessor = rclass.FieldListAccessor() + S2 = lltype.GcStruct('S2', ('x', lltype.Signed), + hints={'immutable_fields': accessor}) + accessor.initialize(S2, ['x']) + test_simple(S2) + + def test_along_link(): S1 = lltype.GcStruct('S1', ('x', lltype.Signed), hints={'immutable': True}) s1 = lltype.malloc(S1) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_escape.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_escape.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_escape.py Thu Nov 5 22:17:18 2009 @@ -1,41 +1,22 @@ -from pypy.translator.translator import TranslationContext, graphof -from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter, malloc_to_stack -from pypy.translator.backendopt.support import find_backedges, find_loop_blocks -from pypy.rpython.llinterp import LLInterpreter +from pypy.translator.interactive import Translation +from pypy.translator.translator import graphof +from pypy.translator.backendopt.escape import AbstractDataFlowInterpreter +from pypy.translator.backendopt.escape import malloc_like_graphs from pypy.rlib.objectmodel import instantiate from pypy import conftest import py def build_adi(function, types): - t = TranslationContext() - t.buildannotator().build_types(function, types) - t.buildrtyper().specialize() + t = Translation(function) + t.rtype(types) if conftest.option.view: t.view() - adi = AbstractDataFlowInterpreter(t) - graph = graphof(t, function) + adi = AbstractDataFlowInterpreter(t.context) + graph = graphof(t.context, function) adi.schedule_function(graph) adi.complete() - return t, adi, graph - -def check_malloc_removal(function, types, args, expected_result, must_remove=True): - t = TranslationContext() - t.buildannotator().build_types(function, types) - t.buildrtyper().specialize() - interp = LLInterpreter(t.rtyper) - graph = graphof(t, function) - res = interp.eval_graph(graph, args) - assert res == expected_result - malloc_to_stack(t) - if must_remove: - for block in graph.iterblocks(): - for op in block.operations: - if op.opname == "malloc": - assert op.args[1].value['flavor'] == 'stack' - res = interp.eval_graph(graph, args) - assert res == expected_result - return t + return t.context, adi, graph def test_simple(): class A(object): @@ -47,9 +28,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_branch(): @@ -66,9 +45,7 @@ t, adi, graph = build_adi(fn2, [int, int]) tvar = graph.startblock.operations[0].result state = adi.getstate(tvar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_loop(): @@ -85,9 +62,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes avarinloop = graph.startblock.exits[0].target.inputargs[1] state1 = adi.getstate(avarinloop) @@ -106,8 +81,7 @@ avar = graph.startblock.operations[0].result state = adi.getstate(avar) assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert crep.escapes def test_classattrs(): @@ -121,9 +95,7 @@ t, adi, graph = build_adi(fn5, []) bvar = graph.startblock.operations[0].result state = adi.getstate(bvar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert not crep.escapes def test_aliasing(): @@ -144,8 +116,7 @@ avar = graph.startblock.exits[0].target.inputargs[1] state = adi.getstate(avar) assert len(state.creation_points) == 2 - for crep in state.creation_points.keys(): - assert crep.changes + for crep in state.creation_points: assert not crep.escapes def test_call(): @@ -161,15 +132,11 @@ g_graph = graphof(t, g) bvar = g_graph.startblock.inputargs[0] bstate = adi.getstate(bvar) - assert len(bstate.creation_points) == 1 - bcrep = bstate.creation_points.keys()[0] - assert not bcrep.changes + bcrep, = bstate.creation_points assert not bcrep.escapes avar = graph.startblock.operations[0].result astate = adi.getstate(avar) - assert len(astate.creation_points) == 1 - acrep = astate.creation_points.keys()[0] - assert acrep.changes + acrep, = astate.creation_points assert not acrep.escapes def test_dependencies(): @@ -197,10 +164,6 @@ reallygraph = resizegraph.startblock.exits[0].target.operations[0].args[0].value._obj.graph reallyarg0 = reallygraph.startblock.inputargs[0] reallystate = adi.getstate(reallyarg0) - assert reallystate.does_change() - assert resizestate.does_change() - assert appendstate.does_change() - assert astate.does_change() def test_substruct(): class A(object): @@ -221,14 +184,10 @@ b0var = graph.startblock.operations[3].result a0state = adi.getstate(a0var) b0state = adi.getstate(b0var) - assert len(a0state.creation_points) == 1 - a0crep = a0state.creation_points.keys()[0] + a0crep, = a0state.creation_points assert not a0crep.escapes - assert a0crep.changes - assert len(b0state.creation_points) == 1 - b0crep = b0state.creation_points.keys()[0] + b0crep, = b0state.creation_points assert b0crep.escapes - assert b0crep.changes def test_multiple_calls(): class A(object): @@ -251,14 +210,11 @@ a1state = adi.getstate(a1var) a2state = adi.getstate(a2var) a3state = adi.getstate(a3var) - assert len(a1state.creation_points) == 1 - assert len(a2state.creation_points) == 1 - assert len(a3state.creation_points) == 1 - a1crep = a1state.creation_points.keys()[0] - a2crep = a2state.creation_points.keys()[0] - a3crep = a3state.creation_points.keys()[0] - assert a1crep.changes and a2crep.changes and a3crep.changes - assert not a1crep.escapes and a2crep.escapes and a3crep.escapes + a1crep, = a1state.creation_points + a2crep, = a2state.creation_points + a3crep, = a3state.creation_points + assert not a1crep.escapes and not a2crep.escapes and not a3crep.escapes + assert not a1crep.returns and a2crep.returns and a3crep.returns def test_indirect_call(): class A(object): @@ -293,12 +249,9 @@ a2var = graph.startblock.operations[3].result a1state = adi.getstate(a1var) a2state = adi.getstate(a2var) - assert len(a1state.creation_points) == 1 - assert len(a2state.creation_points) == 1 - a1crep = a1state.creation_points.keys()[0] - a2crep = a2state.creation_points.keys()[0] - assert a1crep.changes and a2crep.changes - assert not a1crep.escapes and a2crep.escapes + a1crep, = a1state.creation_points + a2crep, = a2state.creation_points + assert not a1crep.escapes and a2crep.returns def test_indirect_call_unknown_graphs(): class A: @@ -324,9 +277,7 @@ t, adi, graph = build_adi(f, []) avar = graph.startblock.operations[0].result state = adi.getstate(avar) - assert len(state.creation_points) == 1 - crep = state.creation_points.keys()[0] - assert crep.changes + crep, = state.creation_points assert crep.escapes def test_flow_blocksonce(): @@ -367,7 +318,6 @@ t, adi, graph = build_adi(createdict, [int, int]) dvar = graph.startblock.operations[0].result dstate = adi.getstate(dvar) - assert dstate.does_change() assert not dstate.does_escape() def test_raise_escapes(): @@ -378,7 +328,18 @@ avar = graph.startblock.operations[0].result state = adi.getstate(avar) assert state.does_escape() - assert state.does_change() + +def test_return(): + class A(object): + pass + def f(): + a = A() + return a + t, adi, graph = build_adi(f, []) + avar = graph.startblock.operations[0].result + state = adi.getstate(avar) + assert not state.does_escape() + assert state.does_return() def test_big(): @@ -387,95 +348,23 @@ # does not crash t, adi, graph = build_adi(entrypoint, [int]) -def test_extfunc_onheaparg(): - py.test.skip("not a valid test anymore") - import os - def f(i): - s = str(i) - os.write(2, s) - return len(s) - t, adi, graph = build_adi(f, [int]) - svar = graph.startblock.operations[0].result - state = adi.getstate(svar) - assert not state.does_escape() - assert state.does_change() - -def test_extfunc_resultonheap(): - py.test.skip("not a valid test anymore") - import os - def f(i): - s = str(i) - return len(s) - t, adi, graph = build_adi(f, [float]) - svar = graph.startblock.operations[0].result - state = adi.getstate(svar) - assert not state.does_escape() - - - -#__________________________________________________________ -# malloc removal tests - -def test_remove_simple(): - class A(object): - pass - def f(): - a = A() - a.x = 1 - return a.x - check_malloc_removal(f, [], [], 1) - -def test_remove_aliasing(): - class A: - pass - def fn6(n): - a1 = A() - a1.x = 5 - a2 = A() - a2.x = 6 - if n > 0: - a = a1 - else: - a = a2 - a.x = 12 - return a1.x - t = check_malloc_removal(fn6, [int], [2], 12) - -def test_remove_call(): +def test_find_malloc_like_graphs(): class A(object): pass - def g(b): - return b.i + 2 - def f(): + def f(x): a = A() - a.i = 2 - return g(a) - t = check_malloc_removal(f, [], [], 4) + a.x = x + return a -def test_dont_alloca_in_loops(): - class A(object): - pass - def f(x): - result = 0 - for i in range(x): - a = A() - a.i = i - result += a.i - return result - t = check_malloc_removal(f, [int], [3], 3, must_remove=False) - graph = graphof(t, f) - assert graph.startblock.exits[0].target.exits[0].target.operations[0].opname == "malloc" + def g(a): + return a -def test_dont_remove_del_objects(): - class A(object): - def __del__(self): - pass - def f(): - a = A() - a.i = 1 - return a.i - t = check_malloc_removal(f, [], [], 1, must_remove=False) - graph = graphof(t, f) - assert graph.startblock.operations[0].opname == "malloc" - + def h(x): + return f(x + 1) + def main(x): + return f(x).x + g(h(x)).x + + t, adi, graph = build_adi(main, [int]) + graphs = malloc_like_graphs(adi) + assert set([g.name for g in graphs]) == set(["f", "h"]) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_mallocprediction.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_mallocprediction.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/backendopt/test/test_mallocprediction.py Thu Nov 5 22:17:18 2009 @@ -54,7 +54,7 @@ assert caller_candidates == {graph: True} assert len(callgraph) == 1 ggraph = graphof(t, g) - assert callgraph[graph] == {ggraph: True} + assert callgraph == {graph: {ggraph: True}} def test_multiple_calls(): class A: @@ -81,9 +81,48 @@ assert len(callgraph) == 1 g1graph = graphof(t, g1) g2graph = graphof(t, g2) - assert callgraph[graph] == {g1graph: True} + assert callgraph == {graph: {g1graph: True}} callgraph, caller_candidates = check_inlining(t, graph, [0], 3 * 42) - assert callgraph[graph] == {g2graph: True} + assert callgraph == {graph: {g2graph: True}} + +def test_malloc_returns(): + class A: + pass + def g(a): + return a.x + def h(x): + return x + 42 + def make_a(x): + a = A() + a.x = x + return a + def fn(i): + a = make_a(h(i)) + return g(a) + t, graph = rtype(fn, [int]) + callgraph, caller_candidates = check_inlining(t, graph, [0], 42) + assert caller_candidates == {graph: True} + assert len(callgraph) == 1 + ggraph = graphof(t, g) + makegraph = graphof(t, make_a) + assert callgraph == {graph: {ggraph: True, makegraph: True}} + +def test_tuple(): + def f(x, y): + return h(x + 1, x * y) + def h(x, y): + return x, y + + def g(x): + a, b = f(x, x*5) + return a + b + t, graph = rtype(g, [int]) + callgraph, caller_candidates = check_inlining(t, graph, [2], 23) + assert caller_candidates == {graph: True} + assert len(callgraph) == 2 + fgraph = graphof(t, f) + hgraph = graphof(t, h) + assert callgraph == {graph: {fgraph: True}, fgraph: {hgraph: True}} def test_indirect_call(): class A(object): @@ -133,9 +172,21 @@ assert total0 + total == 10 def test_richards(): - py.test.skip("Unsure if this makes any sense any more") from pypy.translator.goal.richards import entry_point t, graph = rtype(entry_point, [int]) total0 = preparation(t, t.graphs) total = clever_inlining_and_malloc_removal(t) assert total0 + total == 9 + +def test_loop(): + l = [10, 12, 15, 1] + def f(x): + res = 0 + for i in range(x): + res += i + for i in l: + res += i + return res + t, graph = rtype(f, [int]) + total = clever_inlining_and_malloc_removal(t) + assert total == 3 # range, two iterators Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/database.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/database.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/database.py Thu Nov 5 22:17:18 2009 @@ -288,10 +288,8 @@ finish_callbacks.append(('Stackless transformer: finished', self.stacklesstransformer.finish)) if self.gctransformer: - finish_callbacks.append(('GC transformer: tracking vtables', - self.gctransformer.get_final_dependencies)) finish_callbacks.append(('GC transformer: finished tables', - self.gctransformer.finish_tables)) + self.gctransformer.get_finish_tables())) def add_dependencies(newdependencies): for value in newdependencies: @@ -336,8 +334,18 @@ if finish_callbacks: logmsg, finish = finish_callbacks.pop(0) - newdependencies = finish() - log.database(logmsg) + if not hasattr(finish, 'next'): + newdependencies = finish() + else: + # if 'finish' is a generator, consume the next element + # and put the generator again in the queue + try: + newdependencies = finish.next() + finish_callbacks.insert(0, (None, finish)) + except StopIteration: + newdependencies = None + if logmsg: + log.database(logmsg) if newdependencies: add_dependencies(newdependencies) continue # progress - follow all dependencies again Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/funcgen.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/funcgen.py Thu Nov 5 22:17:18 2009 @@ -714,6 +714,7 @@ from pypy.rpython.lltypesystem.rstr import STR format = [] argv = [] + free_line = "" for arg in op.args: T = arg.concretetype if T == Ptr(STR): @@ -722,6 +723,7 @@ else: format.append('%s') argv.append('RPyString_AsCharP(%s)' % self.expr(arg)) + free_line = "RPyString_FreeCache();" continue elif T == Signed: format.append('%d') @@ -741,9 +743,22 @@ else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) - return "fprintf(stderr, %s%s); RPyString_FreeCache();" % ( - c_string_constant(' '.join(format) + '\n\000'), - ''.join([', ' + s for s in argv])) + argv.insert(0, c_string_constant(' '.join(format) + '\n')) + return ( + "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" + % (', '.join(argv), free_line)) + + def OP_DEBUG_START(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_START(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) + + def OP_DEBUG_STOP(self, op): + arg = op.args[0] + assert isinstance(arg, Constant) + return "PYPY_DEBUG_STOP(%s);" % ( + c_string_constant(''.join(arg.value.chars)),) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gc.py Thu Nov 5 22:17:18 2009 @@ -352,7 +352,7 @@ def need_no_typeptr(self): config = self.db.translator.config - return config.translation.gcconfig.removetypeptr + return config.translation.gcremovetypeptr def OP_GC_GETTYPEPTR_GROUP(self, funcgen, op): # expands to a number of steps, as per rpython/lltypesystem/opimpl.py, Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_asmgcroot.py Thu Nov 5 22:17:18 2009 @@ -20,19 +20,16 @@ return config @classmethod - def _makefunc2(cls, func): + def _makefunc_str_int(cls, func): def main(argv): - arg0 = int(argv[1]) + arg0 = argv[1] arg1 = int(argv[2]) try: res = func(arg0, arg1) except MemoryError: print 'Result: MemoryError' else: - if isinstance(res, int): - print 'Result:', res - else: - print 'Result: "%s"' % (res,) + print 'Result: "%s"' % (res,) return 0 config = cls.make_config() t = TranslationContext(config=config) @@ -56,7 +53,7 @@ redirect = ' 2> NUL' else: redirect = '' - g = os.popen('"%s" %d %d%s' % (exe_name, arg0, arg1, redirect), 'r') + g = os.popen('"%s" %s %d%s' % (exe_name, arg0, arg1, redirect), 'r') for line in g: print >> sys.stderr, 'RUN:', line.rstrip() lines.append(line) @@ -181,3 +178,7 @@ def define_callback_with_collect(cls): return lambda: 0 + +class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, + test_newgc.TestHybridTaggedPointers): + pass Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Nov 5 22:17:18 2009 @@ -350,7 +350,7 @@ # arithmetic operations should not produce GC pointers 'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc', 'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv', - 'bswap', 'bt', + 'bswap', 'bt', 'rdtsc', # zero-extending moves should not produce GC pointers 'movz', ]) @@ -479,6 +479,10 @@ return self._visit_prologue() elif source == self.EBP and target == self.ESP: return self._visit_epilogue() + if source == self.ESP and self.funcname.startswith('VALGRIND_'): + return [] # in VALGRIND_XXX functions, there is a dummy-looking + # mov %esp, %eax. Shows up only when compiling with + # gcc -fno-unit-at-a-time. return self.insns_for_copy(source, target) def visit_pushl(self, line): @@ -1556,6 +1560,7 @@ f = open(fn, 'r') firstline = f.readline() f.seek(0) + assert firstline, "file %r is empty!" % (fn,) if firstline.startswith('seen_main = '): tracker.reload_raw_table(f) f.close() Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Thu Nov 5 22:17:18 2009 @@ -430,12 +430,14 @@ bk = self.translator.annotator.bookkeeper return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph()) - def cmdexec(self, args='', env=None): + def cmdexec(self, args='', env=None, err=False): assert self._compiled res = self.translator.platform.execute(self.executable_name, args, env=env) if res.returncode != 0: raise Exception("Returned %d" % (res.returncode,)) + if err: + return res.out, res.err return res.out def compile(self): @@ -620,10 +622,12 @@ yield self.uniquecname(basecname), subiter() def gen_readable_parts_of_source(self, f): + split_criteria_big = SPLIT_CRITERIA if py.std.sys.platform != "win32": - split_criteria_big = SPLIT_CRITERIA * 4 - else: - split_criteria_big = SPLIT_CRITERIA + if self.database.gcpolicy.need_no_typeptr(): + pass # XXX gcc uses toooooons of memory??? + else: + split_criteria_big = SPLIT_CRITERIA * 4 if self.one_source_file: return gen_readable_parts_of_main_c_file(f, self.database, self.preimpl) @@ -733,7 +737,7 @@ if hasattr(node, 'forward_decl'): if node.forward_decl: print >> f, node.forward_decl - else: + elif node.name is not None: print >> f, '%s %s;' % (node.typetag, node.name) print >> f for node in structdeflist: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/node.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/node.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/node.py Thu Nov 5 22:17:18 2009 @@ -960,8 +960,8 @@ def enum_dependencies(self): # note: for the group used by the GC, it can grow during this phase, - # which means that we might not return all members yet. This is - # fixed by finish_tables() in rpython/memory/gctransform/framework.py + # which means that we might not return all members yet. This is fixed + # by get_finish_tables() in rpython.memory.gctransform.framework. for member in self.obj.members: yield member._as_ptr() Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/asm_gcc_x86.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/asm_gcc_x86.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/asm_gcc_x86.h Thu Nov 5 22:17:18 2009 @@ -50,6 +50,10 @@ : "0"(x), "g"(y) /* inputs */ \ : "cc", "memory") /* clobber */ +/* Pentium only! */ +#define READ_TIMESTAMP(val) \ + asm volatile("rdtsc" : "=A" (val)) + /* prototypes */ Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/g_include.h Thu Nov 5 22:17:18 2009 @@ -51,6 +51,7 @@ /*** modules ***/ #ifdef HAVE_RTYPER /* only if we have an RTyper */ # include "src/rtyper.h" +# include "src/debug.h" #ifndef AVR # include "src/ll_os.h" # include "src/ll_strtod.h" Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py Thu Nov 5 22:17:18 2009 @@ -17,24 +17,45 @@ gcpolicy = "marksweep" should_be_moving = False removetypeptr = False + taggedpointers = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False _isolated_func = None @classmethod - def _makefunc2(cls, f): - t = Translation(f, [int, int], gc=cls.gcpolicy, - policy=annpolicy.StrictAnnotatorPolicy()) - t.config.translation.gcconfig.debugprint = True - t.config.translation.gcconfig.removetypeptr = cls.removetypeptr + def _makefunc_str_int(cls, f): + def main(argv): + arg0 = argv[1] + arg1 = int(argv[2]) + try: + res = f(arg0, arg1) + except MemoryError: + print "MEMORY-ERROR" + else: + print res + return 0 + + t = Translation(main, standalone=True, gc=cls.gcpolicy, + policy=annpolicy.StrictAnnotatorPolicy(), + taggedpointers=cls.taggedpointers, + gcremovetypeptr=cls.removetypeptr) t.disable(['backendopt']) - t.set_backend_extra_options(c_isolated=True, c_debug_defines=True) + t.set_backend_extra_options(c_debug_defines=True) t.rtype() if conftest.option.view: t.viewcg() - isolated_func = t.compile() - return isolated_func + exename = t.compile() + + def run(s, i): + data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + data = data.strip() + if data == 'MEMORY-ERROR': + raise MemoryError + return data + + return run + def setup_class(cls): funcs0 = [] @@ -64,8 +85,8 @@ funcs1.append(func) assert name not in name_to_func name_to_func[name] = len(name_to_func) - def allfuncs(num, arg): - rgc.collect() + def allfuncs(name, arg): + num = name_to_func[name] func0 = funcs0[num] if func0: return str(func0()) @@ -77,7 +98,7 @@ return funcstr(arg) assert 0, 'unreachable' cls.funcsstr = funcsstr - cls.c_allfuncs = staticmethod(cls._makefunc2(allfuncs)) + cls.c_allfuncs = staticmethod(cls._makefunc_str_int(allfuncs)) cls.allfuncs = staticmethod(allfuncs) cls.name_to_func = name_to_func @@ -88,10 +109,9 @@ def run(self, name, *args): if not args: args = (-1, ) + print 'Running %r)' % name + res = self.c_allfuncs(name, *args) num = self.name_to_func[name] - print - print 'Running %r (test number %d)' % (name, num) - res = self.c_allfuncs(num, *args) if self.funcsstr[num]: return res return int(res) @@ -99,8 +119,8 @@ def run_orig(self, name, *args): if not args: args = (-1, ) - num = self.name_to_func[name] - res = self.allfuncs(num, *args) + res = self.allfuncs(name, *args) + num = self.name_to_func[name] if self.funcsstr[num]: return res return int(res) @@ -660,7 +680,6 @@ TP = lltype.GcArray(lltype.Char) def func(): try: - from pypy.rlib import rgc a = rgc.malloc_nonmovable(TP, 3) rgc.collect() if a: @@ -925,9 +944,12 @@ def test_gc_set_max_heap_size(self): py.test.skip("not implemented") + + class TestHybridGCRemoveTypePtr(TestHybridGC): removetypeptr = True + class TestMarkCompactGC(TestSemiSpaceGC): gcpolicy = "markcompact" should_be_moving = True @@ -940,3 +962,52 @@ def test_finalizer_order(self): py.test.skip("not implemented") + +# ____________________________________________________________________ + +class TestHybridTaggedPointers(TestHybridGC): + taggedpointers = True + + + def define_tagged(cls): + class Unrelated(object): + pass + + u = Unrelated() + u.x = UnboxedObject(47) + def fn(n): + rgc.collect() # check that a prebuilt tagged pointer doesn't explode + if n > 0: + x = BoxedObject(n) + else: + x = UnboxedObject(n) + u.x = x # invoke write barrier + rgc.collect() + return x.meth(100) + def func(): + return fn(1000) + fn(-1000) + return func + + def test_tagged(self): + expected = self.run_orig("tagged") + res = self.run("tagged") + assert res == expected + +from pypy.rlib.objectmodel import UnboxedValue + +class TaggedBase(object): + __slots__ = () + def meth(self, x): + raise NotImplementedError + +class BoxedObject(TaggedBase): + attrvalue = 66 + def __init__(self, normalint): + self.normalint = normalint + def meth(self, x): + return self.normalint + x + 2 + +class UnboxedObject(TaggedBase, UnboxedValue): + __slots__ = 'smallint' + def meth(self, x): + return self.smallint + x + 3 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_rtagged.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_rtagged.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_rtagged.py Thu Nov 5 22:17:18 2009 @@ -38,7 +38,7 @@ def entry_point(argv): n = 100 + len(argv) - assert C(n).getvalue() == n + assert C(n).get_untagged_value() == n x = makeint(42) assert isinstance(x, C) @@ -67,7 +67,7 @@ from pypy import conftest def test_tagged_boehm(): - t = Translation(entry_point, standalone=True, gc='boehm') + t = Translation(entry_point, standalone=True, gc='boehm', taggedpointers=True) try: exename = str(t.compile_c()) finally: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_standalone.py Thu Nov 5 22:17:18 2009 @@ -2,7 +2,8 @@ import sys, os, re from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.debug import ll_assert, debug_print +from pypy.rlib.debug import ll_assert, have_debug_prints +from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.translator.translator import TranslationContext from pypy.translator.backendopt import all from pypy.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo @@ -11,9 +12,22 @@ from pypy.tool.autopath import pypydir -class TestStandalone(object): +class StandaloneTests(object): config = None - + + def compile(self, entry_point): + t = TranslationContext(self.config) + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper().specialize() + + cbuilder = CStandaloneBuilder(t, entry_point, t.config) + cbuilder.generate_source() + cbuilder.compile() + return t, cbuilder + + +class TestStandalone(StandaloneTests): + def test_hello_world(self): def entry_point(argv): os.write(1, "hello world\n") @@ -23,13 +37,7 @@ os.write(1, " '" + str(s) + "'\n") return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello world\nargument count: 2\n 'hi'\n 'there'\n''') @@ -43,13 +51,7 @@ print [len(s) for s in argv] return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.startswith('''hello simpler world\n''' '''argument count: 2\n''' @@ -130,13 +132,7 @@ print m, x return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert map(float, data.split()) == [0.0, 0.0] @@ -173,13 +169,8 @@ os.setpgrp() return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) + cbuilder.cmdexec("") def test_profopt_mac_osx_bug(self): @@ -223,12 +214,7 @@ print "BAD POS" os.close(fd) return 0 - t = TranslationContext(self.config) - t.buildannotator().build_types(entry_point, [s_list_of_strings]) - t.buildrtyper().specialize() - cbuilder = CStandaloneBuilder(t, entry_point, t.config) - cbuilder.generate_source() - cbuilder.compile() + t, cbuilder = self.compile(entry_point) data = cbuilder.cmdexec('hi there') assert data.strip() == "OK" @@ -270,6 +256,128 @@ assert " ll_strtod.h" in makefile assert " ll_strtod.o" in makefile + def test_debug_print_start_stop(self): + def entry_point(argv): + x = "got:" + debug_start ("mycat") + if have_debug_prints(): x += "b" + debug_print ("foo", 2, "bar", 3) + debug_start ("cat2") + if have_debug_prints(): x += "c" + debug_print ("baz") + debug_stop ("cat2") + if have_debug_prints(): x += "d" + debug_print ("bok") + debug_stop ("mycat") + if have_debug_prints(): x += "a" + debug_print("toplevel") + os.write(1, x + '.\n') + return 0 + t, cbuilder = self.compile(entry_point) + # check with PYPYLOG undefined + out, err = cbuilder.cmdexec("", err=True, env={}) + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err + # check with PYPYLOG defined to an empty string (same as undefined) + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ''}) + assert out.strip() == 'got:a.' + assert 'toplevel' in err + assert 'mycat' not in err + assert 'foo 2 bar 3' not in err + assert 'cat2' not in err + assert 'baz' not in err + assert 'bok' not in err + # check with PYPYLOG=:- (means print to stderr) + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': ':-'}) + assert out.strip() == 'got:bcda.' + assert 'toplevel' in err + assert '{mycat' in err + assert 'mycat}' in err + assert 'foo 2 bar 3' in err + assert '{cat2' in err + assert 'cat2}' in err + assert 'baz' in err + assert 'bok' in err + # check with PYPYLOG=:somefilename + path = udir.join('test_debug_xxx.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:bcda.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data + assert '{cat2' in data + assert 'cat2}' in data + assert 'baz' in data + assert 'bok' in data + # check with PYPYLOG=somefilename + path = udir.join('test_debug_xxx_prof.log') + out, err = cbuilder.cmdexec("", err=True, env={'PYPYLOG': str(path)}) + assert out.strip() == 'got:a.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' not in data + assert '{cat2' in data + assert 'cat2}' in data + assert 'baz' not in data + assert 'bok' not in data + # check with PYPYLOG=myc:somefilename (includes mycat but not cat2) + path = udir.join('test_debug_xxx_myc.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'myc:%s' % path}) + assert out.strip() == 'got:bda.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data + assert 'cat2' not in data + assert 'baz' not in data + assert 'bok' in data + # check with PYPYLOG=cat:somefilename (includes cat2 but not mycat) + path = udir.join('test_debug_xxx_cat.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'cat:%s' % path}) + assert out.strip() == 'got:a.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in path.read() + assert 'mycat' not in path.read() + assert 'foo 2 bar 3' not in path.read() + assert 'cat2' not in data # because it is nested + assert 'baz' not in data + assert 'bok' not in data + # + # finally, check compiling with logging disabled + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.log = False + self.config = config + t, cbuilder = self.compile(entry_point) + path = udir.join('test_debug_does_not_show_up.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': ':%s' % path}) + assert out.strip() == 'got:.' + assert not err + assert path.check(file=0) + + class TestMaemo(TestStandalone): def setup_class(cls): from pypy.translator.platform.maemo import check_scratchbox Modified: pypy/branch/msvc-asmgcroot/pypy/translator/cli/ilgenerator.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/cli/ilgenerator.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/cli/ilgenerator.py Thu Nov 5 22:17:18 2009 @@ -400,7 +400,10 @@ ilasm.opcode('ldc.i4', ord(value)) elif TYPE is ootype.Float: if isinf(value): - ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') + if value < 0.0: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 ff)') + else: + ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f0 7f)') elif isnan(value): ilasm.opcode('ldc.r8', '(00 00 00 00 00 00 f8 ff)') else: Modified: pypy/branch/msvc-asmgcroot/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/cli/metavm.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/cli/metavm.py Thu Nov 5 22:17:18 2009 @@ -270,6 +270,10 @@ generator.ilasm.call('void [pypylib]pypy.runtime.Utils::debug_print(%s)' % signature) +class _HaveDebugPrints(MicroInstruction): + def render(self, generator, op): + generator.ilasm.load_const(ootype.Bool, True) + OOTYPE_TO_MNEMONIC = { ootype.Bool: 'i1', @@ -306,3 +310,4 @@ SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() DebugPrint = _DebugPrint() +HaveDebugPrints = _HaveDebugPrints() Modified: pypy/branch/msvc-asmgcroot/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/cli/opcodes.py Thu Nov 5 22:17:18 2009 @@ -1,7 +1,8 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, DebugPrint + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, \ + DebugPrint, HaveDebugPrints from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -77,6 +78,9 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'debug_print': [DebugPrint], + 'debug_start': Ignore, + 'debug_stop': Ignore, + 'have_debug_prints': [HaveDebugPrints], 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, Modified: pypy/branch/msvc-asmgcroot/pypy/translator/cli/sdk.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/cli/sdk.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/cli/sdk.py Thu Nov 5 22:17:18 2009 @@ -34,12 +34,35 @@ CSC = 'csc' PEVERIFY = 'peverify' +def get_mono_version(): + from commands import getoutput + lines = getoutput('mono -V').splitlines() + parts = lines[0].split() + # something like ['Mono', 'JIT', 'compiler', 'version', '2.4.2.3', ...] + iversion = parts.index('version') + ver = parts[iversion+1] # '2.4.2.3' + ver = ver.split('.') # ['2', '4', '2', '3'] + return tuple(map(int, ver)) # (2, 4, 2, 3) + + class MonoSDK(AbstractSDK): RUNTIME = ['mono'] ILASM = 'ilasm2' CSC = 'gmcs' PEVERIFY = 'peverify' # it's not part of mono, but we get a meaningful skip message + # this is a workaround for this bug: + # https://bugzilla.novell.com/show_bug.cgi?id=474718 they promised that it + # should be fixed in versions after 2.4.3.x, in the meanwhile pass + # -O=-branch + @classmethod + def runtime(cls): + cls._check_helper('mono') + ver = get_mono_version() + if (2, 1) < ver < (2, 4, 3): + return ['mono', '-O=-branch'] + return ['mono'] + def key_as_dict(handle): import _winreg i = 0 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/jvm/genjvm.py Thu Nov 5 22:17:18 2009 @@ -68,6 +68,7 @@ The following attributes also exist to find the state of the sources: compiled --- True once the sources have been compiled successfully """ + _cached = None def __init__(self, tmpdir, package): """ @@ -117,29 +118,20 @@ def _compile_helper(self): # HACK: compile the Java helper classes. Should eventually # use rte.py + if JvmGeneratedSource._cached == self.classdir: + return + log.red('Compiling java classes') javafiles = self.srcdir.listdir('*.java') - classfiles = self.srcdir.listdir('*.class') - - recompile = True - if len(classfiles) == len(javafiles): - last_modified_java = max([java.mtime() for java in javafiles]) - last_modified_class = max([cls.mtime() for cls in classfiles]) - if last_modified_java < last_modified_class: - recompile = False - - if recompile: - log.red('Compiling java classes') - javasrcs = [str(jf) for jf in javafiles] - self._invoke([getoption('javac'), - '-nowarn', - '-d', str(self.rootdir), - '-classpath', str(self.jnajar) - ] + javasrcs, - True) - - # copy .class files to classdir - for classfile in self.srcdir.listdir('*.class'): - classfile.copy(self.classdir.join('pypy')) + javasrcs = [str(jf) for jf in javafiles] + self._invoke([getoption('javac'), + '-nowarn', + '-d', str(self.classdir), + '-classpath', str(self.jnajar), + ] + javasrcs, + True) + # NOTE if you are trying to add more caching: some .java files + # compile to several .class files of various names. + JvmGeneratedSource._cached = self.classdir def compile(self): """ Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/linux.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/linux.py Thu Nov 5 22:17:18 2009 @@ -8,7 +8,7 @@ class Linux(BasePosix): name = "linux" - link_flags = ['-pthread'] + link_flags = ['-pthread', '-lrt'] cflags = ['-O3', '-pthread', '-fomit-frame-pointer'] standalone_only = [] shared_only = [] From afa at codespeak.net Thu Nov 5 23:13:50 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 5 Nov 2009 23:13:50 +0100 (CET) Subject: [pypy-svn] r69010 - in pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc: . test/msvc Message-ID: <20091105221350.8F8B2168435@codespeak.net> Author: afa Date: Thu Nov 5 23:13:47 2009 New Revision: 69010 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s (contents, props changed) pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s (contents, props changed) Log: fixeol Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py Thu Nov 5 23:13:47 2009 @@ -1,223 +1,223 @@ -LOC_NOWHERE = 0 -LOC_REG = 1 -LOC_EBP_BASED = 2 -LOC_ESP_BASED = 3 -LOC_MASK = 0x03 - -def frameloc(base, offset): - assert base in (LOC_EBP_BASED, LOC_ESP_BASED) - assert offset % 4 == 0 - return base | offset - - -class SomeNewValue(object): - pass -somenewvalue = SomeNewValue() - -class LocalVar(object): - # A local variable location at position 'ofs_from_frame_end', - # which is counted from the end of the stack frame (so it is always - # negative, unless it refers to arguments of the current function). - def __init__(self, ofs_from_frame_end, hint=None): - self.ofs_from_frame_end = ofs_from_frame_end - self.hint = hint - - def __repr__(self): - return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') - - def __hash__(self): - return hash(self.ofs_from_frame_end) - - def __cmp__(self, other): - if isinstance(other, LocalVar): - return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) - else: - return 1 - - def getlocation(self, framesize, uses_frame_pointer): - if (self.hint == 'esp' or not uses_frame_pointer - or self.ofs_from_frame_end % 2 != 0): - # try to use esp-relative addressing - ofs_from_esp = framesize + self.ofs_from_frame_end - if ofs_from_esp % 2 == 0: - return frameloc(LOC_ESP_BASED, ofs_from_esp) - # we can get an odd value if the framesize is marked as bogus - # by visit_andl() - assert uses_frame_pointer - ofs_from_ebp = self.ofs_from_frame_end + 4 - return frameloc(LOC_EBP_BASED, ofs_from_ebp) - - -class Insn(object): - _args_ = [] - _locals_ = [] - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join([str(getattr(self, name)) - for name in self._args_])) - def requestgcroots(self, tracker): - return {} - - def source_of(self, localvar, tag): - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class Label(Insn): - _args_ = ['label', 'lineno'] - def __init__(self, label, lineno): - self.label = label - self.lineno = lineno - self.previous_insns = [] # all insns that jump (or fallthrough) here - -class InsnFunctionStart(Insn): - framesize = 0 - previous_insns = () - def __init__(self, registers): - self.arguments = {} - for reg in registers: - self.arguments[reg] = somenewvalue - - def source_of(self, localvar, tag): - if localvar not in self.arguments: - if localvar in ('%eax', '%edx', '%ecx'): - # xxx this might show a bug in trackgcroot.py failing to - # figure out which instruction stored a value in these - # registers. However, this case also occurs when the - # the function's calling convention was optimized by gcc: - # the 3 registers above are then used to pass arguments - pass - else: - assert (isinstance(localvar, LocalVar) and - localvar.ofs_from_frame_end > 0), ( - "must come from an argument to the function, got %r" % - (localvar,)) - self.arguments[localvar] = somenewvalue - return self.arguments[localvar] - - def all_sources_of(self, localvar): - return [] - -class InsnSetLocal(Insn): - _args_ = ['target', 'sources'] - _locals_ = ['target', 'sources'] - - def __init__(self, target, sources=()): - self.target = target - self.sources = sources - - def source_of(self, localvar, tag): - if localvar == self.target: - return somenewvalue - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return self.sources - return [localvar] - -class InsnCopyLocal(Insn): - _args_ = ['source', 'target'] - _locals_ = ['source', 'target'] - - def __init__(self, source, target): - self.source = source - self.target = target - - def source_of(self, localvar, tag): - if localvar == self.target: - return self.source - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return [self.source] - return [localvar] - -class InsnStackAdjust(Insn): - _args_ = ['delta'] - def __init__(self, delta): - assert delta % 2 == 0 # should be "% 4", but there is the special - self.delta = delta # case of 'pushw' to handle - -class InsnCannotFollowEsp(InsnStackAdjust): - def __init__(self): - self.delta = -7 # use an odd value as marker - -class InsnStop(Insn): - pass - -class InsnRet(InsnStop): - framesize = 0 - def __init__(self, registers): - self.registers = registers - - def requestgcroots(self, tracker): - # no need to track the value of these registers in the caller - # function if we are the main(), or if we are flagged as a - # "bottom" function (a callback from C code) - if tracker.is_stack_bottom: - return {} - else: - return dict(zip(self.registers, self.registers)) - -class InsnCall(Insn): - _args_ = ['lineno', 'gcroots'] - def __init__(self, lineno): - # 'gcroots' is a dict built by side-effect during the call to - # FunctionGcRootTracker.trackgcroots(). Its meaning is as - # follows: the keys are the locations that contain gc roots - # (register names or LocalVar instances). The value - # corresponding to a key is the "tag", which is None for a - # normal gc root, or else the name of a callee-saved register. - # In the latter case it means that this is only a gc root if the - # corresponding register in the caller was really containing a - # gc pointer. A typical example: - # - # InsnCall({LocalVar(-8)': None, - # '%esi': '%esi', - # LocalVar(-12)': '%ebx'}) - # - # means that the value at -8 from the frame end is a gc root - # across this call; that %esi is a gc root if it was in the - # caller (typically because %esi is not modified at all in the - # current function); and that the value at -12 from the frame - # end is a gc root if %ebx was a gc root in the caller - # (typically because the current function saves and restores - # %ebx from there in the prologue and epilogue). - self.gcroots = {} - self.lineno = lineno - - def source_of(self, localvar, tag): - tag1 = self.gcroots.setdefault(localvar, tag) - assert tag1 == tag, ( - "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( - localvar, tag1, tag)) - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class InsnGCROOT(Insn): - _args_ = ['loc'] - _locals_ = ['loc'] - def __init__(self, loc): - self.loc = loc - def requestgcroots(self, tracker): - return {self.loc: None} - -class InsnPrologue(Insn): - def __setattr__(self, attr, value): - if attr == 'framesize': - assert value == 4, ("unrecognized function prologue - " - "only supports push %ebp; movl %esp, %ebp") - Insn.__setattr__(self, attr, value) - -class InsnEpilogue(Insn): - def __init__(self, framesize=None): - if framesize is not None: - self.framesize = framesize - - +LOC_NOWHERE = 0 +LOC_REG = 1 +LOC_EBP_BASED = 2 +LOC_ESP_BASED = 3 +LOC_MASK = 0x03 + +def frameloc(base, offset): + assert base in (LOC_EBP_BASED, LOC_ESP_BASED) + assert offset % 4 == 0 + return base | offset + + +class SomeNewValue(object): + pass +somenewvalue = SomeNewValue() + +class LocalVar(object): + # A local variable location at position 'ofs_from_frame_end', + # which is counted from the end of the stack frame (so it is always + # negative, unless it refers to arguments of the current function). + def __init__(self, ofs_from_frame_end, hint=None): + self.ofs_from_frame_end = ofs_from_frame_end + self.hint = hint + + def __repr__(self): + return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') + + def __hash__(self): + return hash(self.ofs_from_frame_end) + + def __cmp__(self, other): + if isinstance(other, LocalVar): + return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) + else: + return 1 + + def getlocation(self, framesize, uses_frame_pointer): + if (self.hint == 'esp' or not uses_frame_pointer + or self.ofs_from_frame_end % 2 != 0): + # try to use esp-relative addressing + ofs_from_esp = framesize + self.ofs_from_frame_end + if ofs_from_esp % 2 == 0: + return frameloc(LOC_ESP_BASED, ofs_from_esp) + # we can get an odd value if the framesize is marked as bogus + # by visit_andl() + assert uses_frame_pointer + ofs_from_ebp = self.ofs_from_frame_end + 4 + return frameloc(LOC_EBP_BASED, ofs_from_ebp) + + +class Insn(object): + _args_ = [] + _locals_ = [] + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, + ', '.join([str(getattr(self, name)) + for name in self._args_])) + def requestgcroots(self, tracker): + return {} + + def source_of(self, localvar, tag): + return localvar + + def all_sources_of(self, localvar): + return [localvar] + +class Label(Insn): + _args_ = ['label', 'lineno'] + def __init__(self, label, lineno): + self.label = label + self.lineno = lineno + self.previous_insns = [] # all insns that jump (or fallthrough) here + +class InsnFunctionStart(Insn): + framesize = 0 + previous_insns = () + def __init__(self, registers): + self.arguments = {} + for reg in registers: + self.arguments[reg] = somenewvalue + + def source_of(self, localvar, tag): + if localvar not in self.arguments: + if localvar in ('%eax', '%edx', '%ecx'): + # xxx this might show a bug in trackgcroot.py failing to + # figure out which instruction stored a value in these + # registers. However, this case also occurs when the + # the function's calling convention was optimized by gcc: + # the 3 registers above are then used to pass arguments + pass + else: + assert (isinstance(localvar, LocalVar) and + localvar.ofs_from_frame_end > 0), ( + "must come from an argument to the function, got %r" % + (localvar,)) + self.arguments[localvar] = somenewvalue + return self.arguments[localvar] + + def all_sources_of(self, localvar): + return [] + +class InsnSetLocal(Insn): + _args_ = ['target', 'sources'] + _locals_ = ['target', 'sources'] + + def __init__(self, target, sources=()): + self.target = target + self.sources = sources + + def source_of(self, localvar, tag): + if localvar == self.target: + return somenewvalue + return localvar + + def all_sources_of(self, localvar): + if localvar == self.target: + return self.sources + return [localvar] + +class InsnCopyLocal(Insn): + _args_ = ['source', 'target'] + _locals_ = ['source', 'target'] + + def __init__(self, source, target): + self.source = source + self.target = target + + def source_of(self, localvar, tag): + if localvar == self.target: + return self.source + return localvar + + def all_sources_of(self, localvar): + if localvar == self.target: + return [self.source] + return [localvar] + +class InsnStackAdjust(Insn): + _args_ = ['delta'] + def __init__(self, delta): + assert delta % 2 == 0 # should be "% 4", but there is the special + self.delta = delta # case of 'pushw' to handle + +class InsnCannotFollowEsp(InsnStackAdjust): + def __init__(self): + self.delta = -7 # use an odd value as marker + +class InsnStop(Insn): + pass + +class InsnRet(InsnStop): + framesize = 0 + def __init__(self, registers): + self.registers = registers + + def requestgcroots(self, tracker): + # no need to track the value of these registers in the caller + # function if we are the main(), or if we are flagged as a + # "bottom" function (a callback from C code) + if tracker.is_stack_bottom: + return {} + else: + return dict(zip(self.registers, self.registers)) + +class InsnCall(Insn): + _args_ = ['lineno', 'gcroots'] + def __init__(self, lineno): + # 'gcroots' is a dict built by side-effect during the call to + # FunctionGcRootTracker.trackgcroots(). Its meaning is as + # follows: the keys are the locations that contain gc roots + # (register names or LocalVar instances). The value + # corresponding to a key is the "tag", which is None for a + # normal gc root, or else the name of a callee-saved register. + # In the latter case it means that this is only a gc root if the + # corresponding register in the caller was really containing a + # gc pointer. A typical example: + # + # InsnCall({LocalVar(-8)': None, + # '%esi': '%esi', + # LocalVar(-12)': '%ebx'}) + # + # means that the value at -8 from the frame end is a gc root + # across this call; that %esi is a gc root if it was in the + # caller (typically because %esi is not modified at all in the + # current function); and that the value at -12 from the frame + # end is a gc root if %ebx was a gc root in the caller + # (typically because the current function saves and restores + # %ebx from there in the prologue and epilogue). + self.gcroots = {} + self.lineno = lineno + + def source_of(self, localvar, tag): + tag1 = self.gcroots.setdefault(localvar, tag) + assert tag1 == tag, ( + "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( + localvar, tag1, tag)) + return localvar + + def all_sources_of(self, localvar): + return [localvar] + +class InsnGCROOT(Insn): + _args_ = ['loc'] + _locals_ = ['loc'] + def __init__(self, loc): + self.loc = loc + def requestgcroots(self, tracker): + return {self.loc: None} + +class InsnPrologue(Insn): + def __setattr__(self, attr, value): + if attr == 'framesize': + assert value == 4, ("unrecognized function prologue - " + "only supports push %ebp; movl %esp, %ebp") + Insn.__setattr__(self, attr, value) + +class InsnEpilogue(Insn): + def __init__(self, framesize=None): + if framesize is not None: + self.framesize = framesize + + Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s Thu Nov 5 23:13:47 2009 @@ -1,282 +1,282 @@ -; Function compile flags: /Ogtpy -; COMDAT _pypy_g_ll_join_strs__Signed_arrayPtr -_TEXT SEGMENT -_l_result_2$ = -8 ; size = 4 -_l_v405$ = -4 ; size = 4 -_l_num_items_0$ = 8 ; size = 4 -_l_items_2$ = 12 ; size = 4 -_pypy_g_ll_join_strs__Signed_arrayPtr PROC ; COMDAT - -; 1457 : struct pypy_rpy_string0 *pypy_g_ll_join_strs__Signed_arrayPtr(long l_num_items_0, struct pypy_array0 *l_items_2) { - - sub esp, 8 - push ebx - push ebp - push esi - -; 1458 : long l_i_22; long l_i_23; long l_res_index_0; -; 1459 : struct pypy_rpy_string0 *l_result_2; bool_t l_v403; bool_t l_v404; -; 1460 : bool_t l_v409; bool_t l_v410; bool_t l_v411; long l_v402; -; 1461 : long l_v414; long l_v417; long l_v418; long l_v421; long l_v422; -; 1462 : long l_v423; struct pypy_object_vtable0 *l_v408; -; 1463 : struct pypy_rpy_string0 *l_v412; struct pypy_rpy_string0 *l_v413; -; 1464 : struct pypy_rpy_string0 *l_v415; struct pypy_rpy_string0 *l_v419; -; 1465 : struct pypy_rpy_string0 *l_v420; struct pypy_rpy_string0 *l_v424; -; 1466 : void* l_v405; void* l_v406; -; 1467 : -; 1468 : block0: -; 1469 : l_i_23 = 0L; - - xor esi, esi - push edi - -; 1470 : l_v402 = 0L; - - xor edi, edi - -; 1471 : goto block1; -; 1472 : -; 1473 : block1: -; 1474 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); - - cmp DWORD PTR _l_num_items_0$[esp+20], esi -$block0$40039: -$block1$40040: - -; 1475 : l_v404 = l_v403; -; 1476 : while (l_v404) { - - jle SHORT $block2$40046 - mov ebp, DWORD PTR _l_items_2$[esp+20] - add ebp, 8 -$LL5 at pypy_g_ll_@139: -$block6$40044: - -; 1525 : goto block3_back; -; 1526 : -; 1527 : block6: -; 1528 : l_v419 = RPyItem(l_items_2, l_i_23); - - test esi, esi - jl SHORT $LN14 at pypy_g_ll_@139 - mov eax, DWORD PTR _l_items_2$[esp+20] - cmp esi, DWORD PTR [eax+4] - jl SHORT $LN15 at pypy_g_ll_@139 -$LN14 at pypy_g_ll_@139: - call _RPyAbort - ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} -$LN15 at pypy_g_ll_@139: - -; 1529 : l_v420 = l_v419; - - mov ebx, DWORD PTR [ebp] - -; 1530 : l_v421 = RPyField(l_v420, rs_chars).length; - - test ebx, ebx - jne SHORT $LN16 at pypy_g_ll_@139 - call _RPyAbort - ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} -$LN16 at pypy_g_ll_@139: - -; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); -; 1532 : OP_INT_ADD(l_i_23, 1L, l_v423); -; 1533 : l_i_23 = l_v423; - - inc esi - add ebp, 4 - -; 1534 : l_v402 = l_v422; - - add edi, DWORD PTR [ebx+8] - cmp esi, DWORD PTR _l_num_items_0$[esp+20] -$block1_back$40045: - jl SHORT $LL5 at pypy_g_ll_@139 -$block2$40046: - -; 1477 : goto block6; -; 1478 : block1_back: ; -; 1479 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); -; 1480 : l_v404 = l_v403; -; 1481 : } -; 1482 : goto block2; -; 1483 : -; 1484 : block2: -; 1485 : l_result_2 = pypy_g_mallocstr__Signed(l_v402); - - push edi - call _pypy_g_mallocstr__Signed - ;; expected {28(%esp) | 16(%esp), 8(%esp), 4(%esp), 12(%esp) | 36(%esp)} - -; 1486 : l_v405 = (void*)l_items_2; - - mov ecx, DWORD PTR _l_items_2$[esp+24] - add esp, 4 - mov DWORD PTR _l_result_2$[esp+24], eax - mov DWORD PTR _l_v405$[esp+24], ecx - -; 1487 : l_v406 = pypy_asm_gcroot(l_v405); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v405$[esp+24] - -; 1488 : l_items_2 = l_v406; /* for moving GCs */ -; 1489 : l_v408 = RPyField((&pypy_g_ExcData), ed_exc_type); -; 1490 : l_v409 = (l_v408 == NULL); - - cmp DWORD PTR _pypy_g_ExcData, 0 - -; 1491 : if (!l_v409) { - - je SHORT $LN3 at pypy_g_ll_@139 - pop edi - pop esi - pop ebp - -; 1492 : l_v424 = ((struct pypy_rpy_string0 *) NULL); - - xor eax, eax - pop ebx - -; 1535 : goto block1_back; -; 1536 : } - - add esp, 8 - ret 0 -$LN3 at pypy_g_ll_@139: - -; 1493 : goto block4; -; 1494 : } -; 1495 : l_i_22 = 0L; - - xor esi, esi - -; 1496 : l_res_index_0 = 0L; - - xor ebp, ebp - -; 1497 : goto block3; -; 1498 : -; 1499 : block3: -; 1500 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); - - cmp DWORD PTR _l_num_items_0$[esp+20], ebp -$block3$40053: - -; 1501 : l_v411 = l_v410; -; 1502 : while (l_v411) { - - jle SHORT $LN1 at pypy_g_ll_@139 - mov ebx, ecx - add ebx, 8 -$LL2 at pypy_g_ll_@139: -$block5$40057: - -; 1514 : -; 1515 : block5: -; 1516 : l_v412 = RPyItem(l_items_2, l_i_22); - - test esi, esi - jl SHORT $LN9 at pypy_g_ll_@139 - mov edx, DWORD PTR _l_items_2$[esp+20] - cmp esi, DWORD PTR [edx+4] - jl SHORT $LN10 at pypy_g_ll_@139 -$LN9 at pypy_g_ll_@139: - call _RPyAbort - ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } -$LN10 at pypy_g_ll_@139: - -; 1517 : l_v413 = l_v412; - - mov edi, DWORD PTR [ebx] - -; 1518 : l_v414 = RPyField(l_v413, rs_chars).length; - - test edi, edi - jne SHORT $LN11 at pypy_g_ll_@139 - call _RPyAbort - ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } -$LN11 at pypy_g_ll_@139: - mov edi, DWORD PTR [edi+8] - -; 1519 : l_v415 = RPyItem(l_items_2, l_i_22); - - test esi, esi - jl SHORT $LN12 at pypy_g_ll_@139 - mov eax, DWORD PTR _l_items_2$[esp+20] - cmp esi, DWORD PTR [eax+4] - jl SHORT $LN13 at pypy_g_ll_@139 -$LN12 at pypy_g_ll_@139: - call _RPyAbort - ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } -$LN13 at pypy_g_ll_@139: - -; 1520 : pypy_g_copy_string_contents__rpy_stringPtr_rpy_stringPt(l_v415, l_result_2, 0L, l_res_index_0, l_v414); - - mov ecx, DWORD PTR [ebx] - mov edx, DWORD PTR _l_result_2$[esp+24] - push edi - add ecx, 12 ; 0000000cH - push ecx - lea eax, DWORD PTR [edx+ebp+12] - push eax -$block0$80664: -$block0$80634: -$block0$80639: -$block1$80640: -$block0$80644: -$block1$80645: -$block1$80635: -$block0$80659: -$block0$80667: -$block1$80668: -$block0$80673: -$block1$80674: -$block1$80661: -$block0$80678: -$block1$80679: - call _memcpy - ;; expected {36(%esp) | 24(%esp), 16(%esp), 12(%esp), 20(%esp) | } - add esp, 12 ; 0000000cH - -; 1521 : OP_INT_ADD(l_res_index_0, l_v414, l_v417); -; 1522 : OP_INT_ADD(l_i_22, 1L, l_v418); -; 1523 : l_i_22 = l_v418; - - inc esi - add ebx, 4 - -; 1524 : l_res_index_0 = l_v417; - - add ebp, edi - cmp esi, DWORD PTR _l_num_items_0$[esp+20] -$block1$80671: -$block3_back$40058: - jl SHORT $LL2 at pypy_g_ll_@139 -$LN1 at pypy_g_ll_@139: - -; 1503 : goto block5; -; 1504 : block3_back: ; -; 1505 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); -; 1506 : l_v411 = l_v410; -; 1507 : } -; 1508 : l_v424 = l_result_2; -; 1509 : goto block4; -; 1510 : -; 1511 : block4: -; 1512 : RPY_DEBUG_RETURN(); -; 1513 : return l_v424; - - mov eax, DWORD PTR _l_result_2$[esp+24] - pop edi - pop esi - pop ebp -$block4$40052: - pop ebx - -; 1535 : goto block1_back; -; 1536 : } - - add esp, 8 - ret 0 -_pypy_g_ll_join_strs__Signed_arrayPtr ENDP +; Function compile flags: /Ogtpy +; COMDAT _pypy_g_ll_join_strs__Signed_arrayPtr +_TEXT SEGMENT +_l_result_2$ = -8 ; size = 4 +_l_v405$ = -4 ; size = 4 +_l_num_items_0$ = 8 ; size = 4 +_l_items_2$ = 12 ; size = 4 +_pypy_g_ll_join_strs__Signed_arrayPtr PROC ; COMDAT + +; 1457 : struct pypy_rpy_string0 *pypy_g_ll_join_strs__Signed_arrayPtr(long l_num_items_0, struct pypy_array0 *l_items_2) { + + sub esp, 8 + push ebx + push ebp + push esi + +; 1458 : long l_i_22; long l_i_23; long l_res_index_0; +; 1459 : struct pypy_rpy_string0 *l_result_2; bool_t l_v403; bool_t l_v404; +; 1460 : bool_t l_v409; bool_t l_v410; bool_t l_v411; long l_v402; +; 1461 : long l_v414; long l_v417; long l_v418; long l_v421; long l_v422; +; 1462 : long l_v423; struct pypy_object_vtable0 *l_v408; +; 1463 : struct pypy_rpy_string0 *l_v412; struct pypy_rpy_string0 *l_v413; +; 1464 : struct pypy_rpy_string0 *l_v415; struct pypy_rpy_string0 *l_v419; +; 1465 : struct pypy_rpy_string0 *l_v420; struct pypy_rpy_string0 *l_v424; +; 1466 : void* l_v405; void* l_v406; +; 1467 : +; 1468 : block0: +; 1469 : l_i_23 = 0L; + + xor esi, esi + push edi + +; 1470 : l_v402 = 0L; + + xor edi, edi + +; 1471 : goto block1; +; 1472 : +; 1473 : block1: +; 1474 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); + + cmp DWORD PTR _l_num_items_0$[esp+20], esi +$block0$40039: +$block1$40040: + +; 1475 : l_v404 = l_v403; +; 1476 : while (l_v404) { + + jle SHORT $block2$40046 + mov ebp, DWORD PTR _l_items_2$[esp+20] + add ebp, 8 +$LL5 at pypy_g_ll_@139: +$block6$40044: + +; 1525 : goto block3_back; +; 1526 : +; 1527 : block6: +; 1528 : l_v419 = RPyItem(l_items_2, l_i_23); + + test esi, esi + jl SHORT $LN14 at pypy_g_ll_@139 + mov eax, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [eax+4] + jl SHORT $LN15 at pypy_g_ll_@139 +$LN14 at pypy_g_ll_@139: + call _RPyAbort + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} +$LN15 at pypy_g_ll_@139: + +; 1529 : l_v420 = l_v419; + + mov ebx, DWORD PTR [ebp] + +; 1530 : l_v421 = RPyField(l_v420, rs_chars).length; + + test ebx, ebx + jne SHORT $LN16 at pypy_g_ll_@139 + call _RPyAbort + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | 32(%esp)} +$LN16 at pypy_g_ll_@139: + +; 1531 : OP_INT_ADD(l_v402, l_v421, l_v422); +; 1532 : OP_INT_ADD(l_i_23, 1L, l_v423); +; 1533 : l_i_23 = l_v423; + + inc esi + add ebp, 4 + +; 1534 : l_v402 = l_v422; + + add edi, DWORD PTR [ebx+8] + cmp esi, DWORD PTR _l_num_items_0$[esp+20] +$block1_back$40045: + jl SHORT $LL5 at pypy_g_ll_@139 +$block2$40046: + +; 1477 : goto block6; +; 1478 : block1_back: ; +; 1479 : OP_INT_LT(l_i_23, l_num_items_0, l_v403); +; 1480 : l_v404 = l_v403; +; 1481 : } +; 1482 : goto block2; +; 1483 : +; 1484 : block2: +; 1485 : l_result_2 = pypy_g_mallocstr__Signed(l_v402); + + push edi + call _pypy_g_mallocstr__Signed + ;; expected {28(%esp) | 16(%esp), 8(%esp), 4(%esp), 12(%esp) | 36(%esp)} + +; 1486 : l_v405 = (void*)l_items_2; + + mov ecx, DWORD PTR _l_items_2$[esp+24] + add esp, 4 + mov DWORD PTR _l_result_2$[esp+24], eax + mov DWORD PTR _l_v405$[esp+24], ecx + +; 1487 : l_v406 = pypy_asm_gcroot(l_v405); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v405$[esp+24] + +; 1488 : l_items_2 = l_v406; /* for moving GCs */ +; 1489 : l_v408 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1490 : l_v409 = (l_v408 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1491 : if (!l_v409) { + + je SHORT $LN3 at pypy_g_ll_@139 + pop edi + pop esi + pop ebp + +; 1492 : l_v424 = ((struct pypy_rpy_string0 *) NULL); + + xor eax, eax + pop ebx + +; 1535 : goto block1_back; +; 1536 : } + + add esp, 8 + ret 0 +$LN3 at pypy_g_ll_@139: + +; 1493 : goto block4; +; 1494 : } +; 1495 : l_i_22 = 0L; + + xor esi, esi + +; 1496 : l_res_index_0 = 0L; + + xor ebp, ebp + +; 1497 : goto block3; +; 1498 : +; 1499 : block3: +; 1500 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); + + cmp DWORD PTR _l_num_items_0$[esp+20], ebp +$block3$40053: + +; 1501 : l_v411 = l_v410; +; 1502 : while (l_v411) { + + jle SHORT $LN1 at pypy_g_ll_@139 + mov ebx, ecx + add ebx, 8 +$LL2 at pypy_g_ll_@139: +$block5$40057: + +; 1514 : +; 1515 : block5: +; 1516 : l_v412 = RPyItem(l_items_2, l_i_22); + + test esi, esi + jl SHORT $LN9 at pypy_g_ll_@139 + mov edx, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [edx+4] + jl SHORT $LN10 at pypy_g_ll_@139 +$LN9 at pypy_g_ll_@139: + call _RPyAbort + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } +$LN10 at pypy_g_ll_@139: + +; 1517 : l_v413 = l_v412; + + mov edi, DWORD PTR [ebx] + +; 1518 : l_v414 = RPyField(l_v413, rs_chars).length; + + test edi, edi + jne SHORT $LN11 at pypy_g_ll_@139 + call _RPyAbort + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } +$LN11 at pypy_g_ll_@139: + mov edi, DWORD PTR [edi+8] + +; 1519 : l_v415 = RPyItem(l_items_2, l_i_22); + + test esi, esi + jl SHORT $LN12 at pypy_g_ll_@139 + mov eax, DWORD PTR _l_items_2$[esp+20] + cmp esi, DWORD PTR [eax+4] + jl SHORT $LN13 at pypy_g_ll_@139 +$LN12 at pypy_g_ll_@139: + call _RPyAbort + ;; expected {24(%esp) | 12(%esp), 4(%esp), (%esp), 8(%esp) | } +$LN13 at pypy_g_ll_@139: + +; 1520 : pypy_g_copy_string_contents__rpy_stringPtr_rpy_stringPt(l_v415, l_result_2, 0L, l_res_index_0, l_v414); + + mov ecx, DWORD PTR [ebx] + mov edx, DWORD PTR _l_result_2$[esp+24] + push edi + add ecx, 12 ; 0000000cH + push ecx + lea eax, DWORD PTR [edx+ebp+12] + push eax +$block0$80664: +$block0$80634: +$block0$80639: +$block1$80640: +$block0$80644: +$block1$80645: +$block1$80635: +$block0$80659: +$block0$80667: +$block1$80668: +$block0$80673: +$block1$80674: +$block1$80661: +$block0$80678: +$block1$80679: + call _memcpy + ;; expected {36(%esp) | 24(%esp), 16(%esp), 12(%esp), 20(%esp) | } + add esp, 12 ; 0000000cH + +; 1521 : OP_INT_ADD(l_res_index_0, l_v414, l_v417); +; 1522 : OP_INT_ADD(l_i_22, 1L, l_v418); +; 1523 : l_i_22 = l_v418; + + inc esi + add ebx, 4 + +; 1524 : l_res_index_0 = l_v417; + + add ebp, edi + cmp esi, DWORD PTR _l_num_items_0$[esp+20] +$block1$80671: +$block3_back$40058: + jl SHORT $LL2 at pypy_g_ll_@139 +$LN1 at pypy_g_ll_@139: + +; 1503 : goto block5; +; 1504 : block3_back: ; +; 1505 : OP_INT_LT(l_i_22, l_num_items_0, l_v410); +; 1506 : l_v411 = l_v410; +; 1507 : } +; 1508 : l_v424 = l_result_2; +; 1509 : goto block4; +; 1510 : +; 1511 : block4: +; 1512 : RPY_DEBUG_RETURN(); +; 1513 : return l_v424; + + mov eax, DWORD PTR _l_result_2$[esp+24] + pop edi + pop esi + pop ebp +$block4$40052: + pop ebx + +; 1535 : goto block1_back; +; 1536 : } + + add esp, 8 + ret 0 +_pypy_g_ll_join_strs__Signed_arrayPtr ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s Thu Nov 5 23:13:47 2009 @@ -1,31 +1,31 @@ -; Function compile flags: /Odtp -_TEXT SEGMENT -_pypy_g_frameworkgc_setup PROC - -; 46 : void pypy_g_frameworkgc_setup(void) { - - push ebp - mov ebp, esp -$block0$37400: - -; 47 : -; 48 : block0: -; 49 : pypy_g_SemiSpaceGC_setup((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)); - - push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC - call _pypy_g_SemiSpaceGC_setup - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 4 -$block1$37401: - -; 50 : goto block1; -; 51 : -; 52 : block1: -; 53 : RPY_DEBUG_RETURN(); -; 54 : return /* nothing */; -; 55 : } - - pop ebp - ret 0 -_pypy_g_frameworkgc_setup ENDP -_TEXT ENDS +; Function compile flags: /Odtp +_TEXT SEGMENT +_pypy_g_frameworkgc_setup PROC + +; 46 : void pypy_g_frameworkgc_setup(void) { + + push ebp + mov ebp, esp +$block0$37400: + +; 47 : +; 48 : block0: +; 49 : pypy_g_SemiSpaceGC_setup((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC)); + + push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC + call _pypy_g_SemiSpaceGC_setup + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 4 +$block1$37401: + +; 50 : goto block1; +; 51 : +; 52 : block1: +; 53 : RPY_DEBUG_RETURN(); +; 54 : return /* nothing */; +; 55 : } + + pop ebp + ret 0 +_pypy_g_frameworkgc_setup ENDP +_TEXT ENDS Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s Thu Nov 5 23:13:47 2009 @@ -1,747 +1,747 @@ -; Function compile flags: /Odtpy -; COMDAT _pypy_g__inplace_divrem1 -_TEXT SEGMENT -tv158 = -292 ; size = 4 -tv152 = -288 ; size = 4 -tv72 = -284 ; size = 4 -tv65 = -280 ; size = 4 -_l_v30733$ = -273 ; size = 1 -_l_v30737$ = -272 ; size = 8 -_l_v30740$ = -257 ; size = 1 -_l_v30744$ = -256 ; size = 4 -_l_v30753$ = -252 ; size = 4 -_l_v30748$ = -248 ; size = 8 -_l_v30709$ = -233 ; size = 1 -_l_v30710$ = -232 ; size = 1 -_l_v30714$ = -231 ; size = 1 -_l_v30718$ = -230 ; size = 1 -_l_v30729$ = -229 ; size = 1 -_l_v30725$ = -228 ; size = 4 -_l_v30705$ = -224 ; size = 8 -_l_evalue_70$ = -216 ; size = 4 -_l_index_219$ = -212 ; size = 4 -_l_v30738$ = -205 ; size = 1 -_l_v30730$ = -204 ; size = 4 -_l_v30734$ = -200 ; size = 4 -_l_length_82$ = -196 ; size = 4 -_l_v30745$ = -189 ; size = 1 -_l_v30752$ = -188 ; size = 4 -_l_v30749$ = -184 ; size = 8 -_l_l_100$ = -172 ; size = 4 -_l_l_101$ = -168 ; size = 4 -_l_length_83$ = -164 ; size = 4 -_l_v30704$ = -160 ; size = 4 -_l_v30722$ = -156 ; size = 4 -_l_v30715$ = -152 ; size = 8 -_l_v30726$ = -144 ; size = 8 -_l_v30711$ = -132 ; size = 4 -_l_x_97$ = -128 ; size = 4 -_l_v30739$ = -121 ; size = 1 -_l_v30731$ = -120 ; size = 8 -_l_v30735$ = -112 ; size = 8 -_l_v30742$ = -97 ; size = 1 -_l_v30751$ = -96 ; size = 4 -_l_v30707$ = -90 ; size = 1 -_l_v30723$ = -89 ; size = 1 -_l_v30712$ = -88 ; size = 4 -_l_v30716$ = -84 ; size = 4 -_l_v30703$ = -80 ; size = 8 -_l_v30727$ = -72 ; size = 8 -_l_v30732$ = -64 ; size = 8 -_l_v30736$ = -56 ; size = 8 -_l_index_218$ = -44 ; size = 4 -_l_v30750$ = -40 ; size = 4 -_l_v30754$ = -36 ; size = 4 -_l_v30717$ = -30 ; size = 1 -_l_v30720$ = -29 ; size = 1 -_l_v30713$ = -28 ; size = 4 -_l_v30702$ = -24 ; size = 8 -_l_v30706$ = -16 ; size = 8 -_l_v30728$ = -8 ; size = 8 -_l_self_3688$ = 8 ; size = 4 -_l_pin_1$ = 12 ; size = 4 -_l_n_38$ = 16 ; size = 8 -_l_size_53$ = 24 ; size = 4 -_pypy_g__inplace_divrem1 PROC ; COMDAT - -; 16550: long pypy_g__inplace_divrem1(struct pypy_pypy_rlib_rbigint_rbigint0 *l_self_3688, struct pypy_pypy_rlib_rbigint_rbigint0 *l_pin_1, long long l_n_38, long l_size_53) { - - push ebp - mov ebp, esp - sub esp, 292 ; 00000124H -$block0$210880: - -; 16551: struct pypy_object0 *l_evalue_70; long l_index_218; long l_index_219; -; 16552: struct pypy_array5 *l_l_100; struct pypy_array5 *l_l_101; -; 16553: long l_length_82; long l_length_83; bool_t l_v30707; bool_t l_v30709; -; 16554: bool_t l_v30710; bool_t l_v30714; bool_t l_v30717; bool_t l_v30718; -; 16555: bool_t l_v30720; bool_t l_v30723; bool_t l_v30729; bool_t l_v30733; -; 16556: bool_t l_v30738; bool_t l_v30739; bool_t l_v30740; bool_t l_v30742; -; 16557: bool_t l_v30745; long l_v30704; long l_v30712; long l_v30713; -; 16558: long l_v30716; long l_v30722; long l_v30725; long l_v30730; -; 16559: long l_v30734; long l_v30744; long l_v30750; long l_v30751; -; 16560: long l_v30752; long l_v30753; long l_v30754; long long l_v30702; -; 16561: long long l_v30703; long long l_v30705; long long l_v30706; -; 16562: long long l_v30715; long long l_v30726; long long l_v30727; -; 16563: long long l_v30728; long long l_v30731; long long l_v30732; -; 16564: long long l_v30735; long long l_v30736; long long l_v30737; -; 16565: long long l_v30748; long long l_v30749; struct pypy_array5 *l_v30711; -; 16566: long l_x_97; -; 16567: -; 16568: block0: -; 16569: OP_LLONG_GT(l_n_38, 0LL, l_v30707); - - cmp DWORD PTR _l_n_38$[ebp+4], 0 - jl SHORT $LN11 at pypy_g__in - jg SHORT $LN19 at pypy_g__in - cmp DWORD PTR _l_n_38$[ebp], 0 - jbe SHORT $LN11 at pypy_g__in -$LN19 at pypy_g__in: - mov DWORD PTR tv65[ebp], 1 - jmp SHORT $LN12 at pypy_g__in -$LN11 at pypy_g__in: - mov DWORD PTR tv65[ebp], 0 -$LN12 at pypy_g__in: - mov al, BYTE PTR tv65[ebp] - mov BYTE PTR _l_v30707$[ebp], al - -; 16570: if (l_v30707) { - - movzx ecx, BYTE PTR _l_v30707$[ebp] - test ecx, ecx - je SHORT $LN8 at pypy_g__in - -; 16571: goto block3; - - jmp SHORT $block3$210882 -$LN8 at pypy_g__in: - -; 16572: } -; 16573: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); - - mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError -$block1$210883: - -; 16574: goto block1; -; 16575: -; 16576: block1: -; 16577: pypy_g_RPyRaiseException((&pypy_g_exceptions_AssertionError_vtable.ae_super.se_super.e_super), l_evalue_70); - - mov edx, DWORD PTR _l_evalue_70$[ebp] - push edx - push OFFSET _pypy_g_exceptions_AssertionError_vtable - call _pypy_g_RPyRaiseException - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 8 - -; 16578: l_v30753 = -1L; - - mov DWORD PTR _l_v30753$[ebp], -1 -$block2$210884: - -; 16579: goto block2; -; 16580: -; 16581: block2: -; 16582: RPY_DEBUG_RETURN(); -; 16583: return l_v30753; - - mov eax, DWORD PTR _l_v30753$[ebp] - jmp $LN9 at pypy_g__in -$block3$210882: - -; 16584: -; 16585: block3: -; 16586: OP_LLONG_LE(l_n_38, 2147483647LL, l_v30709); - - cmp DWORD PTR _l_n_38$[ebp+4], 0 - jg SHORT $LN13 at pypy_g__in - jl SHORT $LN20 at pypy_g__in - cmp DWORD PTR _l_n_38$[ebp], 2147483647 ; 7fffffffH - ja SHORT $LN13 at pypy_g__in -$LN20 at pypy_g__in: - mov DWORD PTR tv72[ebp], 1 - jmp SHORT $LN14 at pypy_g__in -$LN13 at pypy_g__in: - mov DWORD PTR tv72[ebp], 0 -$LN14 at pypy_g__in: - mov al, BYTE PTR tv72[ebp] - mov BYTE PTR _l_v30709$[ebp], al - -; 16587: if (l_v30709) { - - movzx ecx, BYTE PTR _l_v30709$[ebp] - test ecx, ecx - je SHORT $LN7 at pypy_g__in - -; 16588: goto block4; - - jmp SHORT $block4$210886 -$LN7 at pypy_g__in: - -; 16589: } -; 16590: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); - - mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError - -; 16591: goto block1; - - jmp SHORT $block1$210883 -$block4$210886: - -; 16592: -; 16593: block4: -; 16594: OP_INT_IS_TRUE(l_size_53, l_v30710); - - xor edx, edx - cmp DWORD PTR _l_size_53$[ebp], 0 - setne dl - mov BYTE PTR _l_v30710$[ebp], dl - -; 16595: if (l_v30710) { - - movzx eax, BYTE PTR _l_v30710$[ebp] - test eax, eax - je SHORT $block5$210889 - -; 16596: l_v30754 = l_size_53; - - mov ecx, DWORD PTR _l_size_53$[ebp] - mov DWORD PTR _l_v30754$[ebp], ecx - -; 16597: goto block6; - - jmp SHORT $block6$210888 -$block5$210889: - -; 16598: } -; 16599: goto block5; -; 16600: -; 16601: block5: -; 16602: l_v30711 = RPyField(l_pin_1, prrr_inst_digits); - - mov edx, DWORD PTR _l_pin_1$[ebp] - mov eax, DWORD PTR [edx+8] - mov DWORD PTR _l_v30711$[ebp], eax - -; 16603: l_v30712 = l_v30711->length; - - mov ecx, DWORD PTR _l_v30711$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_v30712$[ebp], edx - -; 16604: l_v30754 = l_v30712; - - mov eax, DWORD PTR _l_v30712$[ebp] - mov DWORD PTR _l_v30754$[ebp], eax -$block6$210888: - -; 16605: goto block6; -; 16606: -; 16607: block6: -; 16608: OP_INT_SUB(l_v30754, 1L, l_v30713); - - mov ecx, DWORD PTR _l_v30754$[ebp] - sub ecx, 1 - mov DWORD PTR _l_v30713$[ebp], ecx - -; 16609: l_v30702 = 0LL; - - mov DWORD PTR _l_v30702$[ebp], 0 - mov DWORD PTR _l_v30702$[ebp+4], 0 - -; 16610: l_x_97 = l_v30713; - - mov edx, DWORD PTR _l_v30713$[ebp] - mov DWORD PTR _l_x_97$[ebp], edx -$block7$210890: - -; 16611: goto block7; -; 16612: -; 16613: block7: -; 16614: OP_INT_GE(l_x_97, 0L, l_v30714); - - xor eax, eax - cmp DWORD PTR _l_x_97$[ebp], 0 - setge al - mov BYTE PTR _l_v30714$[ebp], al -$LN5 at pypy_g__in: - -; 16615: while (l_v30714) { - - movzx ecx, BYTE PTR _l_v30714$[ebp] - test ecx, ecx - je SHORT $block8$210896 - -; 16616: goto block9; - - jmp SHORT $block9$210894 -$block7_back$210895: - -; 16617: block7_back: ; -; 16618: OP_INT_GE(l_x_97, 0L, l_v30714); - - xor edx, edx - cmp DWORD PTR _l_x_97$[ebp], 0 - setge dl - mov BYTE PTR _l_v30714$[ebp], dl - -; 16619: } - - jmp SHORT $LN5 at pypy_g__in -$block8$210896: - -; 16620: goto block8; -; 16621: -; 16622: block8: -; 16623: OP_LLONG_AND(l_v30702, 2147483647LL, l_v30715); - - mov eax, DWORD PTR _l_v30702$[ebp] - and eax, 2147483647 ; 7fffffffH - mov ecx, DWORD PTR _l_v30702$[ebp+4] - and ecx, 0 - mov DWORD PTR _l_v30715$[ebp], eax - mov DWORD PTR _l_v30715$[ebp+4], ecx - -; 16624: OP_TRUNCATE_LONGLONG_TO_INT(l_v30715, l_v30716); - - mov edx, DWORD PTR _l_v30715$[ebp] - mov DWORD PTR _l_v30716$[ebp], edx - -; 16625: l_v30753 = l_v30716; - - mov eax, DWORD PTR _l_v30716$[ebp] - mov DWORD PTR _l_v30753$[ebp], eax - -; 16626: goto block2; - - jmp $block2$210884 -$block9$210894: - -; 16627: -; 16628: block9: -; 16629: OP_LLONG_LSHIFT(l_v30702, 31LL, l_v30703); - - mov eax, DWORD PTR _l_v30702$[ebp] - mov edx, DWORD PTR _l_v30702$[ebp+4] - mov cl, 31 ; 0000001fH - call __allshl - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - mov DWORD PTR _l_v30703$[ebp], eax - mov DWORD PTR _l_v30703$[ebp+4], edx - -; 16630: l_l_100 = RPyField(l_pin_1, prrr_inst_digits); - - mov ecx, DWORD PTR _l_pin_1$[ebp] - mov edx, DWORD PTR [ecx+8] - mov DWORD PTR _l_l_100$[ebp], edx - -; 16631: l_length_83 = l_l_100->length; - - mov eax, DWORD PTR _l_l_100$[ebp] - mov ecx, DWORD PTR [eax+4] - mov DWORD PTR _l_length_83$[ebp], ecx - -; 16632: OP_INT_LT(l_x_97, 0L, l_v30717); - - xor edx, edx - cmp DWORD PTR _l_x_97$[ebp], 0 - setl dl - mov BYTE PTR _l_v30717$[ebp], dl - -; 16633: if (l_v30717) { - - movzx eax, BYTE PTR _l_v30717$[ebp] - test eax, eax - je SHORT $LN3 at pypy_g__in - -; 16634: goto block14; - - jmp $block14$210899 -$LN3 at pypy_g__in: - -; 16635: } -; 16636: l_index_218 = l_x_97; - - mov ecx, DWORD PTR _l_x_97$[ebp] - mov DWORD PTR _l_index_218$[ebp], ecx -$block10$210900: - -; 16637: goto block10; -; 16638: -; 16639: block10: -; 16640: OP_INT_GE(l_index_218, 0L, l_v30718); - - xor edx, edx - cmp DWORD PTR _l_index_218$[ebp], 0 - setge dl - mov BYTE PTR _l_v30718$[ebp], dl - -; 16641: RPyAssert(l_v30718, "negative list getitem index out of bound"); -; 16642: OP_INT_LT(l_index_218, l_length_83, l_v30720); - - mov eax, DWORD PTR _l_index_218$[ebp] - xor ecx, ecx - cmp eax, DWORD PTR _l_length_83$[ebp] - setl cl - mov BYTE PTR _l_v30720$[ebp], cl - -; 16643: RPyAssert(l_v30720, "list getitem index out of bound"); -; 16644: l_v30722 = l_l_100->length; - - mov edx, DWORD PTR _l_l_100$[ebp] - mov eax, DWORD PTR [edx+4] - mov DWORD PTR _l_v30722$[ebp], eax - -; 16645: OP_INT_LT(l_index_218, l_v30722, l_v30723); - - mov ecx, DWORD PTR _l_index_218$[ebp] - xor edx, edx - cmp ecx, DWORD PTR _l_v30722$[ebp] - setl dl - mov BYTE PTR _l_v30723$[ebp], dl - -; 16646: RPyAssert(l_v30723, "fixed getitem out of bounds"); -; 16647: l_v30725 = RPyItem(l_l_100, l_index_218); - - mov eax, DWORD PTR _l_index_218$[ebp] - mov ecx, DWORD PTR _l_l_100$[ebp] - mov edx, DWORD PTR [ecx+eax*4+8] - mov DWORD PTR _l_v30725$[ebp], edx - -; 16648: OP_CAST_INT_TO_LONGLONG(l_v30725, l_v30726); - - mov eax, DWORD PTR _l_v30725$[ebp] - cdq - mov DWORD PTR _l_v30726$[ebp], eax - mov DWORD PTR _l_v30726$[ebp+4], edx - -; 16649: OP_LLONG_ADD(l_v30703, l_v30726, l_v30706); - - mov eax, DWORD PTR _l_v30703$[ebp] - add eax, DWORD PTR _l_v30726$[ebp] - mov ecx, DWORD PTR _l_v30703$[ebp+4] - adc ecx, DWORD PTR _l_v30726$[ebp+4] - mov DWORD PTR _l_v30706$[ebp], eax - mov DWORD PTR _l_v30706$[ebp+4], ecx - -; 16650: OP_LLONG_FLOORDIV(l_v30706, l_n_38, l_v30727); - - mov edx, DWORD PTR _l_n_38$[ebp+4] - push edx - mov eax, DWORD PTR _l_n_38$[ebp] - push eax - mov ecx, DWORD PTR _l_v30706$[ebp+4] - push ecx - mov edx, DWORD PTR _l_v30706$[ebp] - push edx - call __alldiv - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - mov DWORD PTR _l_v30727$[ebp], eax - mov DWORD PTR _l_v30727$[ebp+4], edx - -; 16651: OP_LLONG_XOR(l_v30706, l_n_38, l_v30728); - - mov eax, DWORD PTR _l_v30706$[ebp] - xor eax, DWORD PTR _l_n_38$[ebp] - mov ecx, DWORD PTR _l_v30706$[ebp+4] - xor ecx, DWORD PTR _l_n_38$[ebp+4] - mov DWORD PTR _l_v30728$[ebp], eax - mov DWORD PTR _l_v30728$[ebp+4], ecx - -; 16652: OP_LLONG_LE(l_v30728, 0LL, l_v30729); - - jg SHORT $LN15 at pypy_g__in - jl SHORT $LN21 at pypy_g__in - cmp DWORD PTR _l_v30728$[ebp], 0 - ja SHORT $LN15 at pypy_g__in -$LN21 at pypy_g__in: - mov DWORD PTR tv152[ebp], 1 - jmp SHORT $LN16 at pypy_g__in -$LN15 at pypy_g__in: - mov DWORD PTR tv152[ebp], 0 -$LN16 at pypy_g__in: - mov dl, BYTE PTR tv152[ebp] - mov BYTE PTR _l_v30729$[ebp], dl - -; 16653: OP_CAST_BOOL_TO_INT(l_v30729, l_v30730); - - movzx eax, BYTE PTR _l_v30729$[ebp] - mov DWORD PTR _l_v30730$[ebp], eax - -; 16654: OP_CAST_INT_TO_LONGLONG(l_v30730, l_v30731); - - mov eax, DWORD PTR _l_v30730$[ebp] - cdq - mov DWORD PTR _l_v30731$[ebp], eax - mov DWORD PTR _l_v30731$[ebp+4], edx - -; 16655: OP_LLONG_MOD(l_v30706, l_n_38, l_v30732); - - mov ecx, DWORD PTR _l_n_38$[ebp+4] - push ecx - mov edx, DWORD PTR _l_n_38$[ebp] - push edx - mov eax, DWORD PTR _l_v30706$[ebp+4] - push eax - mov ecx, DWORD PTR _l_v30706$[ebp] - push ecx - call __allrem - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - mov DWORD PTR _l_v30732$[ebp], eax - mov DWORD PTR _l_v30732$[ebp+4], edx - -; 16656: OP_LLONG_NE(l_v30732, 0LL, l_v30733); - - mov edx, DWORD PTR _l_v30732$[ebp] - or edx, DWORD PTR _l_v30732$[ebp+4] - je SHORT $LN17 at pypy_g__in - mov DWORD PTR tv158[ebp], 1 - jmp SHORT $LN18 at pypy_g__in -$LN17 at pypy_g__in: - mov DWORD PTR tv158[ebp], 0 -$LN18 at pypy_g__in: - mov al, BYTE PTR tv158[ebp] - mov BYTE PTR _l_v30733$[ebp], al - -; 16657: OP_CAST_BOOL_TO_INT(l_v30733, l_v30734); - - movzx ecx, BYTE PTR _l_v30733$[ebp] - mov DWORD PTR _l_v30734$[ebp], ecx - -; 16658: OP_CAST_INT_TO_LONGLONG(l_v30734, l_v30735); - - mov eax, DWORD PTR _l_v30734$[ebp] - cdq - mov DWORD PTR _l_v30735$[ebp], eax - mov DWORD PTR _l_v30735$[ebp+4], edx - -; 16659: OP_LLONG_AND(l_v30731, l_v30735, l_v30736); - - mov edx, DWORD PTR _l_v30731$[ebp] - and edx, DWORD PTR _l_v30735$[ebp] - mov eax, DWORD PTR _l_v30731$[ebp+4] - and eax, DWORD PTR _l_v30735$[ebp+4] - mov DWORD PTR _l_v30736$[ebp], edx - mov DWORD PTR _l_v30736$[ebp+4], eax - -; 16660: OP_LLONG_SUB(l_v30727, l_v30736, l_v30705); - - mov ecx, DWORD PTR _l_v30727$[ebp] - sub ecx, DWORD PTR _l_v30736$[ebp] - mov edx, DWORD PTR _l_v30727$[ebp+4] - sbb edx, DWORD PTR _l_v30736$[ebp+4] - mov DWORD PTR _l_v30705$[ebp], ecx - mov DWORD PTR _l_v30705$[ebp+4], edx - -; 16661: OP_LLONG_AND(l_v30705, 2147483647LL, l_v30737); - - mov eax, DWORD PTR _l_v30705$[ebp] - and eax, 2147483647 ; 7fffffffH - mov ecx, DWORD PTR _l_v30705$[ebp+4] - and ecx, 0 - mov DWORD PTR _l_v30737$[ebp], eax - mov DWORD PTR _l_v30737$[ebp+4], ecx - -; 16662: OP_TRUNCATE_LONGLONG_TO_INT(l_v30737, l_v30704); - - mov edx, DWORD PTR _l_v30737$[ebp] - mov DWORD PTR _l_v30704$[ebp], edx - -; 16663: OP_INT_GE(l_v30704, 0L, l_v30738); - - xor eax, eax - cmp DWORD PTR _l_v30704$[ebp], 0 - setge al - mov BYTE PTR _l_v30738$[ebp], al - -; 16664: if (l_v30738) { - - movzx ecx, BYTE PTR _l_v30738$[ebp] - test ecx, ecx - je SHORT $LN2 at pypy_g__in - -; 16665: goto block11; - - jmp SHORT $block11$210908 -$LN2 at pypy_g__in: - -; 16666: } -; 16667: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); - - mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError - -; 16668: goto block1; - - jmp $block1$210883 -$block11$210908: - -; 16669: -; 16670: block11: -; 16671: l_l_101 = RPyField(l_self_3688, prrr_inst_digits); - - mov edx, DWORD PTR _l_self_3688$[ebp] - mov eax, DWORD PTR [edx+8] - mov DWORD PTR _l_l_101$[ebp], eax - -; 16672: l_length_82 = l_l_101->length; - - mov ecx, DWORD PTR _l_l_101$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_length_82$[ebp], edx - -; 16673: OP_INT_LT(l_x_97, 0L, l_v30739); - - xor eax, eax - cmp DWORD PTR _l_x_97$[ebp], 0 - setl al - mov BYTE PTR _l_v30739$[ebp], al - -; 16674: if (l_v30739) { - - movzx ecx, BYTE PTR _l_v30739$[ebp] - test ecx, ecx - je SHORT $LN1 at pypy_g__in - -; 16675: goto block13; - - jmp $block13$210910 -$LN1 at pypy_g__in: - -; 16676: } -; 16677: l_index_219 = l_x_97; - - mov edx, DWORD PTR _l_x_97$[ebp] - mov DWORD PTR _l_index_219$[ebp], edx -$block12$210911: - -; 16678: goto block12; -; 16679: -; 16680: block12: -; 16681: OP_INT_GE(l_index_219, 0L, l_v30740); - - xor eax, eax - cmp DWORD PTR _l_index_219$[ebp], 0 - setge al - mov BYTE PTR _l_v30740$[ebp], al - -; 16682: RPyAssert(l_v30740, "negative list setitem index out of bound"); -; 16683: OP_INT_LT(l_index_219, l_length_82, l_v30742); - - mov ecx, DWORD PTR _l_index_219$[ebp] - xor edx, edx - cmp ecx, DWORD PTR _l_length_82$[ebp] - setl dl - mov BYTE PTR _l_v30742$[ebp], dl - -; 16684: RPyAssert(l_v30742, "list setitem index out of bound"); -; 16685: l_v30744 = l_l_101->length; - - mov eax, DWORD PTR _l_l_101$[ebp] - mov ecx, DWORD PTR [eax+4] - mov DWORD PTR _l_v30744$[ebp], ecx - -; 16686: OP_INT_LT(l_index_219, l_v30744, l_v30745); - - mov edx, DWORD PTR _l_index_219$[ebp] - xor eax, eax - cmp edx, DWORD PTR _l_v30744$[ebp] - setl al - mov BYTE PTR _l_v30745$[ebp], al - -; 16687: RPyAssert(l_v30745, "fixed setitem out of bounds"); -; 16688: RPyItem(l_l_101, l_index_219) = l_v30704; - - mov ecx, DWORD PTR _l_index_219$[ebp] - mov edx, DWORD PTR _l_l_101$[ebp] - mov eax, DWORD PTR _l_v30704$[ebp] - mov DWORD PTR [edx+ecx*4+8], eax - -; 16689: OP_LLONG_MUL(l_v30705, l_n_38, l_v30748); - - mov ecx, DWORD PTR _l_n_38$[ebp+4] - push ecx - mov edx, DWORD PTR _l_n_38$[ebp] - push edx - mov eax, DWORD PTR _l_v30705$[ebp+4] - push eax - mov ecx, DWORD PTR _l_v30705$[ebp] - push ecx - call __allmul - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - mov DWORD PTR _l_v30748$[ebp], eax - mov DWORD PTR _l_v30748$[ebp+4], edx - -; 16690: OP_LLONG_SUB(l_v30706, l_v30748, l_v30749); - - mov edx, DWORD PTR _l_v30706$[ebp] - sub edx, DWORD PTR _l_v30748$[ebp] - mov eax, DWORD PTR _l_v30706$[ebp+4] - sbb eax, DWORD PTR _l_v30748$[ebp+4] - mov DWORD PTR _l_v30749$[ebp], edx - mov DWORD PTR _l_v30749$[ebp+4], eax - -; 16691: OP_INT_SUB(l_x_97, 1L, l_v30750); - - mov ecx, DWORD PTR _l_x_97$[ebp] - sub ecx, 1 - mov DWORD PTR _l_v30750$[ebp], ecx - -; 16692: l_v30702 = l_v30749; - - mov edx, DWORD PTR _l_v30749$[ebp] - mov DWORD PTR _l_v30702$[ebp], edx - mov eax, DWORD PTR _l_v30749$[ebp+4] - mov DWORD PTR _l_v30702$[ebp+4], eax - -; 16693: l_x_97 = l_v30750; - - mov ecx, DWORD PTR _l_v30750$[ebp] - mov DWORD PTR _l_x_97$[ebp], ecx - -; 16694: goto block7_back; - - jmp $block7_back$210895 -$block13$210910: - -; 16695: -; 16696: block13: -; 16697: OP_INT_ADD(l_x_97, l_length_82, l_v30751); - - mov edx, DWORD PTR _l_x_97$[ebp] - add edx, DWORD PTR _l_length_82$[ebp] - mov DWORD PTR _l_v30751$[ebp], edx - -; 16698: l_index_219 = l_v30751; - - mov eax, DWORD PTR _l_v30751$[ebp] - mov DWORD PTR _l_index_219$[ebp], eax - -; 16699: goto block12; - - jmp $block12$210911 -$block14$210899: - -; 16700: -; 16701: block14: -; 16702: OP_INT_ADD(l_x_97, l_length_83, l_v30752); - - mov ecx, DWORD PTR _l_x_97$[ebp] - add ecx, DWORD PTR _l_length_83$[ebp] - mov DWORD PTR _l_v30752$[ebp], ecx - -; 16703: l_index_218 = l_v30752; - - mov edx, DWORD PTR _l_v30752$[ebp] - mov DWORD PTR _l_index_218$[ebp], edx - -; 16704: goto block10; - - jmp $block10$210900 -$LN9 at pypy_g__in: - -; 16705: } - - mov esp, ebp - pop ebp - ret 0 -_pypy_g__inplace_divrem1 ENDP +; Function compile flags: /Odtpy +; COMDAT _pypy_g__inplace_divrem1 +_TEXT SEGMENT +tv158 = -292 ; size = 4 +tv152 = -288 ; size = 4 +tv72 = -284 ; size = 4 +tv65 = -280 ; size = 4 +_l_v30733$ = -273 ; size = 1 +_l_v30737$ = -272 ; size = 8 +_l_v30740$ = -257 ; size = 1 +_l_v30744$ = -256 ; size = 4 +_l_v30753$ = -252 ; size = 4 +_l_v30748$ = -248 ; size = 8 +_l_v30709$ = -233 ; size = 1 +_l_v30710$ = -232 ; size = 1 +_l_v30714$ = -231 ; size = 1 +_l_v30718$ = -230 ; size = 1 +_l_v30729$ = -229 ; size = 1 +_l_v30725$ = -228 ; size = 4 +_l_v30705$ = -224 ; size = 8 +_l_evalue_70$ = -216 ; size = 4 +_l_index_219$ = -212 ; size = 4 +_l_v30738$ = -205 ; size = 1 +_l_v30730$ = -204 ; size = 4 +_l_v30734$ = -200 ; size = 4 +_l_length_82$ = -196 ; size = 4 +_l_v30745$ = -189 ; size = 1 +_l_v30752$ = -188 ; size = 4 +_l_v30749$ = -184 ; size = 8 +_l_l_100$ = -172 ; size = 4 +_l_l_101$ = -168 ; size = 4 +_l_length_83$ = -164 ; size = 4 +_l_v30704$ = -160 ; size = 4 +_l_v30722$ = -156 ; size = 4 +_l_v30715$ = -152 ; size = 8 +_l_v30726$ = -144 ; size = 8 +_l_v30711$ = -132 ; size = 4 +_l_x_97$ = -128 ; size = 4 +_l_v30739$ = -121 ; size = 1 +_l_v30731$ = -120 ; size = 8 +_l_v30735$ = -112 ; size = 8 +_l_v30742$ = -97 ; size = 1 +_l_v30751$ = -96 ; size = 4 +_l_v30707$ = -90 ; size = 1 +_l_v30723$ = -89 ; size = 1 +_l_v30712$ = -88 ; size = 4 +_l_v30716$ = -84 ; size = 4 +_l_v30703$ = -80 ; size = 8 +_l_v30727$ = -72 ; size = 8 +_l_v30732$ = -64 ; size = 8 +_l_v30736$ = -56 ; size = 8 +_l_index_218$ = -44 ; size = 4 +_l_v30750$ = -40 ; size = 4 +_l_v30754$ = -36 ; size = 4 +_l_v30717$ = -30 ; size = 1 +_l_v30720$ = -29 ; size = 1 +_l_v30713$ = -28 ; size = 4 +_l_v30702$ = -24 ; size = 8 +_l_v30706$ = -16 ; size = 8 +_l_v30728$ = -8 ; size = 8 +_l_self_3688$ = 8 ; size = 4 +_l_pin_1$ = 12 ; size = 4 +_l_n_38$ = 16 ; size = 8 +_l_size_53$ = 24 ; size = 4 +_pypy_g__inplace_divrem1 PROC ; COMDAT + +; 16550: long pypy_g__inplace_divrem1(struct pypy_pypy_rlib_rbigint_rbigint0 *l_self_3688, struct pypy_pypy_rlib_rbigint_rbigint0 *l_pin_1, long long l_n_38, long l_size_53) { + + push ebp + mov ebp, esp + sub esp, 292 ; 00000124H +$block0$210880: + +; 16551: struct pypy_object0 *l_evalue_70; long l_index_218; long l_index_219; +; 16552: struct pypy_array5 *l_l_100; struct pypy_array5 *l_l_101; +; 16553: long l_length_82; long l_length_83; bool_t l_v30707; bool_t l_v30709; +; 16554: bool_t l_v30710; bool_t l_v30714; bool_t l_v30717; bool_t l_v30718; +; 16555: bool_t l_v30720; bool_t l_v30723; bool_t l_v30729; bool_t l_v30733; +; 16556: bool_t l_v30738; bool_t l_v30739; bool_t l_v30740; bool_t l_v30742; +; 16557: bool_t l_v30745; long l_v30704; long l_v30712; long l_v30713; +; 16558: long l_v30716; long l_v30722; long l_v30725; long l_v30730; +; 16559: long l_v30734; long l_v30744; long l_v30750; long l_v30751; +; 16560: long l_v30752; long l_v30753; long l_v30754; long long l_v30702; +; 16561: long long l_v30703; long long l_v30705; long long l_v30706; +; 16562: long long l_v30715; long long l_v30726; long long l_v30727; +; 16563: long long l_v30728; long long l_v30731; long long l_v30732; +; 16564: long long l_v30735; long long l_v30736; long long l_v30737; +; 16565: long long l_v30748; long long l_v30749; struct pypy_array5 *l_v30711; +; 16566: long l_x_97; +; 16567: +; 16568: block0: +; 16569: OP_LLONG_GT(l_n_38, 0LL, l_v30707); + + cmp DWORD PTR _l_n_38$[ebp+4], 0 + jl SHORT $LN11 at pypy_g__in + jg SHORT $LN19 at pypy_g__in + cmp DWORD PTR _l_n_38$[ebp], 0 + jbe SHORT $LN11 at pypy_g__in +$LN19 at pypy_g__in: + mov DWORD PTR tv65[ebp], 1 + jmp SHORT $LN12 at pypy_g__in +$LN11 at pypy_g__in: + mov DWORD PTR tv65[ebp], 0 +$LN12 at pypy_g__in: + mov al, BYTE PTR tv65[ebp] + mov BYTE PTR _l_v30707$[ebp], al + +; 16570: if (l_v30707) { + + movzx ecx, BYTE PTR _l_v30707$[ebp] + test ecx, ecx + je SHORT $LN8 at pypy_g__in + +; 16571: goto block3; + + jmp SHORT $block3$210882 +$LN8 at pypy_g__in: + +; 16572: } +; 16573: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError +$block1$210883: + +; 16574: goto block1; +; 16575: +; 16576: block1: +; 16577: pypy_g_RPyRaiseException((&pypy_g_exceptions_AssertionError_vtable.ae_super.se_super.e_super), l_evalue_70); + + mov edx, DWORD PTR _l_evalue_70$[ebp] + push edx + push OFFSET _pypy_g_exceptions_AssertionError_vtable + call _pypy_g_RPyRaiseException + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 8 + +; 16578: l_v30753 = -1L; + + mov DWORD PTR _l_v30753$[ebp], -1 +$block2$210884: + +; 16579: goto block2; +; 16580: +; 16581: block2: +; 16582: RPY_DEBUG_RETURN(); +; 16583: return l_v30753; + + mov eax, DWORD PTR _l_v30753$[ebp] + jmp $LN9 at pypy_g__in +$block3$210882: + +; 16584: +; 16585: block3: +; 16586: OP_LLONG_LE(l_n_38, 2147483647LL, l_v30709); + + cmp DWORD PTR _l_n_38$[ebp+4], 0 + jg SHORT $LN13 at pypy_g__in + jl SHORT $LN20 at pypy_g__in + cmp DWORD PTR _l_n_38$[ebp], 2147483647 ; 7fffffffH + ja SHORT $LN13 at pypy_g__in +$LN20 at pypy_g__in: + mov DWORD PTR tv72[ebp], 1 + jmp SHORT $LN14 at pypy_g__in +$LN13 at pypy_g__in: + mov DWORD PTR tv72[ebp], 0 +$LN14 at pypy_g__in: + mov al, BYTE PTR tv72[ebp] + mov BYTE PTR _l_v30709$[ebp], al + +; 16587: if (l_v30709) { + + movzx ecx, BYTE PTR _l_v30709$[ebp] + test ecx, ecx + je SHORT $LN7 at pypy_g__in + +; 16588: goto block4; + + jmp SHORT $block4$210886 +$LN7 at pypy_g__in: + +; 16589: } +; 16590: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError + +; 16591: goto block1; + + jmp SHORT $block1$210883 +$block4$210886: + +; 16592: +; 16593: block4: +; 16594: OP_INT_IS_TRUE(l_size_53, l_v30710); + + xor edx, edx + cmp DWORD PTR _l_size_53$[ebp], 0 + setne dl + mov BYTE PTR _l_v30710$[ebp], dl + +; 16595: if (l_v30710) { + + movzx eax, BYTE PTR _l_v30710$[ebp] + test eax, eax + je SHORT $block5$210889 + +; 16596: l_v30754 = l_size_53; + + mov ecx, DWORD PTR _l_size_53$[ebp] + mov DWORD PTR _l_v30754$[ebp], ecx + +; 16597: goto block6; + + jmp SHORT $block6$210888 +$block5$210889: + +; 16598: } +; 16599: goto block5; +; 16600: +; 16601: block5: +; 16602: l_v30711 = RPyField(l_pin_1, prrr_inst_digits); + + mov edx, DWORD PTR _l_pin_1$[ebp] + mov eax, DWORD PTR [edx+8] + mov DWORD PTR _l_v30711$[ebp], eax + +; 16603: l_v30712 = l_v30711->length; + + mov ecx, DWORD PTR _l_v30711$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v30712$[ebp], edx + +; 16604: l_v30754 = l_v30712; + + mov eax, DWORD PTR _l_v30712$[ebp] + mov DWORD PTR _l_v30754$[ebp], eax +$block6$210888: + +; 16605: goto block6; +; 16606: +; 16607: block6: +; 16608: OP_INT_SUB(l_v30754, 1L, l_v30713); + + mov ecx, DWORD PTR _l_v30754$[ebp] + sub ecx, 1 + mov DWORD PTR _l_v30713$[ebp], ecx + +; 16609: l_v30702 = 0LL; + + mov DWORD PTR _l_v30702$[ebp], 0 + mov DWORD PTR _l_v30702$[ebp+4], 0 + +; 16610: l_x_97 = l_v30713; + + mov edx, DWORD PTR _l_v30713$[ebp] + mov DWORD PTR _l_x_97$[ebp], edx +$block7$210890: + +; 16611: goto block7; +; 16612: +; 16613: block7: +; 16614: OP_INT_GE(l_x_97, 0L, l_v30714); + + xor eax, eax + cmp DWORD PTR _l_x_97$[ebp], 0 + setge al + mov BYTE PTR _l_v30714$[ebp], al +$LN5 at pypy_g__in: + +; 16615: while (l_v30714) { + + movzx ecx, BYTE PTR _l_v30714$[ebp] + test ecx, ecx + je SHORT $block8$210896 + +; 16616: goto block9; + + jmp SHORT $block9$210894 +$block7_back$210895: + +; 16617: block7_back: ; +; 16618: OP_INT_GE(l_x_97, 0L, l_v30714); + + xor edx, edx + cmp DWORD PTR _l_x_97$[ebp], 0 + setge dl + mov BYTE PTR _l_v30714$[ebp], dl + +; 16619: } + + jmp SHORT $LN5 at pypy_g__in +$block8$210896: + +; 16620: goto block8; +; 16621: +; 16622: block8: +; 16623: OP_LLONG_AND(l_v30702, 2147483647LL, l_v30715); + + mov eax, DWORD PTR _l_v30702$[ebp] + and eax, 2147483647 ; 7fffffffH + mov ecx, DWORD PTR _l_v30702$[ebp+4] + and ecx, 0 + mov DWORD PTR _l_v30715$[ebp], eax + mov DWORD PTR _l_v30715$[ebp+4], ecx + +; 16624: OP_TRUNCATE_LONGLONG_TO_INT(l_v30715, l_v30716); + + mov edx, DWORD PTR _l_v30715$[ebp] + mov DWORD PTR _l_v30716$[ebp], edx + +; 16625: l_v30753 = l_v30716; + + mov eax, DWORD PTR _l_v30716$[ebp] + mov DWORD PTR _l_v30753$[ebp], eax + +; 16626: goto block2; + + jmp $block2$210884 +$block9$210894: + +; 16627: +; 16628: block9: +; 16629: OP_LLONG_LSHIFT(l_v30702, 31LL, l_v30703); + + mov eax, DWORD PTR _l_v30702$[ebp] + mov edx, DWORD PTR _l_v30702$[ebp+4] + mov cl, 31 ; 0000001fH + call __allshl + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30703$[ebp], eax + mov DWORD PTR _l_v30703$[ebp+4], edx + +; 16630: l_l_100 = RPyField(l_pin_1, prrr_inst_digits); + + mov ecx, DWORD PTR _l_pin_1$[ebp] + mov edx, DWORD PTR [ecx+8] + mov DWORD PTR _l_l_100$[ebp], edx + +; 16631: l_length_83 = l_l_100->length; + + mov eax, DWORD PTR _l_l_100$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_length_83$[ebp], ecx + +; 16632: OP_INT_LT(l_x_97, 0L, l_v30717); + + xor edx, edx + cmp DWORD PTR _l_x_97$[ebp], 0 + setl dl + mov BYTE PTR _l_v30717$[ebp], dl + +; 16633: if (l_v30717) { + + movzx eax, BYTE PTR _l_v30717$[ebp] + test eax, eax + je SHORT $LN3 at pypy_g__in + +; 16634: goto block14; + + jmp $block14$210899 +$LN3 at pypy_g__in: + +; 16635: } +; 16636: l_index_218 = l_x_97; + + mov ecx, DWORD PTR _l_x_97$[ebp] + mov DWORD PTR _l_index_218$[ebp], ecx +$block10$210900: + +; 16637: goto block10; +; 16638: +; 16639: block10: +; 16640: OP_INT_GE(l_index_218, 0L, l_v30718); + + xor edx, edx + cmp DWORD PTR _l_index_218$[ebp], 0 + setge dl + mov BYTE PTR _l_v30718$[ebp], dl + +; 16641: RPyAssert(l_v30718, "negative list getitem index out of bound"); +; 16642: OP_INT_LT(l_index_218, l_length_83, l_v30720); + + mov eax, DWORD PTR _l_index_218$[ebp] + xor ecx, ecx + cmp eax, DWORD PTR _l_length_83$[ebp] + setl cl + mov BYTE PTR _l_v30720$[ebp], cl + +; 16643: RPyAssert(l_v30720, "list getitem index out of bound"); +; 16644: l_v30722 = l_l_100->length; + + mov edx, DWORD PTR _l_l_100$[ebp] + mov eax, DWORD PTR [edx+4] + mov DWORD PTR _l_v30722$[ebp], eax + +; 16645: OP_INT_LT(l_index_218, l_v30722, l_v30723); + + mov ecx, DWORD PTR _l_index_218$[ebp] + xor edx, edx + cmp ecx, DWORD PTR _l_v30722$[ebp] + setl dl + mov BYTE PTR _l_v30723$[ebp], dl + +; 16646: RPyAssert(l_v30723, "fixed getitem out of bounds"); +; 16647: l_v30725 = RPyItem(l_l_100, l_index_218); + + mov eax, DWORD PTR _l_index_218$[ebp] + mov ecx, DWORD PTR _l_l_100$[ebp] + mov edx, DWORD PTR [ecx+eax*4+8] + mov DWORD PTR _l_v30725$[ebp], edx + +; 16648: OP_CAST_INT_TO_LONGLONG(l_v30725, l_v30726); + + mov eax, DWORD PTR _l_v30725$[ebp] + cdq + mov DWORD PTR _l_v30726$[ebp], eax + mov DWORD PTR _l_v30726$[ebp+4], edx + +; 16649: OP_LLONG_ADD(l_v30703, l_v30726, l_v30706); + + mov eax, DWORD PTR _l_v30703$[ebp] + add eax, DWORD PTR _l_v30726$[ebp] + mov ecx, DWORD PTR _l_v30703$[ebp+4] + adc ecx, DWORD PTR _l_v30726$[ebp+4] + mov DWORD PTR _l_v30706$[ebp], eax + mov DWORD PTR _l_v30706$[ebp+4], ecx + +; 16650: OP_LLONG_FLOORDIV(l_v30706, l_n_38, l_v30727); + + mov edx, DWORD PTR _l_n_38$[ebp+4] + push edx + mov eax, DWORD PTR _l_n_38$[ebp] + push eax + mov ecx, DWORD PTR _l_v30706$[ebp+4] + push ecx + mov edx, DWORD PTR _l_v30706$[ebp] + push edx + call __alldiv + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30727$[ebp], eax + mov DWORD PTR _l_v30727$[ebp+4], edx + +; 16651: OP_LLONG_XOR(l_v30706, l_n_38, l_v30728); + + mov eax, DWORD PTR _l_v30706$[ebp] + xor eax, DWORD PTR _l_n_38$[ebp] + mov ecx, DWORD PTR _l_v30706$[ebp+4] + xor ecx, DWORD PTR _l_n_38$[ebp+4] + mov DWORD PTR _l_v30728$[ebp], eax + mov DWORD PTR _l_v30728$[ebp+4], ecx + +; 16652: OP_LLONG_LE(l_v30728, 0LL, l_v30729); + + jg SHORT $LN15 at pypy_g__in + jl SHORT $LN21 at pypy_g__in + cmp DWORD PTR _l_v30728$[ebp], 0 + ja SHORT $LN15 at pypy_g__in +$LN21 at pypy_g__in: + mov DWORD PTR tv152[ebp], 1 + jmp SHORT $LN16 at pypy_g__in +$LN15 at pypy_g__in: + mov DWORD PTR tv152[ebp], 0 +$LN16 at pypy_g__in: + mov dl, BYTE PTR tv152[ebp] + mov BYTE PTR _l_v30729$[ebp], dl + +; 16653: OP_CAST_BOOL_TO_INT(l_v30729, l_v30730); + + movzx eax, BYTE PTR _l_v30729$[ebp] + mov DWORD PTR _l_v30730$[ebp], eax + +; 16654: OP_CAST_INT_TO_LONGLONG(l_v30730, l_v30731); + + mov eax, DWORD PTR _l_v30730$[ebp] + cdq + mov DWORD PTR _l_v30731$[ebp], eax + mov DWORD PTR _l_v30731$[ebp+4], edx + +; 16655: OP_LLONG_MOD(l_v30706, l_n_38, l_v30732); + + mov ecx, DWORD PTR _l_n_38$[ebp+4] + push ecx + mov edx, DWORD PTR _l_n_38$[ebp] + push edx + mov eax, DWORD PTR _l_v30706$[ebp+4] + push eax + mov ecx, DWORD PTR _l_v30706$[ebp] + push ecx + call __allrem + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30732$[ebp], eax + mov DWORD PTR _l_v30732$[ebp+4], edx + +; 16656: OP_LLONG_NE(l_v30732, 0LL, l_v30733); + + mov edx, DWORD PTR _l_v30732$[ebp] + or edx, DWORD PTR _l_v30732$[ebp+4] + je SHORT $LN17 at pypy_g__in + mov DWORD PTR tv158[ebp], 1 + jmp SHORT $LN18 at pypy_g__in +$LN17 at pypy_g__in: + mov DWORD PTR tv158[ebp], 0 +$LN18 at pypy_g__in: + mov al, BYTE PTR tv158[ebp] + mov BYTE PTR _l_v30733$[ebp], al + +; 16657: OP_CAST_BOOL_TO_INT(l_v30733, l_v30734); + + movzx ecx, BYTE PTR _l_v30733$[ebp] + mov DWORD PTR _l_v30734$[ebp], ecx + +; 16658: OP_CAST_INT_TO_LONGLONG(l_v30734, l_v30735); + + mov eax, DWORD PTR _l_v30734$[ebp] + cdq + mov DWORD PTR _l_v30735$[ebp], eax + mov DWORD PTR _l_v30735$[ebp+4], edx + +; 16659: OP_LLONG_AND(l_v30731, l_v30735, l_v30736); + + mov edx, DWORD PTR _l_v30731$[ebp] + and edx, DWORD PTR _l_v30735$[ebp] + mov eax, DWORD PTR _l_v30731$[ebp+4] + and eax, DWORD PTR _l_v30735$[ebp+4] + mov DWORD PTR _l_v30736$[ebp], edx + mov DWORD PTR _l_v30736$[ebp+4], eax + +; 16660: OP_LLONG_SUB(l_v30727, l_v30736, l_v30705); + + mov ecx, DWORD PTR _l_v30727$[ebp] + sub ecx, DWORD PTR _l_v30736$[ebp] + mov edx, DWORD PTR _l_v30727$[ebp+4] + sbb edx, DWORD PTR _l_v30736$[ebp+4] + mov DWORD PTR _l_v30705$[ebp], ecx + mov DWORD PTR _l_v30705$[ebp+4], edx + +; 16661: OP_LLONG_AND(l_v30705, 2147483647LL, l_v30737); + + mov eax, DWORD PTR _l_v30705$[ebp] + and eax, 2147483647 ; 7fffffffH + mov ecx, DWORD PTR _l_v30705$[ebp+4] + and ecx, 0 + mov DWORD PTR _l_v30737$[ebp], eax + mov DWORD PTR _l_v30737$[ebp+4], ecx + +; 16662: OP_TRUNCATE_LONGLONG_TO_INT(l_v30737, l_v30704); + + mov edx, DWORD PTR _l_v30737$[ebp] + mov DWORD PTR _l_v30704$[ebp], edx + +; 16663: OP_INT_GE(l_v30704, 0L, l_v30738); + + xor eax, eax + cmp DWORD PTR _l_v30704$[ebp], 0 + setge al + mov BYTE PTR _l_v30738$[ebp], al + +; 16664: if (l_v30738) { + + movzx ecx, BYTE PTR _l_v30738$[ebp] + test ecx, ecx + je SHORT $LN2 at pypy_g__in + +; 16665: goto block11; + + jmp SHORT $block11$210908 +$LN2 at pypy_g__in: + +; 16666: } +; 16667: l_evalue_70 = (&pypy_g_exceptions_AssertionError.ae_super.se_super.e_super); + + mov DWORD PTR _l_evalue_70$[ebp], OFFSET _pypy_g_exceptions_AssertionError + +; 16668: goto block1; + + jmp $block1$210883 +$block11$210908: + +; 16669: +; 16670: block11: +; 16671: l_l_101 = RPyField(l_self_3688, prrr_inst_digits); + + mov edx, DWORD PTR _l_self_3688$[ebp] + mov eax, DWORD PTR [edx+8] + mov DWORD PTR _l_l_101$[ebp], eax + +; 16672: l_length_82 = l_l_101->length; + + mov ecx, DWORD PTR _l_l_101$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_length_82$[ebp], edx + +; 16673: OP_INT_LT(l_x_97, 0L, l_v30739); + + xor eax, eax + cmp DWORD PTR _l_x_97$[ebp], 0 + setl al + mov BYTE PTR _l_v30739$[ebp], al + +; 16674: if (l_v30739) { + + movzx ecx, BYTE PTR _l_v30739$[ebp] + test ecx, ecx + je SHORT $LN1 at pypy_g__in + +; 16675: goto block13; + + jmp $block13$210910 +$LN1 at pypy_g__in: + +; 16676: } +; 16677: l_index_219 = l_x_97; + + mov edx, DWORD PTR _l_x_97$[ebp] + mov DWORD PTR _l_index_219$[ebp], edx +$block12$210911: + +; 16678: goto block12; +; 16679: +; 16680: block12: +; 16681: OP_INT_GE(l_index_219, 0L, l_v30740); + + xor eax, eax + cmp DWORD PTR _l_index_219$[ebp], 0 + setge al + mov BYTE PTR _l_v30740$[ebp], al + +; 16682: RPyAssert(l_v30740, "negative list setitem index out of bound"); +; 16683: OP_INT_LT(l_index_219, l_length_82, l_v30742); + + mov ecx, DWORD PTR _l_index_219$[ebp] + xor edx, edx + cmp ecx, DWORD PTR _l_length_82$[ebp] + setl dl + mov BYTE PTR _l_v30742$[ebp], dl + +; 16684: RPyAssert(l_v30742, "list setitem index out of bound"); +; 16685: l_v30744 = l_l_101->length; + + mov eax, DWORD PTR _l_l_101$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v30744$[ebp], ecx + +; 16686: OP_INT_LT(l_index_219, l_v30744, l_v30745); + + mov edx, DWORD PTR _l_index_219$[ebp] + xor eax, eax + cmp edx, DWORD PTR _l_v30744$[ebp] + setl al + mov BYTE PTR _l_v30745$[ebp], al + +; 16687: RPyAssert(l_v30745, "fixed setitem out of bounds"); +; 16688: RPyItem(l_l_101, l_index_219) = l_v30704; + + mov ecx, DWORD PTR _l_index_219$[ebp] + mov edx, DWORD PTR _l_l_101$[ebp] + mov eax, DWORD PTR _l_v30704$[ebp] + mov DWORD PTR [edx+ecx*4+8], eax + +; 16689: OP_LLONG_MUL(l_v30705, l_n_38, l_v30748); + + mov ecx, DWORD PTR _l_n_38$[ebp+4] + push ecx + mov edx, DWORD PTR _l_n_38$[ebp] + push edx + mov eax, DWORD PTR _l_v30705$[ebp+4] + push eax + mov ecx, DWORD PTR _l_v30705$[ebp] + push ecx + call __allmul + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + mov DWORD PTR _l_v30748$[ebp], eax + mov DWORD PTR _l_v30748$[ebp+4], edx + +; 16690: OP_LLONG_SUB(l_v30706, l_v30748, l_v30749); + + mov edx, DWORD PTR _l_v30706$[ebp] + sub edx, DWORD PTR _l_v30748$[ebp] + mov eax, DWORD PTR _l_v30706$[ebp+4] + sbb eax, DWORD PTR _l_v30748$[ebp+4] + mov DWORD PTR _l_v30749$[ebp], edx + mov DWORD PTR _l_v30749$[ebp+4], eax + +; 16691: OP_INT_SUB(l_x_97, 1L, l_v30750); + + mov ecx, DWORD PTR _l_x_97$[ebp] + sub ecx, 1 + mov DWORD PTR _l_v30750$[ebp], ecx + +; 16692: l_v30702 = l_v30749; + + mov edx, DWORD PTR _l_v30749$[ebp] + mov DWORD PTR _l_v30702$[ebp], edx + mov eax, DWORD PTR _l_v30749$[ebp+4] + mov DWORD PTR _l_v30702$[ebp+4], eax + +; 16693: l_x_97 = l_v30750; + + mov ecx, DWORD PTR _l_v30750$[ebp] + mov DWORD PTR _l_x_97$[ebp], ecx + +; 16694: goto block7_back; + + jmp $block7_back$210895 +$block13$210910: + +; 16695: +; 16696: block13: +; 16697: OP_INT_ADD(l_x_97, l_length_82, l_v30751); + + mov edx, DWORD PTR _l_x_97$[ebp] + add edx, DWORD PTR _l_length_82$[ebp] + mov DWORD PTR _l_v30751$[ebp], edx + +; 16698: l_index_219 = l_v30751; + + mov eax, DWORD PTR _l_v30751$[ebp] + mov DWORD PTR _l_index_219$[ebp], eax + +; 16699: goto block12; + + jmp $block12$210911 +$block14$210899: + +; 16700: +; 16701: block14: +; 16702: OP_INT_ADD(l_x_97, l_length_83, l_v30752); + + mov ecx, DWORD PTR _l_x_97$[ebp] + add ecx, DWORD PTR _l_length_83$[ebp] + mov DWORD PTR _l_v30752$[ebp], ecx + +; 16703: l_index_218 = l_v30752; + + mov edx, DWORD PTR _l_v30752$[ebp] + mov DWORD PTR _l_index_218$[ebp], edx + +; 16704: goto block10; + + jmp $block10$210900 +$LN9 at pypy_g__in: + +; 16705: } + + mov esp, ebp + pop ebp + ret 0 +_pypy_g__inplace_divrem1 ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s Thu Nov 5 23:13:47 2009 @@ -1,547 +1,547 @@ -; Function compile flags: /Ogtpy -; COMDAT _pypy_g_foo -_TEXT SEGMENT -tv419 = -8 ; size = 4 -_l_v416$ = -4 ; size = 4 -_l_v413$ = -4 ; size = 4 -_l_v410$ = -4 ; size = 4 -_l_v407$ = -4 ; size = 4 -_l_v404$ = -4 ; size = 4 -_l_v401$ = -4 ; size = 4 -_l_v394$ = -4 ; size = 4 -_l_v391$ = -4 ; size = 4 -_l_v388$ = -4 ; size = 4 -_l_v385$ = -4 ; size = 4 -_l_v382$ = -4 ; size = 4 -_l_v379$ = -4 ; size = 4 -_l_v376$ = -4 ; size = 4 -_l_v368$ = -4 ; size = 4 -_l_v365$ = -4 ; size = 4 -_l_v362$ = -4 ; size = 4 -_l_v359$ = -4 ; size = 4 -_l_v356$ = -4 ; size = 4 -_l_v353$ = -4 ; size = 4 -_local$40423 = 8 ; size = 1 -_l_rec_1$ = 8 ; size = 4 -_l_a1_1$ = 12 ; size = 4 -_l_a2_1$ = 16 ; size = 4 -_l_a3_1$ = 20 ; size = 4 -_l_a4_1$ = 24 ; size = 4 -_l_a5_1$ = 28 ; size = 4 -_l_a6_1$ = 32 ; size = 4 -_pypy_g_foo PROC ; COMDAT - -; 1026 : bool_t l_v337; bool_t l_v340; bool_t l_v345; bool_t l_v346; -; 1027 : bool_t l_v371; bool_t l_v398; bool_t l_v420; bool_t l_v426; -; 1028 : long l_v342; long l_v344; long l_v374; long l_v399; long l_v421; -; 1029 : struct pypy_header0 *l_v347; struct pypy_object0 *l_v372; -; 1030 : struct pypy_object_vtable0 *l_v339; -; 1031 : struct pypy_object_vtable0 *l_v397; -; 1032 : struct pypy_object_vtable0 *l_v419; -; 1033 : struct pypy_object_vtable0 *l_v425; struct pypy_src8_A0 *l_v335; -; 1034 : void* l_v336; void* l_v341; void* l_v343; void* l_v349; void* l_v351; -; 1035 : void* l_v352; void* l_v353; void* l_v354; void* l_v356; void* l_v357; -; 1036 : void* l_v359; void* l_v360; void* l_v362; void* l_v363; void* l_v365; -; 1037 : void* l_v366; void* l_v368; void* l_v369; void* l_v376; void* l_v377; -; 1038 : void* l_v379; void* l_v380; void* l_v382; void* l_v383; void* l_v385; -; 1039 : void* l_v386; void* l_v388; void* l_v389; void* l_v391; void* l_v392; -; 1040 : void* l_v394; void* l_v395; void* l_v401; void* l_v402; void* l_v404; -; 1041 : void* l_v405; void* l_v407; void* l_v408; void* l_v410; void* l_v411; -; 1042 : void* l_v413; void* l_v414; void* l_v416; void* l_v417; void* l_v424; -; 1043 : void* l_v428; -; 1044 : -; 1045 : block0: -; 1046 : OP_INT_GT(l_rec_1, 0L, l_v337); - - mov eax, DWORD PTR _l_rec_1$[esp-4] - sub esp, 8 - test eax, eax -$block0$34376: - -; 1047 : if (l_v337) { - - jle $block1$34379 - push ebx - mov ebx, DWORD PTR _l_a2_1$[esp+8] - push ebp - mov ebp, DWORD PTR _l_a1_1$[esp+12] - push edi - mov edi, DWORD PTR _l_a3_1$[esp+16] - add eax, -1 - mov DWORD PTR tv419[esp+20], eax - push esi - npad 10 -$LL63 at pypy_g_foo: - -; 1048 : goto block2; -; 1049 : } -; 1050 : goto block1; -; 1051 : -; 1052 : block1: -; 1053 : RPY_DEBUG_RETURN(); -; 1054 : return /* nothing */; -; 1055 : -; 1056 : block2: -; 1057 : pypy_g_stack_check___(); - - lea eax, DWORD PTR _local$40423[esp+20] - sub eax, DWORD PTR __LLstacktoobig_stack_base_pointer -$block0$40413: - cmp eax, DWORD PTR __LLstacktoobig_stack_min -$block2$34378: - jl SHORT $LN16 at pypy_g_foo - cmp eax, DWORD PTR __LLstacktoobig_stack_max - jle SHORT $LN17 at pypy_g_foo -$LN16 at pypy_g_foo: - call _LL_stack_too_big_slowpath - ;; expected {24(%esp) | 12(%esp), (%esp), 4(%esp), 8(%esp) | %ebx, %edi, %ebp, 44(%esp), 48(%esp), 52(%esp)} - test eax, eax - jne $LN71 at pypy_g_foo -$LN17 at pypy_g_foo: -$block1$40416: - -; 1058 : l_v339 = RPyField((&pypy_g_ExcData), ed_exc_type); -; 1059 : l_v340 = (l_v339 == NULL); - - cmp DWORD PTR _pypy_g_ExcData, 0 - -; 1060 : if (!l_v340) { - - jne $LN75 at pypy_g_foo - -; 1061 : goto block1; -; 1062 : } -; 1063 : goto block3; -; 1064 : -; 1065 : block3: -; 1066 : l_v341 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free); - - mov esi, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12 - -; 1067 : OP_RAW_MALLOC_USAGE((0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v342); -; 1068 : l_v343 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_top_of_space); -; 1069 : OP_ADR_DELTA(l_v343, l_v341, l_v344); - - mov eax, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+80 - sub eax, esi - -; 1070 : OP_INT_GT(l_v342, l_v344, l_v345); - - cmp eax, 8 -$block3$34382: - -; 1071 : if (l_v345) { - - jge SHORT $block4$34395 - -; 1184 : goto block1; -; 1185 : -; 1186 : block10: -; 1187 : abort(); /* debug_llinterpcall should be unreachable */ -; 1188 : goto block5; -; 1189 : -; 1190 : block11: -; 1191 : l_v424 = pypy_g_SemiSpaceGC_obtain_free_space((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0)))); - - push 8 - push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC -$block11$34394: - call _pypy_g_SemiSpaceGC_obtain_free_space - ;; expected {32(%esp) | 20(%esp), 8(%esp), 12(%esp), 16(%esp) | %ebx, %edi, %ebp, 52(%esp), 56(%esp), 60(%esp)} - add esp, 8 - -; 1192 : l_v425 = RPyField((&pypy_g_ExcData), ed_exc_type); -; 1193 : l_v426 = (l_v425 == NULL); - - cmp DWORD PTR _pypy_g_ExcData, 0 - -; 1194 : if (!l_v426) { - - je SHORT $LN1 at pypy_g_foo - -; 1195 : l_v428 = NULL; - - xor esi, esi - -; 1196 : goto block6; - - jmp SHORT $block6$34416 -$LN1 at pypy_g_foo: - -; 1197 : } -; 1198 : l_v336 = l_v424; - - mov esi, eax -$block4$34395: - -; 1072 : goto block11; -; 1073 : } -; 1074 : l_v336 = l_v341; -; 1075 : goto block4; -; 1076 : -; 1077 : block4: -; 1078 : OP_INT_IS_TRUE(RUNNING_ON_LLINTERP, l_v346); -; 1079 : if (l_v346) { -; 1080 : goto block10; -; 1081 : } -; 1082 : goto block5; -; 1083 : -; 1084 : block5: -; 1085 : l_v347 = (struct pypy_header0 *)l_v336; -; 1086 : RPyField(l_v347, h_tid) = (GROUP_MEMBER_OFFSET(struct group_pypy_g_typeinfo_s, member1)|0L); -; 1087 : OP_ADR_ADD(l_v336, (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v349); - - lea ecx, DWORD PTR [esi+8] - mov DWORD PTR [esi], 1 -$block5$34398: - -; 1088 : RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free) = l_v349; - - mov DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12, ecx -$block6$34416: - -; 1089 : OP_ADR_ADD(l_v336, 0, l_v351); -; 1090 : l_v352 = (void*)l_v351; -; 1091 : l_v428 = l_v352; -; 1092 : goto block6; -; 1093 : -; 1094 : block6: -; 1095 : l_v353 = (void*)l_a2_1; - - mov DWORD PTR _l_v353$[esp+24], ebx - -; 1096 : l_v354 = pypy_asm_gcroot(l_v353); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v353$[esp+24] - -; 1097 : l_a2_1 = l_v354; /* for moving GCs */ -; 1098 : l_v356 = (void*)l_a5_1; - - mov edx, DWORD PTR _l_a5_1$[esp+20] - mov DWORD PTR _l_v356$[esp+24], edx - -; 1099 : l_v357 = pypy_asm_gcroot(l_v356); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v356$[esp+24] - -; 1100 : l_a5_1 = l_v357; /* for moving GCs */ -; 1101 : l_v359 = (void*)l_a6_1; - - mov eax, DWORD PTR _l_a6_1$[esp+20] - mov DWORD PTR _l_v359$[esp+24], eax - -; 1102 : l_v360 = pypy_asm_gcroot(l_v359); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v359$[esp+24] - -; 1103 : l_a6_1 = l_v360; /* for moving GCs */ -; 1104 : l_v362 = (void*)l_a1_1; - - mov DWORD PTR _l_v362$[esp+24], ebp - -; 1105 : l_v363 = pypy_asm_gcroot(l_v362); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v362$[esp+24] - -; 1106 : l_a1_1 = l_v363; /* for moving GCs */ -; 1107 : l_v365 = (void*)l_a3_1; - - mov DWORD PTR _l_v365$[esp+24], edi - -; 1108 : l_v366 = pypy_asm_gcroot(l_v365); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v365$[esp+24] - -; 1109 : l_a3_1 = l_v366; /* for moving GCs */ -; 1110 : l_v368 = (void*)l_a4_1; - - mov ecx, DWORD PTR _l_a4_1$[esp+20] - mov DWORD PTR _l_v368$[esp+24], ecx - -; 1111 : l_v369 = pypy_asm_gcroot(l_v368); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v368$[esp+24] - -; 1112 : l_a4_1 = l_v369; /* for moving GCs */ -; 1113 : l_v335 = (struct pypy_src8_A0 *)l_v428; -; 1114 : l_v371 = (l_v335 != NULL); - - test esi, esi - -; 1115 : if (!l_v371) { - - je $LN75 at pypy_g_foo - -; 1116 : goto block1; -; 1117 : } -; 1118 : goto block7; -; 1119 : -; 1120 : block7: -; 1121 : l_v372 = (struct pypy_object0 *)l_v335; -; 1122 : RPyField(l_v372, o_typeptr) = (&pypy_g_src8_A_vtable.a_super); -; 1123 : OP_INT_SUB(l_rec_1, 1L, l_v374); -; 1124 : pypy_g_foo(l_v374, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); - - mov edx, DWORD PTR tv419[esp+24] - push esi - push esi - push esi - push esi - push esi - push esi - push edx -$block7$34426: - mov DWORD PTR [esi+4], OFFSET _pypy_g_src8_A_vtable - call _pypy_g_foo - ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %esi, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} - add esp, 28 ; 0000001cH - -; 1125 : l_v376 = (void*)l_a2_1; - - mov DWORD PTR _l_v376$[esp+24], ebx - -; 1126 : l_v377 = pypy_asm_gcroot(l_v376); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v376$[esp+24] - -; 1127 : l_a2_1 = l_v377; /* for moving GCs */ -; 1128 : l_v379 = (void*)l_a6_1; - - mov eax, DWORD PTR _l_a6_1$[esp+20] - mov DWORD PTR _l_v379$[esp+24], eax - -; 1129 : l_v380 = pypy_asm_gcroot(l_v379); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v379$[esp+24] - -; 1130 : l_a6_1 = l_v380; /* for moving GCs */ -; 1131 : l_v382 = (void*)l_a1_1; - - mov DWORD PTR _l_v382$[esp+24], ebp - -; 1132 : l_v383 = pypy_asm_gcroot(l_v382); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v382$[esp+24] - -; 1133 : l_a1_1 = l_v383; /* for moving GCs */ -; 1134 : l_v385 = (void*)l_v335; - - mov DWORD PTR _l_v385$[esp+24], esi - -; 1135 : l_v386 = pypy_asm_gcroot(l_v385); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v385$[esp+24] - -; 1136 : l_v335 = l_v386; /* for moving GCs */ -; 1137 : l_v388 = (void*)l_a3_1; - - mov DWORD PTR _l_v388$[esp+24], edi - -; 1138 : l_v389 = pypy_asm_gcroot(l_v388); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v388$[esp+24] - -; 1139 : l_a3_1 = l_v389; /* for moving GCs */ -; 1140 : l_v391 = (void*)l_a5_1; - - mov ecx, DWORD PTR _l_a5_1$[esp+20] - mov DWORD PTR _l_v391$[esp+24], ecx - -; 1141 : l_v392 = pypy_asm_gcroot(l_v391); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v391$[esp+24] - -; 1142 : l_a5_1 = l_v392; /* for moving GCs */ -; 1143 : l_v394 = (void*)l_a4_1; - - mov edx, DWORD PTR _l_a4_1$[esp+20] - mov DWORD PTR _l_v394$[esp+24], edx - -; 1144 : l_v395 = pypy_asm_gcroot(l_v394); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v394$[esp+24] - -; 1145 : l_a4_1 = l_v395; /* for moving GCs */ -; 1146 : l_v397 = RPyField((&pypy_g_ExcData), ed_exc_type); -; 1147 : l_v398 = (l_v397 == NULL); - - cmp DWORD PTR _pypy_g_ExcData, 0 - -; 1148 : if (!l_v398) { - - jne $LN75 at pypy_g_foo - -; 1149 : goto block1; -; 1150 : } -; 1151 : goto block8; -; 1152 : -; 1153 : block8: -; 1154 : OP_INT_SUB(l_rec_1, 1L, l_v399); -; 1155 : pypy_g_foo(l_v399, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); - - mov eax, DWORD PTR tv419[esp+24] - push esi - push esi - push esi - push esi - push esi - push esi - push eax -$block8$34437: - call _pypy_g_foo - ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} - add esp, 28 ; 0000001cH - -; 1156 : l_v401 = (void*)l_a2_1; - - mov DWORD PTR _l_v401$[esp+24], ebx - -; 1157 : l_v402 = pypy_asm_gcroot(l_v401); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v401$[esp+24] - -; 1158 : l_a2_1 = l_v402; /* for moving GCs */ -; 1159 : l_v404 = (void*)l_a1_1; - - mov DWORD PTR _l_v404$[esp+24], ebp - -; 1160 : l_v405 = pypy_asm_gcroot(l_v404); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v404$[esp+24] - -; 1161 : l_a1_1 = l_v405; /* for moving GCs */ -; 1162 : l_v407 = (void*)l_a3_1; - - mov DWORD PTR _l_v407$[esp+24], edi - -; 1163 : l_v408 = pypy_asm_gcroot(l_v407); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v407$[esp+24] - -; 1164 : l_a3_1 = l_v408; /* for moving GCs */ -; 1165 : l_v410 = (void*)l_a6_1; - - mov ecx, DWORD PTR _l_a6_1$[esp+20] - mov DWORD PTR _l_v410$[esp+24], ecx - -; 1166 : l_v411 = pypy_asm_gcroot(l_v410); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v410$[esp+24] - -; 1167 : l_a6_1 = l_v411; /* for moving GCs */ -; 1168 : l_v413 = (void*)l_a5_1; - - mov edx, DWORD PTR _l_a5_1$[esp+20] - mov DWORD PTR _l_v413$[esp+24], edx - -; 1169 : l_v414 = pypy_asm_gcroot(l_v413); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v413$[esp+24] - -; 1170 : l_a5_1 = l_v414; /* for moving GCs */ -; 1171 : l_v416 = (void*)l_a4_1; - - mov esi, DWORD PTR _l_a4_1$[esp+20] - mov DWORD PTR _l_v416$[esp+24], esi - -; 1172 : l_v417 = pypy_asm_gcroot(l_v416); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v416$[esp+24] - -; 1173 : l_a4_1 = l_v417; /* for moving GCs */ -; 1174 : l_v419 = RPyField((&pypy_g_ExcData), ed_exc_type); -; 1175 : l_v420 = (l_v419 == NULL); - - cmp DWORD PTR _pypy_g_ExcData, 0 - -; 1176 : if (!l_v420) { - - jne SHORT $LN75 at pypy_g_foo - -; 1177 : goto block1; -; 1178 : } -; 1179 : goto block9; -; 1180 : -; 1181 : block9: -; 1182 : OP_INT_SUB(l_rec_1, 1L, l_v421); -; 1183 : pypy_g_foo(l_v421, l_a6_1, l_a5_1, l_a4_1, l_a3_1, l_a2_1, l_a1_1); - - sub DWORD PTR _l_rec_1$[esp+20], 1 - sub DWORD PTR tv419[esp+24], 1 - cmp DWORD PTR _l_rec_1$[esp+20], 0 - mov eax, ebp - mov ebp, DWORD PTR _l_a6_1$[esp+20] - mov ecx, ebx - mov ebx, DWORD PTR _l_a5_1$[esp+20] - mov edx, edi -$block9$34446: - mov edi, esi - mov DWORD PTR _l_a4_1$[esp+20], edx - mov DWORD PTR _l_a5_1$[esp+20], ecx - mov DWORD PTR _l_a6_1$[esp+20], eax -$block0_1$34376: - jg $LL63 at pypy_g_foo - pop esi - pop edi - pop ebp - pop ebx - -; 1199 : goto block4; -; 1200 : } - - add esp, 8 - ret 0 -$LN71 at pypy_g_foo: -$block0$40426: -$block0$40437: -$block0$40434: -$block2$40415: - -; 1048 : goto block2; -; 1049 : } -; 1050 : goto block1; -; 1051 : -; 1052 : block1: -; 1053 : RPY_DEBUG_RETURN(); -; 1054 : return /* nothing */; -; 1055 : -; 1056 : block2: -; 1057 : pypy_g_stack_check___(); - - mov DWORD PTR _pypy_g_ExcData, OFFSET _pypy_g_exceptions_RuntimeError_vtable - mov DWORD PTR _pypy_g_ExcData+4, OFFSET _pypy_g_exceptions_RuntimeError -$block1$40435: -$block1$40438: -$block1$40427: -$LN75 at pypy_g_foo: - pop esi - pop edi - pop ebp - pop ebx -$block1$34379: - -; 1199 : goto block4; -; 1200 : } - - add esp, 8 - ret 0 -_pypy_g_foo ENDP +; Function compile flags: /Ogtpy +; COMDAT _pypy_g_foo +_TEXT SEGMENT +tv419 = -8 ; size = 4 +_l_v416$ = -4 ; size = 4 +_l_v413$ = -4 ; size = 4 +_l_v410$ = -4 ; size = 4 +_l_v407$ = -4 ; size = 4 +_l_v404$ = -4 ; size = 4 +_l_v401$ = -4 ; size = 4 +_l_v394$ = -4 ; size = 4 +_l_v391$ = -4 ; size = 4 +_l_v388$ = -4 ; size = 4 +_l_v385$ = -4 ; size = 4 +_l_v382$ = -4 ; size = 4 +_l_v379$ = -4 ; size = 4 +_l_v376$ = -4 ; size = 4 +_l_v368$ = -4 ; size = 4 +_l_v365$ = -4 ; size = 4 +_l_v362$ = -4 ; size = 4 +_l_v359$ = -4 ; size = 4 +_l_v356$ = -4 ; size = 4 +_l_v353$ = -4 ; size = 4 +_local$40423 = 8 ; size = 1 +_l_rec_1$ = 8 ; size = 4 +_l_a1_1$ = 12 ; size = 4 +_l_a2_1$ = 16 ; size = 4 +_l_a3_1$ = 20 ; size = 4 +_l_a4_1$ = 24 ; size = 4 +_l_a5_1$ = 28 ; size = 4 +_l_a6_1$ = 32 ; size = 4 +_pypy_g_foo PROC ; COMDAT + +; 1026 : bool_t l_v337; bool_t l_v340; bool_t l_v345; bool_t l_v346; +; 1027 : bool_t l_v371; bool_t l_v398; bool_t l_v420; bool_t l_v426; +; 1028 : long l_v342; long l_v344; long l_v374; long l_v399; long l_v421; +; 1029 : struct pypy_header0 *l_v347; struct pypy_object0 *l_v372; +; 1030 : struct pypy_object_vtable0 *l_v339; +; 1031 : struct pypy_object_vtable0 *l_v397; +; 1032 : struct pypy_object_vtable0 *l_v419; +; 1033 : struct pypy_object_vtable0 *l_v425; struct pypy_src8_A0 *l_v335; +; 1034 : void* l_v336; void* l_v341; void* l_v343; void* l_v349; void* l_v351; +; 1035 : void* l_v352; void* l_v353; void* l_v354; void* l_v356; void* l_v357; +; 1036 : void* l_v359; void* l_v360; void* l_v362; void* l_v363; void* l_v365; +; 1037 : void* l_v366; void* l_v368; void* l_v369; void* l_v376; void* l_v377; +; 1038 : void* l_v379; void* l_v380; void* l_v382; void* l_v383; void* l_v385; +; 1039 : void* l_v386; void* l_v388; void* l_v389; void* l_v391; void* l_v392; +; 1040 : void* l_v394; void* l_v395; void* l_v401; void* l_v402; void* l_v404; +; 1041 : void* l_v405; void* l_v407; void* l_v408; void* l_v410; void* l_v411; +; 1042 : void* l_v413; void* l_v414; void* l_v416; void* l_v417; void* l_v424; +; 1043 : void* l_v428; +; 1044 : +; 1045 : block0: +; 1046 : OP_INT_GT(l_rec_1, 0L, l_v337); + + mov eax, DWORD PTR _l_rec_1$[esp-4] + sub esp, 8 + test eax, eax +$block0$34376: + +; 1047 : if (l_v337) { + + jle $block1$34379 + push ebx + mov ebx, DWORD PTR _l_a2_1$[esp+8] + push ebp + mov ebp, DWORD PTR _l_a1_1$[esp+12] + push edi + mov edi, DWORD PTR _l_a3_1$[esp+16] + add eax, -1 + mov DWORD PTR tv419[esp+20], eax + push esi + npad 10 +$LL63 at pypy_g_foo: + +; 1048 : goto block2; +; 1049 : } +; 1050 : goto block1; +; 1051 : +; 1052 : block1: +; 1053 : RPY_DEBUG_RETURN(); +; 1054 : return /* nothing */; +; 1055 : +; 1056 : block2: +; 1057 : pypy_g_stack_check___(); + + lea eax, DWORD PTR _local$40423[esp+20] + sub eax, DWORD PTR __LLstacktoobig_stack_base_pointer +$block0$40413: + cmp eax, DWORD PTR __LLstacktoobig_stack_min +$block2$34378: + jl SHORT $LN16 at pypy_g_foo + cmp eax, DWORD PTR __LLstacktoobig_stack_max + jle SHORT $LN17 at pypy_g_foo +$LN16 at pypy_g_foo: + call _LL_stack_too_big_slowpath + ;; expected {24(%esp) | 12(%esp), (%esp), 4(%esp), 8(%esp) | %ebx, %edi, %ebp, 44(%esp), 48(%esp), 52(%esp)} + test eax, eax + jne $LN71 at pypy_g_foo +$LN17 at pypy_g_foo: +$block1$40416: + +; 1058 : l_v339 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1059 : l_v340 = (l_v339 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1060 : if (!l_v340) { + + jne $LN75 at pypy_g_foo + +; 1061 : goto block1; +; 1062 : } +; 1063 : goto block3; +; 1064 : +; 1065 : block3: +; 1066 : l_v341 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free); + + mov esi, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12 + +; 1067 : OP_RAW_MALLOC_USAGE((0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v342); +; 1068 : l_v343 = RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_top_of_space); +; 1069 : OP_ADR_DELTA(l_v343, l_v341, l_v344); + + mov eax, DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+80 + sub eax, esi + +; 1070 : OP_INT_GT(l_v342, l_v344, l_v345); + + cmp eax, 8 +$block3$34382: + +; 1071 : if (l_v345) { + + jge SHORT $block4$34395 + +; 1184 : goto block1; +; 1185 : +; 1186 : block10: +; 1187 : abort(); /* debug_llinterpcall should be unreachable */ +; 1188 : goto block5; +; 1189 : +; 1190 : block11: +; 1191 : l_v424 = pypy_g_SemiSpaceGC_obtain_free_space((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0)))); + + push 8 + push OFFSET _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC +$block11$34394: + call _pypy_g_SemiSpaceGC_obtain_free_space + ;; expected {32(%esp) | 20(%esp), 8(%esp), 12(%esp), 16(%esp) | %ebx, %edi, %ebp, 52(%esp), 56(%esp), 60(%esp)} + add esp, 8 + +; 1192 : l_v425 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1193 : l_v426 = (l_v425 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1194 : if (!l_v426) { + + je SHORT $LN1 at pypy_g_foo + +; 1195 : l_v428 = NULL; + + xor esi, esi + +; 1196 : goto block6; + + jmp SHORT $block6$34416 +$LN1 at pypy_g_foo: + +; 1197 : } +; 1198 : l_v336 = l_v424; + + mov esi, eax +$block4$34395: + +; 1072 : goto block11; +; 1073 : } +; 1074 : l_v336 = l_v341; +; 1075 : goto block4; +; 1076 : +; 1077 : block4: +; 1078 : OP_INT_IS_TRUE(RUNNING_ON_LLINTERP, l_v346); +; 1079 : if (l_v346) { +; 1080 : goto block10; +; 1081 : } +; 1082 : goto block5; +; 1083 : +; 1084 : block5: +; 1085 : l_v347 = (struct pypy_header0 *)l_v336; +; 1086 : RPyField(l_v347, h_tid) = (GROUP_MEMBER_OFFSET(struct group_pypy_g_typeinfo_s, member1)|0L); +; 1087 : OP_ADR_ADD(l_v336, (0 + ROUND_UP_FOR_ALLOCATION(sizeof(struct pypy_src8_A0), sizeof(struct pypy_forwarding_stub0))), l_v349); + + lea ecx, DWORD PTR [esi+8] + mov DWORD PTR [esi], 1 +$block5$34398: + +; 1088 : RPyField((&pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC), ssgc_inst_free) = l_v349; + + mov DWORD PTR _pypy_g_pypy_rpython_memory_gc_semispace_SemiSpaceGC+12, ecx +$block6$34416: + +; 1089 : OP_ADR_ADD(l_v336, 0, l_v351); +; 1090 : l_v352 = (void*)l_v351; +; 1091 : l_v428 = l_v352; +; 1092 : goto block6; +; 1093 : +; 1094 : block6: +; 1095 : l_v353 = (void*)l_a2_1; + + mov DWORD PTR _l_v353$[esp+24], ebx + +; 1096 : l_v354 = pypy_asm_gcroot(l_v353); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v353$[esp+24] + +; 1097 : l_a2_1 = l_v354; /* for moving GCs */ +; 1098 : l_v356 = (void*)l_a5_1; + + mov edx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v356$[esp+24], edx + +; 1099 : l_v357 = pypy_asm_gcroot(l_v356); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v356$[esp+24] + +; 1100 : l_a5_1 = l_v357; /* for moving GCs */ +; 1101 : l_v359 = (void*)l_a6_1; + + mov eax, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v359$[esp+24], eax + +; 1102 : l_v360 = pypy_asm_gcroot(l_v359); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v359$[esp+24] + +; 1103 : l_a6_1 = l_v360; /* for moving GCs */ +; 1104 : l_v362 = (void*)l_a1_1; + + mov DWORD PTR _l_v362$[esp+24], ebp + +; 1105 : l_v363 = pypy_asm_gcroot(l_v362); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v362$[esp+24] + +; 1106 : l_a1_1 = l_v363; /* for moving GCs */ +; 1107 : l_v365 = (void*)l_a3_1; + + mov DWORD PTR _l_v365$[esp+24], edi + +; 1108 : l_v366 = pypy_asm_gcroot(l_v365); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v365$[esp+24] + +; 1109 : l_a3_1 = l_v366; /* for moving GCs */ +; 1110 : l_v368 = (void*)l_a4_1; + + mov ecx, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v368$[esp+24], ecx + +; 1111 : l_v369 = pypy_asm_gcroot(l_v368); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v368$[esp+24] + +; 1112 : l_a4_1 = l_v369; /* for moving GCs */ +; 1113 : l_v335 = (struct pypy_src8_A0 *)l_v428; +; 1114 : l_v371 = (l_v335 != NULL); + + test esi, esi + +; 1115 : if (!l_v371) { + + je $LN75 at pypy_g_foo + +; 1116 : goto block1; +; 1117 : } +; 1118 : goto block7; +; 1119 : +; 1120 : block7: +; 1121 : l_v372 = (struct pypy_object0 *)l_v335; +; 1122 : RPyField(l_v372, o_typeptr) = (&pypy_g_src8_A_vtable.a_super); +; 1123 : OP_INT_SUB(l_rec_1, 1L, l_v374); +; 1124 : pypy_g_foo(l_v374, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); + + mov edx, DWORD PTR tv419[esp+24] + push esi + push esi + push esi + push esi + push esi + push esi + push edx +$block7$34426: + mov DWORD PTR [esi+4], OFFSET _pypy_g_src8_A_vtable + call _pypy_g_foo + ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %esi, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + add esp, 28 ; 0000001cH + +; 1125 : l_v376 = (void*)l_a2_1; + + mov DWORD PTR _l_v376$[esp+24], ebx + +; 1126 : l_v377 = pypy_asm_gcroot(l_v376); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v376$[esp+24] + +; 1127 : l_a2_1 = l_v377; /* for moving GCs */ +; 1128 : l_v379 = (void*)l_a6_1; + + mov eax, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v379$[esp+24], eax + +; 1129 : l_v380 = pypy_asm_gcroot(l_v379); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v379$[esp+24] + +; 1130 : l_a6_1 = l_v380; /* for moving GCs */ +; 1131 : l_v382 = (void*)l_a1_1; + + mov DWORD PTR _l_v382$[esp+24], ebp + +; 1132 : l_v383 = pypy_asm_gcroot(l_v382); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v382$[esp+24] + +; 1133 : l_a1_1 = l_v383; /* for moving GCs */ +; 1134 : l_v385 = (void*)l_v335; + + mov DWORD PTR _l_v385$[esp+24], esi + +; 1135 : l_v386 = pypy_asm_gcroot(l_v385); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v385$[esp+24] + +; 1136 : l_v335 = l_v386; /* for moving GCs */ +; 1137 : l_v388 = (void*)l_a3_1; + + mov DWORD PTR _l_v388$[esp+24], edi + +; 1138 : l_v389 = pypy_asm_gcroot(l_v388); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v388$[esp+24] + +; 1139 : l_a3_1 = l_v389; /* for moving GCs */ +; 1140 : l_v391 = (void*)l_a5_1; + + mov ecx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v391$[esp+24], ecx + +; 1141 : l_v392 = pypy_asm_gcroot(l_v391); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v391$[esp+24] + +; 1142 : l_a5_1 = l_v392; /* for moving GCs */ +; 1143 : l_v394 = (void*)l_a4_1; + + mov edx, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v394$[esp+24], edx + +; 1144 : l_v395 = pypy_asm_gcroot(l_v394); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v394$[esp+24] + +; 1145 : l_a4_1 = l_v395; /* for moving GCs */ +; 1146 : l_v397 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1147 : l_v398 = (l_v397 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1148 : if (!l_v398) { + + jne $LN75 at pypy_g_foo + +; 1149 : goto block1; +; 1150 : } +; 1151 : goto block8; +; 1152 : +; 1153 : block8: +; 1154 : OP_INT_SUB(l_rec_1, 1L, l_v399); +; 1155 : pypy_g_foo(l_v399, l_v335, l_v335, l_v335, l_v335, l_v335, l_v335); + + mov eax, DWORD PTR tv419[esp+24] + push esi + push esi + push esi + push esi + push esi + push esi + push eax +$block8$34437: + call _pypy_g_foo + ;; expected {52(%esp) | 40(%esp), 28(%esp), 32(%esp), 36(%esp) | %ebx, %edi, %ebp, 72(%esp), 76(%esp), 80(%esp)} + add esp, 28 ; 0000001cH + +; 1156 : l_v401 = (void*)l_a2_1; + + mov DWORD PTR _l_v401$[esp+24], ebx + +; 1157 : l_v402 = pypy_asm_gcroot(l_v401); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v401$[esp+24] + +; 1158 : l_a2_1 = l_v402; /* for moving GCs */ +; 1159 : l_v404 = (void*)l_a1_1; + + mov DWORD PTR _l_v404$[esp+24], ebp + +; 1160 : l_v405 = pypy_asm_gcroot(l_v404); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v404$[esp+24] + +; 1161 : l_a1_1 = l_v405; /* for moving GCs */ +; 1162 : l_v407 = (void*)l_a3_1; + + mov DWORD PTR _l_v407$[esp+24], edi + +; 1163 : l_v408 = pypy_asm_gcroot(l_v407); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v407$[esp+24] + +; 1164 : l_a3_1 = l_v408; /* for moving GCs */ +; 1165 : l_v410 = (void*)l_a6_1; + + mov ecx, DWORD PTR _l_a6_1$[esp+20] + mov DWORD PTR _l_v410$[esp+24], ecx + +; 1166 : l_v411 = pypy_asm_gcroot(l_v410); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v410$[esp+24] + +; 1167 : l_a6_1 = l_v411; /* for moving GCs */ +; 1168 : l_v413 = (void*)l_a5_1; + + mov edx, DWORD PTR _l_a5_1$[esp+20] + mov DWORD PTR _l_v413$[esp+24], edx + +; 1169 : l_v414 = pypy_asm_gcroot(l_v413); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v413$[esp+24] + +; 1170 : l_a5_1 = l_v414; /* for moving GCs */ +; 1171 : l_v416 = (void*)l_a4_1; + + mov esi, DWORD PTR _l_a4_1$[esp+20] + mov DWORD PTR _l_v416$[esp+24], esi + +; 1172 : l_v417 = pypy_asm_gcroot(l_v416); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v416$[esp+24] + +; 1173 : l_a4_1 = l_v417; /* for moving GCs */ +; 1174 : l_v419 = RPyField((&pypy_g_ExcData), ed_exc_type); +; 1175 : l_v420 = (l_v419 == NULL); + + cmp DWORD PTR _pypy_g_ExcData, 0 + +; 1176 : if (!l_v420) { + + jne SHORT $LN75 at pypy_g_foo + +; 1177 : goto block1; +; 1178 : } +; 1179 : goto block9; +; 1180 : +; 1181 : block9: +; 1182 : OP_INT_SUB(l_rec_1, 1L, l_v421); +; 1183 : pypy_g_foo(l_v421, l_a6_1, l_a5_1, l_a4_1, l_a3_1, l_a2_1, l_a1_1); + + sub DWORD PTR _l_rec_1$[esp+20], 1 + sub DWORD PTR tv419[esp+24], 1 + cmp DWORD PTR _l_rec_1$[esp+20], 0 + mov eax, ebp + mov ebp, DWORD PTR _l_a6_1$[esp+20] + mov ecx, ebx + mov ebx, DWORD PTR _l_a5_1$[esp+20] + mov edx, edi +$block9$34446: + mov edi, esi + mov DWORD PTR _l_a4_1$[esp+20], edx + mov DWORD PTR _l_a5_1$[esp+20], ecx + mov DWORD PTR _l_a6_1$[esp+20], eax +$block0_1$34376: + jg $LL63 at pypy_g_foo + pop esi + pop edi + pop ebp + pop ebx + +; 1199 : goto block4; +; 1200 : } + + add esp, 8 + ret 0 +$LN71 at pypy_g_foo: +$block0$40426: +$block0$40437: +$block0$40434: +$block2$40415: + +; 1048 : goto block2; +; 1049 : } +; 1050 : goto block1; +; 1051 : +; 1052 : block1: +; 1053 : RPY_DEBUG_RETURN(); +; 1054 : return /* nothing */; +; 1055 : +; 1056 : block2: +; 1057 : pypy_g_stack_check___(); + + mov DWORD PTR _pypy_g_ExcData, OFFSET _pypy_g_exceptions_RuntimeError_vtable + mov DWORD PTR _pypy_g_ExcData+4, OFFSET _pypy_g_exceptions_RuntimeError +$block1$40435: +$block1$40438: +$block1$40427: +$LN75 at pypy_g_foo: + pop esi + pop edi + pop ebp + pop ebx +$block1$34379: + +; 1199 : goto block4; +; 1200 : } + + add esp, 8 + ret 0 +_pypy_g_foo ENDP Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s Thu Nov 5 23:13:47 2009 @@ -1,420 +1,420 @@ -; Function compile flags: /Odtpy -; COMDAT _pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root -_TEXT SEGMENT -tv138 = -116 ; size = 4 -_l_v271568$ = -109 ; size = 1 -_l_v271567$ = -108 ; size = 4 -_l_v271575$ = -104 ; size = 4 -_l_v271551$ = -97 ; size = 1 -_l_v271579$ = -96 ; size = 4 -_l_v271576$ = -89 ; size = 1 -_l_v271572$ = -88 ; size = 4 -_l_v271583$ = -84 ; size = 4 -_l_v271556$ = -80 ; size = 4 -_l_v271559$ = -76 ; size = 4 -_l_v271544$ = -72 ; size = 4 -_l_v271545$ = -68 ; size = 4 -_l_v271580$ = -64 ; size = 4 -_l_v271557$ = -60 ; size = 4 -_l_v271581$ = -56 ; size = 4 -_l_v271553$ = -52 ; size = 4 -_l_v271570$ = -48 ; size = 4 -_l_v271554$ = -42 ; size = 1 -_l_v271565$ = -41 ; size = 1 -_l_v271558$ = -40 ; size = 4 -_l_v271562$ = -33 ; size = 1 -_l_v271561$ = -32 ; size = 4 -_l_v271547$ = -28 ; size = 4 -_l_v271548$ = -24 ; size = 4 -_l_v271573$ = -18 ; size = 1 -_l_v271546$ = -17 ; size = 1 -_l_v271582$ = -16 ; size = 4 -_l_v271550$ = -12 ; size = 4 -_l_v271564$ = -8 ; size = 4 -_l_v271578$ = -4 ; size = 4 -_l_self_596$ = 8 ; size = 4 -_l_scope_w_259$ = 12 ; size = 4 -_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root PROC ; COMDAT - -; 15629: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root(struct pypy_pypy_interpreter_gateway_BuiltinActivation0 *l_self_596, struct pypy_array1 *l_scope_w_259) { - - push ebp - mov ebp, esp - sub esp, 116 ; 00000074H -$block0$211591: - -; 15630: bool_t l_v271551; bool_t l_v271554; bool_t l_v271562; -; 15631: bool_t l_v271565; bool_t l_v271568; bool_t l_v271573; -; 15632: bool_t l_v271576; char l_v271546; long l_v271550; long l_v271553; -; 15633: long l_v271564; long l_v271567; long l_v271572; long l_v271575; -; 15634: struct pypy_object0 *l_v271556; struct pypy_object0 *l_v271570; -; 15635: struct pypy_object0 *l_v271578; -; 15636: struct pypy_object_vtable0 *l_v271561; -; 15637: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271544; -; 15638: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271545; -; 15639: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271557; -; 15640: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271579; -; 15641: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271580; -; 15642: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271581; -; 15643: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271582; -; 15644: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271583; -; 15645: struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *l_v271548; -; 15646: struct pypy_pypy_module_unicodedata_interp_ucd_UCD0 *l_v271547; -; 15647: void* l_v271558; void* l_v271559; -; 15648: -; 15649: block0: -; 15650: l_v271548 = (struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *)l_self_596; - - mov eax, DWORD PTR _l_self_596$[ebp] - mov DWORD PTR _l_v271548$[ebp], eax - -; 15651: l_v271546 = RPyField(l_v271548, bausucdoswrwr_inst_behavior); - - mov ecx, DWORD PTR _l_v271548$[ebp] - mov dl, BYTE PTR [ecx+8] - mov BYTE PTR _l_v271546$[ebp], dl - -; 15652: RPyAssert(1, "unexpectedly negative list getitem index"); -; 15653: l_v271550 = l_scope_w_259->length; - - mov eax, DWORD PTR _l_scope_w_259$[ebp] - mov ecx, DWORD PTR [eax+4] - mov DWORD PTR _l_v271550$[ebp], ecx - -; 15654: OP_INT_LT(0L, l_v271550, l_v271551); - - xor edx, edx - cmp DWORD PTR _l_v271550$[ebp], 0 - setg dl - mov BYTE PTR _l_v271551$[ebp], dl - -; 15655: RPyAssert(l_v271551, "list getitem index out of bound"); -; 15656: l_v271553 = l_scope_w_259->length; - - mov eax, DWORD PTR _l_scope_w_259$[ebp] - mov ecx, DWORD PTR [eax+4] - mov DWORD PTR _l_v271553$[ebp], ecx - -; 15657: OP_INT_LT(0L, l_v271553, l_v271554); - - xor edx, edx - cmp DWORD PTR _l_v271553$[ebp], 0 - setg dl - mov BYTE PTR _l_v271554$[ebp], dl - -; 15658: RPyAssert(l_v271554, "fixed getitem out of bounds"); -; 15659: l_v271556 = RPyItem(l_scope_w_259, 0L); - - mov eax, DWORD PTR _l_scope_w_259$[ebp] - mov ecx, DWORD PTR [eax+8] - mov DWORD PTR _l_v271556$[ebp], ecx - -; 15660: l_v271557 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271556; - - mov edx, DWORD PTR _l_v271556$[ebp] - mov DWORD PTR _l_v271557$[ebp], edx - -; 15661: l_v271547 = pypy_g_interp_w__UCD(l_v271557, 0); - - push 0 - mov eax, DWORD PTR _l_v271557$[ebp] - push eax - call _pypy_g_interp_w__UCD - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | 12(%ebp)} - add esp, 8 - mov DWORD PTR _l_v271547$[ebp], eax - -; 15662: l_v271558 = (void*)l_scope_w_259; - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov DWORD PTR _l_v271558$[ebp], ecx - -; 15663: l_v271559 = pypy_asm_gcroot(l_v271558); - - mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 - imul eax, DWORD PTR _l_v271558$[ebp] - mov edx, DWORD PTR _l_v271558$[ebp] - mov DWORD PTR _l_v271559$[ebp], edx - -; 15664: l_scope_w_259 = l_v271559; /* for moving GCs */ - - mov eax, DWORD PTR _l_v271559$[ebp] - mov DWORD PTR _l_scope_w_259$[ebp], eax - -; 15665: l_v271561 = RPyField((&pypy_g_ExcData), ed_exc_type); - - mov ecx, DWORD PTR _pypy_g_ExcData - mov DWORD PTR _l_v271561$[ebp], ecx - -; 15666: l_v271562 = (l_v271561 == NULL); - - xor edx, edx - cmp DWORD PTR _l_v271561$[ebp], 0 - sete dl - mov BYTE PTR _l_v271562$[ebp], dl - -; 15667: if (!l_v271562) { - - movzx eax, BYTE PTR _l_v271562$[ebp] - test eax, eax - jne SHORT $block1$211600 - -; 15668: l_v271583 = ((struct pypy_pypy_interpreter_baseobjspace_W_Root0 *) NULL); - - mov DWORD PTR _l_v271583$[ebp], 0 - -; 15669: goto block3; - - jmp $block3$211599 -$block1$211600: - -; 15670: } -; 15671: goto block1; -; 15672: -; 15673: block1: -; 15674: RPyAssert(1, "unexpectedly negative list getitem index"); -; 15675: l_v271564 = l_scope_w_259->length; - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_v271564$[ebp], edx - -; 15676: OP_INT_LT(1L, l_v271564, l_v271565); - - xor eax, eax - cmp DWORD PTR _l_v271564$[ebp], 1 - setg al - mov BYTE PTR _l_v271565$[ebp], al - -; 15677: RPyAssert(l_v271565, "list getitem index out of bound"); -; 15678: l_v271567 = l_scope_w_259->length; - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_v271567$[ebp], edx - -; 15679: OP_INT_LT(1L, l_v271567, l_v271568); - - xor eax, eax - cmp DWORD PTR _l_v271567$[ebp], 1 - setg al - mov BYTE PTR _l_v271568$[ebp], al - -; 15680: RPyAssert(l_v271568, "fixed getitem out of bounds"); -; 15681: l_v271570 = RPyItem(l_scope_w_259, 1L); - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+12] - mov DWORD PTR _l_v271570$[ebp], edx - -; 15682: l_v271544 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271570; - - mov eax, DWORD PTR _l_v271570$[ebp] - mov DWORD PTR _l_v271544$[ebp], eax - -; 15683: RPyAssert(1, "unexpectedly negative list getitem index"); -; 15684: l_v271572 = l_scope_w_259->length; - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_v271572$[ebp], edx - -; 15685: OP_INT_LT(2L, l_v271572, l_v271573); - - xor eax, eax - cmp DWORD PTR _l_v271572$[ebp], 2 - setg al - mov BYTE PTR _l_v271573$[ebp], al - -; 15686: RPyAssert(l_v271573, "list getitem index out of bound"); -; 15687: l_v271575 = l_scope_w_259->length; - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+4] - mov DWORD PTR _l_v271575$[ebp], edx - -; 15688: OP_INT_LT(2L, l_v271575, l_v271576); - - xor eax, eax - cmp DWORD PTR _l_v271575$[ebp], 2 - setg al - mov BYTE PTR _l_v271576$[ebp], al - -; 15689: RPyAssert(l_v271576, "fixed getitem out of bounds"); -; 15690: l_v271578 = RPyItem(l_scope_w_259, 2L); - - mov ecx, DWORD PTR _l_scope_w_259$[ebp] - mov edx, DWORD PTR [ecx+16] - mov DWORD PTR _l_v271578$[ebp], edx - -; 15691: l_v271545 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271578; - - mov eax, DWORD PTR _l_v271578$[ebp] - mov DWORD PTR _l_v271545$[ebp], eax - -; 15692: switch (l_v271546) { - - movsx ecx, BYTE PTR _l_v271546$[ebp] - mov DWORD PTR tv138[ebp], ecx - cmp DWORD PTR tv138[ebp], 3 - ja SHORT $LN1 at pypy_g_Bui@2 - mov edx, DWORD PTR tv138[ebp] - jmp DWORD PTR $LN14 at pypy_g_Bui@2[edx*4] -$LN5 at pypy_g_Bui@2: - -; 15693: case 0: -; 15694: goto block2; - - jmp SHORT $block2$211608 -$LN4 at pypy_g_Bui@2: - -; 15695: case 1: -; 15696: goto block4; - - jmp SHORT $block4$211610 -$LN3 at pypy_g_Bui@2: - -; 15697: case 2: -; 15698: goto block5; - - jmp SHORT $block5$211612 -$LN2 at pypy_g_Bui@2: - -; 15699: case 3: -; 15700: goto block6; - - jmp $block6$211614 -$LN1 at pypy_g_Bui@2: - -; 15701: default: -; 15702: assert(!"bad switch!!"); - - mov eax, OFFSET ??_C at _0N@PGLFNKFI at bad?5switch?$CB?$CB?$AA@ - test eax, eax - je SHORT $block2$211608 - push 15702 ; 00003d56H - push OFFSET ??_C at _1BO@DMBFIACJ@?$AAi?$AAm?$AAp?$AAl?$AAe?$AAm?$AAe?$AAn?$AAt?$AA_?$AA1?$AA1?$AA?4?$AAc?$AA?$AA@ - push OFFSET ??_C at _1CA@EIJBLFPJ@?$AA?$CB?$AA?$CC?$AAb?$AAa?$AAd?$AA?5?$AAs?$AAw?$AAi?$AAt?$AAc?$AAh?$AA?$CB?$AA?$CB?$AA?$CC?$AA?$AA@ - call DWORD PTR __imp___wassert - add esp, 12 ; 0000000cH -$block2$211608: - -; 15703: } -; 15704: -; 15705: block2: -; 15706: l_v271579 = pypy_g_UCD_digit(l_v271547, l_v271544, l_v271545); - - mov edx, DWORD PTR _l_v271545$[ebp] - push edx - mov eax, DWORD PTR _l_v271544$[ebp] - push eax - mov ecx, DWORD PTR _l_v271547$[ebp] - push ecx - call _pypy_g_UCD_digit - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 12 ; 0000000cH - mov DWORD PTR _l_v271579$[ebp], eax - -; 15707: l_v271583 = l_v271579; - - mov edx, DWORD PTR _l_v271579$[ebp] - mov DWORD PTR _l_v271583$[ebp], edx -$block3$211599: - -; 15708: goto block3; -; 15709: -; 15710: block3: -; 15711: RPY_DEBUG_RETURN(); -; 15712: return l_v271583; - - mov eax, DWORD PTR _l_v271583$[ebp] - jmp SHORT $LN9 at pypy_g_Bui@2 -$block4$211610: - -; 15713: -; 15714: block4: -; 15715: l_v271580 = pypy_g_UCD_name(l_v271547, l_v271544, l_v271545); - - mov eax, DWORD PTR _l_v271545$[ebp] - push eax - mov ecx, DWORD PTR _l_v271544$[ebp] - push ecx - mov edx, DWORD PTR _l_v271547$[ebp] - push edx - call _pypy_g_UCD_name - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 12 ; 0000000cH - mov DWORD PTR _l_v271580$[ebp], eax - -; 15716: l_v271583 = l_v271580; - - mov eax, DWORD PTR _l_v271580$[ebp] - mov DWORD PTR _l_v271583$[ebp], eax - -; 15717: goto block3; - - jmp SHORT $block3$211599 -$block5$211612: - -; 15718: -; 15719: block5: -; 15720: l_v271581 = pypy_g_UCD_decimal(l_v271547, l_v271544, l_v271545); - - mov ecx, DWORD PTR _l_v271545$[ebp] - push ecx - mov edx, DWORD PTR _l_v271544$[ebp] - push edx - mov eax, DWORD PTR _l_v271547$[ebp] - push eax - call _pypy_g_UCD_decimal - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 12 ; 0000000cH - mov DWORD PTR _l_v271581$[ebp], eax - -; 15721: l_v271583 = l_v271581; - - mov ecx, DWORD PTR _l_v271581$[ebp] - mov DWORD PTR _l_v271583$[ebp], ecx - -; 15722: goto block3; - - jmp SHORT $block3$211599 -$block6$211614: - -; 15723: -; 15724: block6: -; 15725: l_v271582 = pypy_g_UCD_numeric(l_v271547, l_v271544, l_v271545); - - mov edx, DWORD PTR _l_v271545$[ebp] - push edx - mov eax, DWORD PTR _l_v271544$[ebp] - push eax - mov ecx, DWORD PTR _l_v271547$[ebp] - push ecx - call _pypy_g_UCD_numeric - ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } - add esp, 12 ; 0000000cH - mov DWORD PTR _l_v271582$[ebp], eax - -; 15726: l_v271583 = l_v271582; - - mov edx, DWORD PTR _l_v271582$[ebp] - mov DWORD PTR _l_v271583$[ebp], edx - -; 15727: goto block3; - - jmp SHORT $block3$211599 -$LN9 at pypy_g_Bui@2: - -; 15728: } - - mov esp, ebp - pop ebp - ret 0 - npad 3 -$LN14 at pypy_g_Bui@2: - DD $LN5 at pypy_g_Bui@2 - DD $LN4 at pypy_g_Bui@2 - DD $LN3 at pypy_g_Bui@2 - DD $LN2 at pypy_g_Bui@2 -_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root ENDP +; Function compile flags: /Odtpy +; COMDAT _pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root +_TEXT SEGMENT +tv138 = -116 ; size = 4 +_l_v271568$ = -109 ; size = 1 +_l_v271567$ = -108 ; size = 4 +_l_v271575$ = -104 ; size = 4 +_l_v271551$ = -97 ; size = 1 +_l_v271579$ = -96 ; size = 4 +_l_v271576$ = -89 ; size = 1 +_l_v271572$ = -88 ; size = 4 +_l_v271583$ = -84 ; size = 4 +_l_v271556$ = -80 ; size = 4 +_l_v271559$ = -76 ; size = 4 +_l_v271544$ = -72 ; size = 4 +_l_v271545$ = -68 ; size = 4 +_l_v271580$ = -64 ; size = 4 +_l_v271557$ = -60 ; size = 4 +_l_v271581$ = -56 ; size = 4 +_l_v271553$ = -52 ; size = 4 +_l_v271570$ = -48 ; size = 4 +_l_v271554$ = -42 ; size = 1 +_l_v271565$ = -41 ; size = 1 +_l_v271558$ = -40 ; size = 4 +_l_v271562$ = -33 ; size = 1 +_l_v271561$ = -32 ; size = 4 +_l_v271547$ = -28 ; size = 4 +_l_v271548$ = -24 ; size = 4 +_l_v271573$ = -18 ; size = 1 +_l_v271546$ = -17 ; size = 1 +_l_v271582$ = -16 ; size = 4 +_l_v271550$ = -12 ; size = 4 +_l_v271564$ = -8 ; size = 4 +_l_v271578$ = -4 ; size = 4 +_l_self_596$ = 8 ; size = 4 +_l_scope_w_259$ = 12 ; size = 4 +_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root PROC ; COMDAT + +; 15629: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root(struct pypy_pypy_interpreter_gateway_BuiltinActivation0 *l_self_596, struct pypy_array1 *l_scope_w_259) { + + push ebp + mov ebp, esp + sub esp, 116 ; 00000074H +$block0$211591: + +; 15630: bool_t l_v271551; bool_t l_v271554; bool_t l_v271562; +; 15631: bool_t l_v271565; bool_t l_v271568; bool_t l_v271573; +; 15632: bool_t l_v271576; char l_v271546; long l_v271550; long l_v271553; +; 15633: long l_v271564; long l_v271567; long l_v271572; long l_v271575; +; 15634: struct pypy_object0 *l_v271556; struct pypy_object0 *l_v271570; +; 15635: struct pypy_object0 *l_v271578; +; 15636: struct pypy_object_vtable0 *l_v271561; +; 15637: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271544; +; 15638: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271545; +; 15639: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271557; +; 15640: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271579; +; 15641: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271580; +; 15642: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271581; +; 15643: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271582; +; 15644: struct pypy_pypy_interpreter_baseobjspace_W_Root0 *l_v271583; +; 15645: struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *l_v271548; +; 15646: struct pypy_pypy_module_unicodedata_interp_ucd_UCD0 *l_v271547; +; 15647: void* l_v271558; void* l_v271559; +; 15648: +; 15649: block0: +; 15650: l_v271548 = (struct pypy_pypy_interpreter_gateway_BuiltinActivation_UwS_UCD0 *)l_self_596; + + mov eax, DWORD PTR _l_self_596$[ebp] + mov DWORD PTR _l_v271548$[ebp], eax + +; 15651: l_v271546 = RPyField(l_v271548, bausucdoswrwr_inst_behavior); + + mov ecx, DWORD PTR _l_v271548$[ebp] + mov dl, BYTE PTR [ecx+8] + mov BYTE PTR _l_v271546$[ebp], dl + +; 15652: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15653: l_v271550 = l_scope_w_259->length; + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v271550$[ebp], ecx + +; 15654: OP_INT_LT(0L, l_v271550, l_v271551); + + xor edx, edx + cmp DWORD PTR _l_v271550$[ebp], 0 + setg dl + mov BYTE PTR _l_v271551$[ebp], dl + +; 15655: RPyAssert(l_v271551, "list getitem index out of bound"); +; 15656: l_v271553 = l_scope_w_259->length; + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+4] + mov DWORD PTR _l_v271553$[ebp], ecx + +; 15657: OP_INT_LT(0L, l_v271553, l_v271554); + + xor edx, edx + cmp DWORD PTR _l_v271553$[ebp], 0 + setg dl + mov BYTE PTR _l_v271554$[ebp], dl + +; 15658: RPyAssert(l_v271554, "fixed getitem out of bounds"); +; 15659: l_v271556 = RPyItem(l_scope_w_259, 0L); + + mov eax, DWORD PTR _l_scope_w_259$[ebp] + mov ecx, DWORD PTR [eax+8] + mov DWORD PTR _l_v271556$[ebp], ecx + +; 15660: l_v271557 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271556; + + mov edx, DWORD PTR _l_v271556$[ebp] + mov DWORD PTR _l_v271557$[ebp], edx + +; 15661: l_v271547 = pypy_g_interp_w__UCD(l_v271557, 0); + + push 0 + mov eax, DWORD PTR _l_v271557$[ebp] + push eax + call _pypy_g_interp_w__UCD + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | 12(%ebp)} + add esp, 8 + mov DWORD PTR _l_v271547$[ebp], eax + +; 15662: l_v271558 = (void*)l_scope_w_259; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov DWORD PTR _l_v271558$[ebp], ecx + +; 15663: l_v271559 = pypy_asm_gcroot(l_v271558); + + mov eax, DWORD PTR ?_constant_always_one_@?1??pypy_asm_gcroot@@9 at 9 + imul eax, DWORD PTR _l_v271558$[ebp] + mov edx, DWORD PTR _l_v271558$[ebp] + mov DWORD PTR _l_v271559$[ebp], edx + +; 15664: l_scope_w_259 = l_v271559; /* for moving GCs */ + + mov eax, DWORD PTR _l_v271559$[ebp] + mov DWORD PTR _l_scope_w_259$[ebp], eax + +; 15665: l_v271561 = RPyField((&pypy_g_ExcData), ed_exc_type); + + mov ecx, DWORD PTR _pypy_g_ExcData + mov DWORD PTR _l_v271561$[ebp], ecx + +; 15666: l_v271562 = (l_v271561 == NULL); + + xor edx, edx + cmp DWORD PTR _l_v271561$[ebp], 0 + sete dl + mov BYTE PTR _l_v271562$[ebp], dl + +; 15667: if (!l_v271562) { + + movzx eax, BYTE PTR _l_v271562$[ebp] + test eax, eax + jne SHORT $block1$211600 + +; 15668: l_v271583 = ((struct pypy_pypy_interpreter_baseobjspace_W_Root0 *) NULL); + + mov DWORD PTR _l_v271583$[ebp], 0 + +; 15669: goto block3; + + jmp $block3$211599 +$block1$211600: + +; 15670: } +; 15671: goto block1; +; 15672: +; 15673: block1: +; 15674: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15675: l_v271564 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271564$[ebp], edx + +; 15676: OP_INT_LT(1L, l_v271564, l_v271565); + + xor eax, eax + cmp DWORD PTR _l_v271564$[ebp], 1 + setg al + mov BYTE PTR _l_v271565$[ebp], al + +; 15677: RPyAssert(l_v271565, "list getitem index out of bound"); +; 15678: l_v271567 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271567$[ebp], edx + +; 15679: OP_INT_LT(1L, l_v271567, l_v271568); + + xor eax, eax + cmp DWORD PTR _l_v271567$[ebp], 1 + setg al + mov BYTE PTR _l_v271568$[ebp], al + +; 15680: RPyAssert(l_v271568, "fixed getitem out of bounds"); +; 15681: l_v271570 = RPyItem(l_scope_w_259, 1L); + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+12] + mov DWORD PTR _l_v271570$[ebp], edx + +; 15682: l_v271544 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271570; + + mov eax, DWORD PTR _l_v271570$[ebp] + mov DWORD PTR _l_v271544$[ebp], eax + +; 15683: RPyAssert(1, "unexpectedly negative list getitem index"); +; 15684: l_v271572 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271572$[ebp], edx + +; 15685: OP_INT_LT(2L, l_v271572, l_v271573); + + xor eax, eax + cmp DWORD PTR _l_v271572$[ebp], 2 + setg al + mov BYTE PTR _l_v271573$[ebp], al + +; 15686: RPyAssert(l_v271573, "list getitem index out of bound"); +; 15687: l_v271575 = l_scope_w_259->length; + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+4] + mov DWORD PTR _l_v271575$[ebp], edx + +; 15688: OP_INT_LT(2L, l_v271575, l_v271576); + + xor eax, eax + cmp DWORD PTR _l_v271575$[ebp], 2 + setg al + mov BYTE PTR _l_v271576$[ebp], al + +; 15689: RPyAssert(l_v271576, "fixed getitem out of bounds"); +; 15690: l_v271578 = RPyItem(l_scope_w_259, 2L); + + mov ecx, DWORD PTR _l_scope_w_259$[ebp] + mov edx, DWORD PTR [ecx+16] + mov DWORD PTR _l_v271578$[ebp], edx + +; 15691: l_v271545 = (struct pypy_pypy_interpreter_baseobjspace_W_Root0 *)l_v271578; + + mov eax, DWORD PTR _l_v271578$[ebp] + mov DWORD PTR _l_v271545$[ebp], eax + +; 15692: switch (l_v271546) { + + movsx ecx, BYTE PTR _l_v271546$[ebp] + mov DWORD PTR tv138[ebp], ecx + cmp DWORD PTR tv138[ebp], 3 + ja SHORT $LN1 at pypy_g_Bui@2 + mov edx, DWORD PTR tv138[ebp] + jmp DWORD PTR $LN14 at pypy_g_Bui@2[edx*4] +$LN5 at pypy_g_Bui@2: + +; 15693: case 0: +; 15694: goto block2; + + jmp SHORT $block2$211608 +$LN4 at pypy_g_Bui@2: + +; 15695: case 1: +; 15696: goto block4; + + jmp SHORT $block4$211610 +$LN3 at pypy_g_Bui@2: + +; 15697: case 2: +; 15698: goto block5; + + jmp SHORT $block5$211612 +$LN2 at pypy_g_Bui@2: + +; 15699: case 3: +; 15700: goto block6; + + jmp $block6$211614 +$LN1 at pypy_g_Bui@2: + +; 15701: default: +; 15702: assert(!"bad switch!!"); + + mov eax, OFFSET ??_C at _0N@PGLFNKFI at bad?5switch?$CB?$CB?$AA@ + test eax, eax + je SHORT $block2$211608 + push 15702 ; 00003d56H + push OFFSET ??_C at _1BO@DMBFIACJ@?$AAi?$AAm?$AAp?$AAl?$AAe?$AAm?$AAe?$AAn?$AAt?$AA_?$AA1?$AA1?$AA?4?$AAc?$AA?$AA@ + push OFFSET ??_C at _1CA@EIJBLFPJ@?$AA?$CB?$AA?$CC?$AAb?$AAa?$AAd?$AA?5?$AAs?$AAw?$AAi?$AAt?$AAc?$AAh?$AA?$CB?$AA?$CB?$AA?$CC?$AA?$AA@ + call DWORD PTR __imp___wassert + add esp, 12 ; 0000000cH +$block2$211608: + +; 15703: } +; 15704: +; 15705: block2: +; 15706: l_v271579 = pypy_g_UCD_digit(l_v271547, l_v271544, l_v271545); + + mov edx, DWORD PTR _l_v271545$[ebp] + push edx + mov eax, DWORD PTR _l_v271544$[ebp] + push eax + mov ecx, DWORD PTR _l_v271547$[ebp] + push ecx + call _pypy_g_UCD_digit + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271579$[ebp], eax + +; 15707: l_v271583 = l_v271579; + + mov edx, DWORD PTR _l_v271579$[ebp] + mov DWORD PTR _l_v271583$[ebp], edx +$block3$211599: + +; 15708: goto block3; +; 15709: +; 15710: block3: +; 15711: RPY_DEBUG_RETURN(); +; 15712: return l_v271583; + + mov eax, DWORD PTR _l_v271583$[ebp] + jmp SHORT $LN9 at pypy_g_Bui@2 +$block4$211610: + +; 15713: +; 15714: block4: +; 15715: l_v271580 = pypy_g_UCD_name(l_v271547, l_v271544, l_v271545); + + mov eax, DWORD PTR _l_v271545$[ebp] + push eax + mov ecx, DWORD PTR _l_v271544$[ebp] + push ecx + mov edx, DWORD PTR _l_v271547$[ebp] + push edx + call _pypy_g_UCD_name + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271580$[ebp], eax + +; 15716: l_v271583 = l_v271580; + + mov eax, DWORD PTR _l_v271580$[ebp] + mov DWORD PTR _l_v271583$[ebp], eax + +; 15717: goto block3; + + jmp SHORT $block3$211599 +$block5$211612: + +; 15718: +; 15719: block5: +; 15720: l_v271581 = pypy_g_UCD_decimal(l_v271547, l_v271544, l_v271545); + + mov ecx, DWORD PTR _l_v271545$[ebp] + push ecx + mov edx, DWORD PTR _l_v271544$[ebp] + push edx + mov eax, DWORD PTR _l_v271547$[ebp] + push eax + call _pypy_g_UCD_decimal + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271581$[ebp], eax + +; 15721: l_v271583 = l_v271581; + + mov ecx, DWORD PTR _l_v271581$[ebp] + mov DWORD PTR _l_v271583$[ebp], ecx + +; 15722: goto block3; + + jmp SHORT $block3$211599 +$block6$211614: + +; 15723: +; 15724: block6: +; 15725: l_v271582 = pypy_g_UCD_numeric(l_v271547, l_v271544, l_v271545); + + mov edx, DWORD PTR _l_v271545$[ebp] + push edx + mov eax, DWORD PTR _l_v271544$[ebp] + push eax + mov ecx, DWORD PTR _l_v271547$[ebp] + push ecx + call _pypy_g_UCD_numeric + ;; expected {4(%ebp) | %ebx, %esi, %edi, (%ebp) | } + add esp, 12 ; 0000000cH + mov DWORD PTR _l_v271582$[ebp], eax + +; 15726: l_v271583 = l_v271582; + + mov edx, DWORD PTR _l_v271582$[ebp] + mov DWORD PTR _l_v271583$[ebp], edx + +; 15727: goto block3; + + jmp SHORT $block3$211599 +$LN9 at pypy_g_Bui@2: + +; 15728: } + + mov esp, ebp + pop ebp + ret 0 + npad 3 +$LN14 at pypy_g_Bui@2: + DD $LN5 at pypy_g_Bui@2 + DD $LN4 at pypy_g_Bui@2 + DD $LN3 at pypy_g_Bui@2 + DD $LN2 at pypy_g_Bui@2 +_pypy_g_BuiltinActivation_UwS_UCD_ObjSpace_W_Root_W_Root ENDP From fijal at codespeak.net Thu Nov 5 23:44:44 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 5 Nov 2009 23:44:44 +0100 (CET) Subject: [pypy-svn] r69011 - pypy/extradoc/talk/rupy2009 Message-ID: <20091105224444.60ECC168435@codespeak.net> Author: fijal Date: Thu Nov 5 23:44:41 2009 New Revision: 69011 Modified: pypy/extradoc/talk/rupy2009/talk.pdf pypy/extradoc/talk/rupy2009/talk.tex Log: update Modified: pypy/extradoc/talk/rupy2009/talk.pdf ============================================================================== Files pypy/extradoc/talk/rupy2009/talk.pdf (original) and pypy/extradoc/talk/rupy2009/talk.pdf Thu Nov 5 23:44:41 2009 differ Modified: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- pypy/extradoc/talk/rupy2009/talk.tex (original) +++ pypy/extradoc/talk/rupy2009/talk.tex Thu Nov 5 23:44:41 2009 @@ -67,7 +67,9 @@ \begin{itemize} \item Python - a programming language \item CPython - main implementation of Python - xxx + \item JVM - Java Virtual Machine - VM used to run Java, among others + \item JIT - Just in time compiler + \item Psyco - JIT for Python \end{itemize} \end{frame} @@ -106,7 +108,33 @@ \pause \begin{itemize} \item So, it's CPython that is slow on this particular benchmark + \pause + \item Same example, using numpy and vectorization about 3x faster than JVM + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Python's speed} + \begin{itemize} + \item Instead of: ``Why is Python slow?'' + \pause + \item Better: ``Why is Python hard to optimize?'' + \pause + \item Even better: ``How are we going to fix it?'' \end{itemize} \end{frame} +\begin{frame} + \frametitle{Why is Python hard to optimize?} + \begin{itemize} + \item Duck typing (dynamic dispatch) + \item Frames + \item Object encapsulation + \item Dictionaries of instances + \end{itemize} +\end{frame} + +\begin{frame} +\end{frame} + \end{document} From benjamin at codespeak.net Thu Nov 5 23:49:12 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 5 Nov 2009 23:49:12 +0100 (CET) Subject: [pypy-svn] r69012 - pypy/extradoc/talk/rupy2009 Message-ID: <20091105224912.373D5168435@codespeak.net> Author: benjamin Date: Thu Nov 5 23:49:10 2009 New Revision: 69012 Modified: pypy/extradoc/talk/rupy2009/talk.tex Log: something else I thought of Modified: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- pypy/extradoc/talk/rupy2009/talk.tex (original) +++ pypy/extradoc/talk/rupy2009/talk.tex Thu Nov 5 23:49:10 2009 @@ -131,6 +131,7 @@ \item Frames \item Object encapsulation \item Dictionaries of instances + \item Ability to dynamically change builtins \end{itemize} \end{frame} From magcius at codespeak.net Thu Nov 5 23:58:16 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 5 Nov 2009 23:58:16 +0100 (CET) Subject: [pypy-svn] r69013 - pypy/branch/avm Message-ID: <20091105225816.4A25C168435@codespeak.net> Author: magcius Date: Thu Nov 5 23:58:15 2009 New Revision: 69013 Removed: pypy/branch/avm/ Log: This is broken. Starting anew. From magcius at codespeak.net Thu Nov 5 23:58:42 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Thu, 5 Nov 2009 23:58:42 +0100 (CET) Subject: [pypy-svn] r69014 - pypy/branch/avm Message-ID: <20091105225842.3EF0F168435@codespeak.net> Author: magcius Date: Thu Nov 5 23:58:41 2009 New Revision: 69014 Added: pypy/branch/avm/ - copied from r69013, pypy/trunk/ Log: Starting anew. From afa at codespeak.net Fri Nov 6 00:09:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 6 Nov 2009 00:09:17 +0100 (CET) Subject: [pypy-svn] r69015 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091105230917.A00C1168435@codespeak.net> Author: afa Date: Fri Nov 6 00:09:17 2009 New Revision: 69015 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: Make trackgcroot.py really platform-independant: don't use sys.platform, but the given "format": elf, mingw32, msvc Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Fri Nov 6 00:09:17 2009 @@ -630,9 +630,9 @@ assert self.r_unaryinsn_star.match(line) # indirect call else: target = match.group(1) - if target in FUNCTIONS_NOT_RETURNING: + if target in self.FUNCTIONS_NOT_RETURNING: return [InsnStop(), InsnCannotFollowEsp()] - if sys.platform == 'win32' and target == '__alloca': + if self.format == 'mingw32' and target == '__alloca': # in functions with large stack requirements, windows # needs a call to _alloca(), to turn reserved pages # into committed memory. @@ -647,7 +647,7 @@ return [InsnStackAdjust(-4)] insns = [InsnCall(self.currentlineno), InsnSetLocal(self.EAX)] # the result is there - if sys.platform == 'win32': + if self.format in ('mingw32', 'msvc'): # On win32, the address of a foreign function must be # computed, the optimizer may store it in a register. We # could ignore this, except when the function has a @@ -716,6 +716,14 @@ r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") + FUNCTIONS_NOT_RETURNING = { + 'abort': None, + '_exit': None, + '__assert_fail': None, + '___assert_rtn': None, + 'L___assert_rtn$stub': None + } + def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) funcname = match.group(1) @@ -779,6 +787,17 @@ r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") r_jmptable_end = re.compile(r"[^\t]") + FUNCTIONS_NOT_RETURNING = { + '_abort': None, + '__exit': None, + '__assert': None, + '__wassert': None, + '__imp__abort': None, + '__imp___wassert': None, + 'DWORD PTR __imp__abort': None, + 'DWORD PTR __imp___wassert': None, + } + @classmethod def init_regexp(cls): super(MsvcFunctionGcRootTracker, cls).init_regexp() @@ -1375,26 +1394,6 @@ pass -if sys.platform != 'win32': - FUNCTIONS_NOT_RETURNING = { - 'abort': None, - '_exit': None, - '__assert_fail': None, - '___assert_rtn': None, - 'L___assert_rtn$stub': None - } -else: - FUNCTIONS_NOT_RETURNING = { - '_abort': None, - '__exit': None, - '__assert': None, - '__wassert': None, - '__imp__abort': None, - '__imp___wassert': None, - 'DWORD PTR __imp__abort': None, - 'DWORD PTR __imp___wassert': None, - } - # __________ debugging output __________ def format_location(loc): From afa at codespeak.net Fri Nov 6 01:04:24 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 6 Nov 2009 01:04:24 +0100 (CET) Subject: [pypy-svn] r69016 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091106000424.E9095168436@codespeak.net> Author: afa Date: Fri Nov 6 01:04:24 2009 New Revision: 69016 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: This hack is not a good idea, and fails on Linux Let's see if it is still necessary on Windows Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Fri Nov 6 01:04:24 2009 @@ -196,8 +196,6 @@ for insn in self.insns: if isinstance(insn, (InsnRet, InsnEpilogue, InsnGCROOT)): deltas = {} - if isinstance(insn, InsnRet): - deltas[insn] = 0 self.walk_instructions_backwards(walker, insn, 0) size_at_insn = [] for insn1, delta1 in deltas.items(): From arigo at codespeak.net Fri Nov 6 05:51:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Nov 2009 05:51:01 +0100 (CET) Subject: [pypy-svn] r69017 - in pypy/branch/gc-dump-malloc/pypy/rpython: . lltypesystem Message-ID: <20091106045101.E90E4168437@codespeak.net> Author: arigo Date: Fri Nov 6 05:50:59 2009 New Revision: 69017 Modified: pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-dump-malloc/pypy/rpython/rmodel.py Log: Two hacks to improve the precision of the result. Modified: pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/lltypesystem/lltype.py Fri Nov 6 05:50:59 2009 @@ -258,8 +258,8 @@ def __str__(self): # -- long version -- - #return "%s %s { %s }" % (self.__class__.__name__, - # self._name, self._str_fields()) + return "%s %s { %s }" % (self.__class__.__name__, + self._name, self._str_fields()) # -- short version -- return "%s %s { %s }" % (self.__class__.__name__, self._name, ', '.join(self._names)) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/rmodel.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/rmodel.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/rmodel.py Fri Nov 6 05:50:59 2009 @@ -432,7 +432,8 @@ def externalvsinternal(rtyper, item_repr): # -> external_item_repr, (internal_)item_repr from pypy.rpython import rclass - if (isinstance(item_repr, rclass.AbstractInstanceRepr) and + if (0 and # XXX XXX XXX ONLY TO GET BETTER MEMORY DUMPS + isinstance(item_repr, rclass.AbstractInstanceRepr) and getattr(item_repr, 'gcflavor', 'gc') == 'gc'): return item_repr, rclass.getinstancerepr(rtyper, None) else: From arigo at codespeak.net Fri Nov 6 06:21:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Nov 2009 06:21:22 +0100 (CET) Subject: [pypy-svn] r69018 - in pypy/trunk/pypy: annotation rpython/test Message-ID: <20091106052122.0FE83168437@codespeak.net> Author: arigo Date: Fri Nov 6 06:21:22 2009 New Revision: 69018 Modified: pypy/trunk/pypy/annotation/builtin.py pypy/trunk/pypy/rpython/test/test_rlist.py Log: Test and fix for 'lst2 = list(lst1)'. Must use offspring() in the annotator. Modified: pypy/trunk/pypy/annotation/builtin.py ============================================================================== --- pypy/trunk/pypy/annotation/builtin.py (original) +++ pypy/trunk/pypy/annotation/builtin.py Fri Nov 6 06:21:22 2009 @@ -7,7 +7,7 @@ from pypy.annotation.model import SomeString, SomeTuple, s_Bool, SomeBuiltin from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString -from pypy.annotation.model import SomePBC, SomeInstance, SomeDict +from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef from pypy.annotation.model import SomeOOObject from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation @@ -218,6 +218,8 @@ return SomeObject() def builtin_list(s_iterable): + if isinstance(s_iterable, SomeList): + return s_iterable.listdef.offspring() s_iter = s_iterable.iter() return getbookkeeper().newlist(s_iter.next()) Modified: pypy/trunk/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rlist.py (original) +++ pypy/trunk/pypy/rpython/test/test_rlist.py Fri Nov 6 06:21:22 2009 @@ -1306,6 +1306,22 @@ res = self.interpret(f, [3]) assert res == 0 + def test_make_new_list(self): + class A: + def _freeze_(self): + return True + a1 = A() + a2 = A() + def f(i): + lst = [a1, a1] + lst2 = list(lst) + lst2.append(a2) + return lst2[i] is a2 + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [2]) + assert res == True + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist From arigo at codespeak.net Fri Nov 6 06:22:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Nov 2009 06:22:19 +0100 (CET) Subject: [pypy-svn] r69019 - in pypy/branch/gc-dump-malloc/pypy: annotation rpython/test Message-ID: <20091106052219.C8405168437@codespeak.net> Author: arigo Date: Fri Nov 6 06:22:19 2009 New Revision: 69019 Modified: pypy/branch/gc-dump-malloc/pypy/annotation/builtin.py pypy/branch/gc-dump-malloc/pypy/rpython/test/test_rlist.py Log: Merge r69018 from trunk. Modified: pypy/branch/gc-dump-malloc/pypy/annotation/builtin.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/annotation/builtin.py (original) +++ pypy/branch/gc-dump-malloc/pypy/annotation/builtin.py Fri Nov 6 06:22:19 2009 @@ -7,7 +7,7 @@ from pypy.annotation.model import SomeString, SomeTuple, s_Bool, SomeBuiltin from pypy.annotation.model import SomeUnicodeCodePoint, SomeAddress from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString -from pypy.annotation.model import SomePBC, SomeInstance, SomeDict +from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef from pypy.annotation.model import SomeOOObject from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation @@ -218,6 +218,8 @@ return SomeObject() def builtin_list(s_iterable): + if isinstance(s_iterable, SomeList): + return s_iterable.listdef.offspring() s_iter = s_iterable.iter() return getbookkeeper().newlist(s_iter.next()) Modified: pypy/branch/gc-dump-malloc/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/gc-dump-malloc/pypy/rpython/test/test_rlist.py Fri Nov 6 06:22:19 2009 @@ -1306,6 +1306,22 @@ res = self.interpret(f, [3]) assert res == 0 + def test_make_new_list(self): + class A: + def _freeze_(self): + return True + a1 = A() + a2 = A() + def f(i): + lst = [a1, a1] + lst2 = list(lst) + lst2.append(a2) + return lst2[i] is a2 + res = self.interpret(f, [1]) + assert res == False + res = self.interpret(f, [2]) + assert res == True + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist From arigo at codespeak.net Fri Nov 6 10:07:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Nov 2009 10:07:39 +0100 (CET) Subject: [pypy-svn] r69025 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091106090739.9F54F16800D@codespeak.net> Author: arigo Date: Fri Nov 6 10:07:38 2009 New Revision: 69025 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_slist.py Log: Fix the "xxx inefficient" in pyjitpl by turning the guard_value into guard_true or guard_false in optimizeopt.py. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Nov 6 10:07:38 2009 @@ -367,6 +367,7 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) + self.bool_boxes = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(OPT_FORCINGS) @@ -513,6 +514,8 @@ self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True + elif op.returns_bool_result(): + self.bool_boxes[op.result] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -566,6 +569,16 @@ def optimize_GUARD_VALUE(self, op): constbox = op.args[1] assert isinstance(constbox, Const) + if op.args[0] in self.bool_boxes: + if constbox.value == 0: + opnum = rop.GUARD_FALSE + elif constbox.value == 1: + opnum = rop.GUARD_TRUE + else: + raise InvalidLoop + newop = ResOperation(opnum, [op.args[0]], None, op.descr) + newop.fail_args = op.fail_args + op = newop self.optimize_guard(op, constbox) def optimize_GUARD_TRUE(self, op): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Nov 6 10:07:38 2009 @@ -354,7 +354,6 @@ def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -394,7 +393,6 @@ indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -408,7 +406,6 @@ def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) - # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): return False @@ -426,7 +423,6 @@ rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? - # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): return False @@ -442,7 +438,6 @@ def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) - # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): self.make_result_box(box) Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Fri Nov 6 10:07:38 2009 @@ -93,6 +93,14 @@ def is_final(self): return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST + def returns_bool_result(self): + opnum = self.opnum + if we_are_translated(): + assert opnum >= 0 + elif opnum < 0: + return False # for tests + return opboolresult[opnum] + # ____________________________________________________________ _oplist = [ @@ -137,40 +145,40 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1', + 'FLOAT_IS_TRUE/1b', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # '_COMPARISON_FIRST', - 'INT_LT/2', - 'INT_LE/2', - 'INT_EQ/2', - 'INT_NE/2', - 'INT_GT/2', - 'INT_GE/2', - 'UINT_LT/2', - 'UINT_LE/2', - 'UINT_GT/2', - 'UINT_GE/2', + 'INT_LT/2b', + 'INT_LE/2b', + 'INT_EQ/2b', + 'INT_NE/2b', + 'INT_GT/2b', + 'INT_GE/2b', + 'UINT_LT/2b', + 'UINT_LE/2b', + 'UINT_GT/2b', + 'UINT_GE/2b', '_COMPARISON_LAST', - 'FLOAT_LT/2', # maybe these ones should be comparisons too - 'FLOAT_LE/2', - 'FLOAT_EQ/2', - 'FLOAT_NE/2', - 'FLOAT_GT/2', - 'FLOAT_GE/2', + 'FLOAT_LT/2b', # maybe these ones should be comparisons too + 'FLOAT_LE/2b', + 'FLOAT_EQ/2b', + 'FLOAT_NE/2b', + 'FLOAT_GT/2b', + 'FLOAT_GE/2b', # - 'INT_IS_TRUE/1', + 'INT_IS_TRUE/1b', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1', + 'BOOL_NOT/1b', # 'SAME_AS/1', # gets a Const, turns it into a Box # - 'OONONNULL/1', - 'OOISNULL/1', - 'OOIS/2', - 'OOISNOT/2', + 'OONONNULL/1b', + 'OOISNULL/1b', + 'OOIS/2b', + 'OOISNOT/2b', # 'ARRAYLEN_GC/1d', 'STRLEN/1', @@ -182,8 +190,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1d', - 'SUBCLASSOF/2', + 'INSTANCEOF/1db', + 'SUBCLASSOF/2b', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -231,6 +239,7 @@ opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" +opboolresult= [] # mapping numbers to a flag "returns a boolean" def setup(debug_print=False): @@ -239,16 +248,18 @@ print '%30s = %d' % (name, i) if '/' in name: name, arity = name.split('/') - withdescr = arity.endswith('d') - arity = int(arity.rstrip('d')) + withdescr = 'd' in arity + boolresult = 'b' in arity + arity = int(arity.rstrip('db')) else: - arity, withdescr = -1, True # default + arity, withdescr, boolresult = -1, True, False # default setattr(rop, name, i) if not name.startswith('_'): opname[i] = name oparity.append(arity) opwithdescr.append(withdescr) - assert len(oparity) == len(opwithdescr) == len(_oplist) + opboolresult.append(boolresult) + assert len(oparity)==len(opwithdescr)==len(opboolresult)==len(_oplist) setup(__name__ == '__main__') # print out the table when run directly del _oplist Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Fri Nov 6 10:07:38 2009 @@ -524,6 +524,45 @@ """ self.optimize_loop(ops, '', ops) + def test_guard_value_to_guard_true(self): + ops = """ + [i] + i1 = int_lt(i, 3) + guard_value(i1, 1) [i] + jump(i) + """ + expected = """ + [i] + i1 = int_lt(i, 3) + guard_true(i1) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_to_guard_false(self): + ops = """ + [p] + i1 = ooisnull(p) + guard_value(i1, 0) [p] + jump(p) + """ + expected = """ + [p] + i1 = ooisnull(p) + guard_false(i1) [p] + jump(p) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_on_nonbool(self): + ops = """ + [i] + i1 = int_add(i, 3) + guard_value(i1, 0) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', ops) + def test_p123_simple(self): ops = """ Modified: pypy/trunk/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_slist.py Fri Nov 6 10:07:38 2009 @@ -82,13 +82,20 @@ self.check_loops(call=0) def test_getitem_neg(self): + myjitdriver = JitDriver(greens = [], reds = ['i', 'n']) def f(n): - lst = [41] - lst.append(42) - return lst[n] - res = self.interp_operations(f, [-2], listops=True) + x = i = 0 + while i < 10: + myjitdriver.can_enter_jit(n=n, i=i) + myjitdriver.jit_merge_point(n=n, i=i) + lst = [41] + lst.append(42) + x = lst[n] + i += 1 + return x + res = self.meta_interp(f, [-2], listops=True) assert res == 41 - self.check_history_(call=1) + self.check_loops(call=1, guard_value=0) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): From afa at codespeak.net Fri Nov 6 10:29:20 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 6 Nov 2009 10:29:20 +0100 (CET) Subject: [pypy-svn] r69026 - pypy/branch/msvc-asmgcroot/pypy/translator/c/src Message-ID: <20091106092920.20A10168013@codespeak.net> Author: afa Date: Fri Nov 6 10:29:20 2009 New Revision: 69026 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/debug.h Log: Fix a crash in mingw32 builds, when logs are used Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/src/debug.h (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/src/debug.h Fri Nov 6 10:29:20 2009 @@ -125,12 +125,18 @@ return 1; } +#if defined(_MSC_VER) || defined(__MINGW32__) +#define PYPY_LONG_LONG_PRINTF_FORMAT "I64" +#else +#define PYPY_LONG_LONG_PRINTF_FORMAT "ll" +#endif + static void display_startstop(const char *prefix, const char *postfix, const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); - fprintf(pypy_debug_file, "%s[%llx] %s%s%s\n%s", + fprintf(pypy_debug_file, "%s[%"PYPY_LONG_LONG_PRINTF_FORMAT"x] %s%s%s\n%s", colors, timestamp, prefix, category, postfix, debug_stop_colors); From cfbolz at codespeak.net Fri Nov 6 10:47:45 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 10:47:45 +0100 (CET) Subject: [pypy-svn] r69027 - pypy/branch/jit-less-inlining Message-ID: <20091106094745.4D756168439@codespeak.net> Author: cfbolz Date: Fri Nov 6 10:47:44 2009 New Revision: 69027 Added: pypy/branch/jit-less-inlining/ - copied from r69026, pypy/trunk/ Log: (pedronis, cfbolz): a branch where to play with being less aggressive when inlining in the JIT. From arigo at codespeak.net Fri Nov 6 10:52:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 6 Nov 2009 10:52:26 +0100 (CET) Subject: [pypy-svn] r69028 - pypy/trunk/pypy/translator/c/src Message-ID: <20091106095226.03909168439@codespeak.net> Author: arigo Date: Fri Nov 6 10:52:26 2009 New Revision: 69028 Modified: pypy/trunk/pypy/translator/c/src/debug.h Log: Use sched_setaffinity() on Linux to restrict the process to a single CPU, to make sure that the times reported by the TSC are consistent. (Only when running in profiling.) Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Fri Nov 6 10:52:26 2009 @@ -45,6 +45,19 @@ #ifndef PYPY_NOT_MAIN_FILE #include +#if defined(__GNUC__) && !defined(WIN32) +# include + static void pypy_setup_profiling() + { + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(0, &set); /* restrict to a single cpu */ + sched_setaffinity(0, sizeof(cpu_set_t), &set); + } +#else +static void pypy_setup_profiling() { } +#endif + long pypy_have_debug_prints = -1; FILE *pypy_debug_file = NULL; static bool_t debug_ready = 0; @@ -64,6 +77,7 @@ { /* PYPYLOG=filename --- profiling version */ debug_profile = 1; + pypy_setup_profiling(); } else { From cfbolz at codespeak.net Fri Nov 6 13:44:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 13:44:25 +0100 (CET) Subject: [pypy-svn] r69029 - in pypy/branch/jit-less-inlining/pypy/jit/metainterp: . test Message-ID: <20091106124425.E82391683DF@codespeak.net> Author: cfbolz Date: Fri Nov 6 13:44:24 2009 New Revision: 69029 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/optimizeopt.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/resoperation.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py Log: revert 69025, it breaks tests. no cookie. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/optimizeopt.py Fri Nov 6 13:44:24 2009 @@ -367,7 +367,6 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) - self.bool_boxes = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(OPT_FORCINGS) @@ -514,8 +513,6 @@ self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True - elif op.returns_bool_result(): - self.bool_boxes[op.result] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -569,16 +566,6 @@ def optimize_GUARD_VALUE(self, op): constbox = op.args[1] assert isinstance(constbox, Const) - if op.args[0] in self.bool_boxes: - if constbox.value == 0: - opnum = rop.GUARD_FALSE - elif constbox.value == 1: - opnum = rop.GUARD_TRUE - else: - raise InvalidLoop - newop = ResOperation(opnum, [op.args[0]], None, op.descr) - newop.fail_args = op.fail_args - op = newop self.optimize_guard(op, constbox) def optimize_GUARD_TRUE(self, op): Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py Fri Nov 6 13:44:24 2009 @@ -354,6 +354,7 @@ def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) + # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -393,6 +394,7 @@ indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) + # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -406,6 +408,7 @@ def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) + # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): return False @@ -423,6 +426,7 @@ rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? + # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): return False @@ -438,6 +442,7 @@ def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) + # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): self.make_result_box(box) Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/resoperation.py Fri Nov 6 13:44:24 2009 @@ -93,14 +93,6 @@ def is_final(self): return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - def returns_bool_result(self): - opnum = self.opnum - if we_are_translated(): - assert opnum >= 0 - elif opnum < 0: - return False # for tests - return opboolresult[opnum] - # ____________________________________________________________ _oplist = [ @@ -145,40 +137,40 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1b', + 'FLOAT_IS_TRUE/1', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # '_COMPARISON_FIRST', - 'INT_LT/2b', - 'INT_LE/2b', - 'INT_EQ/2b', - 'INT_NE/2b', - 'INT_GT/2b', - 'INT_GE/2b', - 'UINT_LT/2b', - 'UINT_LE/2b', - 'UINT_GT/2b', - 'UINT_GE/2b', + 'INT_LT/2', + 'INT_LE/2', + 'INT_EQ/2', + 'INT_NE/2', + 'INT_GT/2', + 'INT_GE/2', + 'UINT_LT/2', + 'UINT_LE/2', + 'UINT_GT/2', + 'UINT_GE/2', '_COMPARISON_LAST', - 'FLOAT_LT/2b', # maybe these ones should be comparisons too - 'FLOAT_LE/2b', - 'FLOAT_EQ/2b', - 'FLOAT_NE/2b', - 'FLOAT_GT/2b', - 'FLOAT_GE/2b', + 'FLOAT_LT/2', # maybe these ones should be comparisons too + 'FLOAT_LE/2', + 'FLOAT_EQ/2', + 'FLOAT_NE/2', + 'FLOAT_GT/2', + 'FLOAT_GE/2', # - 'INT_IS_TRUE/1b', + 'INT_IS_TRUE/1', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1b', + 'BOOL_NOT/1', # 'SAME_AS/1', # gets a Const, turns it into a Box # - 'OONONNULL/1b', - 'OOISNULL/1b', - 'OOIS/2b', - 'OOISNOT/2b', + 'OONONNULL/1', + 'OOISNULL/1', + 'OOIS/2', + 'OOISNOT/2', # 'ARRAYLEN_GC/1d', 'STRLEN/1', @@ -190,8 +182,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1db', - 'SUBCLASSOF/2b', + 'INSTANCEOF/1d', + 'SUBCLASSOF/2', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -239,7 +231,6 @@ opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" -opboolresult= [] # mapping numbers to a flag "returns a boolean" def setup(debug_print=False): @@ -248,18 +239,16 @@ print '%30s = %d' % (name, i) if '/' in name: name, arity = name.split('/') - withdescr = 'd' in arity - boolresult = 'b' in arity - arity = int(arity.rstrip('db')) + withdescr = arity.endswith('d') + arity = int(arity.rstrip('d')) else: - arity, withdescr, boolresult = -1, True, False # default + arity, withdescr = -1, True # default setattr(rop, name, i) if not name.startswith('_'): opname[i] = name oparity.append(arity) opwithdescr.append(withdescr) - opboolresult.append(boolresult) - assert len(oparity)==len(opwithdescr)==len(opboolresult)==len(_oplist) + assert len(oparity) == len(opwithdescr) == len(_oplist) setup(__name__ == '__main__') # print out the table when run directly del _oplist Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_optimizeopt.py Fri Nov 6 13:44:24 2009 @@ -524,45 +524,6 @@ """ self.optimize_loop(ops, '', ops) - def test_guard_value_to_guard_true(self): - ops = """ - [i] - i1 = int_lt(i, 3) - guard_value(i1, 1) [i] - jump(i) - """ - expected = """ - [i] - i1 = int_lt(i, 3) - guard_true(i1) [i] - jump(i) - """ - self.optimize_loop(ops, 'Not', expected) - - def test_guard_value_to_guard_false(self): - ops = """ - [p] - i1 = ooisnull(p) - guard_value(i1, 0) [p] - jump(p) - """ - expected = """ - [p] - i1 = ooisnull(p) - guard_false(i1) [p] - jump(p) - """ - self.optimize_loop(ops, 'Not', expected) - - def test_guard_value_on_nonbool(self): - ops = """ - [i] - i1 = int_add(i, 3) - guard_value(i1, 0) [i] - jump(i) - """ - self.optimize_loop(ops, 'Not', ops) - def test_p123_simple(self): ops = """ Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py Fri Nov 6 13:44:24 2009 @@ -82,20 +82,13 @@ self.check_loops(call=0) def test_getitem_neg(self): - myjitdriver = JitDriver(greens = [], reds = ['i', 'n']) def f(n): - x = i = 0 - while i < 10: - myjitdriver.can_enter_jit(n=n, i=i) - myjitdriver.jit_merge_point(n=n, i=i) - lst = [41] - lst.append(42) - x = lst[n] - i += 1 - return x - res = self.meta_interp(f, [-2], listops=True) + lst = [41] + lst.append(42) + return lst[n] + res = self.interp_operations(f, [-2], listops=True) assert res == 41 - self.check_loops(call=1, guard_value=0) + self.check_history_(call=1) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): From cfbolz at codespeak.net Fri Nov 6 13:45:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 13:45:42 +0100 (CET) Subject: [pypy-svn] r69030 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091106124542.44313168432@codespeak.net> Author: cfbolz Date: Fri Nov 6 13:45:41 2009 New Revision: 69030 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_slist.py Log: revert 69025, it breaks tests. no cookies on trunk either. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Fri Nov 6 13:45:41 2009 @@ -367,7 +367,6 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) - self.bool_boxes = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(OPT_FORCINGS) @@ -514,8 +513,6 @@ self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True - elif op.returns_bool_result(): - self.bool_boxes[op.result] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -569,16 +566,6 @@ def optimize_GUARD_VALUE(self, op): constbox = op.args[1] assert isinstance(constbox, Const) - if op.args[0] in self.bool_boxes: - if constbox.value == 0: - opnum = rop.GUARD_FALSE - elif constbox.value == 1: - opnum = rop.GUARD_TRUE - else: - raise InvalidLoop - newop = ResOperation(opnum, [op.args[0]], None, op.descr) - newop.fail_args = op.fail_args - op = newop self.optimize_guard(op, constbox) def optimize_GUARD_TRUE(self, op): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Fri Nov 6 13:45:41 2009 @@ -354,6 +354,7 @@ def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) + # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -393,6 +394,7 @@ indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) + # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -406,6 +408,7 @@ def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) + # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): return False @@ -423,6 +426,7 @@ rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? + # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): return False @@ -438,6 +442,7 @@ def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) + # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): self.make_result_box(box) Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Fri Nov 6 13:45:41 2009 @@ -93,14 +93,6 @@ def is_final(self): return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - def returns_bool_result(self): - opnum = self.opnum - if we_are_translated(): - assert opnum >= 0 - elif opnum < 0: - return False # for tests - return opboolresult[opnum] - # ____________________________________________________________ _oplist = [ @@ -145,40 +137,40 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1b', + 'FLOAT_IS_TRUE/1', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # '_COMPARISON_FIRST', - 'INT_LT/2b', - 'INT_LE/2b', - 'INT_EQ/2b', - 'INT_NE/2b', - 'INT_GT/2b', - 'INT_GE/2b', - 'UINT_LT/2b', - 'UINT_LE/2b', - 'UINT_GT/2b', - 'UINT_GE/2b', + 'INT_LT/2', + 'INT_LE/2', + 'INT_EQ/2', + 'INT_NE/2', + 'INT_GT/2', + 'INT_GE/2', + 'UINT_LT/2', + 'UINT_LE/2', + 'UINT_GT/2', + 'UINT_GE/2', '_COMPARISON_LAST', - 'FLOAT_LT/2b', # maybe these ones should be comparisons too - 'FLOAT_LE/2b', - 'FLOAT_EQ/2b', - 'FLOAT_NE/2b', - 'FLOAT_GT/2b', - 'FLOAT_GE/2b', + 'FLOAT_LT/2', # maybe these ones should be comparisons too + 'FLOAT_LE/2', + 'FLOAT_EQ/2', + 'FLOAT_NE/2', + 'FLOAT_GT/2', + 'FLOAT_GE/2', # - 'INT_IS_TRUE/1b', + 'INT_IS_TRUE/1', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1b', + 'BOOL_NOT/1', # 'SAME_AS/1', # gets a Const, turns it into a Box # - 'OONONNULL/1b', - 'OOISNULL/1b', - 'OOIS/2b', - 'OOISNOT/2b', + 'OONONNULL/1', + 'OOISNULL/1', + 'OOIS/2', + 'OOISNOT/2', # 'ARRAYLEN_GC/1d', 'STRLEN/1', @@ -190,8 +182,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1db', - 'SUBCLASSOF/2b', + 'INSTANCEOF/1d', + 'SUBCLASSOF/2', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -239,7 +231,6 @@ opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" -opboolresult= [] # mapping numbers to a flag "returns a boolean" def setup(debug_print=False): @@ -248,18 +239,16 @@ print '%30s = %d' % (name, i) if '/' in name: name, arity = name.split('/') - withdescr = 'd' in arity - boolresult = 'b' in arity - arity = int(arity.rstrip('db')) + withdescr = arity.endswith('d') + arity = int(arity.rstrip('d')) else: - arity, withdescr, boolresult = -1, True, False # default + arity, withdescr = -1, True # default setattr(rop, name, i) if not name.startswith('_'): opname[i] = name oparity.append(arity) opwithdescr.append(withdescr) - opboolresult.append(boolresult) - assert len(oparity)==len(opwithdescr)==len(opboolresult)==len(_oplist) + assert len(oparity) == len(opwithdescr) == len(_oplist) setup(__name__ == '__main__') # print out the table when run directly del _oplist Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Fri Nov 6 13:45:41 2009 @@ -524,45 +524,6 @@ """ self.optimize_loop(ops, '', ops) - def test_guard_value_to_guard_true(self): - ops = """ - [i] - i1 = int_lt(i, 3) - guard_value(i1, 1) [i] - jump(i) - """ - expected = """ - [i] - i1 = int_lt(i, 3) - guard_true(i1) [i] - jump(i) - """ - self.optimize_loop(ops, 'Not', expected) - - def test_guard_value_to_guard_false(self): - ops = """ - [p] - i1 = ooisnull(p) - guard_value(i1, 0) [p] - jump(p) - """ - expected = """ - [p] - i1 = ooisnull(p) - guard_false(i1) [p] - jump(p) - """ - self.optimize_loop(ops, 'Not', expected) - - def test_guard_value_on_nonbool(self): - ops = """ - [i] - i1 = int_add(i, 3) - guard_value(i1, 0) [i] - jump(i) - """ - self.optimize_loop(ops, 'Not', ops) - def test_p123_simple(self): ops = """ Modified: pypy/trunk/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_slist.py Fri Nov 6 13:45:41 2009 @@ -82,20 +82,13 @@ self.check_loops(call=0) def test_getitem_neg(self): - myjitdriver = JitDriver(greens = [], reds = ['i', 'n']) def f(n): - x = i = 0 - while i < 10: - myjitdriver.can_enter_jit(n=n, i=i) - myjitdriver.jit_merge_point(n=n, i=i) - lst = [41] - lst.append(42) - x = lst[n] - i += 1 - return x - res = self.meta_interp(f, [-2], listops=True) + lst = [41] + lst.append(42) + return lst[n] + res = self.interp_operations(f, [-2], listops=True) assert res == 41 - self.check_loops(call=1, guard_value=0) + self.check_history_(call=1) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): From cfbolz at codespeak.net Fri Nov 6 14:02:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 14:02:46 +0100 (CET) Subject: [pypy-svn] r69031 - in pypy/branch/jit-less-inlining/pypy/jit/metainterp: . test Message-ID: <20091106130246.0838E168433@codespeak.net> Author: cfbolz Date: Fri Nov 6 14:02:46 2009 New Revision: 69031 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Log: (pedronis, cfbolz): a first step towards nirvana: start tracing from the beginning of functions that cannot be inlined. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py Fri Nov 6 14:02:46 2009 @@ -828,6 +828,9 @@ def add_new_loop(self, loop): pass + def add_new_bridge(self, bridge): + pass + def view(self, **kwds): pass @@ -840,6 +843,7 @@ def __init__(self): self.loops = [] + self.bridges = [] self.locations = [] def set_history(self, history): @@ -863,6 +867,9 @@ def add_new_loop(self, loop): self.loops.append(loop) + def add_new_bridge(self, bridge): + self.bridges.append(bridge) + # test read interface def get_all_loops(self): Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_basic.py Fri Nov 6 14:02:46 2009 @@ -109,7 +109,12 @@ else: raise Exception("FAILED") - def check_history_(self, expected=None, **isns): + def check_history(self, expected=None, **isns): + # this can be used after calling meta_interp + get_stats().check_history(expected, **isns) + + def check_operations_history(self, expected=None, **isns): + # this can be used after interp_operations self.metainterp.staticdata.stats.check_history(expected, **isns) @@ -304,7 +309,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=1, int_mul=0, call=1, guard_no_exception=0) + self.check_operations_history(int_add=1, int_mul=0, call=1, guard_no_exception=0) def test_residual_call_pure(self): def externfn(x, y): @@ -315,7 +320,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=0, int_mul=0, call=0) + self.check_operations_history(int_add=0, int_mul=0, call=0) def test_constant_across_mp(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -417,7 +422,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=1) + self.check_operations_history(getfield_gc=1) def test_getfield_immutable(self): class A: @@ -434,7 +439,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=0) + self.check_operations_history(getfield_gc=0) def test_setfield_bool(self): class A: @@ -748,7 +753,7 @@ return isinstance(obj, B) res = self.interp_operations(fn, [0]) assert res - self.check_history_(guard_class=1) + self.check_operations_history(guard_class=1) res = self.interp_operations(fn, [1]) assert not res @@ -769,7 +774,7 @@ return obj.a res = self.interp_operations(fn, [1]) assert res == 1 - self.check_history_(guard_class=0, instanceof=0) + self.check_operations_history(guard_class=0, instanceof=0) def test_r_dict(self): from pypy.rlib.objectmodel import r_dict @@ -901,7 +906,7 @@ return g(a, b) res = self.interp_operations(f, [3, 5]) assert res == 8 - self.check_history_(int_add=0, call=1) + self.check_operations_history(int_add=0, call=1) def test_listcomp(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'lst']) @@ -925,7 +930,7 @@ return tup[1] res = self.interp_operations(f, [3, 5]) assert res == 5 - self.check_history_(setfield_gc=2, getfield_gc_pure=1) + self.check_operations_history(setfield_gc=2, getfield_gc_pure=1) def test_oosend_look_inside_only_one(self): class A: Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py Fri Nov 6 14:02:46 2009 @@ -564,6 +564,48 @@ res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) assert res == main(100) + def test_trace_from_start(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "+": + n += 7 + if op == "-": + n -= 1 + if op == "c": + n = f('---', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=1) + pc = 1 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('+-cl--', i) + self.meta_interp(g, [50], backendopt=True) + self.check_tree_loop_count(3) + self.check_history(int_add=1) + + + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Fri Nov 6 14:02:46 2009 @@ -294,9 +294,15 @@ def maybe_enter_jit(*args): maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True - self.maybe_enter_jit_fn = maybe_enter_jit + def maybe_enter_from_start(*args): + if not self.jitdriver.can_inline(*args[:self.num_green_args]): + maybe_compile_and_run(*args) + maybe_enter_from_start._always_inline_ = True + self.maybe_enter_from_start_fn = maybe_enter_from_start + + def make_leave_jit_graph(self): self.leave_graph = None if self.jitdriver.leave: @@ -505,6 +511,7 @@ def ll_portal_runner(*args): while 1: try: + self.maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) except ContinueRunningNormally, e: From cfbolz at codespeak.net Fri Nov 6 14:07:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 14:07:06 +0100 (CET) Subject: [pypy-svn] r69032 - pypy/branch/jit-less-inlining/pypy/jit/metainterp Message-ID: <20091106130706.F1D96168435@codespeak.net> Author: cfbolz Date: Fri Nov 6 14:07:06 2009 New Revision: 69032 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Log: (pedronis, cfbolz): oops, can_inline can be None Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Fri Nov 6 14:07:06 2009 @@ -296,8 +296,9 @@ maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit + can_inline = self.jitdriver.can_inline def maybe_enter_from_start(*args): - if not self.jitdriver.can_inline(*args[:self.num_green_args]): + if can_inline is not None and not can_inline(*args[:self.num_green_args]): maybe_compile_and_run(*args) maybe_enter_from_start._always_inline_ = True self.maybe_enter_from_start_fn = maybe_enter_from_start From cfbolz at codespeak.net Fri Nov 6 14:18:04 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 14:18:04 +0100 (CET) Subject: [pypy-svn] r69033 - pypy/branch/jit-less-inlining/pypy/jit/metainterp Message-ID: <20091106131804.68E99168435@codespeak.net> Author: cfbolz Date: Fri Nov 6 14:18:03 2009 New Revision: 69033 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py Log: (pedronis, cfbolz): this was not meant to be checked in at all. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py Fri Nov 6 14:18:03 2009 @@ -828,9 +828,6 @@ def add_new_loop(self, loop): pass - def add_new_bridge(self, bridge): - pass - def view(self, **kwds): pass @@ -843,7 +840,6 @@ def __init__(self): self.loops = [] - self.bridges = [] self.locations = [] def set_history(self, history): @@ -867,9 +863,6 @@ def add_new_loop(self, loop): self.loops.append(loop) - def add_new_bridge(self, bridge): - self.bridges.append(bridge) - # test read interface def get_all_loops(self): From cfbolz at codespeak.net Fri Nov 6 15:10:52 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 15:10:52 +0100 (CET) Subject: [pypy-svn] r69034 - pypy/branch/jit-less-inlining/pypy/jit/metainterp/test Message-ID: <20091106141052.B386E168437@codespeak.net> Author: cfbolz Date: Fri Nov 6 15:10:51 2009 New Revision: 69034 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_codewriter.py Log: (pedronis, cfbolz): forgot to fix those Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_codewriter.py Fri Nov 6 15:10:51 2009 @@ -356,7 +356,7 @@ return y.x + 5 res = self.interp_operations(f, [23]) assert res == 28 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, int_add=1) + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) def test_array(self): class X(object): @@ -371,7 +371,7 @@ return a.y[index] res = self.interp_operations(f, [2], listops=True) assert res == 30 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, getarrayitem_gc=0, getarrayitem_gc_pure=1) @@ -389,7 +389,7 @@ return y.lst[index] + y.y + 5 res = self.interp_operations(f, [23, 0], listops=True) assert res == 23 + 24 + 5 - self.check_history_(getfield_gc=0, getfield_gc_pure=2, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, getarrayitem_gc=0, getarrayitem_gc_pure=1, int_add=3) From cfbolz at codespeak.net Fri Nov 6 15:11:26 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 15:11:26 +0100 (CET) Subject: [pypy-svn] r69035 - pypy/branch/jit-less-inlining/pypy/jit/metainterp Message-ID: <20091106141126.C62E7168437@codespeak.net> Author: cfbolz Date: Fri Nov 6 15:11:26 2009 New Revision: 69035 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Log: (pedronis, cfbolz): push and pull to order things in such a way that the translation tests work again. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Fri Nov 6 15:11:26 2009 @@ -161,8 +161,10 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() - self.rewrite_jit_merge_point(policy) + self.make_exception_classes() self.make_driverhook_graphs() + self.make_enter_function() + self.rewrite_jit_merge_point(policy) if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) @@ -172,7 +174,6 @@ self.leave_graph, self.portal_runner_ptr ) - self.make_enter_function() self.rewrite_can_enter_jit() self.rewrite_set_param() self.add_profiler_finish() @@ -262,7 +263,68 @@ self.stats, opt, ProfilerClass=ProfilerClass, warmrunnerdesc=self) - + + def make_exception_classes(self): + portalfunc_ARGS = unrolling_iterable( + [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(self.PORTAL_FUNCTYPE.ARGS)]) + class DoneWithThisFrameVoid(JitException): + def __str__(self): + return 'DoneWithThisFrameVoid()' + + class DoneWithThisFrameInt(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Signed + self.result = result + def __str__(self): + return 'DoneWithThisFrameInt(%s)' % (self.result,) + + class DoneWithThisFrameRef(JitException): + def __init__(self, cpu, result): + assert lltype.typeOf(result) == cpu.ts.BASETYPE + self.result = result + def __str__(self): + return 'DoneWithThisFrameRef(%s)' % (self.result,) + + class DoneWithThisFrameFloat(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Float + self.result = result + def __str__(self): + return 'DoneWithThisFrameFloat(%s)' % (self.result,) + + class ExitFrameWithExceptionRef(JitException): + def __init__(self, cpu, value): + assert lltype.typeOf(value) == cpu.ts.BASETYPE + self.value = value + def __str__(self): + return 'ExitFrameWithExceptionRef(%s)' % (self.value,) + + class ContinueRunningNormally(ContinueRunningNormallyBase): + def __init__(self, argboxes): + # accepts boxes as argument, but unpacks them immediately + # before we raise the exception -- the boxes' values will + # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap + for i, name, ARG in portalfunc_ARGS: + v = unwrap(ARG, argboxes[i]) + setattr(self, name, v) + + def __str__(self): + return 'ContinueRunningNormally(%s)' % ( + ', '.join(map(str, self.args)),) + + self.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.DoneWithThisFrameInt = DoneWithThisFrameInt + self.DoneWithThisFrameRef = DoneWithThisFrameRef + self.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.ContinueRunningNormally = ContinueRunningNormally + self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt + self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef + self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -446,64 +508,7 @@ portalfunc_ARGS = unrolling_iterable( [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(PORTALFUNC.ARGS)]) - class DoneWithThisFrameVoid(JitException): - def __str__(self): - return 'DoneWithThisFrameVoid()' - - class DoneWithThisFrameInt(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Signed - self.result = result - def __str__(self): - return 'DoneWithThisFrameInt(%s)' % (self.result,) - class DoneWithThisFrameRef(JitException): - def __init__(self, cpu, result): - assert lltype.typeOf(result) == cpu.ts.BASETYPE - self.result = result - def __str__(self): - return 'DoneWithThisFrameRef(%s)' % (self.result,) - - class DoneWithThisFrameFloat(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Float - self.result = result - def __str__(self): - return 'DoneWithThisFrameFloat(%s)' % (self.result,) - - class ExitFrameWithExceptionRef(JitException): - def __init__(self, cpu, value): - assert lltype.typeOf(value) == cpu.ts.BASETYPE - self.value = value - def __str__(self): - return 'ExitFrameWithExceptionRef(%s)' % (self.value,) - - class ContinueRunningNormally(ContinueRunningNormallyBase): - def __init__(self, argboxes): - # accepts boxes as argument, but unpacks them immediately - # before we raise the exception -- the boxes' values will - # be modified in a 'finally' by restore_patched_boxes(). - from pypy.jit.metainterp.warmstate import unwrap - for i, name, ARG in portalfunc_ARGS: - v = unwrap(ARG, argboxes[i]) - setattr(self, name, v) - - def __str__(self): - return 'ContinueRunningNormally(%s)' % ( - ', '.join(map(str, self.args)),) - - self.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.DoneWithThisFrameInt = DoneWithThisFrameInt - self.DoneWithThisFrameRef = DoneWithThisFrameRef - self.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.ContinueRunningNormally = ContinueRunningNormally - self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt - self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef - self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) @@ -515,24 +520,24 @@ self.maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) - except ContinueRunningNormally, e: + except self.ContinueRunningNormally, e: args = () for _, name, _ in portalfunc_ARGS: v = getattr(e, name) args = args + (v,) - except DoneWithThisFrameVoid: + except self.DoneWithThisFrameVoid: assert result_kind == 'void' return - except DoneWithThisFrameInt, e: + except self.DoneWithThisFrameInt, e: assert result_kind == 'int' return lltype.cast_primitive(RESULT, e.result) - except DoneWithThisFrameRef, e: + except self.DoneWithThisFrameRef, e: assert result_kind == 'ref' return ts.cast_from_ref(RESULT, e.result) - except DoneWithThisFrameFloat, e: + except self.DoneWithThisFrameFloat, e: assert result_kind == 'float' return e.result - except ExitFrameWithExceptionRef, e: + except self.ExitFrameWithExceptionRef, e: value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) From cfbolz at codespeak.net Fri Nov 6 15:12:48 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 15:12:48 +0100 (CET) Subject: [pypy-svn] r69036 - in pypy/branch/jit-less-inlining/pypy/jit/metainterp: . test Message-ID: <20091106141248.D2BE4168437@codespeak.net> Author: cfbolz Date: Fri Nov 6 15:12:48 2009 New Revision: 69036 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py Log: (pedronis, cfbolz): make the metainterp keep track of where in the history the high-level functions start and return. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py Fri Nov 6 15:12:48 2009 @@ -106,7 +106,7 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - def __init__(self, metainterp, jitcode): + def __init__(self, metainterp, jitcode, greenkey=None): assert isinstance(jitcode, codewriter.JitCode) self.metainterp = metainterp self.jitcode = jitcode @@ -114,6 +114,8 @@ self.constants = jitcode.constants self.exception_target = -1 self.name = jitcode.name # purely for having name attribute + # this is not None for frames that are recursive portal calls + self.greenkey = greenkey # ------------------------------ # Decoding of the JitCode @@ -589,7 +591,7 @@ result = vinfo.get_array_length(virtualizable, arrayindex) self.make_result_box(ConstInt(result)) - def perform_call(self, jitcode, varargs): + def perform_call(self, jitcode, varargs, greenkey=None): if (self.metainterp.is_blackholing() and jitcode.calldescr is not None): # when producing only a BlackHole, we can implement this by @@ -613,7 +615,7 @@ return res else: # when tracing, this bytecode causes the subfunction to be entered - f = self.metainterp.newframe(jitcode) + f = self.metainterp.newframe(jitcode, greenkey) f.setup_call(varargs) return True @@ -1126,14 +1128,18 @@ def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu + self.portal_trace_positions = [] def is_blackholing(self): return self.history is None - def newframe(self, jitcode): + def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 - f = MIFrame(self, jitcode) + if greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (greenkey, len(self.history.operations))) + f = MIFrame(self, jitcode, greenkey) self.framestack.append(f) return f @@ -1141,6 +1147,9 @@ frame = self.framestack.pop() if frame.jitcode is self.staticdata.portal_code: self.in_recursion -= 1 + if frame.greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (None, len(self.history.operations))) return frame def finishframe(self, resultbox): Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py Fri Nov 6 15:12:48 2009 @@ -62,3 +62,31 @@ # doesn't provide interning on its own n1_1 = gd.get_fail_descr_number(fail_descr1) assert n1_1 != n1 + +def test_portal_trace_positions(): + jitcode = codewriter.JitCode("f") + jitcode.code = jitcode.constants = None + portal = codewriter.JitCode("portal") + portal.code = portal.constants = None + class FakeStaticData: + cpu = None + portal_code = portal + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp.framestack = [] + class FakeHistory: + operations = [] + history = metainterp.history = FakeHistory() + metainterp.newframe(portal, "green1") + history.operations.append(1) + metainterp.newframe(jitcode) + history.operations.append(2) + metainterp.newframe(portal, "green2") + history.operations.append(3) + metainterp.popframe() + history.operations.append(4) + metainterp.popframe() + history.operations.append(5) + metainterp.popframe() + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5)] From cfbolz at codespeak.net Fri Nov 6 16:25:55 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 16:25:55 +0100 (CET) Subject: [pypy-svn] r69037 - in pypy/branch/jit-less-inlining/pypy/jit/metainterp: . test Message-ID: <20091106152555.1FC40168432@codespeak.net> Author: cfbolz Date: Fri Nov 6 16:25:53 2009 New Revision: 69037 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py Log: (pedronis, cfbolz, antocuni): If we abort due to a too long trace, we mark the longest function as non-inlinable. Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py Fri Nov 6 16:25:53 2009 @@ -648,7 +648,7 @@ portal_code = self.metainterp.staticdata.portal_code greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): - return self.perform_call(portal_code, varargs[1:]) + return self.perform_call(portal_code, varargs[1:], greenkey) return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") @@ -1129,6 +1129,7 @@ self.staticdata = staticdata self.cpu = staticdata.cpu self.portal_trace_positions = [] + self.greenkey_of_huge_function = None def is_blackholing(self): return self.history is None @@ -1343,6 +1344,8 @@ warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: self.staticdata.profiler.count(ABORT_TOO_LONG) + self.greenkey_of_huge_function = self.find_biggest_function() + self.portal_trace_positions = None self.switch_to_blackhole() def _interpret(self): @@ -1436,7 +1439,7 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(key) + warmrunnerstate.reset_counter_from_failure(key, self) raise def forget_consts(self, boxes, startindex=0): @@ -1826,6 +1829,30 @@ if boxes[i] is oldbox: boxes[i] = newbox + def find_biggest_function(self): + assert not self.is_blackholing() + + start_stack = [] + max_size = 0 + max_key = None + for pair in self.portal_trace_positions: + key, pos = pair + if key is not None: + start_stack.append(pair) + else: + greenkey, startpos = start_stack.pop() + size = pos - startpos + if size > max_size: + max_size = size + max_key = greenkey + if start_stack: + key, pos = start_stack[0] + size = len(self.history.operations) - pos + if size > max_size: + max_size = size + max_key = key + return max_key + class GenerateMergePoint(Exception): def __init__(self, args, target_loop_token): Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py Fri Nov 6 16:25:53 2009 @@ -88,5 +88,18 @@ metainterp.popframe() history.operations.append(5) metainterp.popframe() + history.operations.append(6) assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), (None, 3), (None, 5)] + assert metainterp.find_biggest_function() == "green1" + + metainterp.newframe(portal, "green3") + history.operations.append(7) + metainterp.newframe(jitcode) + history.operations.append(8) + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5), ("green3", 6)] + assert metainterp.find_biggest_function() == "green1" + + history.operations.extend([9, 10, 11, 12]) + assert metainterp.find_biggest_function() == "green3" Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py Fri Nov 6 16:25:53 2009 @@ -604,6 +604,49 @@ self.check_tree_loop_count(3) self.check_history(int_add=1) + def test_dont_inline_huge_stuff(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "-": + n -= 1 + elif op == "c": + f('--------------------', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=0) + pc = 0 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + myjitdriver.set_param('inlining', True) + # carefully chosen threshold to make sure that the inner function + # cannot be inlined, but the inner function on its own is small + # enough + myjitdriver.set_param('trace_limit', 40) + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('-c-----------l-', i+100) + self.meta_interp(g, [10], backendopt=True) + self.check_aborted_count(1) + self.check_history(call=1) + self.check_tree_loop_count(3) class TestLLtype(RecursiveTests, LLJitMixin): Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Fri Nov 6 16:25:53 2009 @@ -358,7 +358,7 @@ maybe_enter_jit._always_inline_ = True self.maybe_enter_jit_fn = maybe_enter_jit - can_inline = self.jitdriver.can_inline + can_inline = self.state.can_inline_greenargs def maybe_enter_from_start(*args): if can_inline is not None and not can_inline(*args[:self.num_green_args]): maybe_compile_and_run(*args) Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py Fri Nov 6 16:25:53 2009 @@ -161,8 +161,16 @@ key.counter += 1 return key.counter >= self.trace_eagerness - def reset_counter_from_failure(self, key): + def reset_counter_from_failure(self, key, metainterp): key.counter = 0 + self.disable_noninlinable_function(metainterp) + + def disable_noninlinable_function(self, metainterp): + greenkey = metainterp.greenkey_of_huge_function + if greenkey is not None: + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -220,6 +228,7 @@ except ContinueRunningNormally: # the trace got too long, reset the counter cell.counter = 0 + self.disable_noninlinable_function(metainterp) raise else: # machine code was already compiled for these greenargs @@ -274,6 +283,7 @@ class JitCell(BaseJitCell): counter = 0 compiled_merge_points = None + dont_trace_here = False # if self.warmrunnerdesc.get_jitcell_at_ptr is None: jit_getter = self._make_jitcell_getter_default(JitCell) @@ -428,18 +438,27 @@ return # can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + unwrap_greenkey = self.make_unwrap_greenkey() if can_inline_ptr is None: - def can_inline_callable(greenkey): + def can_inline_callable(*greenargs): return True else: rtyper = self.warmrunnerdesc.rtyper - unwrap_greenkey = self.make_unwrap_greenkey() # - def can_inline_callable(greenkey): - greenargs = unwrap_greenkey(greenkey) + def can_inline_callable(*greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) return fn(*greenargs) - self.can_inline_callable = can_inline_callable + def can_inline(*greenargs): + cell = self.jit_getter(*greenargs) + if cell.dont_trace_here: + return False + return can_inline_callable(*greenargs) + self.can_inline_greenargs = can_inline + def can_inline_greenkey(greenkey): + greenargs = unwrap_greenkey(greenkey) + return can_inline(*greenargs) + self.can_inline_callable = can_inline_greenkey + # get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr if get_location_ptr is None: From cfbolz at codespeak.net Fri Nov 6 16:30:51 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 16:30:51 +0100 (CET) Subject: [pypy-svn] r69038 - pypy/branch/jit-less-inlining/pypy/jit/metainterp/test Message-ID: <20091106153051.C1528168005@codespeak.net> Author: cfbolz Date: Fri Nov 6 16:30:51 2009 New Revision: 69038 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_blackhole.py Log: fix test_blackhole Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_blackhole.py Fri Nov 6 16:30:51 2009 @@ -6,8 +6,8 @@ class BlackholeTests(object): def meta_interp(self, *args): - def counting_init(frame, metainterp, jitcode): - previnit(frame, metainterp, jitcode) + def counting_init(frame, metainterp, jitcode, greenkey=None): + previnit(frame, metainterp, jitcode, greenkey) self.seen_frames.append(jitcode.name) # previnit = pyjitpl.MIFrame.__init__.im_func From cfbolz at codespeak.net Fri Nov 6 16:32:42 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 16:32:42 +0100 (CET) Subject: [pypy-svn] r69039 - pypy/branch/jit-less-inlining/pypy/jit/metainterp/test Message-ID: <20091106153242.71C09168432@codespeak.net> Author: cfbolz Date: Fri Nov 6 16:32:41 2009 New Revision: 69039 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py Log: fix test_slist Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py Fri Nov 6 16:32:41 2009 @@ -35,7 +35,7 @@ return m res = self.interp_operations(f, [11], listops=True) assert res == 49 - self.check_history_(call=5) + self.check_operations_history(call=5) def test_list_of_voids(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'lst']) @@ -88,7 +88,7 @@ return lst[n] res = self.interp_operations(f, [-2], listops=True) assert res == 41 - self.check_history_(call=1) + self.check_operations_history(call=1) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): From cfbolz at codespeak.net Fri Nov 6 16:51:18 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 6 Nov 2009 16:51:18 +0100 (CET) Subject: [pypy-svn] r69040 - in pypy/branch/jit-less-inlining/pypy/jit/metainterp: . test Message-ID: <20091106155118.9EF02168432@codespeak.net> Author: cfbolz Date: Fri Nov 6 16:51:18 2009 New Revision: 69040 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Log: (cfbolz, antocuni): fix some more tests Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_warmstate.py Fri Nov 6 16:51:18 2009 @@ -165,7 +165,13 @@ class FakeWarmRunnerDesc: can_inline_ptr = None get_printable_location_ptr = None + green_args_spec = [lltype.Signed, lltype.Float] + class FakeCell: + dont_trace_here = False state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is True @@ -179,12 +185,17 @@ return False CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], lltype.Bool)) + class FakeCell: + dont_trace_here = False class FakeWarmRunnerDesc: rtyper = None green_args_spec = [lltype.Signed, lltype.Float] can_inline_ptr = llhelper(CAN_INLINE, can_inline) get_printable_location_ptr = None state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is False Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py Fri Nov 6 16:51:18 2009 @@ -161,13 +161,13 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() + if self.jitdriver.virtualizables: + from pypy.jit.metainterp.virtualizable import VirtualizableInfo + self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) self.make_exception_classes() self.make_driverhook_graphs() self.make_enter_function() self.rewrite_jit_merge_point(policy) - if self.jitdriver.virtualizables: - from pypy.jit.metainterp.virtualizable import VirtualizableInfo - self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, From fijal at codespeak.net Fri Nov 6 22:14:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 6 Nov 2009 22:14:33 +0100 (CET) Subject: [pypy-svn] r69041 - in pypy/extradoc/talk/rupy2009: . examples Message-ID: <20091106211433.D0A2449843C@codespeak.net> Author: fijal Date: Fri Nov 6 22:14:30 2009 New Revision: 69041 Added: pypy/extradoc/talk/rupy2009/examples/escaping.py (contents, props changed) pypy/extradoc/talk/rupy2009/examples/instance.py (contents, props changed) pypy/extradoc/talk/rupy2009/examples/interp.py (contents, props changed) pypy/extradoc/talk/rupy2009/examples/loop.py (contents, props changed) pypy/extradoc/talk/rupy2009/time.png (contents, props changed) Modified: pypy/extradoc/talk/rupy2009/talk.pdf pypy/extradoc/talk/rupy2009/talk.tex Log: update + some examples Added: pypy/extradoc/talk/rupy2009/examples/escaping.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/examples/escaping.py Fri Nov 6 22:14:30 2009 @@ -0,0 +1,12 @@ + +[x, y, z] +v0 = IntObject(x.value % y.value) +return IntObject(z.value + v0.value) + + + + + +[x, y, z] +i0 = x.value % y.value +return IntObject(z.value + i0) Added: pypy/extradoc/talk/rupy2009/examples/instance.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/examples/instance.py Fri Nov 6 22:14:30 2009 @@ -0,0 +1,15 @@ + +class A(object): + def __init__(self, next, v): + self.next = next + self.v = v + +def f(): + a = A(None, 0) + i = 0 + while i < 1000000: + a = A(a, i) + i += 1 + return a + +f() Added: pypy/extradoc/talk/rupy2009/examples/interp.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/examples/interp.py Fri Nov 6 22:14:30 2009 @@ -0,0 +1,30 @@ + + +[x, y] +# LOAD_FAST +frame.valuestack.push(frame.locals[0]) +# LOAD_FAST +frame.valuestack.push(frame.locals[1]) +# BINARY_ADD +v1 = frame.valuestack.pop() +v0 = frame.valuestack.pop() +v2 = v0.add(v1) + return IntObject(self.value + other.value) +frame.valustack.push(v2) +# RETURN_VALUE +return frame.valuestack[-1] + + + +[x, y] +v0 = IntObject(x.value + y.value) +return v0 + + + + +[x, y] +return x + y + + + Added: pypy/extradoc/talk/rupy2009/examples/loop.py ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rupy2009/examples/loop.py Fri Nov 6 22:14:30 2009 @@ -0,0 +1,8 @@ +def f(): + i = 0 + s = 0 + while i < 1000000: + s += (i % 10) + i += 1 + +f() Modified: pypy/extradoc/talk/rupy2009/talk.pdf ============================================================================== Files pypy/extradoc/talk/rupy2009/talk.pdf (original) and pypy/extradoc/talk/rupy2009/talk.pdf Fri Nov 6 22:14:30 2009 differ Modified: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- pypy/extradoc/talk/rupy2009/talk.tex (original) +++ pypy/extradoc/talk/rupy2009/talk.tex Fri Nov 6 22:14:30 2009 @@ -13,6 +13,7 @@ \usepackage{times} \usepackage[T1]{fontenc} +\usepackage{color} \title{The speed of PyPy} @@ -55,7 +56,7 @@ \begin{itemize} \item Is Python really slow? \pause - \item Sometimes + \item Sometimes, for some usecases \pause \item Let's have a look at some examples \end{itemize} @@ -81,7 +82,7 @@ \vspace{.5cm} \begin{tabular}{| l | c | r |} \hline - & CPython & JVM (client mode) \\ + & CPython & Java (hotspot client mode) \\ Average of 10 runs: & 7.6s & 0.77s \\ \hline \end{tabular} @@ -125,17 +126,291 @@ \end{frame} \begin{frame} + \frametitle{Some evidence} + \begin{figure} + \includegraphics[width=1.0\textwidth]{time.png} + \end{figure} +\end{frame} + +\begin{frame} \frametitle{Why is Python hard to optimize?} \begin{itemize} \item Duck typing (dynamic dispatch) \item Frames \item Object encapsulation \item Dictionaries of instances + \item Changing globals \item Ability to dynamically change builtins \end{itemize} \end{frame} \begin{frame} + \frametitle{Duck typing} + \begin{itemize} + \item Dispatching over item type + \item {\tt z = x + y} + \item Needs to check what the type of {\tt x} and {\tt y} is + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Frames} + \begin{itemize} + \item Python interpreters use frames on heap (instead of stack) + \item Locals are stored on those frames + \item Intermediate results are store on valuestack of frames + \item In fact, you can access frames via {\tt sys.\_getframe() or + traceback} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Object encapsulation} + \begin{itemize} + \item Also called boxing + \item Each object, even {\tt int}, has to be boxed + \item Requires allocations and indirection + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Example - addition} + \begin{itemize} + \item {\tt z = x + y} + \item read value for {\tt x} from frame, store on valuestack + \item read value for {\tt x} from frame, store on valuestack + \item allocate new integer + \item read two values from valuestack, add and store the result in + freshly allocated integer + \item move the result from valuestack to locals + \pause + \item {\color{red} in fact, should be one assembler instruction} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dictionaries of instances} + \begin{itemize} + \item Need to perform a dictionary lookup for {\tt x.y} + \item There are {\bf three} lookups per method call + (descriptor, object, type) + \item {\bf Two} for attribute access + (descriptor, object) + \item Looks like list lookup should be enough + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Changing globals} + \begin{itemize} + \item Global symbols can change at any moment in time + \item Makes for example inlining hard + \item Requires global dict lookups, even for constants + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Ability to dynamically change builtins} + \begin{itemize} + \item You can say {\tt int = my\_function} + \item But you can't {\tt int.\_\_add\_\_ = my\_method} + \item Still messes up optimizations + \item Global lookup is also a dictionary lookup, even if globals + don't change + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Interpreting vs compiling} + \begin{itemize} + \item Apparently, processors are good at branch prediction + \item We didn't measure much of a difference, less than 2x overall + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{CPython specific problems} + \begin{itemize} + \item In general, CPython is fairly well optimized + \item refcounting is an inefficient garbage collection scheme + \item GIL + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dynamic compilation to the rescue} + \begin{itemize} + \item You don't pay for feature, until you actually use it + \item In static compilation, compiler has to prove that bad + things can't happen + \item Impossible in Python + \item With dynamic compilation, you just throw away compiled + code in case things go wrong or start over + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dealing with frames} + \begin{itemize} + \item Allocate frame + \pause + \item Use C stack, but remember where frame fields are + living on the stack + \pause + \item Be able to reconstruct frame on demand (for example + {\tt sys.\_getframe() was called}) + \pause + \item The effect is that you don't pay for frames, unless + you really use them + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dealing with dynamic dispatch} + \begin{itemize} + \item The answer is to simply specialize over types + \item Provides possibly multiple versions of compiled code for + single Python code + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dealing with object encapsulation} + \begin{itemize} + \item ``Virtual objects'' + \item Also known as escape analysis + \item If object does not ``escape'', don't allocate it at all + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How does it work in practice?} + \begin{itemize} + \pause + \item Pretty well + \pause + \item XXXX speedup over CPython + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Dealing with attribute access} + \begin{itemize} + \item Fairly complex task + \item Sharing dict, more or less the same effect as V8's hidden + classes + \pause + \item Python is a very complex language + \item Shadowing methods with attributes + \item Descriptors before attributes + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Other important optimizations} + \begin{itemize} + \item Caching globals + \item Caching builtins + \item A lot of smaller ones + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{CPython vs PyPy} + \begin{itemize} + \item I use CPython for everyday usage + \pause + \item But personally, I hope to change it in next months + \pause + \item ... in places where performance matters, but that don't + depend on third party C modules (like numpy) + \pause + \item ... like building and developing PyPy + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Status of PyPy} + \begin{itemize} + \item Very compliant Python interpreter + \item Most of important stdlib modules + \item Differencies are agreed to be implementation details + \pause + \item {\color{red} or bugs} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Examples of working programs} + \begin{itemize} + \item Django (sqlite only) + \item Twisted + \item PyPy's translation toolchain + \item ctypes + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Status of JIT} + \begin{itemize} + \item Because the way it's constructed, handles all Python language + features (unlike for example {\bf Psyco}) + \item Changes very quickly these days + \item Ready for cautious tests + \item Not ready as a drop-in replacement of CPython + \pause + \item {\color{green} yet!} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Adapt today!} + \begin{itemize} + \item A fact: People rely on deep obscure features of language + \item Examples: + \pause + \begin{itemize} + \item {\tt except ImportError, e: \\ \quad if str(e) != ...: + raise} + \pause + \item Exact naming of list comprehension variable + \pause + \item Immediate finalization + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Profit tomorrow!} + \begin{itemize} + \item We plan to release JIT-ready version somewhere early 2010 + xxx + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{How you can help} + \begin{itemize} + \item It's all open source after all ... + \item Try running existing programs + \item Profile, report bugs + \pause + \item Talk to your boss (XXX) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Thank you!} + \begin{itemize} + \item This talk is already online (with all examples): + {\tt http://codespeak.net/svn/pypy/dist/extradoc/talk/rupy2009/talk.pdf} + \item {\tt http://morepypy.blogspot.com} + \item \#pypy on freenode + \item If you want to know more about PyPy, feel free to bug me + around (like, how does the JIT work?) + \item Any questions? + \end{itemize} \end{frame} \end{document} + Added: pypy/extradoc/talk/rupy2009/time.png ============================================================================== Binary file. No diff available. From benjamin at codespeak.net Fri Nov 6 22:29:41 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 6 Nov 2009 22:29:41 +0100 (CET) Subject: [pypy-svn] r69042 - pypy/extradoc/talk/rupy2009 Message-ID: <20091106212941.7E66C49843C@codespeak.net> Author: benjamin Date: Fri Nov 6 22:29:40 2009 New Revision: 69042 Modified: pypy/extradoc/talk/rupy2009/talk.tex Log: rephrase in 2 places Modified: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- pypy/extradoc/talk/rupy2009/talk.tex (original) +++ pypy/extradoc/talk/rupy2009/talk.tex Fri Nov 6 22:29:40 2009 @@ -204,7 +204,7 @@ \frametitle{Changing globals} \begin{itemize} \item Global symbols can change at any moment in time - \item Makes for example inlining hard + \item Makes inlining hard \item Requires global dict lookups, even for constants \end{itemize} \end{frame} @@ -375,7 +375,7 @@ \pause \item Exact naming of list comprehension variable \pause - \item Immediate finalization + \item Reliance on reference counting \end{itemize} \end{itemize} \end{frame} From fijal at codespeak.net Sat Nov 7 06:54:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 7 Nov 2009 06:54:56 +0100 (CET) Subject: [pypy-svn] r69043 - in pypy/extradoc/talk/rupy2009: . examples Message-ID: <20091107055456.195001684EA@codespeak.net> Author: fijal Date: Sat Nov 7 06:54:55 2009 New Revision: 69043 Modified: pypy/extradoc/talk/rupy2009/examples/escaping.py pypy/extradoc/talk/rupy2009/examples/interp.py pypy/extradoc/talk/rupy2009/examples/loop.py pypy/extradoc/talk/rupy2009/talk.pdf pypy/extradoc/talk/rupy2009/talk.tex Log: last minute changes Modified: pypy/extradoc/talk/rupy2009/examples/escaping.py ============================================================================== --- pypy/extradoc/talk/rupy2009/examples/escaping.py (original) +++ pypy/extradoc/talk/rupy2009/examples/escaping.py Sat Nov 7 06:54:55 2009 @@ -1,4 +1,6 @@ + + [x, y, z] v0 = IntObject(x.value % y.value) return IntObject(z.value + v0.value) @@ -7,6 +9,24 @@ + + + + + + + + + + + + + + + + + + [x, y, z] i0 = x.value % y.value return IntObject(z.value + i0) Modified: pypy/extradoc/talk/rupy2009/examples/interp.py ============================================================================== --- pypy/extradoc/talk/rupy2009/examples/interp.py (original) +++ pypy/extradoc/talk/rupy2009/examples/interp.py Sat Nov 7 06:54:55 2009 @@ -1,5 +1,10 @@ + + + + + [x, y] # LOAD_FAST frame.valuestack.push(frame.locals[0]) @@ -16,6 +21,18 @@ + + + + + + + + + + + + [x, y] v0 = IntObject(x.value + y.value) return v0 @@ -23,6 +40,26 @@ + + + + + + + + + + + + + + + + + + + + [x, y] return x + y Modified: pypy/extradoc/talk/rupy2009/examples/loop.py ============================================================================== --- pypy/extradoc/talk/rupy2009/examples/loop.py (original) +++ pypy/extradoc/talk/rupy2009/examples/loop.py Sat Nov 7 06:54:55 2009 @@ -1,7 +1,7 @@ def f(): i = 0 s = 0 - while i < 1000000: + while i < 10000000: s += (i % 10) i += 1 Modified: pypy/extradoc/talk/rupy2009/talk.pdf ============================================================================== Files pypy/extradoc/talk/rupy2009/talk.pdf (original) and pypy/extradoc/talk/rupy2009/talk.pdf Sat Nov 7 06:54:55 2009 differ Modified: pypy/extradoc/talk/rupy2009/talk.tex ============================================================================== --- pypy/extradoc/talk/rupy2009/talk.tex (original) +++ pypy/extradoc/talk/rupy2009/talk.tex Sat Nov 7 06:54:55 2009 @@ -289,7 +289,7 @@ \pause \item Pretty well \pause - \item XXXX speedup over CPython + \item A reasonable speedup over CPython \end{itemize} \end{frame} @@ -316,6 +316,15 @@ \end{frame} \begin{frame} + \frametitle{Bird's view of JIT} + \begin{itemize} + \item Mixed mode - interpreter \& JIT for hot paths + \item Tracing JIT (like TraceMonkey), not up-front + \item + \end{itemize} +\end{frame} + +\begin{frame} \frametitle{CPython vs PyPy} \begin{itemize} \item I use CPython for everyday usage @@ -384,7 +393,8 @@ \frametitle{Profit tomorrow!} \begin{itemize} \item We plan to release JIT-ready version somewhere early 2010 - xxx + \pause + \item It should be able to speed up real-world programs \end{itemize} \end{frame} @@ -395,7 +405,7 @@ \item Try running existing programs \item Profile, report bugs \pause - \item Talk to your boss (XXX) + \item Talk to your boss \end{itemize} \end{frame} From cfbolz at codespeak.net Sat Nov 7 11:13:56 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 7 Nov 2009 11:13:56 +0100 (CET) Subject: [pypy-svn] r69044 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091107101356.CE4FA168539@codespeak.net> Author: cfbolz Date: Sat Nov 7 11:13:54 2009 New Revision: 69044 Added: pypy/extradoc/sprintinfo/ddorf2009/planning.txt (contents, props changed) Log: (all): planning for today Added: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Sat Nov 7 11:13:54 2009 @@ -0,0 +1,30 @@ + +people present: + - Anto + - Armin + - Samuele + - Carl Friedrich + + +TASKS +====== + + - sanitize inlining IN-PROGRESS (Samuele, Carl Friedrich) + - directly call assembler for residual portal calls + - making the CLI backend working with logging + - compress the virtuals part of resume data more + - understand the memory behaviour of pystone with the JIT (Armin, Anto) + - fix the guard_value(bool, 1) -> guard_true hack (Armin) + - try to do something non-insane about Python-level exceptions + - make the assembler produced by generate_failure smaller + - lose less information across residual calls (Samuele, Carl Friedrich) + - we should think about merging several consecutive guards in the + optimizer, to make the assembler smaller and to save resume data space + - put the class into the structure to get only one promote when using an + instance + - look into failing pypy-c-jit apptests + + + + + - at the end of the sprint, migrate tasks to jit.txt From cfbolz at codespeak.net Sat Nov 7 12:27:09 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 7 Nov 2009 12:27:09 +0100 (CET) Subject: [pypy-svn] r69045 - pypy/branch/jit-less-inlining/pypy/jit/metainterp Message-ID: <20091107112709.5C20616849E@codespeak.net> Author: cfbolz Date: Sat Nov 7 12:27:08 2009 New Revision: 69045 Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py Log: (pedronis, cfbolz): add a debug print for disabling inlining of a function Modified: pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py Sat Nov 7 12:27:08 2009 @@ -10,7 +10,7 @@ from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell -from pypy.rlib.debug import debug_start, debug_stop +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -170,7 +170,11 @@ if greenkey is not None: cell = self.jit_cell_at_key(greenkey) cell.dont_trace_here = True - + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): From arigo at codespeak.net Sat Nov 7 14:02:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 7 Nov 2009 14:02:07 +0100 (CET) Subject: [pypy-svn] r69046 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091107130207.6C39716849E@codespeak.net> Author: arigo Date: Sat Nov 7 14:02:06 2009 New Revision: 69046 Modified: pypy/trunk/pypy/jit/backend/x86/support.py Log: Bah. Force inlining here, to avoid allocating the tuple. Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Sat Nov 7 14:02:06 2009 @@ -22,6 +22,7 @@ new_item = lltype.malloc(ATP, CHUNK_SIZE, zero=True) self.chunks.append(new_item) self.lgt += 1 + _grow._dont_inline_ = True def get_addr_for_num(self, i): chunk_no, ofs = self._no_of(i) @@ -34,6 +35,7 @@ while i >= len(self.chunks) * CHUNK_SIZE: self._grow() return i / CHUNK_SIZE, i % CHUNK_SIZE + _no_of._always_inline_ = True def setitem(self, i, v): chunk_no, ofs = self._no_of(i) From cfbolz at codespeak.net Sat Nov 7 16:02:19 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 7 Nov 2009 16:02:19 +0100 (CET) Subject: [pypy-svn] r69047 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091107150219.E2E3C16850B@codespeak.net> Author: cfbolz Date: Sat Nov 7 16:02:16 2009 New Revision: 69047 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_blackhole.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_recursive.py pypy/trunk/pypy/jit/metainterp/test/test_slist.py pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py pypy/trunk/pypy/jit/metainterp/warmspot.py pypy/trunk/pypy/jit/metainterp/warmstate.py Log: (pedronis, cfbolz): merge the jit-less-inlining branch: ------------------------------------------------------------------------ r69045 | cfbolz | 2009-11-07 12:27:08 +0100 (Sat, 07 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py (pedronis, cfbolz): add a debug print for disabling inlining of a function ------------------------------------------------------------------------ r69040 | cfbolz | 2009-11-06 16:51:18 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_warmstate.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (cfbolz, antocuni): fix some more tests ------------------------------------------------------------------------ r69039 | cfbolz | 2009-11-06 16:32:41 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_slist.py fix test_slist ------------------------------------------------------------------------ r69038 | cfbolz | 2009-11-06 16:30:51 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_blackhole.py fix test_blackhole ------------------------------------------------------------------------ r69037 | cfbolz | 2009-11-06 16:25:53 +0100 (Fri, 06 Nov 2009) | 3 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmstate.py (pedronis, cfbolz, antocuni): If we abort due to a too long trace, we mark the longest function as non-inlinable. ------------------------------------------------------------------------ r69036 | cfbolz | 2009-11-06 15:12:48 +0100 (Fri, 06 Nov 2009) | 3 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/pyjitpl.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_pyjitpl.py (pedronis, cfbolz): make the metainterp keep track of where in the history the high-level functions start and return. ------------------------------------------------------------------------ r69035 | cfbolz | 2009-11-06 15:11:26 +0100 (Fri, 06 Nov 2009) | 3 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (pedronis, cfbolz): push and pull to order things in such a way that the translation tests work again. ------------------------------------------------------------------------ r69034 | cfbolz | 2009-11-06 15:10:51 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_codewriter.py (pedronis, cfbolz): forgot to fix those ------------------------------------------------------------------------ r69033 | cfbolz | 2009-11-06 14:18:03 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py (pedronis, cfbolz): this was not meant to be checked in at all. ------------------------------------------------------------------------ r69032 | cfbolz | 2009-11-06 14:07:06 +0100 (Fri, 06 Nov 2009) | 2 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (pedronis, cfbolz): oops, can_inline can be None ------------------------------------------------------------------------ r69031 | cfbolz | 2009-11-06 14:02:46 +0100 (Fri, 06 Nov 2009) | 3 lines Changed paths: M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/history.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_basic.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/test/test_recursive.py M /pypy/branch/jit-less-inlining/pypy/jit/metainterp/warmspot.py (pedronis, cfbolz): a first step towards nirvana: start tracing from the beginning of functions that cannot be inlined. Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sat Nov 7 16:02:16 2009 @@ -106,7 +106,7 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - def __init__(self, metainterp, jitcode): + def __init__(self, metainterp, jitcode, greenkey=None): assert isinstance(jitcode, codewriter.JitCode) self.metainterp = metainterp self.jitcode = jitcode @@ -114,6 +114,8 @@ self.constants = jitcode.constants self.exception_target = -1 self.name = jitcode.name # purely for having name attribute + # this is not None for frames that are recursive portal calls + self.greenkey = greenkey # ------------------------------ # Decoding of the JitCode @@ -589,7 +591,7 @@ result = vinfo.get_array_length(virtualizable, arrayindex) self.make_result_box(ConstInt(result)) - def perform_call(self, jitcode, varargs): + def perform_call(self, jitcode, varargs, greenkey=None): if (self.metainterp.is_blackholing() and jitcode.calldescr is not None): # when producing only a BlackHole, we can implement this by @@ -613,7 +615,7 @@ return res else: # when tracing, this bytecode causes the subfunction to be entered - f = self.metainterp.newframe(jitcode) + f = self.metainterp.newframe(jitcode, greenkey) f.setup_call(varargs) return True @@ -646,7 +648,7 @@ portal_code = self.metainterp.staticdata.portal_code greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): - return self.perform_call(portal_code, varargs[1:]) + return self.perform_call(portal_code, varargs[1:], greenkey) return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") @@ -1126,14 +1128,19 @@ def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu + self.portal_trace_positions = [] + self.greenkey_of_huge_function = None def is_blackholing(self): return self.history is None - def newframe(self, jitcode): + def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 - f = MIFrame(self, jitcode) + if greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (greenkey, len(self.history.operations))) + f = MIFrame(self, jitcode, greenkey) self.framestack.append(f) return f @@ -1141,6 +1148,9 @@ frame = self.framestack.pop() if frame.jitcode is self.staticdata.portal_code: self.in_recursion -= 1 + if frame.greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (None, len(self.history.operations))) return frame def finishframe(self, resultbox): @@ -1334,6 +1344,8 @@ warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: self.staticdata.profiler.count(ABORT_TOO_LONG) + self.greenkey_of_huge_function = self.find_biggest_function() + self.portal_trace_positions = None self.switch_to_blackhole() def _interpret(self): @@ -1427,7 +1439,7 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(key) + warmrunnerstate.reset_counter_from_failure(key, self) raise def forget_consts(self, boxes, startindex=0): @@ -1817,6 +1829,30 @@ if boxes[i] is oldbox: boxes[i] = newbox + def find_biggest_function(self): + assert not self.is_blackholing() + + start_stack = [] + max_size = 0 + max_key = None + for pair in self.portal_trace_positions: + key, pos = pair + if key is not None: + start_stack.append(pair) + else: + greenkey, startpos = start_stack.pop() + size = pos - startpos + if size > max_size: + max_size = size + max_key = greenkey + if start_stack: + key, pos = start_stack[0] + size = len(self.history.operations) - pos + if size > max_size: + max_size = size + max_key = key + return max_key + class GenerateMergePoint(Exception): def __init__(self, args, target_loop_token): Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sat Nov 7 16:02:16 2009 @@ -109,7 +109,12 @@ else: raise Exception("FAILED") - def check_history_(self, expected=None, **isns): + def check_history(self, expected=None, **isns): + # this can be used after calling meta_interp + get_stats().check_history(expected, **isns) + + def check_operations_history(self, expected=None, **isns): + # this can be used after interp_operations self.metainterp.staticdata.stats.check_history(expected, **isns) @@ -304,7 +309,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=1, int_mul=0, call=1, guard_no_exception=0) + self.check_operations_history(int_add=1, int_mul=0, call=1, guard_no_exception=0) def test_residual_call_pure(self): def externfn(x, y): @@ -315,7 +320,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=0, int_mul=0, call=0) + self.check_operations_history(int_add=0, int_mul=0, call=0) def test_constant_across_mp(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -417,7 +422,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=1) + self.check_operations_history(getfield_gc=1) def test_getfield_immutable(self): class A: @@ -434,7 +439,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=0) + self.check_operations_history(getfield_gc=0) def test_setfield_bool(self): class A: @@ -748,7 +753,7 @@ return isinstance(obj, B) res = self.interp_operations(fn, [0]) assert res - self.check_history_(guard_class=1) + self.check_operations_history(guard_class=1) res = self.interp_operations(fn, [1]) assert not res @@ -769,7 +774,7 @@ return obj.a res = self.interp_operations(fn, [1]) assert res == 1 - self.check_history_(guard_class=0, instanceof=0) + self.check_operations_history(guard_class=0, instanceof=0) def test_r_dict(self): from pypy.rlib.objectmodel import r_dict @@ -901,7 +906,7 @@ return g(a, b) res = self.interp_operations(f, [3, 5]) assert res == 8 - self.check_history_(int_add=0, call=1) + self.check_operations_history(int_add=0, call=1) def test_listcomp(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'lst']) @@ -925,7 +930,7 @@ return tup[1] res = self.interp_operations(f, [3, 5]) assert res == 5 - self.check_history_(setfield_gc=2, getfield_gc_pure=1) + self.check_operations_history(setfield_gc=2, getfield_gc_pure=1) def test_oosend_look_inside_only_one(self): class A: Modified: pypy/trunk/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_blackhole.py Sat Nov 7 16:02:16 2009 @@ -6,8 +6,8 @@ class BlackholeTests(object): def meta_interp(self, *args): - def counting_init(frame, metainterp, jitcode): - previnit(frame, metainterp, jitcode) + def counting_init(frame, metainterp, jitcode, greenkey=None): + previnit(frame, metainterp, jitcode, greenkey) self.seen_frames.append(jitcode.name) # previnit = pyjitpl.MIFrame.__init__.im_func Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Sat Nov 7 16:02:16 2009 @@ -356,7 +356,7 @@ return y.x + 5 res = self.interp_operations(f, [23]) assert res == 28 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, int_add=1) + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) def test_array(self): class X(object): @@ -371,7 +371,7 @@ return a.y[index] res = self.interp_operations(f, [2], listops=True) assert res == 30 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, getarrayitem_gc=0, getarrayitem_gc_pure=1) @@ -389,7 +389,7 @@ return y.lst[index] + y.y + 5 res = self.interp_operations(f, [23, 0], listops=True) assert res == 23 + 24 + 5 - self.check_history_(getfield_gc=0, getfield_gc_pure=2, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, getarrayitem_gc=0, getarrayitem_gc_pure=1, int_add=3) Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Sat Nov 7 16:02:16 2009 @@ -62,3 +62,44 @@ # doesn't provide interning on its own n1_1 = gd.get_fail_descr_number(fail_descr1) assert n1_1 != n1 + +def test_portal_trace_positions(): + jitcode = codewriter.JitCode("f") + jitcode.code = jitcode.constants = None + portal = codewriter.JitCode("portal") + portal.code = portal.constants = None + class FakeStaticData: + cpu = None + portal_code = portal + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp.framestack = [] + class FakeHistory: + operations = [] + history = metainterp.history = FakeHistory() + metainterp.newframe(portal, "green1") + history.operations.append(1) + metainterp.newframe(jitcode) + history.operations.append(2) + metainterp.newframe(portal, "green2") + history.operations.append(3) + metainterp.popframe() + history.operations.append(4) + metainterp.popframe() + history.operations.append(5) + metainterp.popframe() + history.operations.append(6) + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5)] + assert metainterp.find_biggest_function() == "green1" + + metainterp.newframe(portal, "green3") + history.operations.append(7) + metainterp.newframe(jitcode) + history.operations.append(8) + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5), ("green3", 6)] + assert metainterp.find_biggest_function() == "green1" + + history.operations.extend([9, 10, 11, 12]) + assert metainterp.find_biggest_function() == "green3" Modified: pypy/trunk/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_recursive.py Sat Nov 7 16:02:16 2009 @@ -564,6 +564,91 @@ res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) assert res == main(100) + def test_trace_from_start(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "+": + n += 7 + if op == "-": + n -= 1 + if op == "c": + n = f('---', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=1) + pc = 1 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('+-cl--', i) + self.meta_interp(g, [50], backendopt=True) + self.check_tree_loop_count(3) + self.check_history(int_add=1) + + def test_dont_inline_huge_stuff(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "-": + n -= 1 + elif op == "c": + f('--------------------', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=0) + pc = 0 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + myjitdriver.set_param('inlining', True) + # carefully chosen threshold to make sure that the inner function + # cannot be inlined, but the inner function on its own is small + # enough + myjitdriver.set_param('trace_limit', 40) + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('-c-----------l-', i+100) + self.meta_interp(g, [10], backendopt=True) + self.check_aborted_count(1) + self.check_history(call=1) + self.check_tree_loop_count(3) + + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_slist.py Sat Nov 7 16:02:16 2009 @@ -35,7 +35,7 @@ return m res = self.interp_operations(f, [11], listops=True) assert res == 49 - self.check_history_(call=5) + self.check_operations_history(call=5) def test_list_of_voids(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'lst']) @@ -88,7 +88,7 @@ return lst[n] res = self.interp_operations(f, [-2], listops=True) assert res == 41 - self.check_history_(call=1) + self.check_operations_history(call=1) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): Modified: pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_warmstate.py Sat Nov 7 16:02:16 2009 @@ -165,7 +165,13 @@ class FakeWarmRunnerDesc: can_inline_ptr = None get_printable_location_ptr = None + green_args_spec = [lltype.Signed, lltype.Float] + class FakeCell: + dont_trace_here = False state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is True @@ -179,12 +185,17 @@ return False CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], lltype.Bool)) + class FakeCell: + dont_trace_here = False class FakeWarmRunnerDesc: rtyper = None green_args_spec = [lltype.Signed, lltype.Float] can_inline_ptr = llhelper(CAN_INLINE, can_inline) get_printable_location_ptr = None state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is False Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Sat Nov 7 16:02:16 2009 @@ -161,18 +161,19 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() - self.rewrite_jit_merge_point(policy) - self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) + self.make_exception_classes() + self.make_driverhook_graphs() + self.make_enter_function() + self.rewrite_jit_merge_point(policy) self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, self.leave_graph, self.portal_runner_ptr ) - self.make_enter_function() self.rewrite_can_enter_jit() self.rewrite_set_param() self.add_profiler_finish() @@ -262,7 +263,68 @@ self.stats, opt, ProfilerClass=ProfilerClass, warmrunnerdesc=self) - + + def make_exception_classes(self): + portalfunc_ARGS = unrolling_iterable( + [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(self.PORTAL_FUNCTYPE.ARGS)]) + class DoneWithThisFrameVoid(JitException): + def __str__(self): + return 'DoneWithThisFrameVoid()' + + class DoneWithThisFrameInt(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Signed + self.result = result + def __str__(self): + return 'DoneWithThisFrameInt(%s)' % (self.result,) + + class DoneWithThisFrameRef(JitException): + def __init__(self, cpu, result): + assert lltype.typeOf(result) == cpu.ts.BASETYPE + self.result = result + def __str__(self): + return 'DoneWithThisFrameRef(%s)' % (self.result,) + + class DoneWithThisFrameFloat(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Float + self.result = result + def __str__(self): + return 'DoneWithThisFrameFloat(%s)' % (self.result,) + + class ExitFrameWithExceptionRef(JitException): + def __init__(self, cpu, value): + assert lltype.typeOf(value) == cpu.ts.BASETYPE + self.value = value + def __str__(self): + return 'ExitFrameWithExceptionRef(%s)' % (self.value,) + + class ContinueRunningNormally(ContinueRunningNormallyBase): + def __init__(self, argboxes): + # accepts boxes as argument, but unpacks them immediately + # before we raise the exception -- the boxes' values will + # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap + for i, name, ARG in portalfunc_ARGS: + v = unwrap(ARG, argboxes[i]) + setattr(self, name, v) + + def __str__(self): + return 'ContinueRunningNormally(%s)' % ( + ', '.join(map(str, self.args)),) + + self.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.DoneWithThisFrameInt = DoneWithThisFrameInt + self.DoneWithThisFrameRef = DoneWithThisFrameRef + self.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.ContinueRunningNormally = ContinueRunningNormally + self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt + self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef + self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -294,9 +356,16 @@ def maybe_enter_jit(*args): maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True - self.maybe_enter_jit_fn = maybe_enter_jit + can_inline = self.state.can_inline_greenargs + def maybe_enter_from_start(*args): + if can_inline is not None and not can_inline(*args[:self.num_green_args]): + maybe_compile_and_run(*args) + maybe_enter_from_start._always_inline_ = True + self.maybe_enter_from_start_fn = maybe_enter_from_start + + def make_leave_jit_graph(self): self.leave_graph = None if self.jitdriver.leave: @@ -439,64 +508,7 @@ portalfunc_ARGS = unrolling_iterable( [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(PORTALFUNC.ARGS)]) - class DoneWithThisFrameVoid(JitException): - def __str__(self): - return 'DoneWithThisFrameVoid()' - - class DoneWithThisFrameInt(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Signed - self.result = result - def __str__(self): - return 'DoneWithThisFrameInt(%s)' % (self.result,) - - class DoneWithThisFrameRef(JitException): - def __init__(self, cpu, result): - assert lltype.typeOf(result) == cpu.ts.BASETYPE - self.result = result - def __str__(self): - return 'DoneWithThisFrameRef(%s)' % (self.result,) - - class DoneWithThisFrameFloat(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Float - self.result = result - def __str__(self): - return 'DoneWithThisFrameFloat(%s)' % (self.result,) - - class ExitFrameWithExceptionRef(JitException): - def __init__(self, cpu, value): - assert lltype.typeOf(value) == cpu.ts.BASETYPE - self.value = value - def __str__(self): - return 'ExitFrameWithExceptionRef(%s)' % (self.value,) - - class ContinueRunningNormally(ContinueRunningNormallyBase): - def __init__(self, argboxes): - # accepts boxes as argument, but unpacks them immediately - # before we raise the exception -- the boxes' values will - # be modified in a 'finally' by restore_patched_boxes(). - from pypy.jit.metainterp.warmstate import unwrap - for i, name, ARG in portalfunc_ARGS: - v = unwrap(ARG, argboxes[i]) - setattr(self, name, v) - def __str__(self): - return 'ContinueRunningNormally(%s)' % ( - ', '.join(map(str, self.args)),) - - self.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.DoneWithThisFrameInt = DoneWithThisFrameInt - self.DoneWithThisFrameRef = DoneWithThisFrameRef - self.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.ContinueRunningNormally = ContinueRunningNormally - self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt - self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef - self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) @@ -505,26 +517,27 @@ def ll_portal_runner(*args): while 1: try: + self.maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) - except ContinueRunningNormally, e: + except self.ContinueRunningNormally, e: args = () for _, name, _ in portalfunc_ARGS: v = getattr(e, name) args = args + (v,) - except DoneWithThisFrameVoid: + except self.DoneWithThisFrameVoid: assert result_kind == 'void' return - except DoneWithThisFrameInt, e: + except self.DoneWithThisFrameInt, e: assert result_kind == 'int' return lltype.cast_primitive(RESULT, e.result) - except DoneWithThisFrameRef, e: + except self.DoneWithThisFrameRef, e: assert result_kind == 'ref' return ts.cast_from_ref(RESULT, e.result) - except DoneWithThisFrameFloat, e: + except self.DoneWithThisFrameFloat, e: assert result_kind == 'float' return e.result - except ExitFrameWithExceptionRef, e: + except self.ExitFrameWithExceptionRef, e: value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) Modified: pypy/trunk/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmstate.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmstate.py Sat Nov 7 16:02:16 2009 @@ -10,7 +10,7 @@ from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell -from pypy.rlib.debug import debug_start, debug_stop +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -161,8 +161,20 @@ key.counter += 1 return key.counter >= self.trace_eagerness - def reset_counter_from_failure(self, key): + def reset_counter_from_failure(self, key, metainterp): key.counter = 0 + self.disable_noninlinable_function(metainterp) + + def disable_noninlinable_function(self, metainterp): + greenkey = metainterp.greenkey_of_huge_function + if greenkey is not None: + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -220,6 +232,7 @@ except ContinueRunningNormally: # the trace got too long, reset the counter cell.counter = 0 + self.disable_noninlinable_function(metainterp) raise else: # machine code was already compiled for these greenargs @@ -274,6 +287,7 @@ class JitCell(BaseJitCell): counter = 0 compiled_merge_points = None + dont_trace_here = False # if self.warmrunnerdesc.get_jitcell_at_ptr is None: jit_getter = self._make_jitcell_getter_default(JitCell) @@ -428,18 +442,27 @@ return # can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + unwrap_greenkey = self.make_unwrap_greenkey() if can_inline_ptr is None: - def can_inline_callable(greenkey): + def can_inline_callable(*greenargs): return True else: rtyper = self.warmrunnerdesc.rtyper - unwrap_greenkey = self.make_unwrap_greenkey() # - def can_inline_callable(greenkey): - greenargs = unwrap_greenkey(greenkey) + def can_inline_callable(*greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) return fn(*greenargs) - self.can_inline_callable = can_inline_callable + def can_inline(*greenargs): + cell = self.jit_getter(*greenargs) + if cell.dont_trace_here: + return False + return can_inline_callable(*greenargs) + self.can_inline_greenargs = can_inline + def can_inline_greenkey(greenkey): + greenargs = unwrap_greenkey(greenkey) + return can_inline(*greenargs) + self.can_inline_callable = can_inline_greenkey + # get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr if get_location_ptr is None: From cfbolz at codespeak.net Sat Nov 7 16:02:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 7 Nov 2009 16:02:46 +0100 (CET) Subject: [pypy-svn] r69048 - pypy/branch/jit-less-inlining Message-ID: <20091107150246.C22EB16850B@codespeak.net> Author: cfbolz Date: Sat Nov 7 16:02:46 2009 New Revision: 69048 Removed: pypy/branch/jit-less-inlining/ Log: remove merged branch From arigo at codespeak.net Sat Nov 7 17:12:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 7 Nov 2009 17:12:58 +0100 (CET) Subject: [pypy-svn] r69050 - in pypy/trunk/pypy/jit: backend/test metainterp metainterp/test Message-ID: <20091107161258.5ED35318137@codespeak.net> Author: arigo Date: Sat Nov 7 17:12:57 2009 New Revision: 69050 Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/test/test_random.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Log: Fix for reached_can_enter_jit: check for duplicate boxes and remove them. Fixes a rare crash in the nightly run of pypy-c-jit. Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Sat Nov 7 17:12:57 2009 @@ -673,13 +673,19 @@ def test_same_as(self): r = self.execute_operation(rop.SAME_AS, [ConstInt(5)], 'int') assert r.value == 5 + r = self.execute_operation(rop.SAME_AS, [BoxInt(5)], 'int') + assert r.value == 5 u_box = self.alloc_unicode(u"hello\u1234") r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref') assert r.value == u_box.value + r = self.execute_operation(rop.SAME_AS, [u_box], 'ref') + assert r.value == u_box.value if self.cpu.supports_floats: r = self.execute_operation(rop.SAME_AS, [ConstFloat(5.5)], 'float') assert r.value == 5.5 + r = self.execute_operation(rop.SAME_AS, [BoxFloat(5.5)], 'float') + assert r.value == 5.5 def test_jump(self): # this test generates small loops where the JUMP passes many Modified: pypy/trunk/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_random.py Sat Nov 7 17:12:57 2009 @@ -246,7 +246,9 @@ class ConstUnaryOperation(UnaryOperation): def produce_into(self, builder, r): - if r.random() < 0.75 or not builder.cpu.supports_floats: + if r.random() < 0.4: + UnaryOperation.produce_into(self, builder, r) + elif r.random() < 0.75 or not builder.cpu.supports_floats: self.put(builder, [ConstInt(r.random_integer())]) else: self.put(builder, [ConstFloat(r.random_float())]) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sat Nov 7 17:12:57 2009 @@ -1442,22 +1442,33 @@ warmrunnerstate.reset_counter_from_failure(key, self) raise - def forget_consts(self, boxes, startindex=0): - for i in range(startindex, len(boxes)): + def remove_consts_and_duplicates(self, boxes, startindex, endindex, + duplicates): + for i in range(startindex, endindex): box = boxes[i] - if isinstance(box, Const): - constbox = box - box = constbox.clonebox() + if isinstance(box, Const) or box in duplicates: + oldbox = box + box = oldbox.clonebox() boxes[i] = box - self.history.record(rop.SAME_AS, [constbox], box) + self.history.record(rop.SAME_AS, [oldbox], box) + else: + duplicates[box] = None def reached_can_enter_jit(self, live_arg_boxes): - self.forget_consts(live_arg_boxes, self.staticdata.num_green_args) + num_green_args = self.staticdata.num_green_args + duplicates = {} + self.remove_consts_and_duplicates(live_arg_boxes, + num_green_args, + len(live_arg_boxes), + duplicates) live_arg_boxes = live_arg_boxes[:] if self.staticdata.virtualizable_info is not None: # we use ':-1' to remove the last item, which is the virtualizable # itself - self.forget_consts(self.virtualizable_boxes) + self.remove_consts_and_duplicates(self.virtualizable_boxes, + 0, + len(self.virtualizable_boxes)-1, + duplicates) live_arg_boxes += self.virtualizable_boxes[:-1] # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Sat Nov 7 17:12:57 2009 @@ -165,7 +165,7 @@ 'INT_INVERT/1', 'BOOL_NOT/1', # - 'SAME_AS/1', # gets a Const, turns it into a Box + 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # 'OONONNULL/1', 'OOISNULL/1', Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Sat Nov 7 17:12:57 2009 @@ -2,7 +2,10 @@ # some unit tests for the bytecode decoding from pypy.jit.metainterp import pyjitpl, codewriter, resoperation -from pypy.jit.metainterp.history import AbstractFailDescr +from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt +from pypy.jit.metainterp.history import History +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.metainterp.test.test_optimizeopt import equaloplists def make_frame(code): bytecode = codewriter.JitCode("hello") @@ -103,3 +106,39 @@ history.operations.extend([9, 10, 11, 12]) assert metainterp.find_biggest_function() == "green3" + +def test_remove_consts_and_duplicates(): + class FakeStaticData: + cpu = None + def is_another_box_like(box, referencebox): + assert box is not referencebox + assert isinstance(box, referencebox.clonebox().__class__) + assert box.value == referencebox.value + return True + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp.history = History(None) + b1 = BoxInt(1) + b2 = BoxInt(2) + c3 = ConstInt(3) + boxes = [b1, b2, b1, c3] + dup = {} + metainterp.remove_consts_and_duplicates(boxes, 0, 4, dup) + assert boxes[0] is b1 + assert boxes[1] is b2 + assert is_another_box_like(boxes[2], b1) + assert is_another_box_like(boxes[3], c3) + assert equaloplists(metainterp.history.operations, [ + ResOperation(rop.SAME_AS, [b1], boxes[2]), + ResOperation(rop.SAME_AS, [c3], boxes[3]), + ]) + assert dup == {b1: None, b2: None} + # + del metainterp.history.operations[:] + b4 = BoxInt(4) + boxes = ["something random", b2, b4, "something else"] + metainterp.remove_consts_and_duplicates(boxes, 1, 3, dup) + assert is_another_box_like(boxes[1], b2) + assert boxes[2] is b4 + assert equaloplists(metainterp.history.operations, [ + ResOperation(rop.SAME_AS, [b2], boxes[1]), + ]) From arigo at codespeak.net Sat Nov 7 17:57:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 7 Nov 2009 17:57:19 +0100 (CET) Subject: [pypy-svn] r69052 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091107165719.E39E51684AF@codespeak.net> Author: arigo Date: Sat Nov 7 17:57:19 2009 New Revision: 69052 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_slist.py Log: Redo r69025, this time running all tests. Sorry sorry. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sat Nov 7 17:57:19 2009 @@ -367,6 +367,7 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) + self.bool_boxes = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(OPT_FORCINGS) @@ -513,6 +514,8 @@ self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True + elif op.returns_bool_result(): + self.bool_boxes[op.result] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -523,6 +526,22 @@ if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp descr.store_final_boxes(op, newboxes) + # + # Hack: turn guard_value(bool) into guard_true/guard_false. + # This is done after the operation is emitted, to let + # store_final_boxes_in_guard set the guard_opnum field + # of the descr to the original rop.GUARD_VALUE. + if op.opnum == rop.GUARD_VALUE and op.args[0] in self.bool_boxes: + constvalue = op.args[1].getint() + if constvalue == 0: + opnum = rop.GUARD_FALSE + elif constvalue == 1: + opnum = rop.GUARD_TRUE + else: + raise AssertionError("uh?") + op.opnum = opnum + assert len(op.args) == 2 + op.args.pop() def optimize_default(self, op): if op.is_always_pure(): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sat Nov 7 17:57:19 2009 @@ -356,7 +356,6 @@ def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -396,7 +395,6 @@ indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -410,7 +408,6 @@ def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) - # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): return False @@ -428,7 +425,6 @@ rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? - # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): return False @@ -444,7 +440,6 @@ def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) - # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): self.make_result_box(box) Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Sat Nov 7 17:57:19 2009 @@ -93,6 +93,14 @@ def is_final(self): return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST + def returns_bool_result(self): + opnum = self.opnum + if we_are_translated(): + assert opnum >= 0 + elif opnum < 0: + return False # for tests + return opboolresult[opnum] + # ____________________________________________________________ _oplist = [ @@ -137,40 +145,40 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1', + 'FLOAT_IS_TRUE/1b', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # '_COMPARISON_FIRST', - 'INT_LT/2', - 'INT_LE/2', - 'INT_EQ/2', - 'INT_NE/2', - 'INT_GT/2', - 'INT_GE/2', - 'UINT_LT/2', - 'UINT_LE/2', - 'UINT_GT/2', - 'UINT_GE/2', + 'INT_LT/2b', + 'INT_LE/2b', + 'INT_EQ/2b', + 'INT_NE/2b', + 'INT_GT/2b', + 'INT_GE/2b', + 'UINT_LT/2b', + 'UINT_LE/2b', + 'UINT_GT/2b', + 'UINT_GE/2b', '_COMPARISON_LAST', - 'FLOAT_LT/2', # maybe these ones should be comparisons too - 'FLOAT_LE/2', - 'FLOAT_EQ/2', - 'FLOAT_NE/2', - 'FLOAT_GT/2', - 'FLOAT_GE/2', + 'FLOAT_LT/2b', # maybe these ones should be comparisons too + 'FLOAT_LE/2b', + 'FLOAT_EQ/2b', + 'FLOAT_NE/2b', + 'FLOAT_GT/2b', + 'FLOAT_GE/2b', # - 'INT_IS_TRUE/1', + 'INT_IS_TRUE/1b', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1', + 'BOOL_NOT/1b', # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'OONONNULL/1', - 'OOISNULL/1', - 'OOIS/2', - 'OOISNOT/2', + 'OONONNULL/1b', + 'OOISNULL/1b', + 'OOIS/2b', + 'OOISNOT/2b', # 'ARRAYLEN_GC/1d', 'STRLEN/1', @@ -182,8 +190,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1d', - 'SUBCLASSOF/2', + 'INSTANCEOF/1db', + 'SUBCLASSOF/2b', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -231,6 +239,7 @@ opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" +opboolresult= [] # mapping numbers to a flag "returns a boolean" def setup(debug_print=False): @@ -239,16 +248,18 @@ print '%30s = %d' % (name, i) if '/' in name: name, arity = name.split('/') - withdescr = arity.endswith('d') - arity = int(arity.rstrip('d')) + withdescr = 'd' in arity + boolresult = 'b' in arity + arity = int(arity.rstrip('db')) else: - arity, withdescr = -1, True # default + arity, withdescr, boolresult = -1, True, False # default setattr(rop, name, i) if not name.startswith('_'): opname[i] = name oparity.append(arity) opwithdescr.append(withdescr) - assert len(oparity) == len(opwithdescr) == len(_oplist) + opboolresult.append(boolresult) + assert len(oparity)==len(opwithdescr)==len(opboolresult)==len(_oplist) setup(__name__ == '__main__') # print out the table when run directly del _oplist Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sat Nov 7 17:57:19 2009 @@ -524,6 +524,45 @@ """ self.optimize_loop(ops, '', ops) + def test_guard_value_to_guard_true(self): + ops = """ + [i] + i1 = int_lt(i, 3) + guard_value(i1, 1) [i] + jump(i) + """ + expected = """ + [i] + i1 = int_lt(i, 3) + guard_true(i1) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_to_guard_false(self): + ops = """ + [p] + i1 = ooisnull(p) + guard_value(i1, 0) [p] + jump(p) + """ + expected = """ + [p] + i1 = ooisnull(p) + guard_false(i1) [p] + jump(p) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_on_nonbool(self): + ops = """ + [i] + i1 = int_add(i, 3) + guard_value(i1, 0) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', ops) + def test_p123_simple(self): ops = """ Modified: pypy/trunk/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_slist.py Sat Nov 7 17:57:19 2009 @@ -82,13 +82,20 @@ self.check_loops(call=0) def test_getitem_neg(self): + myjitdriver = JitDriver(greens = [], reds = ['i', 'n']) def f(n): - lst = [41] - lst.append(42) - return lst[n] - res = self.interp_operations(f, [-2], listops=True) + x = i = 0 + while i < 10: + myjitdriver.can_enter_jit(n=n, i=i) + myjitdriver.jit_merge_point(n=n, i=i) + lst = [41] + lst.append(42) + x = lst[n] + i += 1 + return x + res = self.meta_interp(f, [-2], listops=True) assert res == 41 - self.check_operations_history(call=1) + self.check_loops(call=1, guard_value=0) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): From arigo at codespeak.net Sat Nov 7 19:10:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 7 Nov 2009 19:10:40 +0100 (CET) Subject: [pypy-svn] r69053 - pypy/extradoc/planning Message-ID: <20091107181040.2352349843D@codespeak.net> Author: arigo Date: Sat Nov 7 19:10:40 2009 New Revision: 69053 Modified: pypy/extradoc/planning/jit.txt Log: Check in the results of today's digging. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sat Nov 7 19:10:40 2009 @@ -106,3 +106,19 @@ - try to unify interfaces to make doing the right thing for ootype easier - different constraints for different groups of people - what to do with ootype jit support after Anto finished his PhD? + + +memory usage +------------ + +- we use too much memory during jitting. Notably of the following + types (in decreasing order of total size, for Pystone): + - rpy_string + - GcArray of AbstractValue (unknown if fixedsize or not) + - DoneWithThisFrameRef + - dict {AbstractValue: SHORT} + - GcArray of ResOperation + - resizable list of AbstractValue + - ResOperation + - ResumeGuardDescr + - ConstInt From cfbolz at codespeak.net Sun Nov 8 11:31:15 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 11:31:15 +0100 (CET) Subject: [pypy-svn] r69054 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091108103115.3BDD8168539@codespeak.net> Author: cfbolz Date: Sun Nov 8 11:31:12 2009 New Revision: 69054 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Sun Nov 8 11:31:12 2009 @@ -9,15 +9,16 @@ TASKS ====== - - sanitize inlining IN-PROGRESS (Samuele, Carl Friedrich) - - directly call assembler for residual portal calls + - make the size of ResOperation instances less insane (Armin, Anto) + - set up the buildbot on the Mac Mini (Samuele, Carl Friedrich) + - directly call assembler for residual portal calls - making the CLI backend working with logging - compress the virtuals part of resume data more - - understand the memory behaviour of pystone with the JIT (Armin, Anto) - - fix the guard_value(bool, 1) -> guard_true hack (Armin) + - understand the memory behaviour of pystone with the JIT DONE + - fix the guard_value(bool, 1) -> guard_true hack DONE - try to do something non-insane about Python-level exceptions - make the assembler produced by generate_failure smaller - - lose less information across residual calls (Samuele, Carl Friedrich) + - lose less information across residual calls IN-PROGRESS (Samuele, Carl Friedrich) - we should think about merging several consecutive guards in the optimizer, to make the assembler smaller and to save resume data space - put the class into the structure to get only one promote when using an From cfbolz at codespeak.net Sun Nov 8 12:16:30 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 12:16:30 +0100 (CET) Subject: [pypy-svn] r69055 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108111630.9B88B168539@codespeak.net> Author: cfbolz Date: Sun Nov 8 12:16:29 2009 New Revision: 69055 Added: pypy/trunk/pypy/jit/metainterp/effectinfo.py (contents, props changed) Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Log: (pedronis, cfbolz): don't delete all of the cached information about structure fields and array items. Instead, use the write analyzer to figure out which fields an external function can actually write to and invalidate the caches of those fields. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Sun Nov 8 12:16:29 2009 @@ -11,7 +11,9 @@ from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer +from pypy.translator.backendopt.writeanalyze import WriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType +from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze import py, sys from pypy.tool.ansi_print import ansi_log @@ -177,6 +179,7 @@ self.cpu = metainterp_sd.cpu self.portal_runner_ptr = portal_runner_ptr self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) + self.write_analyzer = WriteAnalyzer(self.rtyper.annotator.translator) def make_portal_bytecode(self, graph): log.info("making JitCodes...") @@ -302,7 +305,7 @@ jitcodes[oocls] = jitcode methdescr.setup(jitcodes) - def getcalldescr(self, v_func, args, result): + def getcalldescr(self, v_func, args, result, consider_effects_of=None): non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -312,7 +315,13 @@ assert NON_VOID_ARGS == [T for T in ARGS if T is not lltype.Void] assert RESULT == FUNC.RESULT # ok - calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) + if (self.rtyper.type_system.name == 'lltypesystem' and + consider_effects_of is not None): + effectinfo = effectinfo_from_writeanalyze( + self.write_analyzer.analyze(consider_effects_of), self.cpu) + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) + else: + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) return calldescr, non_void_args def register_known_gctype(self, vtable, STRUCT): @@ -1147,9 +1156,8 @@ args = op.args[1:-1] else: args = op.args[1:] - calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], - args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + op.args[0], args, op.result, consider_effects_of=op) pure = False if op.opname == "direct_call": func = getattr(get_funcobj(op.args[0].value), '_callable', None) Added: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Sun Nov 8 12:16:29 2009 @@ -0,0 +1,39 @@ +from pypy.rpython.lltypesystem import lltype + +class EffectInfo(object): + _cache = {} + + def __new__(cls, write_descrs_fields, write_descrs_arrays): + key = frozenset(write_descrs_fields), frozenset(write_descrs_arrays) + if key in cls._cache: + return cls._cache[key] + result = object.__new__(cls) + result.write_descrs_fields = write_descrs_fields + result.write_descrs_arrays = write_descrs_arrays + cls._cache[key] = result + return result + +def effectinfo_from_writeanalyze(effects, cpu): + from pypy.translator.backendopt.writeanalyze import top_set + if effects is top_set: + return None + write_descrs_fields = [] + write_descrs_arrays = [] + for tup in effects: + if tup[0] == "struct": + _, T, fieldname = tup + if not isinstance(T.TO, lltype.GcStruct): # can be a non-GC-struct + continue + descr = cpu.fielddescrof(T.TO, fieldname) + write_descrs_fields.append(descr) + elif tup[0] == "array": + _, T = tup + if not isinstance(T.TO, lltype.GcArray): # can be a non-GC-array + continue + descr = cpu.arraydescrof(T.TO) + write_descrs_arrays.append(descr) + else: + assert 0 + return EffectInfo(write_descrs_fields, write_descrs_arrays) + + Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sun Nov 8 12:16:29 2009 @@ -852,6 +852,20 @@ opnum == rop.SETARRAYITEM_GC or opnum == rop.DEBUG_MERGE_POINT): return + if opnum == rop.CALL: + effectinfo = op.descr.get_extra_info() + if effectinfo is not None: + for fielddescr in effectinfo.write_descrs_fields: + try: + del self.cached_fields[fielddescr] + except KeyError: + pass + for arraydescr in effectinfo.write_descrs_arrays: + try: + del self.cached_arrayitems[arraydescr] + except KeyError: + pass + return self.clean_caches() def optimize_GETFIELD_GC(self, op, value): Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sun Nov 8 12:16:29 2009 @@ -972,7 +972,6 @@ self.check_loop_count(1) self.check_loops(call=1) - class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): @@ -1104,5 +1103,65 @@ history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value assert res == expected + def test_residual_call_doesnt_lose_info(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) + + class A(object): + pass + + globall = [""] + @dont_look_inside + def g(x): + globall[0] = str(x) + return x + + def f(x): + y = A() + y.v = x + l = [0] + while y.v > 0: + myjitdriver.can_enter_jit(x=x, y=y, l=l) + myjitdriver.jit_merge_point(x=x, y=y, l=l) + l[0] = y.v + lc = l[0] + y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 + return y.v + res = self.meta_interp(f, [20], listops=True) + self.check_loops(getfield_gc=1, getarrayitem_gc=0) + + def test_writeanalyzer_top_set(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) + + class A(object): + pass + class B(A): + pass + + @dont_look_inside + def g(x): + # instantiate cannot be followed by the writeanalyzer + if x % 2: + C = A + else: + C = B + a = instantiate(C) + a.v = x + return a.v + + def f(x): + y = A() + y.v = x + l = [0] + while y.v > 0: + myjitdriver.can_enter_jit(x=x, y=y, l=l) + myjitdriver.jit_merge_point(x=x, y=y, l=l) + l[0] = y.v + lc = l[0] + y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 + return y.v + res = self.meta_interp(f, [20], listops=True) + self.check_loops(getfield_gc=2, getarrayitem_gc=1) + class TestLLtype(BaseLLtypeTests, LLJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Nov 8 12:16:29 2009 @@ -16,6 +16,7 @@ from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.specnode import ConstantSpecNode +from pypy.jit.metainterp.effectinfo import EffectInfo from pypy.jit.metainterp.test.oparser import parse def test_sort_descrs(): @@ -93,6 +94,11 @@ usize = cpu.sizeof(U) onedescr = cpu.fielddescrof(U, 'one') + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) + writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) + writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), cpu.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U)} @@ -159,6 +165,9 @@ adescr.sort_key() bdescr.sort_key() + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) # XXX fix ootype + cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), node_vtable_adr2: cpu.typedescrof(NODE2), u_vtable_adr: cpu.typedescrof(U)} Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sun Nov 8 12:16:29 2009 @@ -616,18 +616,18 @@ guard_no_exception() [] i1 = int_add(i, 3) guard_no_exception() [] - i2 = call(i1) + i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] guard_no_exception() [] - i3 = call(i2) + i3 = call(i2, descr=nonwritedescr) jump(i1) # the exception is considered lost when we loop back """ expected = """ [i] i1 = int_add(i, 3) - i2 = call(i1) + i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] - i3 = call(i2) + i3 = call(i2, descr=nonwritedescr) jump(i1) """ self.optimize_loop(ops, 'Not', expected) @@ -1841,7 +1841,117 @@ class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): - pass + + def test_residual_call_does_not_invalidate_caches(self): + ops = """ + [p1, p2] + i1 = getfield_gc(p1, descr=valuedescr) + i2 = call(i1, descr=nonwritedescr) + i3 = getfield_gc(p1, descr=valuedescr) + escape(i1) + escape(i3) + jump(p1, p2) + """ + expected = """ + [p1, p2] + i1 = getfield_gc(p1, descr=valuedescr) + i2 = call(i1, descr=nonwritedescr) + escape(i1) + escape(i1) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_residual_call_invalidate_some_caches(self): + ops = """ + [p1, p2] + i1 = getfield_gc(p1, descr=adescr) + i2 = getfield_gc(p1, descr=bdescr) + i3 = call(i1, descr=writeadescr) + i4 = getfield_gc(p1, descr=adescr) + i5 = getfield_gc(p1, descr=bdescr) + escape(i1) + escape(i2) + escape(i4) + escape(i5) + jump(p1, p2) + """ + expected = """ + [p1, p2] + i1 = getfield_gc(p1, descr=adescr) + i2 = getfield_gc(p1, descr=bdescr) + i3 = call(i1, descr=writeadescr) + i4 = getfield_gc(p1, descr=adescr) + escape(i1) + escape(i2) + escape(i4) + escape(i2) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_residual_call_invalidate_arrays(self): + ops = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i3 = call(i1, descr=writeadescr) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p6 = getarrayitem_gc(p2, 1, descr=arraydescr2) + escape(p3) + escape(p4) + escape(p5) + escape(p6) + jump(p1, p2, i1) + """ + expected = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i3 = call(i1, descr=writeadescr) + escape(p3) + escape(p4) + escape(p3) + escape(p4) + jump(p1, p2, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_residual_call_invalidate_some_arrays(self): + ops = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i3 = call(i1, descr=writearraydescr) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p6 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i4 = getarrayitem_gc(p1, 1, descr=arraydescr) + escape(p3) + escape(p4) + escape(p5) + escape(p6) + escape(i2) + escape(i4) + jump(p1, p2, i1) + """ + expected = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i3 = call(i1, descr=writearraydescr) + i4 = getarrayitem_gc(p1, 1, descr=arraydescr) + escape(p3) + escape(p4) + escape(p3) + escape(p4) + escape(i2) + escape(i4) + jump(p1, p2, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 8 12:16:29 2009 @@ -228,7 +228,8 @@ assert f(20) == 10000*20 + (20*21)/2 res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) assert res == 10000*20 + (20*21)/2 - self.check_loops(call=1, getfield_gc=2, setfield_gc=2) + # there are no getfields because the optimizer gets rid of them + self.check_loops(call=1, getfield_gc=0, setfield_gc=2) # xxx for now a call that forces the virtualizable during tracing # is supposed to always force it later too. @@ -259,7 +260,9 @@ return m res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) assert res == f(20) - self.check_loops(call=1, getfield_gc=2, setfield_gc=2) + # the getfield_gc of inst_node is optimized away, because ext does not + # write to it + self.check_loops(call=1, getfield_gc=1, setfield_gc=2) # xxx for now a call that forces the virtualizable during tracing # is supposed to always force it later too. From cfbolz at codespeak.net Sun Nov 8 12:19:48 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 12:19:48 +0100 (CET) Subject: [pypy-svn] r69056 - in pypy/trunk/pypy/jit/backend: cli llgraph llsupport llsupport/test Message-ID: <20091108111948.86FBF168541@codespeak.net> Author: cfbolz Date: Sun Nov 8 12:19:47 2009 New Revision: 69056 Modified: pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Log: grumble, this belongs to revision 69055. Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Sun Nov 8 12:19:47 2009 @@ -414,6 +414,9 @@ def get_meth_info(self): clitype = self.get_delegate_clitype() return clitype.GetMethod('Invoke') + + def get_extra_info(self): + return None # XXX fix me class MethDescr(AbstractMethDescr): Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Sun Nov 8 12:19:47 2009 @@ -24,9 +24,13 @@ ofs = -1 typeinfo = '?' - def __init__(self, ofs, typeinfo='?'): + def __init__(self, ofs, typeinfo='?', extrainfo=None): self.ofs = ofs self.typeinfo = typeinfo + self.extrainfo = extrainfo + + def get_extra_info(self): + return self.extrainfo def __hash__(self): return hash((self.ofs, self.typeinfo)) @@ -284,9 +288,9 @@ return res @staticmethod - def calldescrof(FUNC, ARGS, RESULT): + def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): token = history.getkind(RESULT) - return Descr(0, token[0]) + return Descr(0, token[0], extrainfo=extrainfo) def get_exception(self): return self.cast_adr_to_int(llimpl.get_exception()) @@ -682,6 +686,9 @@ return boxresult(RESULT, res) self.callfunc = callfunc + def get_extra_info(self): + return None # XXX + class MethDescr(history.AbstractMethDescr): callmeth = None Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Sun Nov 8 12:19:47 2009 @@ -180,8 +180,12 @@ loop_token = None arg_classes = '' # <-- annotation hack - def __init__(self, arg_classes): + def __init__(self, arg_classes, extrainfo=None): self.arg_classes = arg_classes # string of "r" and "i" (ref/int) + self.extrainfo = extrainfo + + def get_extra_info(self): + return self.extrainfo def instantiate_arg_classes(self): result = [] @@ -256,7 +260,7 @@ NonGcPtrCallDescr, 'Call', 'get_result_size', '_returns_a_float') -def get_call_descr(gccache, ARGS, RESULT): +def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): arg_classes = [] for ARG in ARGS: kind = getkind(ARG) @@ -267,12 +271,12 @@ raise NotImplementedError('ARG = %r' % (ARG,)) arg_classes = ''.join(arg_classes) cls = getCallDescrClass(RESULT) - key = (cls, arg_classes) + key = (cls, arg_classes, extrainfo) cache = gccache._cache_call try: return cache[key] except KeyError: - calldescr = cls(arg_classes) + calldescr = cls(arg_classes, extrainfo) cache[key] = calldescr return calldescr Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Sun Nov 8 12:19:47 2009 @@ -225,8 +225,8 @@ return ofs, size, ptr, float unpack_arraydescr._always_inline_ = True - def calldescrof(self, FUNC, ARGS, RESULT): - return get_call_descr(self.gc_ll_descr, ARGS, RESULT) + def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): + return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_descr.py Sun Nov 8 12:19:47 2009 @@ -180,6 +180,20 @@ assert descr4.returns_a_float() assert descr4.arg_classes == "ff" +def test_call_descr_extra_info(): + c1 = GcCache(True) + T = lltype.GcStruct('T') + U = lltype.GcStruct('U', ('x', lltype.Signed)) + descr1 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U), "hello") + extrainfo = descr1.get_extra_info() + assert extrainfo == "hello" + descr2 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U), "hello") + assert descr1 is descr2 + descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) + extrainfo = descr3.get_extra_info() + assert extrainfo is None + + def test_repr_of_descr(): c0 = GcCache(False) T = lltype.GcStruct('T') From cfbolz at codespeak.net Sun Nov 8 12:33:49 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 12:33:49 +0100 (CET) Subject: [pypy-svn] r69057 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091108113349.2AB6216854B@codespeak.net> Author: cfbolz Date: Sun Nov 8 12:33:48 2009 New Revision: 69057 Added: pypy/trunk/pypy/jit/metainterp/test/test_list.py - copied unchanged from r69054, pypy/trunk/pypy/jit/metainterp/test/test_vlist.py Removed: pypy/trunk/pypy/jit/metainterp/test/test_vlist.py Log: rename the test, the only non-skipped tests in there are not actually about virtual lists From cfbolz at codespeak.net Sun Nov 8 12:41:36 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 12:41:36 +0100 (CET) Subject: [pypy-svn] r69058 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108114136.ACAE816854B@codespeak.net> Author: cfbolz Date: Sun Nov 8 12:41:35 2009 New Revision: 69058 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_list.py Log: attach effects info to oopspec operations as well Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Sun Nov 8 12:41:35 2009 @@ -1240,8 +1240,8 @@ else: opname = 'residual_call' - calldescr, non_void_args = self.codewriter.getcalldescr(c_func, args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + c_func, args, op.result, consider_effects_of=op) self.emit(opname) self.emit(self.get_position(calldescr)) self.emit_varargs([c_func] + non_void_args) Modified: pypy/trunk/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_list.py Sun Nov 8 12:41:35 2009 @@ -378,8 +378,30 @@ self.check_all_virtualized() + class TestOOtype(ListTests, OOJitMixin): pass class TestLLtype(ListTests, LLJitMixin): - pass + def test_listops_dont_invalidate_caches(self): + class A(object): + pass + jitdriver = JitDriver(greens = [], reds = ['n', 'a', 'lst']) + def f(n): + a = A() + a.x = 1 + if n < 1091212: + a.x = 2 # fool the annotator + lst = [n * 5, n * 10, n * 20] + while n > 0: + jitdriver.can_enter_jit(n=n, a=a, lst=lst) + jitdriver.jit_merge_point(n=n, a=a, lst=lst) + n += a.x + n = lst.pop() + lst.append(n - 10 + a.x) + a = lst.pop() + b = lst.pop() + return a * b + res = self.meta_interp(f, [37]) + assert res == f(37) + self.check_loops(getfield_gc=1) From arigo at codespeak.net Sun Nov 8 14:54:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 8 Nov 2009 14:54:39 +0100 (CET) Subject: [pypy-svn] r69059 - in pypy/trunk/pypy/jit: backend/llsupport backend/x86 metainterp Message-ID: <20091108135439.2C72A168016@codespeak.net> Author: arigo Date: Sun Nov 8 14:54:36 2009 New Revision: 69059 Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: Make the list ResOperation.args non-resizable. Fight a bit to understand how that list is unified with inputargs, which is a definitely resizable list. Modified: pypy/trunk/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/descr.py Sun Nov 8 14:54:36 2009 @@ -224,13 +224,17 @@ result = BoxInt() result_list = [result] operations = [ - ResOperation(rop.CALL, args, result, self), + ResOperation(rop.CALL, args[:], result, self), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=BasicFailDescr()), ResOperation(rop.FINISH, result_list, None, descr=BasicFailDescr())] operations[1].fail_args = [] loop_token = LoopToken() + # note: the 'args' that we pass below is not the same object as the + # 'args[:]' that was passed above to ResOperation, because we want + # the argument to ResOperation to be non-resizable, but the argument + # to compile_loop to be resizable. cpu.compile_loop(args, operations, loop_token) self.loop_token = loop_token return loop_token Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Sun Nov 8 14:54:36 2009 @@ -190,7 +190,9 @@ self.xrm.free_regs.insert(0, xmmtmp) assert tmpreg not in nonfloatlocs assert xmmtmp not in floatlocs - self.possibly_free_vars(inputargs) + # note: we need to make a copy of inputargs because possibly_free_vars + # is also used on op.args, which is a non-resizable list + self.possibly_free_vars(list(inputargs)) return nonfloatlocs, floatlocs def possibly_free_var(self, var): @@ -234,11 +236,11 @@ loop_consts[inputargs[i]] = i return loop_consts - def _update_bindings(self, locs, args): + def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py used = {} - for i in range(len(args)): - arg = args[i] + for i in range(len(inputargs)): + arg = inputargs[i] loc = locs[i] if arg.type == FLOAT: if isinstance(loc, REG): @@ -260,7 +262,9 @@ for reg in X86XMMRegisterManager.all_regs: if reg not in used: self.xrm.free_regs.append(reg) - self.possibly_free_vars(args) + # note: we need to make a copy of inputargs because possibly_free_vars + # is also used on op.args, which is a non-resizable list + self.possibly_free_vars(list(inputargs)) self.rm._check_invariants() self.xrm._check_invariants() @@ -851,7 +855,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result, []) + result_loc = self.rm.force_allocate_reg(op.result) self.Perform(op, [base_loc, imm(ofs)], result_loc) def consider_strgetitem(self, op, ignored): Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sun Nov 8 14:54:36 2009 @@ -540,8 +540,7 @@ else: raise AssertionError("uh?") op.opnum = opnum - assert len(op.args) == 2 - op.args.pop() + op.args = [op.args[0]] def optimize_default(self, op): if op.is_always_pure(): @@ -568,7 +567,7 @@ value = self.getvalue(op.args[i]) specnodes[i].teardown_virtual_node(self, value, exitargs) op2 = op.clone() - op2.args = exitargs + op2.args = exitargs[:] self.emit_operation(op2, must_clone=False) def optimize_guard(self, op, constbox): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sun Nov 8 14:54:36 2009 @@ -1283,6 +1283,7 @@ if require_attention: require_attention = self.after_residual_call() # check if the operation can be constant-folded away + argboxes = list(argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Sun Nov 8 14:54:36 2009 @@ -1,4 +1,5 @@ from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized class ResOperation(object): @@ -12,9 +13,11 @@ pc = 0 def __init__(self, opnum, args, result, descr=None): + make_sure_not_resized(args) assert isinstance(opnum, int) self.opnum = opnum self.args = list(args) + make_sure_not_resized(self.args) assert not isinstance(result, list) self.result = result self.setdescr(descr) Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Sun Nov 8 14:54:36 2009 @@ -66,6 +66,7 @@ debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator translator.config.translation.gc = "boehm" + translator.config.translation.list_comprehension_operations = True warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) warmrunnerdesc.state.set_param_threshold(3) # for tests warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests From cfbolz at codespeak.net Sun Nov 8 15:13:52 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 15:13:52 +0100 (CET) Subject: [pypy-svn] r69060 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108141352.CB423168020@codespeak.net> Author: cfbolz Date: Sun Nov 8 15:13:51 2009 New Revision: 69060 Added: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py (contents, props changed) Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py Log: (cfbolz, pedronis): fix problems with removetypeptr Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Sun Nov 8 15:13:51 2009 @@ -1,3 +1,4 @@ +from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype class EffectInfo(object): @@ -24,6 +25,11 @@ _, T, fieldname = tup if not isinstance(T.TO, lltype.GcStruct): # can be a non-GC-struct continue + if fieldname == "typeptr" and T.TO is OBJECT: + # filter out the typeptr, because + # a) it is optimized in different ways + # b) it might not be there in C if removetypeptr is specified + continue descr = cpu.fielddescrof(T.TO, fieldname) write_descrs_fields.append(descr) elif tup[0] == "array": Added: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Sun Nov 8 15:13:51 2009 @@ -0,0 +1,9 @@ +from pypy.rpython.lltypesystem.rclass import OBJECT +from pypy.rpython.lltypesystem import lltype +from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze + +def test_filter_out_typeptr(): + effects = frozenset([("struct", lltype.Ptr(OBJECT), "typeptr")]) + effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays From cfbolz at codespeak.net Sun Nov 8 15:19:53 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 15:19:53 +0100 (CET) Subject: [pypy-svn] r69061 - pypy/branch/merge-guards Message-ID: <20091108141953.4BFF1168020@codespeak.net> Author: cfbolz Date: Sun Nov 8 15:19:52 2009 New Revision: 69061 Added: pypy/branch/merge-guards/ - copied from r69060, pypy/trunk/ Log: (pedronis, cfbolz): a branch where we want to try to merge consecutive guards From cfbolz at codespeak.net Sun Nov 8 15:29:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 15:29:39 +0100 (CET) Subject: [pypy-svn] r69062 - in pypy/trunk/pypy/jit/backend: cli/test x86/test Message-ID: <20091108142939.6138B16801F@codespeak.net> Author: cfbolz Date: Sun Nov 8 15:29:39 2009 New Revision: 69062 Added: pypy/trunk/pypy/jit/backend/cli/test/test_list.py - copied, changed from r69046, pypy/trunk/pypy/jit/backend/cli/test/test_vlist.py pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_list.py - copied, changed from r69046, pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py pypy/trunk/pypy/jit/backend/x86/test/test_list.py - copied, changed from r69046, pypy/trunk/pypy/jit/backend/x86/test/test_vlist.py Removed: pypy/trunk/pypy/jit/backend/cli/test/test_vlist.py pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py pypy/trunk/pypy/jit/backend/x86/test/test_vlist.py Log: oops, there were places that import this thingie Copied: pypy/trunk/pypy/jit/backend/cli/test/test_list.py (from r69046, pypy/trunk/pypy/jit/backend/cli/test/test_vlist.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_vlist.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_list.py Sun Nov 8 15:29:39 2009 @@ -1,11 +1,11 @@ import py -from pypy.jit.metainterp.test import test_vlist +from pypy.jit.metainterp.test import test_list from pypy.jit.backend.cli.test.test_basic import CliJitMixin -class TestVlist(CliJitMixin, test_vlist.TestOOtype): +class TestVlist(CliJitMixin, test_list.TestOOtype): # for the individual tests see - # ====> ../../../metainterp/test/test_vlist.py + # ====> ../../../metainterp/test/test_list.py def skip(self): py.test.skip("works only after translation") Copied: pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_list.py (from r69046, pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_vlist.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_list.py Sun Nov 8 15:29:39 2009 @@ -1,10 +1,10 @@ import py from pypy.jit.backend.cli.test.test_zrpy_basic import CliTranslatedJitMixin -from pypy.jit.metainterp.test import test_vlist +from pypy.jit.metainterp.test import test_list -class TestVList(CliTranslatedJitMixin, test_vlist.TestOOtype): +class TestVList(CliTranslatedJitMixin, test_list.TestOOtype): # for the individual tests see - # ====> ../../../metainterp/test/test_vlist.py + # ====> ../../../metainterp/test/test_list.py pass Copied: pypy/trunk/pypy/jit/backend/x86/test/test_list.py (from r69046, pypy/trunk/pypy/jit/backend/x86/test/test_vlist.py) ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_vlist.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_list.py Sun Nov 8 15:29:39 2009 @@ -1,8 +1,8 @@ -from pypy.jit.metainterp.test.test_vlist import ListTests +from pypy.jit.metainterp.test.test_list import ListTests from pypy.jit.backend.x86.test.test_basic import Jit386Mixin -class TestVList(Jit386Mixin, ListTests): +class TestList(Jit386Mixin, ListTests): # for individual tests see - # ====> ../../../metainterp/test/test_vlist.py + # ====> ../../../metainterp/test/test_list.py pass From arigo at codespeak.net Sun Nov 8 15:52:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 8 Nov 2009 15:52:07 +0100 (CET) Subject: [pypy-svn] r69063 - in pypy/trunk/pypy/jit/backend/x86: . test Message-ID: <20091108145207.58E58168068@codespeak.net> Author: arigo Date: Sun Nov 8 15:52:06 2009 New Revision: 69063 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/jit/backend/x86/ri386.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: In class MODRM, replace the 'extradata' string with a list of characters. Should help a bit because we can e.g. build a list of 4 characters easily, whereas making a string from 4 characters leads to a mess of mallocs. Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Sun Nov 8 15:52:06 2009 @@ -864,7 +864,7 @@ loc_mask = arglocs[1] mc = self.mc._mc mc.TEST(loc_cond, loc_mask) - mc.write('\x74\x00') # JZ after_the_call + mc.write(constlistofchars('\x74\x00')) # JZ after_the_call jz_location = mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. @@ -884,7 +884,7 @@ # patch the JZ above offset = mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.overwrite(jz_location-1, [chr(offset)]) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -920,7 +920,7 @@ mc.MOV(eax, heap(nursery_free_adr)) mc.LEA(edx, addr_add(eax, imm(size))) mc.CMP(edx, heap(nursery_top_adr)) - mc.write('\x76\x00') # JNA after the block + mc.write(constlistofchars('\x76\x00')) # JNA after the block jmp_adr = mc.get_relative_pos() mc.PUSH(imm(size)) mc.CALL(rel32(slowpath_addr)) @@ -932,7 +932,7 @@ mc.ADD(esp, imm(4)) offset = mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, chr(offset)) + mc.overwrite(jmp_adr-1, [chr(offset)]) mc.MOV(addr_add(eax, imm(0)), imm(tid)) mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Sun Nov 8 15:52:06 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free +from pypy.rlib.debug import make_sure_not_resized class InMemoryCodeBuilder(I386CodeBuilder): @@ -18,18 +19,19 @@ self._size = map_size self._pos = 0 - def overwrite(self, pos, data): - assert pos + len(data) <= self._size - for c in data: + def overwrite(self, pos, listofchars): + make_sure_not_resized(listofchars) + assert pos + len(listofchars) <= self._size + for c in listofchars: self._data[pos] = c pos += 1 return pos - def write(self, data): - self._pos = self.overwrite(self._pos, data) + def write(self, listofchars): + self._pos = self.overwrite(self._pos, listofchars) def writechr(self, n): - # purely for performance: don't convert chr(n) to a str + # purely for performance: don't make the one-element list [chr(n)] pos = self._pos assert pos + 1 <= self._size self._data[pos] = chr(n) Modified: pypy/trunk/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386.py Sun Nov 8 15:52:06 2009 @@ -1,5 +1,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated +from pypy.rlib.debug import make_sure_not_resized class OPERAND(object): _attrs_ = [] @@ -112,6 +113,7 @@ def __init__(self, byte, extradata): self.byte = byte self.extradata = extradata + make_sure_not_resized(extradata) def lowest8bits(self): return MODRM8(self.byte, self.extradata) @@ -282,7 +284,7 @@ def memregister(register): assert register.width == 4 - return MODRM(0xC0 | register.op, '') + return MODRM(0xC0 | register.op, constlistofchars('')) def mem(basereg, offset=0): return memSIB(basereg, None, 0, offset) @@ -307,11 +309,11 @@ def memregister8(register): assert register.width == 1 - return MODRM8(0xC0 | register.op, '') + return MODRM8(0xC0 | register.op, constlistofchars('')) def memregister64(register): assert register.width == 8 - return MODRM64(0xC0 | register.op, '') + return MODRM64(0xC0 | register.op, constlistofchars('')) def mem8(basereg, offset=0): return memSIB8(basereg, None, 0, offset) @@ -328,17 +330,17 @@ if index is None: return cls(0x05, packimm32(offset)) if scaleshift > 0: - return cls(0x04, chr((scaleshift<<6) | (index.op<<3) | 0x05) + + return cls(0x04, [chr((scaleshift<<6) | (index.op<<3) | 0x05)] + packimm32(offset)) base = index index = None if index is not None: - SIB = chr((scaleshift<<6) | (index.op<<3) | base.op) + SIB = [chr((scaleshift<<6) | (index.op<<3) | base.op)] elif base is esp: - SIB = '\x24' + SIB = constlistofchars('\x24') elif offset == 0 and base is not ebp: - return cls(base.op, '') + return cls(base.op, constlistofchars('')) elif single_byte(offset): return cls(0x40 | base.op, packimm8(offset)) else: @@ -358,19 +360,25 @@ return -128 <= value < 128 def packimm32(i): - return (chr(i & 0xFF) + - chr((i >> 8) & 0xFF) + - chr((i >> 16) & 0xFF) + - chr((i >> 24) & 0xFF)) + lst = [chr(i & 0xFF), + chr((i >> 8) & 0xFF), + chr((i >> 16) & 0xFF), + chr((i >> 24) & 0xFF)] + make_sure_not_resized(lst) + return lst def packimm8(i): if i < 0: i += 256 - return chr(i) + lst = [chr(i)] + make_sure_not_resized(lst) + return lst def packimm16(i): - return (chr(i & 0xFF) + - chr((i >> 8) & 0xFF)) + lst = [chr(i & 0xFF), + chr((i >> 8) & 0xFF)] + make_sure_not_resized(lst) + return lst def unpack(s): assert len(s) in (1, 2, 4) @@ -388,6 +396,11 @@ a = intmask(a) return a +def constlistofchars(s): + assert isinstance(s, str) + return [c for c in s] +constlistofchars._annspecialcase_ = 'specialize:memo' + missing = MISSING() # __________________________________________________________ @@ -395,11 +408,11 @@ class I386CodeBuilder(object): - def write(self, data): + def write(self, listofchars): raise NotImplementedError def writechr(self, n): - self.write(chr(n)) + self.write([chr(n)]) def tell(self): raise NotImplementedError Modified: pypy/trunk/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386setup.py Sun Nov 8 15:52:06 2009 @@ -164,7 +164,7 @@ op = op[1:] if op: if len(op) > 1: - lines.append('builder.write(%r)' % (op,)) + lines.append('builder.write(constlistofchars(%r))' % (op,)) else: lines.append('builder.writechr(%d)' % (ord(op),)) else: @@ -180,6 +180,7 @@ 'packimm32': packimm32, 'packimm8': packimm8, 'packimm16': packimm16, + 'constlistofchars': constlistofchars, } exec compile2(source) in miniglobals return miniglobals['encode'] Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Sun Nov 8 15:52:06 2009 @@ -256,7 +256,8 @@ self.op = op self.instrindex = self.index - def write(self, data): + def write(self, listofchars): + data = ''.join(listofchars) end = self.index+len(data) if data != self.expected[self.index:end]: print self.op From cfbolz at codespeak.net Sun Nov 8 16:13:15 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 16:13:15 +0100 (CET) Subject: [pypy-svn] r69064 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108151315.B48E116806A@codespeak.net> Author: cfbolz Date: Sun Nov 8 16:13:14 2009 New Revision: 69064 Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Log: (pedronis, cfbolz): fix test_slist. yay, arrays of voids! Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Sun Nov 8 16:13:14 2009 @@ -36,6 +36,8 @@ _, T = tup if not isinstance(T.TO, lltype.GcArray): # can be a non-GC-array continue + if T.TO.OF is lltype.Void: + continue descr = cpu.arraydescrof(T.TO) write_descrs_arrays.append(descr) else: Modified: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Sun Nov 8 16:13:14 2009 @@ -7,3 +7,9 @@ effectinfo = effectinfo_from_writeanalyze(effects, None) assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays + +def test_filter_out_array_of_void(): + effects = frozenset([("array", lltype.Ptr(lltype.GcArray(lltype.Void)))]) + effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays From cfbolz at codespeak.net Sun Nov 8 16:18:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 16:18:39 +0100 (CET) Subject: [pypy-svn] r69065 - in pypy/branch/merge-guards/pypy/jit/metainterp: . test Message-ID: <20091108151839.12A4716802D@codespeak.net> Author: cfbolz Date: Sun Nov 8 16:18:38 2009 New Revision: 69065 Modified: pypy/branch/merge-guards/pypy/jit/metainterp/optimizeopt.py pypy/branch/merge-guards/pypy/jit/metainterp/test/test_optimizeopt.py Log: (pedronis, cfbolz): replace guard_class(x, ...), guard_value(x, ...) by just a guard_value. Modified: pypy/branch/merge-guards/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/merge-guards/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/merge-guards/pypy/jit/metainterp/optimizeopt.py Sun Nov 8 16:18:38 2009 @@ -42,7 +42,9 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'level') + _attrs_ = ('box', 'known_class', 'guard_class_index', 'level') + guard_class_index = -1 + level = LEVEL_UNKNOWN def __init__(self, box): @@ -85,10 +87,11 @@ else: return None - def make_constant_class(self, classbox): + def make_constant_class(self, classbox, opindex): if self.level < LEVEL_KNOWNCLASS: self.known_class = classbox self.level = LEVEL_KNOWNCLASS + self.guard_class_index = opindex def is_nonnull(self): level = self.level @@ -570,7 +573,7 @@ op2.args = exitargs[:] self.emit_operation(op2, must_clone=False) - def optimize_guard(self, op, constbox): + def optimize_guard(self, op, constbox, emit_operation=True): value = self.getvalue(op.args[0]) if value.is_constant(): box = value.box @@ -578,13 +581,23 @@ if not box.same_constant(constbox): raise InvalidLoop return - self.emit_operation(op) + if emit_operation: + self.emit_operation(op) value.make_constant(constbox) def optimize_GUARD_VALUE(self, op): + value = self.getvalue(op.args[0]) + emit_operation = True + if value.guard_class_index != -1: + # there already has been a guard_class on this value, which is + # rather silly. replace the original guard_class with a guard_value + guard_class_op = self.newoperations[value.guard_class_index] + guard_class_op.opnum = op.opnum + guard_class_op.args[1] = op.args[1] + emit_operation = False constbox = op.args[1] assert isinstance(constbox, Const) - self.optimize_guard(op, constbox) + self.optimize_guard(op, constbox, emit_operation) def optimize_GUARD_TRUE(self, op): self.optimize_guard(op, CONST_1) @@ -604,7 +617,7 @@ assert realclassbox.same_constant(expectedclassbox) return self.emit_operation(op) - value.make_constant_class(expectedclassbox) + value.make_constant_class(expectedclassbox, len(self.newoperations) - 1) def optimize_GUARD_NO_EXCEPTION(self, op): if not self.exception_might_have_happened: Modified: pypy/branch/merge-guards/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/merge-guards/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/merge-guards/pypy/jit/metainterp/test/test_optimizeopt.py Sun Nov 8 16:18:38 2009 @@ -1513,6 +1513,23 @@ 'Virtual(node_vtable, nextdescr=Virtual(node_vtable))', None) + def test_merge_guard_class_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_class(p1, ConstClass(node_vtable)) [i0] + i3 = int_add(i1, i2) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + # ---------- def make_fail_descr(self): From cfbolz at codespeak.net Sun Nov 8 16:32:43 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 16:32:43 +0100 (CET) Subject: [pypy-svn] r69067 - pypy/branch/merge-guards/pypy/jit/metainterp/test Message-ID: <20091108153243.2A87316802D@codespeak.net> Author: cfbolz Date: Sun Nov 8 16:32:41 2009 New Revision: 69067 Modified: pypy/branch/merge-guards/pypy/jit/metainterp/test/test_basic.py Log: (arigo around, cfbolz): a more functional test for the last commit Modified: pypy/branch/merge-guards/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/merge-guards/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/merge-guards/pypy/jit/metainterp/test/test_basic.py Sun Nov 8 16:32:41 2009 @@ -1163,5 +1163,33 @@ res = self.meta_interp(f, [20], listops=True) self.check_loops(getfield_gc=2, getarrayitem_gc=1) + def test_merge_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 1 + class B(A): + def g(self, y): + return y - 2 + + a1 = A() + a2 = A() + b = B() + def f(x): + l = [a1] * 100 + [a2] * 100 + [b] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + x = a.g(x) + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + self.check_loops(guard_class=0, guard_value=3) + + + class TestLLtype(BaseLLtypeTests, LLJitMixin): pass From cfbolz at codespeak.net Sun Nov 8 17:09:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 17:09:06 +0100 (CET) Subject: [pypy-svn] r69068 - pypy/trunk/pypy/interpreter Message-ID: <20091108160906.E0A67168056@codespeak.net> Author: cfbolz Date: Sun Nov 8 17:09:06 2009 New Revision: 69068 Modified: pypy/trunk/pypy/interpreter/gateway.py Log: (pedronis, arigo, cfbolz): make BuiltinActivations immutable Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Sun Nov 8 17:09:06 2009 @@ -257,6 +257,7 @@ activation_cls = type("BuiltinActivation_UwS_%s" % label, (BuiltinActivation,), d) + activation_cls._immutable_ = True cache[key] = activation_cls, self.run_args return activation_cls @@ -270,6 +271,7 @@ class BuiltinActivation(object): + _immutable_ = True def __init__(self, behavior): """NOT_RPYTHON""" From cfbolz at codespeak.net Sun Nov 8 17:17:53 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 17:17:53 +0100 (CET) Subject: [pypy-svn] r69069 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108161753.3D3B8168056@codespeak.net> Author: cfbolz Date: Sun Nov 8 17:17:52 2009 New Revision: 69069 Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Log: there are Void fields too. It's very funny how this thing with Voids goes on and on and on. Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Sun Nov 8 17:17:52 2009 @@ -25,6 +25,8 @@ _, T, fieldname = tup if not isinstance(T.TO, lltype.GcStruct): # can be a non-GC-struct continue + if getattr(T.TO, fieldname) is lltype.Void: + continue if fieldname == "typeptr" and T.TO is OBJECT: # filter out the typeptr, because # a) it is optimized in different ways Modified: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Sun Nov 8 17:17:52 2009 @@ -13,3 +13,9 @@ effectinfo = effectinfo_from_writeanalyze(effects, None) assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays + +def test_filter_out_struct_with_void(): + effects = frozenset([("struct", lltype.Ptr(lltype.GcStruct("x", ("a", lltype.Void))), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays From cfbolz at codespeak.net Sun Nov 8 17:20:23 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 17:20:23 +0100 (CET) Subject: [pypy-svn] r69070 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091108162023.5ECA7168056@codespeak.net> Author: cfbolz Date: Sun Nov 8 17:20:22 2009 New Revision: 69070 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: merge the merge-guards branch: ------------------------------------------------------------------------ r69061 | cfbolz | 2009-11-08 15:19:52 +0100 (Sun, 08 Nov 2009) | 2 lines Changed paths: A /pypy/branch/merge-guards (from /pypy/trunk:69060) (pedronis, cfbolz): a branch where we want to try to merge consecutive guards ------------------------------------------------------------------------ r69065 | cfbolz | 2009-11-08 16:18:38 +0100 (Sun, 08 Nov 2009) | 2 lines Changed paths: M /pypy/branch/merge-guards/pypy/jit/metainterp/optimizeopt.py M /pypy/branch/merge-guards/pypy/jit/metainterp/test/test_optimizeopt.py (pedronis, cfbolz): replace guard_class(x, ...), guard_value(x, ...) by just a guard_value. ------------------------------------------------------------------------ r69067 | cfbolz | 2009-11-08 16:32:41 +0100 (Sun, 08 Nov 2009) | 2 lines Changed paths: M /pypy/branch/merge-guards/pypy/jit/metainterp/test/test_basic.py (arigo around, cfbolz): a more functional test for the last commit ------------------------------------------------------------------------ Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Sun Nov 8 17:20:22 2009 @@ -42,7 +42,9 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'level') + _attrs_ = ('box', 'known_class', 'guard_class_index', 'level') + guard_class_index = -1 + level = LEVEL_UNKNOWN def __init__(self, box): @@ -85,10 +87,11 @@ else: return None - def make_constant_class(self, classbox): + def make_constant_class(self, classbox, opindex): if self.level < LEVEL_KNOWNCLASS: self.known_class = classbox self.level = LEVEL_KNOWNCLASS + self.guard_class_index = opindex def is_nonnull(self): level = self.level @@ -570,7 +573,7 @@ op2.args = exitargs[:] self.emit_operation(op2, must_clone=False) - def optimize_guard(self, op, constbox): + def optimize_guard(self, op, constbox, emit_operation=True): value = self.getvalue(op.args[0]) if value.is_constant(): box = value.box @@ -578,13 +581,23 @@ if not box.same_constant(constbox): raise InvalidLoop return - self.emit_operation(op) + if emit_operation: + self.emit_operation(op) value.make_constant(constbox) def optimize_GUARD_VALUE(self, op): + value = self.getvalue(op.args[0]) + emit_operation = True + if value.guard_class_index != -1: + # there already has been a guard_class on this value, which is + # rather silly. replace the original guard_class with a guard_value + guard_class_op = self.newoperations[value.guard_class_index] + guard_class_op.opnum = op.opnum + guard_class_op.args[1] = op.args[1] + emit_operation = False constbox = op.args[1] assert isinstance(constbox, Const) - self.optimize_guard(op, constbox) + self.optimize_guard(op, constbox, emit_operation) def optimize_GUARD_TRUE(self, op): self.optimize_guard(op, CONST_1) @@ -604,7 +617,7 @@ assert realclassbox.same_constant(expectedclassbox) return self.emit_operation(op) - value.make_constant_class(expectedclassbox) + value.make_constant_class(expectedclassbox, len(self.newoperations) - 1) def optimize_GUARD_NO_EXCEPTION(self, op): if not self.exception_might_have_happened: Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sun Nov 8 17:20:22 2009 @@ -1163,5 +1163,33 @@ res = self.meta_interp(f, [20], listops=True) self.check_loops(getfield_gc=2, getarrayitem_gc=1) + def test_merge_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 1 + class B(A): + def g(self, y): + return y - 2 + + a1 = A() + a2 = A() + b = B() + def f(x): + l = [a1] * 100 + [a2] * 100 + [b] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + x = a.g(x) + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + self.check_loops(guard_class=0, guard_value=3) + + + class TestLLtype(BaseLLtypeTests, LLJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Sun Nov 8 17:20:22 2009 @@ -1513,6 +1513,23 @@ 'Virtual(node_vtable, nextdescr=Virtual(node_vtable))', None) + def test_merge_guard_class_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_class(p1, ConstClass(node_vtable)) [i0] + i3 = int_add(i1, i2) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + # ---------- def make_fail_descr(self): From cfbolz at codespeak.net Sun Nov 8 17:20:41 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 17:20:41 +0100 (CET) Subject: [pypy-svn] r69071 - pypy/branch/merge-guards Message-ID: <20091108162041.1EE13168056@codespeak.net> Author: cfbolz Date: Sun Nov 8 17:20:40 2009 New Revision: 69071 Removed: pypy/branch/merge-guards/ Log: kill merged branch From arigo at codespeak.net Sun Nov 8 17:59:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 8 Nov 2009 17:59:50 +0100 (CET) Subject: [pypy-svn] r69073 - pypy/dist/pypy/doc Message-ID: <20091108165950.DD5AF16801D@codespeak.net> Author: arigo Date: Sun Nov 8 17:59:50 2009 New Revision: 69073 Modified: pypy/dist/pypy/doc/index.txt Log: Warn users again downloading release 1.1 if they are interested in the JIT. Modified: pypy/dist/pypy/doc/index.txt ============================================================================== --- pypy/dist/pypy/doc/index.txt (original) +++ pypy/dist/pypy/doc/index.txt Sun Nov 8 17:59:50 2009 @@ -8,7 +8,8 @@ Getting into PyPy ... ============================================= -* `Release 1.1`_: the latest official release +* `Release 1.1`_: the latest official release (oldish now, does not + contain the JIT_) * `PyPy Blog`_: news and status info about PyPy @@ -57,3 +58,4 @@ .. _`Getting Started`: getting-started.html .. _papers: extradoc.html .. _`Release 1.1`: release-1.1.0.html +.. _JIT: http://codespeak.net/pypy/trunk/pypy/doc/jit/ From cfbolz at codespeak.net Sun Nov 8 18:17:28 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 8 Nov 2009 18:17:28 +0100 (CET) Subject: [pypy-svn] r69074 - pypy/trunk/pypy/module/__builtin__/test Message-ID: <20091108171728.BCD95168020@codespeak.net> Author: cfbolz Date: Sun Nov 8 18:17:27 2009 New Revision: 69074 Modified: pypy/trunk/pypy/module/__builtin__/test/test_classobj.py Log: (pedronis, cfbolz): this is not a valid appdirect test Modified: pypy/trunk/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_classobj.py Sun Nov 8 18:17:27 2009 @@ -1,4 +1,5 @@ -from pypy.conftest import gettestobjspace +import py +from pypy.conftest import gettestobjspace, option from pypy.interpreter import gateway @@ -769,6 +770,8 @@ class AppTestOldStyleSharing(AppTestOldstyle): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withsharingdict": True}) + if option.runappdirect: + py.test.skip("can only be run on py.py") def is_sharing(space, w_inst): from pypy.objspace.std.sharingdict import SharedDictImplementation, W_DictMultiObject w_d = w_inst.getdict() From arigo at codespeak.net Sun Nov 8 18:34:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 8 Nov 2009 18:34:34 +0100 (CET) Subject: [pypy-svn] r69076 - in pypy/branch/gc-dump-malloc/pypy: interpreter jit/backend/cli jit/backend/cli/test jit/backend/llgraph jit/backend/llsupport jit/backend/llsupport/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/__builtin__/test translator/c/src Message-ID: <20091108173434.3C8B016804C@codespeak.net> Author: arigo Date: Sun Nov 8 18:34:33 2009 New Revision: 69076 Added: pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/test/test_list.py - copied unchanged from r69075, pypy/trunk/pypy/jit/backend/cli/test/test_list.py pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/test/test_zrpy_list.py - copied unchanged from r69075, pypy/trunk/pypy/jit/backend/cli/test/test_zrpy_list.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_list.py - copied unchanged from r69075, pypy/trunk/pypy/jit/backend/x86/test/test_list.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/effectinfo.py - copied unchanged from r69075, pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_effectinfo.py - copied unchanged from r69075, pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_list.py - copied unchanged from r69075, pypy/trunk/pypy/jit/metainterp/test/test_list.py Removed: pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/test/test_vlist.py pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/test/test_zrpy_vlist.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_vlist.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_vlist.py Modified: pypy/branch/gc-dump-malloc/pypy/interpreter/gateway.py pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/runner.py pypy/branch/gc-dump-malloc/pypy/jit/backend/llgraph/runner.py pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/descr.py pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/llmodel.py pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/test/test_descr.py pypy/branch/gc-dump-malloc/pypy/jit/backend/test/runner_test.py pypy/branch/gc-dump-malloc/pypy/jit/backend/test/test_random.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/assembler.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/codebuf.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/regalloc.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386setup.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/support.py pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/codewriter.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/optimizeopt.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/pyjitpl.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/resoperation.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_basic.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_blackhole.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_recursive.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_slist.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmspot.py pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmstate.py pypy/branch/gc-dump-malloc/pypy/module/__builtin__/test/test_classobj.py pypy/branch/gc-dump-malloc/pypy/translator/c/src/debug.h Log: svn merge -r68999:69075 svn+ssh://codespeak.net/svn/pypy/trunk Modified: pypy/branch/gc-dump-malloc/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/interpreter/gateway.py (original) +++ pypy/branch/gc-dump-malloc/pypy/interpreter/gateway.py Sun Nov 8 18:34:33 2009 @@ -257,6 +257,7 @@ activation_cls = type("BuiltinActivation_UwS_%s" % label, (BuiltinActivation,), d) + activation_cls._immutable_ = True cache[key] = activation_cls, self.run_args return activation_cls @@ -270,6 +271,7 @@ class BuiltinActivation(object): + _immutable_ = True def __init__(self, behavior): """NOT_RPYTHON""" Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/cli/runner.py Sun Nov 8 18:34:33 2009 @@ -414,6 +414,9 @@ def get_meth_info(self): clitype = self.get_delegate_clitype() return clitype.GetMethod('Invoke') + + def get_extra_info(self): + return None # XXX fix me class MethDescr(AbstractMethDescr): Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/llgraph/runner.py Sun Nov 8 18:34:33 2009 @@ -24,9 +24,13 @@ ofs = -1 typeinfo = '?' - def __init__(self, ofs, typeinfo='?'): + def __init__(self, ofs, typeinfo='?', extrainfo=None): self.ofs = ofs self.typeinfo = typeinfo + self.extrainfo = extrainfo + + def get_extra_info(self): + return self.extrainfo def __hash__(self): return hash((self.ofs, self.typeinfo)) @@ -284,9 +288,9 @@ return res @staticmethod - def calldescrof(FUNC, ARGS, RESULT): + def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): token = history.getkind(RESULT) - return Descr(0, token[0]) + return Descr(0, token[0], extrainfo=extrainfo) def get_exception(self): return self.cast_adr_to_int(llimpl.get_exception()) @@ -682,6 +686,9 @@ return boxresult(RESULT, res) self.callfunc = callfunc + def get_extra_info(self): + return None # XXX + class MethDescr(history.AbstractMethDescr): callmeth = None Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/descr.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/descr.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/descr.py Sun Nov 8 18:34:33 2009 @@ -180,8 +180,12 @@ loop_token = None arg_classes = '' # <-- annotation hack - def __init__(self, arg_classes): + def __init__(self, arg_classes, extrainfo=None): self.arg_classes = arg_classes # string of "r" and "i" (ref/int) + self.extrainfo = extrainfo + + def get_extra_info(self): + return self.extrainfo def instantiate_arg_classes(self): result = [] @@ -220,13 +224,17 @@ result = BoxInt() result_list = [result] operations = [ - ResOperation(rop.CALL, args, result, self), + ResOperation(rop.CALL, args[:], result, self), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=BasicFailDescr()), ResOperation(rop.FINISH, result_list, None, descr=BasicFailDescr())] operations[1].fail_args = [] loop_token = LoopToken() + # note: the 'args' that we pass below is not the same object as the + # 'args[:]' that was passed above to ResOperation, because we want + # the argument to ResOperation to be non-resizable, but the argument + # to compile_loop to be resizable. cpu.compile_loop(args, operations, loop_token) self.loop_token = loop_token return loop_token @@ -256,7 +264,7 @@ NonGcPtrCallDescr, 'Call', 'get_result_size', '_returns_a_float') -def get_call_descr(gccache, ARGS, RESULT): +def get_call_descr(gccache, ARGS, RESULT, extrainfo=None): arg_classes = [] for ARG in ARGS: kind = getkind(ARG) @@ -267,12 +275,12 @@ raise NotImplementedError('ARG = %r' % (ARG,)) arg_classes = ''.join(arg_classes) cls = getCallDescrClass(RESULT) - key = (cls, arg_classes) + key = (cls, arg_classes, extrainfo) cache = gccache._cache_call try: return cache[key] except KeyError: - calldescr = cls(arg_classes) + calldescr = cls(arg_classes, extrainfo) cache[key] = calldescr return calldescr Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/llmodel.py Sun Nov 8 18:34:33 2009 @@ -225,8 +225,8 @@ return ofs, size, ptr, float unpack_arraydescr._always_inline_ = True - def calldescrof(self, FUNC, ARGS, RESULT): - return get_call_descr(self.gc_ll_descr, ARGS, RESULT) + def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): + return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/test/test_descr.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/test/test_descr.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/llsupport/test/test_descr.py Sun Nov 8 18:34:33 2009 @@ -180,6 +180,20 @@ assert descr4.returns_a_float() assert descr4.arg_classes == "ff" +def test_call_descr_extra_info(): + c1 = GcCache(True) + T = lltype.GcStruct('T') + U = lltype.GcStruct('U', ('x', lltype.Signed)) + descr1 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U), "hello") + extrainfo = descr1.get_extra_info() + assert extrainfo == "hello" + descr2 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U), "hello") + assert descr1 is descr2 + descr3 = get_call_descr(c1, [lltype.Ptr(T)], lltype.Ptr(U)) + extrainfo = descr3.get_extra_info() + assert extrainfo is None + + def test_repr_of_descr(): c0 = GcCache(False) T = lltype.GcStruct('T') Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/test/runner_test.py Sun Nov 8 18:34:33 2009 @@ -673,13 +673,19 @@ def test_same_as(self): r = self.execute_operation(rop.SAME_AS, [ConstInt(5)], 'int') assert r.value == 5 + r = self.execute_operation(rop.SAME_AS, [BoxInt(5)], 'int') + assert r.value == 5 u_box = self.alloc_unicode(u"hello\u1234") r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ref') assert r.value == u_box.value + r = self.execute_operation(rop.SAME_AS, [u_box], 'ref') + assert r.value == u_box.value if self.cpu.supports_floats: r = self.execute_operation(rop.SAME_AS, [ConstFloat(5.5)], 'float') assert r.value == 5.5 + r = self.execute_operation(rop.SAME_AS, [BoxFloat(5.5)], 'float') + assert r.value == 5.5 def test_jump(self): # this test generates small loops where the JUMP passes many Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/test/test_random.py Sun Nov 8 18:34:33 2009 @@ -246,7 +246,9 @@ class ConstUnaryOperation(UnaryOperation): def produce_into(self, builder, r): - if r.random() < 0.75 or not builder.cpu.supports_floats: + if r.random() < 0.4: + UnaryOperation.produce_into(self, builder, r) + elif r.random() < 0.75 or not builder.cpu.supports_floats: self.put(builder, [ConstInt(r.random_integer())]) else: self.put(builder, [ConstFloat(r.random_float())]) Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/assembler.py Sun Nov 8 18:34:33 2009 @@ -864,7 +864,7 @@ loc_mask = arglocs[1] mc = self.mc._mc mc.TEST(loc_cond, loc_mask) - mc.write('\x74\x00') # JZ after_the_call + mc.write(constlistofchars('\x74\x00')) # JZ after_the_call jz_location = mc.get_relative_pos() # the following is supposed to be the slow path, so whenever possible # we choose the most compact encoding over the most efficient one. @@ -884,7 +884,7 @@ # patch the JZ above offset = mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.overwrite(jz_location-1, [chr(offset)]) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() @@ -920,7 +920,7 @@ mc.MOV(eax, heap(nursery_free_adr)) mc.LEA(edx, addr_add(eax, imm(size))) mc.CMP(edx, heap(nursery_top_adr)) - mc.write('\x76\x00') # JNA after the block + mc.write(constlistofchars('\x76\x00')) # JNA after the block jmp_adr = mc.get_relative_pos() mc.PUSH(imm(size)) mc.CALL(rel32(slowpath_addr)) @@ -932,7 +932,7 @@ mc.ADD(esp, imm(4)) offset = mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - mc.overwrite(jmp_adr-1, chr(offset)) + mc.overwrite(jmp_adr-1, [chr(offset)]) mc.MOV(addr_add(eax, imm(0)), imm(tid)) mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/codebuf.py Sun Nov 8 18:34:33 2009 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.backend.x86.ri386 import I386CodeBuilder from pypy.rlib.rmmap import PTR, alloc, free +from pypy.rlib.debug import make_sure_not_resized class InMemoryCodeBuilder(I386CodeBuilder): @@ -18,18 +19,19 @@ self._size = map_size self._pos = 0 - def overwrite(self, pos, data): - assert pos + len(data) <= self._size - for c in data: + def overwrite(self, pos, listofchars): + make_sure_not_resized(listofchars) + assert pos + len(listofchars) <= self._size + for c in listofchars: self._data[pos] = c pos += 1 return pos - def write(self, data): - self._pos = self.overwrite(self._pos, data) + def write(self, listofchars): + self._pos = self.overwrite(self._pos, listofchars) def writechr(self, n): - # purely for performance: don't convert chr(n) to a str + # purely for performance: don't make the one-element list [chr(n)] pos = self._pos assert pos + 1 <= self._size self._data[pos] = chr(n) Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/regalloc.py Sun Nov 8 18:34:33 2009 @@ -190,7 +190,9 @@ self.xrm.free_regs.insert(0, xmmtmp) assert tmpreg not in nonfloatlocs assert xmmtmp not in floatlocs - self.possibly_free_vars(inputargs) + # note: we need to make a copy of inputargs because possibly_free_vars + # is also used on op.args, which is a non-resizable list + self.possibly_free_vars(list(inputargs)) return nonfloatlocs, floatlocs def possibly_free_var(self, var): @@ -234,11 +236,11 @@ loop_consts[inputargs[i]] = i return loop_consts - def _update_bindings(self, locs, args): + def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py used = {} - for i in range(len(args)): - arg = args[i] + for i in range(len(inputargs)): + arg = inputargs[i] loc = locs[i] if arg.type == FLOAT: if isinstance(loc, REG): @@ -260,7 +262,9 @@ for reg in X86XMMRegisterManager.all_regs: if reg not in used: self.xrm.free_regs.append(reg) - self.possibly_free_vars(args) + # note: we need to make a copy of inputargs because possibly_free_vars + # is also used on op.args, which is a non-resizable list + self.possibly_free_vars(list(inputargs)) self.rm._check_invariants() self.xrm._check_invariants() @@ -851,7 +855,7 @@ ofs = arraydescr.get_ofs_length(self.translate_support_code) base_loc = self.rm.make_sure_var_in_reg(op.args[0], op.args) self.rm.possibly_free_vars(op.args) - result_loc = self.rm.force_allocate_reg(op.result, []) + result_loc = self.rm.force_allocate_reg(op.result) self.Perform(op, [base_loc, imm(ofs)], result_loc) def consider_strgetitem(self, op, ignored): Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386.py Sun Nov 8 18:34:33 2009 @@ -1,5 +1,6 @@ from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated +from pypy.rlib.debug import make_sure_not_resized class OPERAND(object): _attrs_ = [] @@ -112,6 +113,7 @@ def __init__(self, byte, extradata): self.byte = byte self.extradata = extradata + make_sure_not_resized(extradata) def lowest8bits(self): return MODRM8(self.byte, self.extradata) @@ -282,7 +284,7 @@ def memregister(register): assert register.width == 4 - return MODRM(0xC0 | register.op, '') + return MODRM(0xC0 | register.op, constlistofchars('')) def mem(basereg, offset=0): return memSIB(basereg, None, 0, offset) @@ -307,11 +309,11 @@ def memregister8(register): assert register.width == 1 - return MODRM8(0xC0 | register.op, '') + return MODRM8(0xC0 | register.op, constlistofchars('')) def memregister64(register): assert register.width == 8 - return MODRM64(0xC0 | register.op, '') + return MODRM64(0xC0 | register.op, constlistofchars('')) def mem8(basereg, offset=0): return memSIB8(basereg, None, 0, offset) @@ -328,17 +330,17 @@ if index is None: return cls(0x05, packimm32(offset)) if scaleshift > 0: - return cls(0x04, chr((scaleshift<<6) | (index.op<<3) | 0x05) + + return cls(0x04, [chr((scaleshift<<6) | (index.op<<3) | 0x05)] + packimm32(offset)) base = index index = None if index is not None: - SIB = chr((scaleshift<<6) | (index.op<<3) | base.op) + SIB = [chr((scaleshift<<6) | (index.op<<3) | base.op)] elif base is esp: - SIB = '\x24' + SIB = constlistofchars('\x24') elif offset == 0 and base is not ebp: - return cls(base.op, '') + return cls(base.op, constlistofchars('')) elif single_byte(offset): return cls(0x40 | base.op, packimm8(offset)) else: @@ -358,19 +360,25 @@ return -128 <= value < 128 def packimm32(i): - return (chr(i & 0xFF) + - chr((i >> 8) & 0xFF) + - chr((i >> 16) & 0xFF) + - chr((i >> 24) & 0xFF)) + lst = [chr(i & 0xFF), + chr((i >> 8) & 0xFF), + chr((i >> 16) & 0xFF), + chr((i >> 24) & 0xFF)] + make_sure_not_resized(lst) + return lst def packimm8(i): if i < 0: i += 256 - return chr(i) + lst = [chr(i)] + make_sure_not_resized(lst) + return lst def packimm16(i): - return (chr(i & 0xFF) + - chr((i >> 8) & 0xFF)) + lst = [chr(i & 0xFF), + chr((i >> 8) & 0xFF)] + make_sure_not_resized(lst) + return lst def unpack(s): assert len(s) in (1, 2, 4) @@ -388,6 +396,11 @@ a = intmask(a) return a +def constlistofchars(s): + assert isinstance(s, str) + return [c for c in s] +constlistofchars._annspecialcase_ = 'specialize:memo' + missing = MISSING() # __________________________________________________________ @@ -395,11 +408,11 @@ class I386CodeBuilder(object): - def write(self, data): + def write(self, listofchars): raise NotImplementedError def writechr(self, n): - self.write(chr(n)) + self.write([chr(n)]) def tell(self): raise NotImplementedError Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/ri386setup.py Sun Nov 8 18:34:33 2009 @@ -164,7 +164,7 @@ op = op[1:] if op: if len(op) > 1: - lines.append('builder.write(%r)' % (op,)) + lines.append('builder.write(constlistofchars(%r))' % (op,)) else: lines.append('builder.writechr(%d)' % (ord(op),)) else: @@ -180,6 +180,7 @@ 'packimm32': packimm32, 'packimm8': packimm8, 'packimm16': packimm16, + 'constlistofchars': constlistofchars, } exec compile2(source) in miniglobals return miniglobals['encode'] Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/support.py Sun Nov 8 18:34:33 2009 @@ -22,6 +22,7 @@ new_item = lltype.malloc(ATP, CHUNK_SIZE, zero=True) self.chunks.append(new_item) self.lgt += 1 + _grow._dont_inline_ = True def get_addr_for_num(self, i): chunk_no, ofs = self._no_of(i) @@ -34,6 +35,7 @@ while i >= len(self.chunks) * CHUNK_SIZE: self._grow() return i / CHUNK_SIZE, i % CHUNK_SIZE + _no_of._always_inline_ = True def setitem(self, i, v): chunk_no, ofs = self._no_of(i) Modified: pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Sun Nov 8 18:34:33 2009 @@ -256,7 +256,8 @@ self.op = op self.instrindex = self.index - def write(self, data): + def write(self, listofchars): + data = ''.join(listofchars) end = self.index+len(data) if data != self.expected[self.index:end]: print self.op Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/codewriter.py Sun Nov 8 18:34:33 2009 @@ -11,7 +11,9 @@ from pypy.tool.udir import udir from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.backendopt.canraise import RaiseAnalyzer +from pypy.translator.backendopt.writeanalyze import WriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType +from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze import py, sys from pypy.tool.ansi_print import ansi_log @@ -177,6 +179,7 @@ self.cpu = metainterp_sd.cpu self.portal_runner_ptr = portal_runner_ptr self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) + self.write_analyzer = WriteAnalyzer(self.rtyper.annotator.translator) def make_portal_bytecode(self, graph): log.info("making JitCodes...") @@ -302,7 +305,7 @@ jitcodes[oocls] = jitcode methdescr.setup(jitcodes) - def getcalldescr(self, v_func, args, result): + def getcalldescr(self, v_func, args, result, consider_effects_of=None): non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -312,7 +315,13 @@ assert NON_VOID_ARGS == [T for T in ARGS if T is not lltype.Void] assert RESULT == FUNC.RESULT # ok - calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) + if (self.rtyper.type_system.name == 'lltypesystem' and + consider_effects_of is not None): + effectinfo = effectinfo_from_writeanalyze( + self.write_analyzer.analyze(consider_effects_of), self.cpu) + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) + else: + calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) return calldescr, non_void_args def register_known_gctype(self, vtable, STRUCT): @@ -1147,9 +1156,8 @@ args = op.args[1:-1] else: args = op.args[1:] - calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], - args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + op.args[0], args, op.result, consider_effects_of=op) pure = False if op.opname == "direct_call": func = getattr(get_funcobj(op.args[0].value), '_callable', None) @@ -1232,8 +1240,8 @@ else: opname = 'residual_call' - calldescr, non_void_args = self.codewriter.getcalldescr(c_func, args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + c_func, args, op.result, consider_effects_of=op) self.emit(opname) self.emit(self.get_position(calldescr)) self.emit_varargs([c_func] + non_void_args) Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/optimizeopt.py Sun Nov 8 18:34:33 2009 @@ -42,7 +42,9 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'level') + _attrs_ = ('box', 'known_class', 'guard_class_index', 'level') + guard_class_index = -1 + level = LEVEL_UNKNOWN def __init__(self, box): @@ -85,10 +87,11 @@ else: return None - def make_constant_class(self, classbox): + def make_constant_class(self, classbox, opindex): if self.level < LEVEL_KNOWNCLASS: self.known_class = classbox self.level = LEVEL_KNOWNCLASS + self.guard_class_index = opindex def is_nonnull(self): level = self.level @@ -367,6 +370,7 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) self.heap_op_optimizer = HeapOpOptimizer(self) + self.bool_boxes = {} def forget_numberings(self, virtualbox): self.metainterp_sd.profiler.count(OPT_FORCINGS) @@ -513,6 +517,8 @@ self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True + elif op.returns_bool_result(): + self.bool_boxes[op.result] = None self.newoperations.append(op) def store_final_boxes_in_guard(self, op): @@ -523,6 +529,21 @@ if len(newboxes) > self.metainterp_sd.options.failargs_limit: raise compile.GiveUp descr.store_final_boxes(op, newboxes) + # + # Hack: turn guard_value(bool) into guard_true/guard_false. + # This is done after the operation is emitted, to let + # store_final_boxes_in_guard set the guard_opnum field + # of the descr to the original rop.GUARD_VALUE. + if op.opnum == rop.GUARD_VALUE and op.args[0] in self.bool_boxes: + constvalue = op.args[1].getint() + if constvalue == 0: + opnum = rop.GUARD_FALSE + elif constvalue == 1: + opnum = rop.GUARD_TRUE + else: + raise AssertionError("uh?") + op.opnum = opnum + op.args = [op.args[0]] def optimize_default(self, op): if op.is_always_pure(): @@ -549,10 +570,10 @@ value = self.getvalue(op.args[i]) specnodes[i].teardown_virtual_node(self, value, exitargs) op2 = op.clone() - op2.args = exitargs + op2.args = exitargs[:] self.emit_operation(op2, must_clone=False) - def optimize_guard(self, op, constbox): + def optimize_guard(self, op, constbox, emit_operation=True): value = self.getvalue(op.args[0]) if value.is_constant(): box = value.box @@ -560,13 +581,23 @@ if not box.same_constant(constbox): raise InvalidLoop return - self.emit_operation(op) + if emit_operation: + self.emit_operation(op) value.make_constant(constbox) def optimize_GUARD_VALUE(self, op): + value = self.getvalue(op.args[0]) + emit_operation = True + if value.guard_class_index != -1: + # there already has been a guard_class on this value, which is + # rather silly. replace the original guard_class with a guard_value + guard_class_op = self.newoperations[value.guard_class_index] + guard_class_op.opnum = op.opnum + guard_class_op.args[1] = op.args[1] + emit_operation = False constbox = op.args[1] assert isinstance(constbox, Const) - self.optimize_guard(op, constbox) + self.optimize_guard(op, constbox, emit_operation) def optimize_GUARD_TRUE(self, op): self.optimize_guard(op, CONST_1) @@ -586,7 +617,7 @@ assert realclassbox.same_constant(expectedclassbox) return self.emit_operation(op) - value.make_constant_class(expectedclassbox) + value.make_constant_class(expectedclassbox, len(self.newoperations) - 1) def optimize_GUARD_NO_EXCEPTION(self, op): if not self.exception_might_have_happened: @@ -833,6 +864,20 @@ opnum == rop.SETARRAYITEM_GC or opnum == rop.DEBUG_MERGE_POINT): return + if opnum == rop.CALL: + effectinfo = op.descr.get_extra_info() + if effectinfo is not None: + for fielddescr in effectinfo.write_descrs_fields: + try: + del self.cached_fields[fielddescr] + except KeyError: + pass + for arraydescr in effectinfo.write_descrs_arrays: + try: + del self.cached_arrayitems[arraydescr] + except KeyError: + pass + return self.clean_caches() def optimize_GETFIELD_GC(self, op, value): Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/pyjitpl.py Sun Nov 8 18:34:33 2009 @@ -106,7 +106,7 @@ parent_resumedata_snapshot = None parent_resumedata_frame_info_list = None - def __init__(self, metainterp, jitcode): + def __init__(self, metainterp, jitcode, greenkey=None): assert isinstance(jitcode, codewriter.JitCode) self.metainterp = metainterp self.jitcode = jitcode @@ -114,6 +114,8 @@ self.constants = jitcode.constants self.exception_target = -1 self.name = jitcode.name # purely for having name attribute + # this is not None for frames that are recursive portal calls + self.greenkey = greenkey # ------------------------------ # Decoding of the JitCode @@ -354,7 +356,6 @@ def opimpl_check_neg_index(self, pc, arraybox, arraydesc, indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -394,7 +395,6 @@ indexbox): negbox = self.metainterp.execute_and_record( rop.INT_LT, None, indexbox, ConstInt(0)) - # xxx inefficient negbox = self.implement_guard_value(pc, negbox) if negbox.getint(): # the index is < 0; add the array length to it @@ -408,7 +408,6 @@ def opimpl_check_zerodivisionerror(self, pc, box): nonzerobox = self.metainterp.execute_and_record( rop.INT_NE, None, box, ConstInt(0)) - # xxx inefficient nonzerobox = self.implement_guard_value(pc, nonzerobox) if nonzerobox.getint(): return False @@ -426,7 +425,6 @@ rop.INT_AND, None, tmp1, box2) # tmp2=-1 tmp3 = self.metainterp.execute_and_record( rop.INT_EQ, None, tmp2, ConstInt(-1)) # tmp3? - # xxx inefficient tmp4 = self.implement_guard_value(pc, tmp3) # tmp4? if not tmp4.getint(): return False @@ -442,7 +440,6 @@ def opimpl_int_abs(self, pc, box): nonneg = self.metainterp.execute_and_record( rop.INT_GE, None, box, ConstInt(0)) - # xxx inefficient nonneg = self.implement_guard_value(pc, nonneg) if nonneg.getint(): self.make_result_box(box) @@ -589,7 +586,7 @@ result = vinfo.get_array_length(virtualizable, arrayindex) self.make_result_box(ConstInt(result)) - def perform_call(self, jitcode, varargs): + def perform_call(self, jitcode, varargs, greenkey=None): if (self.metainterp.is_blackholing() and jitcode.calldescr is not None): # when producing only a BlackHole, we can implement this by @@ -613,7 +610,7 @@ return res else: # when tracing, this bytecode causes the subfunction to be entered - f = self.metainterp.newframe(jitcode) + f = self.metainterp.newframe(jitcode, greenkey) f.setup_call(varargs) return True @@ -646,7 +643,7 @@ portal_code = self.metainterp.staticdata.portal_code greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): - return self.perform_call(portal_code, varargs[1:]) + return self.perform_call(portal_code, varargs[1:], greenkey) return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") @@ -1126,14 +1123,19 @@ def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu + self.portal_trace_positions = [] + self.greenkey_of_huge_function = None def is_blackholing(self): return self.history is None - def newframe(self, jitcode): + def newframe(self, jitcode, greenkey=None): if jitcode is self.staticdata.portal_code: self.in_recursion += 1 - f = MIFrame(self, jitcode) + if greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (greenkey, len(self.history.operations))) + f = MIFrame(self, jitcode, greenkey) self.framestack.append(f) return f @@ -1141,6 +1143,9 @@ frame = self.framestack.pop() if frame.jitcode is self.staticdata.portal_code: self.in_recursion -= 1 + if frame.greenkey is not None and not self.is_blackholing(): + self.portal_trace_positions.append( + (None, len(self.history.operations))) return frame def finishframe(self, resultbox): @@ -1278,6 +1283,7 @@ if require_attention: require_attention = self.after_residual_call() # check if the operation can be constant-folded away + argboxes = list(argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: @@ -1334,6 +1340,8 @@ warmrunnerstate = self.staticdata.state if len(self.history.operations) > warmrunnerstate.trace_limit: self.staticdata.profiler.count(ABORT_TOO_LONG) + self.greenkey_of_huge_function = self.find_biggest_function() + self.portal_trace_positions = None self.switch_to_blackhole() def _interpret(self): @@ -1427,25 +1435,36 @@ except ContinueRunningNormallyBase: if not started_as_blackhole: warmrunnerstate = self.staticdata.state - warmrunnerstate.reset_counter_from_failure(key) + warmrunnerstate.reset_counter_from_failure(key, self) raise - def forget_consts(self, boxes, startindex=0): - for i in range(startindex, len(boxes)): + def remove_consts_and_duplicates(self, boxes, startindex, endindex, + duplicates): + for i in range(startindex, endindex): box = boxes[i] - if isinstance(box, Const): - constbox = box - box = constbox.clonebox() + if isinstance(box, Const) or box in duplicates: + oldbox = box + box = oldbox.clonebox() boxes[i] = box - self.history.record(rop.SAME_AS, [constbox], box) + self.history.record(rop.SAME_AS, [oldbox], box) + else: + duplicates[box] = None def reached_can_enter_jit(self, live_arg_boxes): - self.forget_consts(live_arg_boxes, self.staticdata.num_green_args) + num_green_args = self.staticdata.num_green_args + duplicates = {} + self.remove_consts_and_duplicates(live_arg_boxes, + num_green_args, + len(live_arg_boxes), + duplicates) live_arg_boxes = live_arg_boxes[:] if self.staticdata.virtualizable_info is not None: # we use ':-1' to remove the last item, which is the virtualizable # itself - self.forget_consts(self.virtualizable_boxes) + self.remove_consts_and_duplicates(self.virtualizable_boxes, + 0, + len(self.virtualizable_boxes)-1, + duplicates) live_arg_boxes += self.virtualizable_boxes[:-1] # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: @@ -1817,6 +1836,30 @@ if boxes[i] is oldbox: boxes[i] = newbox + def find_biggest_function(self): + assert not self.is_blackholing() + + start_stack = [] + max_size = 0 + max_key = None + for pair in self.portal_trace_positions: + key, pos = pair + if key is not None: + start_stack.append(pair) + else: + greenkey, startpos = start_stack.pop() + size = pos - startpos + if size > max_size: + max_size = size + max_key = greenkey + if start_stack: + key, pos = start_stack[0] + size = len(self.history.operations) - pos + if size > max_size: + max_size = size + max_key = key + return max_key + class GenerateMergePoint(Exception): def __init__(self, args, target_loop_token): Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/resoperation.py Sun Nov 8 18:34:33 2009 @@ -1,4 +1,5 @@ from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.debug import make_sure_not_resized class ResOperation(object): @@ -12,9 +13,11 @@ pc = 0 def __init__(self, opnum, args, result, descr=None): + make_sure_not_resized(args) assert isinstance(opnum, int) self.opnum = opnum self.args = list(args) + make_sure_not_resized(self.args) assert not isinstance(result, list) self.result = result self.setdescr(descr) @@ -93,6 +96,14 @@ def is_final(self): return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST + def returns_bool_result(self): + opnum = self.opnum + if we_are_translated(): + assert opnum >= 0 + elif opnum < 0: + return False # for tests + return opboolresult[opnum] + # ____________________________________________________________ _oplist = [ @@ -137,40 +148,40 @@ 'FLOAT_TRUEDIV/2', 'FLOAT_NEG/1', 'FLOAT_ABS/1', - 'FLOAT_IS_TRUE/1', + 'FLOAT_IS_TRUE/1b', 'CAST_FLOAT_TO_INT/1', 'CAST_INT_TO_FLOAT/1', # '_COMPARISON_FIRST', - 'INT_LT/2', - 'INT_LE/2', - 'INT_EQ/2', - 'INT_NE/2', - 'INT_GT/2', - 'INT_GE/2', - 'UINT_LT/2', - 'UINT_LE/2', - 'UINT_GT/2', - 'UINT_GE/2', + 'INT_LT/2b', + 'INT_LE/2b', + 'INT_EQ/2b', + 'INT_NE/2b', + 'INT_GT/2b', + 'INT_GE/2b', + 'UINT_LT/2b', + 'UINT_LE/2b', + 'UINT_GT/2b', + 'UINT_GE/2b', '_COMPARISON_LAST', - 'FLOAT_LT/2', # maybe these ones should be comparisons too - 'FLOAT_LE/2', - 'FLOAT_EQ/2', - 'FLOAT_NE/2', - 'FLOAT_GT/2', - 'FLOAT_GE/2', + 'FLOAT_LT/2b', # maybe these ones should be comparisons too + 'FLOAT_LE/2b', + 'FLOAT_EQ/2b', + 'FLOAT_NE/2b', + 'FLOAT_GT/2b', + 'FLOAT_GE/2b', # - 'INT_IS_TRUE/1', + 'INT_IS_TRUE/1b', 'INT_NEG/1', 'INT_INVERT/1', - 'BOOL_NOT/1', + 'BOOL_NOT/1b', # - 'SAME_AS/1', # gets a Const, turns it into a Box + 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'OONONNULL/1', - 'OOISNULL/1', - 'OOIS/2', - 'OOISNOT/2', + 'OONONNULL/1b', + 'OOISNULL/1b', + 'OOIS/2b', + 'OOISNOT/2b', # 'ARRAYLEN_GC/1d', 'STRLEN/1', @@ -182,8 +193,8 @@ 'UNICODEGETITEM/2', # # ootype operations - 'INSTANCEOF/1d', - 'SUBCLASSOF/2', + 'INSTANCEOF/1db', + 'SUBCLASSOF/2b', # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- @@ -231,6 +242,7 @@ opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" +opboolresult= [] # mapping numbers to a flag "returns a boolean" def setup(debug_print=False): @@ -239,16 +251,18 @@ print '%30s = %d' % (name, i) if '/' in name: name, arity = name.split('/') - withdescr = arity.endswith('d') - arity = int(arity.rstrip('d')) + withdescr = 'd' in arity + boolresult = 'b' in arity + arity = int(arity.rstrip('db')) else: - arity, withdescr = -1, True # default + arity, withdescr, boolresult = -1, True, False # default setattr(rop, name, i) if not name.startswith('_'): opname[i] = name oparity.append(arity) opwithdescr.append(withdescr) - assert len(oparity) == len(opwithdescr) == len(_oplist) + opboolresult.append(boolresult) + assert len(oparity)==len(opwithdescr)==len(opboolresult)==len(_oplist) setup(__name__ == '__main__') # print out the table when run directly del _oplist Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_basic.py Sun Nov 8 18:34:33 2009 @@ -109,7 +109,12 @@ else: raise Exception("FAILED") - def check_history_(self, expected=None, **isns): + def check_history(self, expected=None, **isns): + # this can be used after calling meta_interp + get_stats().check_history(expected, **isns) + + def check_operations_history(self, expected=None, **isns): + # this can be used after interp_operations self.metainterp.staticdata.stats.check_history(expected, **isns) @@ -304,7 +309,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=1, int_mul=0, call=1, guard_no_exception=0) + self.check_operations_history(int_add=1, int_mul=0, call=1, guard_no_exception=0) def test_residual_call_pure(self): def externfn(x, y): @@ -315,7 +320,7 @@ return externfn(n, n+1) res = self.interp_operations(f, [6]) assert res == 42 - self.check_history_(int_add=0, int_mul=0, call=0) + self.check_operations_history(int_add=0, int_mul=0, call=0) def test_constant_across_mp(self): myjitdriver = JitDriver(greens = [], reds = ['n']) @@ -417,7 +422,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=1) + self.check_operations_history(getfield_gc=1) def test_getfield_immutable(self): class A: @@ -434,7 +439,7 @@ return a.foo * x res = self.interp_operations(f, [42]) assert res == 210 - self.check_history_(getfield_gc=0) + self.check_operations_history(getfield_gc=0) def test_setfield_bool(self): class A: @@ -748,7 +753,7 @@ return isinstance(obj, B) res = self.interp_operations(fn, [0]) assert res - self.check_history_(guard_class=1) + self.check_operations_history(guard_class=1) res = self.interp_operations(fn, [1]) assert not res @@ -769,7 +774,7 @@ return obj.a res = self.interp_operations(fn, [1]) assert res == 1 - self.check_history_(guard_class=0, instanceof=0) + self.check_operations_history(guard_class=0, instanceof=0) def test_r_dict(self): from pypy.rlib.objectmodel import r_dict @@ -901,7 +906,7 @@ return g(a, b) res = self.interp_operations(f, [3, 5]) assert res == 8 - self.check_history_(int_add=0, call=1) + self.check_operations_history(int_add=0, call=1) def test_listcomp(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'lst']) @@ -925,7 +930,7 @@ return tup[1] res = self.interp_operations(f, [3, 5]) assert res == 5 - self.check_history_(setfield_gc=2, getfield_gc_pure=1) + self.check_operations_history(setfield_gc=2, getfield_gc_pure=1) def test_oosend_look_inside_only_one(self): class A: @@ -967,7 +972,6 @@ self.check_loop_count(1) self.check_loops(call=1) - class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): @@ -1099,5 +1103,93 @@ history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value assert res == expected + def test_residual_call_doesnt_lose_info(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) + + class A(object): + pass + + globall = [""] + @dont_look_inside + def g(x): + globall[0] = str(x) + return x + + def f(x): + y = A() + y.v = x + l = [0] + while y.v > 0: + myjitdriver.can_enter_jit(x=x, y=y, l=l) + myjitdriver.jit_merge_point(x=x, y=y, l=l) + l[0] = y.v + lc = l[0] + y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 + return y.v + res = self.meta_interp(f, [20], listops=True) + self.check_loops(getfield_gc=1, getarrayitem_gc=0) + + def test_writeanalyzer_top_set(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) + + class A(object): + pass + class B(A): + pass + + @dont_look_inside + def g(x): + # instantiate cannot be followed by the writeanalyzer + if x % 2: + C = A + else: + C = B + a = instantiate(C) + a.v = x + return a.v + + def f(x): + y = A() + y.v = x + l = [0] + while y.v > 0: + myjitdriver.can_enter_jit(x=x, y=y, l=l) + myjitdriver.jit_merge_point(x=x, y=y, l=l) + l[0] = y.v + lc = l[0] + y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 + return y.v + res = self.meta_interp(f, [20], listops=True) + self.check_loops(getfield_gc=2, getarrayitem_gc=1) + + def test_merge_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 1 + class B(A): + def g(self, y): + return y - 2 + + a1 = A() + a2 = A() + b = B() + def f(x): + l = [a1] * 100 + [a2] * 100 + [b] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + x = a.g(x) + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + self.check_loops(guard_class=0, guard_value=3) + + + class TestLLtype(BaseLLtypeTests, LLJitMixin): pass Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_blackhole.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_blackhole.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_blackhole.py Sun Nov 8 18:34:33 2009 @@ -6,8 +6,8 @@ class BlackholeTests(object): def meta_interp(self, *args): - def counting_init(frame, metainterp, jitcode): - previnit(frame, metainterp, jitcode) + def counting_init(frame, metainterp, jitcode, greenkey=None): + previnit(frame, metainterp, jitcode, greenkey) self.seen_frames.append(jitcode.name) # previnit = pyjitpl.MIFrame.__init__.im_func Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_codewriter.py Sun Nov 8 18:34:33 2009 @@ -356,7 +356,7 @@ return y.x + 5 res = self.interp_operations(f, [23]) assert res == 28 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, int_add=1) + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, int_add=1) def test_array(self): class X(object): @@ -371,7 +371,7 @@ return a.y[index] res = self.interp_operations(f, [2], listops=True) assert res == 30 - self.check_history_(getfield_gc=0, getfield_gc_pure=1, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=1, getarrayitem_gc=0, getarrayitem_gc_pure=1) @@ -389,7 +389,7 @@ return y.lst[index] + y.y + 5 res = self.interp_operations(f, [23, 0], listops=True) assert res == 23 + 24 + 5 - self.check_history_(getfield_gc=0, getfield_gc_pure=2, + self.check_operations_history(getfield_gc=0, getfield_gc_pure=2, getarrayitem_gc=0, getarrayitem_gc_pure=1, int_add=3) Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Nov 8 18:34:33 2009 @@ -16,6 +16,7 @@ from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.specnode import ConstantSpecNode +from pypy.jit.metainterp.effectinfo import EffectInfo from pypy.jit.metainterp.test.oparser import parse def test_sort_descrs(): @@ -93,6 +94,11 @@ usize = cpu.sizeof(U) onedescr = cpu.fielddescrof(U, 'one') + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([], [])) + writeadescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [])) + writearraydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo([adescr], [arraydescr])) + cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2), cpu.cast_adr_to_int(u_vtable_adr): cpu.sizeof(U)} @@ -159,6 +165,9 @@ adescr.sort_key() bdescr.sort_key() + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + nonwritedescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) # XXX fix ootype + cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), node_vtable_adr2: cpu.typedescrof(NODE2), u_vtable_adr: cpu.typedescrof(U)} Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_optimizeopt.py Sun Nov 8 18:34:33 2009 @@ -524,6 +524,45 @@ """ self.optimize_loop(ops, '', ops) + def test_guard_value_to_guard_true(self): + ops = """ + [i] + i1 = int_lt(i, 3) + guard_value(i1, 1) [i] + jump(i) + """ + expected = """ + [i] + i1 = int_lt(i, 3) + guard_true(i1) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_to_guard_false(self): + ops = """ + [p] + i1 = ooisnull(p) + guard_value(i1, 0) [p] + jump(p) + """ + expected = """ + [p] + i1 = ooisnull(p) + guard_false(i1) [p] + jump(p) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_guard_value_on_nonbool(self): + ops = """ + [i] + i1 = int_add(i, 3) + guard_value(i1, 0) [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', ops) + def test_p123_simple(self): ops = """ @@ -577,18 +616,18 @@ guard_no_exception() [] i1 = int_add(i, 3) guard_no_exception() [] - i2 = call(i1) + i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] guard_no_exception() [] - i3 = call(i2) + i3 = call(i2, descr=nonwritedescr) jump(i1) # the exception is considered lost when we loop back """ expected = """ [i] i1 = int_add(i, 3) - i2 = call(i1) + i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] - i3 = call(i2) + i3 = call(i2, descr=nonwritedescr) jump(i1) """ self.optimize_loop(ops, 'Not', expected) @@ -1474,6 +1513,23 @@ 'Virtual(node_vtable, nextdescr=Virtual(node_vtable))', None) + def test_merge_guard_class_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_class(p1, ConstClass(node_vtable)) [i0] + i3 = int_add(i1, i2) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + # ---------- def make_fail_descr(self): @@ -1802,7 +1858,117 @@ class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): - pass + + def test_residual_call_does_not_invalidate_caches(self): + ops = """ + [p1, p2] + i1 = getfield_gc(p1, descr=valuedescr) + i2 = call(i1, descr=nonwritedescr) + i3 = getfield_gc(p1, descr=valuedescr) + escape(i1) + escape(i3) + jump(p1, p2) + """ + expected = """ + [p1, p2] + i1 = getfield_gc(p1, descr=valuedescr) + i2 = call(i1, descr=nonwritedescr) + escape(i1) + escape(i1) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_residual_call_invalidate_some_caches(self): + ops = """ + [p1, p2] + i1 = getfield_gc(p1, descr=adescr) + i2 = getfield_gc(p1, descr=bdescr) + i3 = call(i1, descr=writeadescr) + i4 = getfield_gc(p1, descr=adescr) + i5 = getfield_gc(p1, descr=bdescr) + escape(i1) + escape(i2) + escape(i4) + escape(i5) + jump(p1, p2) + """ + expected = """ + [p1, p2] + i1 = getfield_gc(p1, descr=adescr) + i2 = getfield_gc(p1, descr=bdescr) + i3 = call(i1, descr=writeadescr) + i4 = getfield_gc(p1, descr=adescr) + escape(i1) + escape(i2) + escape(i4) + escape(i2) + jump(p1, p2) + """ + self.optimize_loop(ops, 'Not, Not', expected) + + def test_residual_call_invalidate_arrays(self): + ops = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i3 = call(i1, descr=writeadescr) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p6 = getarrayitem_gc(p2, 1, descr=arraydescr2) + escape(p3) + escape(p4) + escape(p5) + escape(p6) + jump(p1, p2, i1) + """ + expected = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i3 = call(i1, descr=writeadescr) + escape(p3) + escape(p4) + escape(p3) + escape(p4) + jump(p1, p2, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + + def test_residual_call_invalidate_some_arrays(self): + ops = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i3 = call(i1, descr=writearraydescr) + p5 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p6 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i4 = getarrayitem_gc(p1, 1, descr=arraydescr) + escape(p3) + escape(p4) + escape(p5) + escape(p6) + escape(i2) + escape(i4) + jump(p1, p2, i1) + """ + expected = """ + [p1, p2, i1] + p3 = getarrayitem_gc(p1, 0, descr=arraydescr2) + p4 = getarrayitem_gc(p2, 1, descr=arraydescr2) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i3 = call(i1, descr=writearraydescr) + i4 = getarrayitem_gc(p1, 1, descr=arraydescr) + escape(p3) + escape(p4) + escape(p3) + escape(p4) + escape(i2) + escape(i4) + jump(p1, p2, i1) + """ + self.optimize_loop(ops, 'Not, Not, Not', expected) + class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_pyjitpl.py Sun Nov 8 18:34:33 2009 @@ -2,7 +2,10 @@ # some unit tests for the bytecode decoding from pypy.jit.metainterp import pyjitpl, codewriter, resoperation -from pypy.jit.metainterp.history import AbstractFailDescr +from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt +from pypy.jit.metainterp.history import History +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.jit.metainterp.test.test_optimizeopt import equaloplists def make_frame(code): bytecode = codewriter.JitCode("hello") @@ -62,3 +65,80 @@ # doesn't provide interning on its own n1_1 = gd.get_fail_descr_number(fail_descr1) assert n1_1 != n1 + +def test_portal_trace_positions(): + jitcode = codewriter.JitCode("f") + jitcode.code = jitcode.constants = None + portal = codewriter.JitCode("portal") + portal.code = portal.constants = None + class FakeStaticData: + cpu = None + portal_code = portal + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp.framestack = [] + class FakeHistory: + operations = [] + history = metainterp.history = FakeHistory() + metainterp.newframe(portal, "green1") + history.operations.append(1) + metainterp.newframe(jitcode) + history.operations.append(2) + metainterp.newframe(portal, "green2") + history.operations.append(3) + metainterp.popframe() + history.operations.append(4) + metainterp.popframe() + history.operations.append(5) + metainterp.popframe() + history.operations.append(6) + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5)] + assert metainterp.find_biggest_function() == "green1" + + metainterp.newframe(portal, "green3") + history.operations.append(7) + metainterp.newframe(jitcode) + history.operations.append(8) + assert metainterp.portal_trace_positions == [("green1", 0), ("green2", 2), + (None, 3), (None, 5), ("green3", 6)] + assert metainterp.find_biggest_function() == "green1" + + history.operations.extend([9, 10, 11, 12]) + assert metainterp.find_biggest_function() == "green3" + +def test_remove_consts_and_duplicates(): + class FakeStaticData: + cpu = None + def is_another_box_like(box, referencebox): + assert box is not referencebox + assert isinstance(box, referencebox.clonebox().__class__) + assert box.value == referencebox.value + return True + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + metainterp.history = History(None) + b1 = BoxInt(1) + b2 = BoxInt(2) + c3 = ConstInt(3) + boxes = [b1, b2, b1, c3] + dup = {} + metainterp.remove_consts_and_duplicates(boxes, 0, 4, dup) + assert boxes[0] is b1 + assert boxes[1] is b2 + assert is_another_box_like(boxes[2], b1) + assert is_another_box_like(boxes[3], c3) + assert equaloplists(metainterp.history.operations, [ + ResOperation(rop.SAME_AS, [b1], boxes[2]), + ResOperation(rop.SAME_AS, [c3], boxes[3]), + ]) + assert dup == {b1: None, b2: None} + # + del metainterp.history.operations[:] + b4 = BoxInt(4) + boxes = ["something random", b2, b4, "something else"] + metainterp.remove_consts_and_duplicates(boxes, 1, 3, dup) + assert is_another_box_like(boxes[1], b2) + assert boxes[2] is b4 + assert equaloplists(metainterp.history.operations, [ + ResOperation(rop.SAME_AS, [b2], boxes[1]), + ]) Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_recursive.py Sun Nov 8 18:34:33 2009 @@ -564,6 +564,91 @@ res = self.meta_interp(main, [100], optimizer=OPTIMIZER_SIMPLE, inline=True) assert res == main(100) + def test_trace_from_start(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "+": + n += 7 + if op == "-": + n -= 1 + if op == "c": + n = f('---', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=1) + pc = 1 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('+-cl--', i) + self.meta_interp(g, [50], backendopt=True) + self.check_tree_loop_count(3) + self.check_history(int_add=1) + + def test_dont_inline_huge_stuff(self): + def p(code, pc): + code = hlstr(code) + return "%s %d %s" % (code, pc, code[pc]) + def c(code, pc): + return "l" not in hlstr(code) + myjitdriver = JitDriver(greens=['code', 'pc'], reds=['n'], + get_printable_location=p, can_inline=c) + + def f(code, n): + pc = 0 + while pc < len(code): + + myjitdriver.jit_merge_point(n=n, code=code, pc=pc) + op = code[pc] + if op == "-": + n -= 1 + elif op == "c": + f('--------------------', n) + elif op == "l": + if n > 0: + myjitdriver.can_enter_jit(n=n, code=code, pc=0) + pc = 0 + continue + else: + assert 0 + pc += 1 + return n + def g(m): + myjitdriver.set_param('inlining', True) + # carefully chosen threshold to make sure that the inner function + # cannot be inlined, but the inner function on its own is small + # enough + myjitdriver.set_param('trace_limit', 40) + if m > 1000000: + f('', 0) + result = 0 + for i in range(m): + result += f('-c-----------l-', i+100) + self.meta_interp(g, [10], backendopt=True) + self.check_aborted_count(1) + self.check_history(call=1) + self.check_tree_loop_count(3) + + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_slist.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_slist.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_slist.py Sun Nov 8 18:34:33 2009 @@ -35,7 +35,7 @@ return m res = self.interp_operations(f, [11], listops=True) assert res == 49 - self.check_history_(call=5) + self.check_operations_history(call=5) def test_list_of_voids(self): myjitdriver = JitDriver(greens = [], reds = ['n', 'lst']) @@ -82,13 +82,20 @@ self.check_loops(call=0) def test_getitem_neg(self): + myjitdriver = JitDriver(greens = [], reds = ['i', 'n']) def f(n): - lst = [41] - lst.append(42) - return lst[n] - res = self.interp_operations(f, [-2], listops=True) + x = i = 0 + while i < 10: + myjitdriver.can_enter_jit(n=n, i=i) + myjitdriver.jit_merge_point(n=n, i=i) + lst = [41] + lst.append(42) + x = lst[n] + i += 1 + return x + res = self.meta_interp(f, [-2], listops=True) assert res == 41 - self.check_history_(call=1) + self.check_loops(call=1, guard_value=0) # we don't support resizable lists on ootype #class TestOOtype(ListTests, OOJitMixin): Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 8 18:34:33 2009 @@ -228,7 +228,8 @@ assert f(20) == 10000*20 + (20*21)/2 res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) assert res == 10000*20 + (20*21)/2 - self.check_loops(call=1, getfield_gc=2, setfield_gc=2) + # there are no getfields because the optimizer gets rid of them + self.check_loops(call=1, getfield_gc=0, setfield_gc=2) # xxx for now a call that forces the virtualizable during tracing # is supposed to always force it later too. @@ -259,7 +260,9 @@ return m res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) assert res == f(20) - self.check_loops(call=1, getfield_gc=2, setfield_gc=2) + # the getfield_gc of inst_node is optimized away, because ext does not + # write to it + self.check_loops(call=1, getfield_gc=1, setfield_gc=2) # xxx for now a call that forces the virtualizable during tracing # is supposed to always force it later too. Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/test/test_warmstate.py Sun Nov 8 18:34:33 2009 @@ -165,7 +165,13 @@ class FakeWarmRunnerDesc: can_inline_ptr = None get_printable_location_ptr = None + green_args_spec = [lltype.Signed, lltype.Float] + class FakeCell: + dont_trace_here = False state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is True @@ -179,12 +185,17 @@ return False CAN_INLINE = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Float], lltype.Bool)) + class FakeCell: + dont_trace_here = False class FakeWarmRunnerDesc: rtyper = None green_args_spec = [lltype.Signed, lltype.Float] can_inline_ptr = llhelper(CAN_INLINE, can_inline) get_printable_location_ptr = None state = WarmEnterState(FakeWarmRunnerDesc()) + def jit_getter(*args): + return FakeCell() + state.jit_getter = jit_getter state.make_jitdriver_callbacks() res = state.can_inline_callable([BoxInt(5), BoxFloat(42.5)]) assert res is False Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmspot.py Sun Nov 8 18:34:33 2009 @@ -66,6 +66,7 @@ debug_level=DEBUG_STEPS, inline=False, **kwds): translator = interp.typer.annotator.translator translator.config.translation.gc = "boehm" + translator.config.translation.list_comprehension_operations = True warmrunnerdesc = WarmRunnerDesc(translator, backendopt=backendopt, **kwds) warmrunnerdesc.state.set_param_threshold(3) # for tests warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests @@ -161,18 +162,19 @@ self.build_meta_interp(CPUClass, **kwds) self.make_args_specification() - self.rewrite_jit_merge_point(policy) - self.make_driverhook_graphs() if self.jitdriver.virtualizables: from pypy.jit.metainterp.virtualizable import VirtualizableInfo self.metainterp_sd.virtualizable_info = VirtualizableInfo(self) + self.make_exception_classes() + self.make_driverhook_graphs() + self.make_enter_function() + self.rewrite_jit_merge_point(policy) self.codewriter.generate_bytecode(self.metainterp_sd, self.portal_graph, self.leave_graph, self.portal_runner_ptr ) - self.make_enter_function() self.rewrite_can_enter_jit() self.rewrite_set_param() self.add_profiler_finish() @@ -262,7 +264,68 @@ self.stats, opt, ProfilerClass=ProfilerClass, warmrunnerdesc=self) - + + def make_exception_classes(self): + portalfunc_ARGS = unrolling_iterable( + [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(self.PORTAL_FUNCTYPE.ARGS)]) + class DoneWithThisFrameVoid(JitException): + def __str__(self): + return 'DoneWithThisFrameVoid()' + + class DoneWithThisFrameInt(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Signed + self.result = result + def __str__(self): + return 'DoneWithThisFrameInt(%s)' % (self.result,) + + class DoneWithThisFrameRef(JitException): + def __init__(self, cpu, result): + assert lltype.typeOf(result) == cpu.ts.BASETYPE + self.result = result + def __str__(self): + return 'DoneWithThisFrameRef(%s)' % (self.result,) + + class DoneWithThisFrameFloat(JitException): + def __init__(self, result): + assert lltype.typeOf(result) is lltype.Float + self.result = result + def __str__(self): + return 'DoneWithThisFrameFloat(%s)' % (self.result,) + + class ExitFrameWithExceptionRef(JitException): + def __init__(self, cpu, value): + assert lltype.typeOf(value) == cpu.ts.BASETYPE + self.value = value + def __str__(self): + return 'ExitFrameWithExceptionRef(%s)' % (self.value,) + + class ContinueRunningNormally(ContinueRunningNormallyBase): + def __init__(self, argboxes): + # accepts boxes as argument, but unpacks them immediately + # before we raise the exception -- the boxes' values will + # be modified in a 'finally' by restore_patched_boxes(). + from pypy.jit.metainterp.warmstate import unwrap + for i, name, ARG in portalfunc_ARGS: + v = unwrap(ARG, argboxes[i]) + setattr(self, name, v) + + def __str__(self): + return 'ContinueRunningNormally(%s)' % ( + ', '.join(map(str, self.args)),) + + self.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.DoneWithThisFrameInt = DoneWithThisFrameInt + self.DoneWithThisFrameRef = DoneWithThisFrameRef + self.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.ContinueRunningNormally = ContinueRunningNormally + self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid + self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt + self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef + self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat + self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef + self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally def make_enter_function(self): from pypy.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self) @@ -294,9 +357,16 @@ def maybe_enter_jit(*args): maybe_compile_and_run(*args) maybe_enter_jit._always_inline_ = True - self.maybe_enter_jit_fn = maybe_enter_jit + can_inline = self.state.can_inline_greenargs + def maybe_enter_from_start(*args): + if can_inline is not None and not can_inline(*args[:self.num_green_args]): + maybe_compile_and_run(*args) + maybe_enter_from_start._always_inline_ = True + self.maybe_enter_from_start_fn = maybe_enter_from_start + + def make_leave_jit_graph(self): self.leave_graph = None if self.jitdriver.leave: @@ -439,64 +509,7 @@ portalfunc_ARGS = unrolling_iterable( [(i, 'arg%d' % i, ARG) for i, ARG in enumerate(PORTALFUNC.ARGS)]) - class DoneWithThisFrameVoid(JitException): - def __str__(self): - return 'DoneWithThisFrameVoid()' - - class DoneWithThisFrameInt(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Signed - self.result = result - def __str__(self): - return 'DoneWithThisFrameInt(%s)' % (self.result,) - - class DoneWithThisFrameRef(JitException): - def __init__(self, cpu, result): - assert lltype.typeOf(result) == cpu.ts.BASETYPE - self.result = result - def __str__(self): - return 'DoneWithThisFrameRef(%s)' % (self.result,) - - class DoneWithThisFrameFloat(JitException): - def __init__(self, result): - assert lltype.typeOf(result) is lltype.Float - self.result = result - def __str__(self): - return 'DoneWithThisFrameFloat(%s)' % (self.result,) - - class ExitFrameWithExceptionRef(JitException): - def __init__(self, cpu, value): - assert lltype.typeOf(value) == cpu.ts.BASETYPE - self.value = value - def __str__(self): - return 'ExitFrameWithExceptionRef(%s)' % (self.value,) - - class ContinueRunningNormally(ContinueRunningNormallyBase): - def __init__(self, argboxes): - # accepts boxes as argument, but unpacks them immediately - # before we raise the exception -- the boxes' values will - # be modified in a 'finally' by restore_patched_boxes(). - from pypy.jit.metainterp.warmstate import unwrap - for i, name, ARG in portalfunc_ARGS: - v = unwrap(ARG, argboxes[i]) - setattr(self, name, v) - def __str__(self): - return 'ContinueRunningNormally(%s)' % ( - ', '.join(map(str, self.args)),) - - self.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.DoneWithThisFrameInt = DoneWithThisFrameInt - self.DoneWithThisFrameRef = DoneWithThisFrameRef - self.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.ContinueRunningNormally = ContinueRunningNormally - self.metainterp_sd.DoneWithThisFrameVoid = DoneWithThisFrameVoid - self.metainterp_sd.DoneWithThisFrameInt = DoneWithThisFrameInt - self.metainterp_sd.DoneWithThisFrameRef = DoneWithThisFrameRef - self.metainterp_sd.DoneWithThisFrameFloat = DoneWithThisFrameFloat - self.metainterp_sd.ExitFrameWithExceptionRef = ExitFrameWithExceptionRef - self.metainterp_sd.ContinueRunningNormally = ContinueRunningNormally rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) @@ -505,26 +518,27 @@ def ll_portal_runner(*args): while 1: try: + self.maybe_enter_from_start_fn(*args) return support.maybe_on_top_of_llinterp(rtyper, portal_ptr)(*args) - except ContinueRunningNormally, e: + except self.ContinueRunningNormally, e: args = () for _, name, _ in portalfunc_ARGS: v = getattr(e, name) args = args + (v,) - except DoneWithThisFrameVoid: + except self.DoneWithThisFrameVoid: assert result_kind == 'void' return - except DoneWithThisFrameInt, e: + except self.DoneWithThisFrameInt, e: assert result_kind == 'int' return lltype.cast_primitive(RESULT, e.result) - except DoneWithThisFrameRef, e: + except self.DoneWithThisFrameRef, e: assert result_kind == 'ref' return ts.cast_from_ref(RESULT, e.result) - except DoneWithThisFrameFloat, e: + except self.DoneWithThisFrameFloat, e: assert result_kind == 'float' return e.result - except ExitFrameWithExceptionRef, e: + except self.ExitFrameWithExceptionRef, e: value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) Modified: pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/gc-dump-malloc/pypy/jit/metainterp/warmstate.py Sun Nov 8 18:34:33 2009 @@ -10,7 +10,7 @@ from pypy.rlib.jit import PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell -from pypy.rlib.debug import debug_start, debug_stop +from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import support, history # ____________________________________________________________ @@ -161,8 +161,20 @@ key.counter += 1 return key.counter >= self.trace_eagerness - def reset_counter_from_failure(self, key): + def reset_counter_from_failure(self, key, metainterp): key.counter = 0 + self.disable_noninlinable_function(metainterp) + + def disable_noninlinable_function(self, metainterp): + greenkey = metainterp.greenkey_of_huge_function + if greenkey is not None: + cell = self.jit_cell_at_key(greenkey) + cell.dont_trace_here = True + debug_start("jit-disableinlining") + sd = self.warmrunnerdesc.metainterp_sd + loc = sd.state.get_location_str(greenkey) + debug_print("disabled inlining", loc) + debug_stop("jit-disableinlining") def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): @@ -220,6 +232,7 @@ except ContinueRunningNormally: # the trace got too long, reset the counter cell.counter = 0 + self.disable_noninlinable_function(metainterp) raise else: # machine code was already compiled for these greenargs @@ -274,6 +287,7 @@ class JitCell(BaseJitCell): counter = 0 compiled_merge_points = None + dont_trace_here = False # if self.warmrunnerdesc.get_jitcell_at_ptr is None: jit_getter = self._make_jitcell_getter_default(JitCell) @@ -428,18 +442,27 @@ return # can_inline_ptr = self.warmrunnerdesc.can_inline_ptr + unwrap_greenkey = self.make_unwrap_greenkey() if can_inline_ptr is None: - def can_inline_callable(greenkey): + def can_inline_callable(*greenargs): return True else: rtyper = self.warmrunnerdesc.rtyper - unwrap_greenkey = self.make_unwrap_greenkey() # - def can_inline_callable(greenkey): - greenargs = unwrap_greenkey(greenkey) + def can_inline_callable(*greenargs): fn = support.maybe_on_top_of_llinterp(rtyper, can_inline_ptr) return fn(*greenargs) - self.can_inline_callable = can_inline_callable + def can_inline(*greenargs): + cell = self.jit_getter(*greenargs) + if cell.dont_trace_here: + return False + return can_inline_callable(*greenargs) + self.can_inline_greenargs = can_inline + def can_inline_greenkey(greenkey): + greenargs = unwrap_greenkey(greenkey) + return can_inline(*greenargs) + self.can_inline_callable = can_inline_greenkey + # get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr if get_location_ptr is None: Modified: pypy/branch/gc-dump-malloc/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/gc-dump-malloc/pypy/module/__builtin__/test/test_classobj.py Sun Nov 8 18:34:33 2009 @@ -1,4 +1,5 @@ -from pypy.conftest import gettestobjspace +import py +from pypy.conftest import gettestobjspace, option from pypy.interpreter import gateway @@ -769,6 +770,8 @@ class AppTestOldStyleSharing(AppTestOldstyle): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withsharingdict": True}) + if option.runappdirect: + py.test.skip("can only be run on py.py") def is_sharing(space, w_inst): from pypy.objspace.std.sharingdict import SharedDictImplementation, W_DictMultiObject w_d = w_inst.getdict() Modified: pypy/branch/gc-dump-malloc/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/gc-dump-malloc/pypy/translator/c/src/debug.h (original) +++ pypy/branch/gc-dump-malloc/pypy/translator/c/src/debug.h Sun Nov 8 18:34:33 2009 @@ -45,6 +45,19 @@ #ifndef PYPY_NOT_MAIN_FILE #include +#if defined(__GNUC__) && !defined(WIN32) +# include + static void pypy_setup_profiling() + { + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(0, &set); /* restrict to a single cpu */ + sched_setaffinity(0, sizeof(cpu_set_t), &set); + } +#else +static void pypy_setup_profiling() { } +#endif + long pypy_have_debug_prints = -1; FILE *pypy_debug_file = NULL; static bool_t debug_ready = 0; @@ -64,6 +77,7 @@ { /* PYPYLOG=filename --- profiling version */ debug_profile = 1; + pypy_setup_profiling(); } else { From cfbolz at codespeak.net Mon Nov 9 10:50:00 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 10:50:00 +0100 (CET) Subject: [pypy-svn] r69079 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091109095000.9F7BA168050@codespeak.net> Author: cfbolz Date: Mon Nov 9 10:49:58 2009 New Revision: 69079 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Mon Nov 9 10:49:58 2009 @@ -9,21 +9,22 @@ TASKS ====== - - make the size of ResOperation instances less insane (Armin, Anto) + - make the size of ResOperation instances less insane SEMI-DONE - set up the buildbot on the Mac Mini (Samuele, Carl Friedrich) - - directly call assembler for residual portal calls + - directly call assembler for residual portal calls - making the CLI backend working with logging - - compress the virtuals part of resume data more + - compress the virtuals part of resume data more (Samuele, Carl Friedrich) - understand the memory behaviour of pystone with the JIT DONE - fix the guard_value(bool, 1) -> guard_true hack DONE - - try to do something non-insane about Python-level exceptions + - try to do something non-insane about Python-level exceptions (Maciek, Armin) - make the assembler produced by generate_failure smaller - - lose less information across residual calls IN-PROGRESS (Samuele, Carl Friedrich) - - we should think about merging several consecutive guards in the - optimizer, to make the assembler smaller and to save resume data space + - lose less information across residual calls DONE + - merge guard_nonnull(x, ...), guard_class(x, ...) (Armin, Maciek) + - merge guard_class(x, ...), guard_value(x, ...) DONE - put the class into the structure to get only one promote when using an instance - look into failing pypy-c-jit apptests + - port writeanalyze to ootype (Anto, Carl Friedrich around) From fijal at codespeak.net Mon Nov 9 11:16:02 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 11:16:02 +0100 (CET) Subject: [pypy-svn] r69080 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091109101602.6CDD1168050@codespeak.net> Author: fijal Date: Mon Nov 9 11:16:01 2009 New Revision: 69080 Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py Log: (arigo, fijal) Fix instanceof with test Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Mon Nov 9 11:16:01 2009 @@ -320,12 +320,11 @@ def opimpl_runtimenew(self, classbox): self.execute(rop.RUNTIMENEW, classbox) - @arguments("box", "descr") - def opimpl_instanceof(self, objbox, typedescr): - frame = self.metainterp.framestack[-1] + @arguments("orgpc", "box", "descr") + def opimpl_instanceof(self, pc, objbox, typedescr): clsbox = self.cls_of_box(objbox) if isinstance(objbox, Box): - self.generate_guard(frame.pc, rop.GUARD_CLASS, objbox, [clsbox]) + self.generate_guard(pc, rop.GUARD_CLASS, objbox, [clsbox]) self.execute_with_descr(rop.INSTANCEOF, typedescr, objbox) @arguments("box", "box") Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Mon Nov 9 11:16:01 2009 @@ -757,6 +757,31 @@ res = self.interp_operations(fn, [1]) assert not res + def test_isinstance_2(self): + driver = JitDriver(greens = [], reds = ['x', 'n', 'sum']) + class A: + pass + class B(A): + pass + class C(B): + pass + + def main(): + return f(5, B()) * 10 + f(5, C()) + f(5, A()) * 100 + + def f(n, x): + sum = 0 + while n > 0: + driver.can_enter_jit(x=x, n=n, sum=sum) + driver.jit_merge_point(x=x, n=n, sum=sum) + if isinstance(x, B): + sum += 1 + n -= 1 + return sum + + res = self.meta_interp(main, []) + assert res == 55 + def test_assert_isinstance(self): class A: pass From antocuni at codespeak.net Mon Nov 9 11:22:43 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 9 Nov 2009 11:22:43 +0100 (CET) Subject: [pypy-svn] r69082 - in pypy/trunk/pypy/translator/backendopt: . test Message-ID: <20091109102243.3AAC5168050@codespeak.net> Author: antocuni Date: Mon Nov 9 11:22:42 2009 New Revision: 69082 Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py pypy/trunk/pypy/translator/backendopt/writeanalyze.py Log: (antocuni, cfbolz) port the writeanalizer to ootype Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Mon Nov 9 11:22:42 2009 @@ -1,5 +1,6 @@ import py from pypy.rpython.lltypesystem import lltype +from pypy.rpython.ootypesystem import ootype from pypy.translator.translator import TranslationContext, graphof from pypy.translator.simplify import get_funcobj from pypy.translator.backendopt.writeanalyze import WriteAnalyzer, top_set @@ -7,17 +8,19 @@ from pypy.conftest import option -class TestCanRaise(object): +class BaseTestCanRaise(object): + type_system = None + + def translate(self, func, sig): t = TranslationContext() t.buildannotator().build_types(func, sig) - t.buildrtyper().specialize() + t.buildrtyper(type_system=self.type_system).specialize() if option.view: t.view() return t, WriteAnalyzer(t) - def test_writes_simple(self): def g(x): return True @@ -108,7 +111,10 @@ t, wa = self.translate(f, [int]) fgraph = graphof(t, f) result = wa.analyze(fgraph.startblock.operations[0]) - assert result is top_set + if self.type_system == 'lltype': + assert result is top_set + else: + assert not result # ootype is more precise in this case def test_llexternal(self): from pypy.rpython.lltypesystem.rffi import llexternal @@ -124,6 +130,10 @@ result = wa.analyze(fgraph.startblock.operations[0]) assert not result + +class TestLLtype(BaseTestCanRaise): + type_system = 'lltype' + def test_list(self): def g(x, y, z): return f(x, y, z) @@ -152,3 +162,40 @@ assert struct == "struct" assert name == "length" assert S1 is S2 + + +class TestOOtype(BaseTestCanRaise): + type_system = 'ootype' + + def test_array(self): + def g(x, y, z): + return f(x, y, z) + def f(x, y, z): + l = [0] * x + l[1] = 42 + return len(l) + z + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[0].opname == 'direct_call' + + result = sorted(wa.analyze(ggraph.startblock.operations[0])) + assert len(result) == 1 + array, A = result[0] + assert array == 'array' + assert A.ITEM is ootype.Signed + + def test_list(self): + def g(x, y, z): + return f(x, y, z) + def f(x, y, z): + l = [0] * x + l.append(z) + return len(l) + z + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[0].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[0]) + assert result is top_set Modified: pypy/trunk/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/writeanalyze.py Mon Nov 9 11:22:42 2009 @@ -1,4 +1,5 @@ from pypy.translator.backendopt import graphanalyze +from pypy.rpython.ootypesystem import ootype reload(graphanalyze) top_set = object() @@ -27,13 +28,24 @@ return result is top_set def analyze_simple_operation(self, op): - if op.opname == "setfield": + if op.opname in ("setfield", "oosetfield"): return frozenset([ ("struct", op.args[0].concretetype, op.args[1].value)]) elif op.opname == "setarrayitem": - return frozenset([("array", op.args[0].concretetype)]) + return self._array_result(op.args[0].concretetype) return empty_set + def _array_result(self, TYPE): + return frozenset([("array", TYPE)]) + def analyze_external_call(self, op): return self.bottom_result() # an external call cannot change anything + def analyze_external_method(self, op, TYPE, meth): + if isinstance(TYPE, ootype.Array): + methname = op.args[0].value + if methname == 'll_setitem_fast': + return self._array_result(op.args[1].concretetype) + elif methname in ('ll_getitem_fast', 'll_length'): + return self.bottom_result() + return graphanalyze.GraphAnalyzer.analyze_external_method(self, op, TYPE, meth) From antocuni at codespeak.net Mon Nov 9 11:40:15 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 9 Nov 2009 11:40:15 +0100 (CET) Subject: [pypy-svn] r69083 - in pypy/trunk/pypy/jit/backend/cli: . test Message-ID: <20091109104015.A2FCC168050@codespeak.net> Author: antocuni Date: Mon Nov 9 11:40:13 2009 New Revision: 69083 Added: pypy/trunk/pypy/jit/backend/cli/test/test_descr.py (contents, props changed) Modified: pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Log: (antocuni, cfbolz) start a new test_descr.py and implement get_extra_info for calldescrs Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Mon Nov 9 11:40:13 2009 @@ -71,8 +71,8 @@ return self.inputargs @staticmethod - def calldescrof(FUNC, ARGS, RESULT): - return StaticMethDescr.new(FUNC, ARGS, RESULT) + def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): + return StaticMethDescr.new(FUNC, ARGS, RESULT, extrainfo) @staticmethod def methdescrof(SELFTYPE, methname): @@ -384,7 +384,7 @@ funcclass = ootype.nullruntimeclass has_result = False - def __init__(self, FUNC, ARGS, RESULT): + def __init__(self, FUNC, ARGS, RESULT, extrainfo=None): DescrWithKey.__init__(self, (FUNC, ARGS, RESULT)) from pypy.jit.backend.llgraph.runner import boxresult, make_getargs getargs = make_getargs(FUNC.ARGS) @@ -397,6 +397,7 @@ self.callfunc = callfunc self.funcclass = dotnet.classof(FUNC) self.has_result = (FUNC.RESULT != ootype.Void) + self.extrainfo = extrainfo if RESULT is ootype.Void: def get_errbox(): return None @@ -416,7 +417,7 @@ return clitype.GetMethod('Invoke') def get_extra_info(self): - return None # XXX fix me + return self.extrainfo class MethDescr(AbstractMethDescr): Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Mon Nov 9 11:40:13 2009 @@ -44,13 +44,3 @@ test_instantiate_does_not_call = skip test_listcomp = skip test_tuple_immutable = skip - - -def test_fielddescr_ootype(): - from pypy.rpython.ootypesystem import ootype - from pypy.jit.backend.cli.runner import CliCPU - A = ootype.Instance("A", ootype.ROOT, {"foo": ootype.Signed}) - B = ootype.Instance("B", A) - descr1 = CliCPU.fielddescrof(A, "foo") - descr2 = CliCPU.fielddescrof(B, "foo") - assert descr1 is descr2 Added: pypy/trunk/pypy/jit/backend/cli/test/test_descr.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/backend/cli/test/test_descr.py Mon Nov 9 11:40:13 2009 @@ -0,0 +1,24 @@ +from pypy.rpython.ootypesystem import ootype +from pypy.jit.backend.cli.runner import CliCPU + + +def test_fielddescr_ootype(): + A = ootype.Instance("A", ootype.ROOT, {"foo": ootype.Signed}) + B = ootype.Instance("B", A) + descr1 = CliCPU.fielddescrof(A, "foo") + descr2 = CliCPU.fielddescrof(B, "foo") + assert descr1 is descr2 + +def test_call_descr_extra_info(): + FUNC = ootype.StaticMethod([], ootype.Signed) + ARGS = () + descr1 = CliCPU.calldescrof(FUNC, ARGS, ootype.Signed, "hello") + extrainfo = descr1.get_extra_info() + assert extrainfo == "hello" + + descr2 = CliCPU.calldescrof(FUNC, ARGS, ootype.Signed, "hello") + assert descr2 is descr1 + + descr3 = CliCPU.calldescrof(FUNC, ARGS, ootype.Signed) + assert descr3 is not descr1 + assert descr3.get_extra_info() is None From antocuni at codespeak.net Mon Nov 9 11:45:01 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 9 Nov 2009 11:45:01 +0100 (CET) Subject: [pypy-svn] r69084 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091109104501.7DE5F168050@codespeak.net> Author: antocuni Date: Mon Nov 9 11:45:01 2009 New Revision: 69084 Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py Log: (antocuni, cfbolz) move this test to the base class, as it's not lltype specific Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Mon Nov 9 11:45:01 2009 @@ -997,6 +997,33 @@ self.check_loop_count(1) self.check_loops(call=1) + def test_merge_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 1 + class B(A): + def g(self, y): + return y - 2 + + a1 = A() + a2 = A() + b = B() + def f(x): + l = [a1] * 100 + [a2] * 100 + [b] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + x = a.g(x) + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + self.check_loops(guard_class=0, guard_value=3) + + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): @@ -1128,91 +1155,6 @@ history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value assert res == expected - def test_residual_call_doesnt_lose_info(self): - myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) - - class A(object): - pass - - globall = [""] - @dont_look_inside - def g(x): - globall[0] = str(x) - return x - - def f(x): - y = A() - y.v = x - l = [0] - while y.v > 0: - myjitdriver.can_enter_jit(x=x, y=y, l=l) - myjitdriver.jit_merge_point(x=x, y=y, l=l) - l[0] = y.v - lc = l[0] - y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 - return y.v - res = self.meta_interp(f, [20], listops=True) - self.check_loops(getfield_gc=1, getarrayitem_gc=0) - - def test_writeanalyzer_top_set(self): - from pypy.rlib.objectmodel import instantiate - myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) - - class A(object): - pass - class B(A): - pass - - @dont_look_inside - def g(x): - # instantiate cannot be followed by the writeanalyzer - if x % 2: - C = A - else: - C = B - a = instantiate(C) - a.v = x - return a.v - - def f(x): - y = A() - y.v = x - l = [0] - while y.v > 0: - myjitdriver.can_enter_jit(x=x, y=y, l=l) - myjitdriver.jit_merge_point(x=x, y=y, l=l) - l[0] = y.v - lc = l[0] - y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 - return y.v - res = self.meta_interp(f, [20], listops=True) - self.check_loops(getfield_gc=2, getarrayitem_gc=1) - - def test_merge_guardclass_guardvalue(self): - from pypy.rlib.objectmodel import instantiate - myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) - - class A(object): - def g(self, x): - return x - 1 - class B(A): - def g(self, y): - return y - 2 - - a1 = A() - a2 = A() - b = B() - def f(x): - l = [a1] * 100 + [a2] * 100 + [b] * 100 - while x > 0: - myjitdriver.can_enter_jit(x=x, l=l) - myjitdriver.jit_merge_point(x=x, l=l) - a = l[x] - x = a.g(x) - hint(a, promote=True) - return x - res = self.meta_interp(f, [299], listops=True) - self.check_loops(guard_class=0, guard_value=3) From antocuni at codespeak.net Mon Nov 9 12:06:54 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 9 Nov 2009 12:06:54 +0100 (CET) Subject: [pypy-svn] r69085 - in pypy/trunk/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091109110654.1369A168050@codespeak.net> Author: antocuni Date: Mon Nov 9 12:06:53 2009 New Revision: 69085 Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/effectinfo.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Log: (antocuni, cfbolz) make effectinfo work on ootype too, and restore test_residual_call_doesnt_lose_info which was killed by mistake by r69084 Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Mon Nov 9 12:06:53 2009 @@ -506,8 +506,8 @@ return FieldDescr.new(T1, fieldname) @staticmethod - def calldescrof(FUNC, ARGS, RESULT): - return StaticMethDescr.new(FUNC, ARGS, RESULT) + def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): + return StaticMethDescr.new(FUNC, ARGS, RESULT, extrainfo) @staticmethod def methdescrof(SELFTYPE, methname): @@ -675,7 +675,7 @@ class StaticMethDescr(OODescr): - def __init__(self, FUNC, ARGS, RESULT): + def __init__(self, FUNC, ARGS, RESULT, extrainfo=None): self.FUNC = FUNC getargs = make_getargs(FUNC.ARGS) def callfunc(funcbox, argboxes): @@ -685,9 +685,10 @@ if RESULT is not ootype.Void: return boxresult(RESULT, res) self.callfunc = callfunc + self.extrainfo = extrainfo def get_extra_info(self): - return None # XXX + return self.extrainfo class MethDescr(history.AbstractMethDescr): Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Mon Nov 9 12:06:53 2009 @@ -315,8 +315,7 @@ assert NON_VOID_ARGS == [T for T in ARGS if T is not lltype.Void] assert RESULT == FUNC.RESULT # ok - if (self.rtyper.type_system.name == 'lltypesystem' and - consider_effects_of is not None): + if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( self.write_analyzer.analyze(consider_effects_of), self.cpu) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) Modified: pypy/trunk/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/effectinfo.py Mon Nov 9 12:06:53 2009 @@ -1,5 +1,7 @@ +from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype +from pypy.rpython.ootypesystem import ootype class EffectInfo(object): _cache = {} @@ -23,27 +25,42 @@ for tup in effects: if tup[0] == "struct": _, T, fieldname = tup - if not isinstance(T.TO, lltype.GcStruct): # can be a non-GC-struct + T = deref(T) + if not consider_struct(T, fieldname): continue - if getattr(T.TO, fieldname) is lltype.Void: - continue - if fieldname == "typeptr" and T.TO is OBJECT: - # filter out the typeptr, because - # a) it is optimized in different ways - # b) it might not be there in C if removetypeptr is specified - continue - descr = cpu.fielddescrof(T.TO, fieldname) + descr = cpu.fielddescrof(T, fieldname) write_descrs_fields.append(descr) elif tup[0] == "array": _, T = tup - if not isinstance(T.TO, lltype.GcArray): # can be a non-GC-array + ARRAY = deref(T) + if not consider_array(ARRAY): continue - if T.TO.OF is lltype.Void: - continue - descr = cpu.arraydescrof(T.TO) + descr = cpu.arraydescrof(ARRAY) write_descrs_arrays.append(descr) else: assert 0 return EffectInfo(write_descrs_fields, write_descrs_arrays) +def consider_struct(TYPE, fieldname): + if fieldType(TYPE, fieldname) is lltype.Void: + return False + if isinstance(TYPE, ootype.OOType): + return True + if not isinstance(TYPE, lltype.GcStruct): # can be a non-GC-struct + return False + if fieldname == "typeptr" and TYPE is OBJECT: + # filter out the typeptr, because + # a) it is optimized in different ways + # b) it might not be there in C if removetypeptr is specified + return False + return True + +def consider_array(ARRAY): + if arrayItem(ARRAY) is lltype.Void: + return False + if isinstance(ARRAY, ootype.Array): + return True + if not isinstance(ARRAY, lltype.GcArray): # can be a non-GC-array + return False + return True Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Mon Nov 9 12:06:53 2009 @@ -1023,6 +1023,31 @@ res = self.meta_interp(f, [299], listops=True) self.check_loops(guard_class=0, guard_value=3) + def test_residual_call_doesnt_lose_info(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) + + class A(object): + pass + + globall = [""] + @dont_look_inside + def g(x): + globall[0] = str(x) + return x + + def f(x): + y = A() + y.v = x + l = [0] + while y.v > 0: + myjitdriver.can_enter_jit(x=x, y=y, l=l) + myjitdriver.jit_merge_point(x=x, y=y, l=l) + l[0] = y.v + lc = l[0] + y.v = g(y.v) - y.v/y.v + lc/l[0] - 1 + return y.v + res = self.meta_interp(f, [20], listops=True) + self.check_loops(getfield_gc=1, getarrayitem_gc=0) class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_effectinfo.py Mon Nov 9 12:06:53 2009 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype +from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze def test_filter_out_typeptr(): @@ -19,3 +20,15 @@ effectinfo = effectinfo_from_writeanalyze(effects, None) assert not effectinfo.write_descrs_fields assert not effectinfo.write_descrs_arrays + +def test_filter_out_ooarray_of_void(): + effects = frozenset([("array", ootype.Array(ootype.Void))]) + effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays + +def test_filter_out_instance_with_void(): + effects = frozenset([("struct", ootype.Instance("x", ootype.ROOT, {"a": ootype.Void}), "a")]) + effectinfo = effectinfo_from_writeanalyze(effects, None) + assert not effectinfo.write_descrs_fields + assert not effectinfo.write_descrs_arrays From cfbolz at codespeak.net Mon Nov 9 13:37:23 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 13:37:23 +0100 (CET) Subject: [pypy-svn] r69086 - pypy/branch/compress-virtuals-resumedata Message-ID: <20091109123723.21C5F16804D@codespeak.net> Author: cfbolz Date: Mon Nov 9 13:37:22 2009 New Revision: 69086 Added: pypy/branch/compress-virtuals-resumedata/ - copied from r69085, pypy/trunk/ Log: (pedronis, cfbolz): make a branch for trying to compress the virtuals part of resume data. From fijal at codespeak.net Mon Nov 9 13:45:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 13:45:45 +0100 (CET) Subject: [pypy-svn] r69087 - pypy/branch/guard-nonnull Message-ID: <20091109124545.E0E2E16804D@codespeak.net> Author: fijal Date: Mon Nov 9 13:45:45 2009 New Revision: 69087 Added: pypy/branch/guard-nonnull/ - copied from r69086, pypy/trunk/ Log: (arigo, fijal) A bit of nullness refactoring. From fijal at codespeak.net Mon Nov 9 13:46:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 13:46:57 +0100 (CET) Subject: [pypy-svn] r69088 - in pypy/branch/guard-nonnull/pypy/jit: backend/llgraph backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20091109124657.06D8516804D@codespeak.net> Author: fijal Date: Mon Nov 9 13:46:57 2009 New Revision: 69088 Modified: pypy/branch/guard-nonnull/pypy/jit/backend/llgraph/llimpl.py pypy/branch/guard-nonnull/pypy/jit/backend/test/runner_test.py pypy/branch/guard-nonnull/pypy/jit/backend/test/test_random.py pypy/branch/guard-nonnull/pypy/jit/backend/x86/assembler.py pypy/branch/guard-nonnull/pypy/jit/backend/x86/regalloc.py pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_runner.py pypy/branch/guard-nonnull/pypy/jit/metainterp/executor.py pypy/branch/guard-nonnull/pypy/jit/metainterp/history.py pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizefindnode.py pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizeopt.py pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py pypy/branch/guard-nonnull/pypy/jit/metainterp/resoperation.py pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizeopt.py Log: (arigo, fijal) Remove oononnull and ooisnull operations, replace with guard_nonnull/isnull Kills a bit of stuff here and there Modified: pypy/branch/guard-nonnull/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/llgraph/llimpl.py Mon Nov 9 13:46:57 2009 @@ -104,8 +104,6 @@ 'new_with_vtable' : (('ref',), 'ref'), 'new' : ((), 'ref'), 'new_array' : (('int',), 'ref'), - 'oononnull' : (('ref',), 'bool'), - 'ooisnull' : (('ref',), 'bool'), 'oois' : (('ref', 'ref'), 'bool'), 'ooisnot' : (('ref', 'ref'), 'bool'), 'instanceof' : (('ref',), 'bool'), @@ -134,6 +132,8 @@ 'guard_exception' : (('ref',), 'ref'), 'guard_no_overflow' : ((), None), 'guard_overflow' : ((), None), + 'guard_nonnull' : (('ref',), None), + 'guard_isnull' : (('ref',), None), 'newstr' : (('int',), 'ref'), 'strlen' : (('ref',), 'int'), 'strgetitem' : (('ref', 'int'), 'int'), @@ -551,6 +551,9 @@ if value: raise GuardFailed + op_guard_nonnull = op_guard_true + op_guard_isnull = op_guard_false + def op_guard_class(self, _, value, expected_class): value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) expected_class = llmemory.cast_adr_to_ptr( Modified: pypy/branch/guard-nonnull/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/test/runner_test.py Mon Nov 9 13:46:57 2009 @@ -441,9 +441,13 @@ def test_passing_guards(self): + t_box, T_box = self.alloc_instance(self.T) + nullbox = self.null_instance() all = [(rop.GUARD_TRUE, [BoxInt(1)]), (rop.GUARD_FALSE, [BoxInt(0)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)])] + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), + (rop.GUARD_NONNULL, [t_box]), + (rop.GUARD_ISNULL, [nullbox])] if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) for (opname, args) in all: @@ -460,9 +464,13 @@ # 'void') def test_failing_guards(self): + t_box, T_box = self.alloc_instance(self.T) + nullbox = self.null_instance() all = [(rop.GUARD_TRUE, [BoxInt(0)]), (rop.GUARD_FALSE, [BoxInt(1)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)])] + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), + (rop.GUARD_NONNULL, [nullbox]), + (rop.GUARD_ISNULL, [t_box])] if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) for opname, args in all: @@ -493,10 +501,6 @@ assert r.value == 0 r = self.execute_operation(rop.OOISNOT, [u2_box, u1_box], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNULL, [u1_box], 'int') - assert r.value == 0 - r = self.execute_operation(rop.OONONNULL, [u2_box], 'int') - assert r.value == 1 # null_box = self.null_instance() r = self.execute_operation(rop.OOIS, [null_box, @@ -513,10 +517,6 @@ assert r.value == 1 r = self.execute_operation(rop.OOISNOT, [null_box, u1_box], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNULL, [null_box], 'int') - assert r.value == 1 - r = self.execute_operation(rop.OONONNULL, [null_box], 'int') - assert r.value == 0 def test_array_basic(self): a_box, A = self.alloc_array_of(lltype.Signed, 342) @@ -962,10 +962,6 @@ assert r.value == 1 r = self.execute_operation(rop.OOISNOT, [BoxInt(v), BoxInt(v)], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOISNULL, [BoxInt(v)], 'int') - assert r.value == 0 - r = self.execute_operation(rop.OONONNULL, [BoxInt(v)], 'int') - assert r.value == 1 lltype.free(x, flavor='raw') def test_new_plain_struct(self): @@ -1335,6 +1331,22 @@ [value, chr1, chr2]) assert len(dict.fromkeys([value, chr1, chr2]).keys()) == 3 + def test_guards_nongc(self): + x = lltype.malloc(lltype.Struct('x'), flavor='raw') + v = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + vbox = BoxInt(v) + ops = [ + (rop.GUARD_NONNULL, vbox, False), + (rop.GUARD_ISNULL, vbox, True), + (rop.GUARD_NONNULL, BoxInt(0), True), + (rop.GUARD_ISNULL, BoxInt(0), False), + ] + for opname, arg, res in ops: + self.execute_operation(opname, [arg], 'void') + assert self.guard_failed == res + + lltype.free(x, flavor='raw') + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' Modified: pypy/branch/guard-nonnull/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/test/test_random.py Mon Nov 9 13:46:57 2009 @@ -57,27 +57,18 @@ def get_bool_var(self, r): if self.boolvars and r.random() < 0.8: - v = r.choice(self.boolvars) + return r.choice(self.boolvars) elif self.ptrvars and r.random() < 0.4: v, S = r.choice(self.ptrvars + self.prebuilt_ptr_consts)[:2] v2, S2 = r.choice(self.ptrvars + self.prebuilt_ptr_consts)[:2] if S == S2 and not (isinstance(v, ConstPtr) and isinstance(v2, ConstPtr)): if r.random() < 0.5: - v = self.do(rop.OOIS, [v, v2]) + return self.do(rop.OOIS, [v, v2]) else: - v = self.do(rop.OOISNOT, [v, v2]) - else: - if isinstance(v, ConstPtr): - v, S = r.choice(self.ptrvars) - if r.random() < 0.5: - v = self.do(rop.OONONNULL, [v]) - else: - v = self.do(rop.OOISNULL, [v]) - else: - v = r.choice(self.intvars) - v = self.do(rop.INT_IS_TRUE, [v]) - return v + return self.do(rop.OOISNOT, [v, v2]) + v = r.choice(self.intvars) + return self.do(rop.INT_IS_TRUE, [v]) def subset_of_intvars(self, r): subset = [] @@ -358,6 +349,16 @@ builder.should_fail_by = op builder.guard_op = op +class GuardPtrOperation(GuardOperation): + def gen_guard(self, builder, r): + if not builder.ptrvars: + raise CannotProduceOperation + box = r.choice(builder.ptrvars)[0] + op = ResOperation(self.opnum, [box], None) + passing = ((self.opnum == rop.GUARD_NONNULL and box.value) or + (self.opnum == rop.GUARD_ISNULL and not box.value)) + return op, passing + class GuardValueOperation(GuardOperation): def gen_guard(self, builder, r): v = r.choice(builder.intvars) @@ -410,6 +411,8 @@ OPERATIONS.append(GuardOperation(rop.GUARD_TRUE)) OPERATIONS.append(GuardOperation(rop.GUARD_FALSE)) OPERATIONS.append(GuardOperation(rop.GUARD_FALSE)) +OPERATIONS.append(GuardPtrOperation(rop.GUARD_NONNULL)) +OPERATIONS.append(GuardPtrOperation(rop.GUARD_ISNULL)) OPERATIONS.append(GuardValueOperation(rop.GUARD_VALUE)) for _op in [rop.INT_NEG, Modified: pypy/branch/guard-nonnull/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/x86/assembler.py Mon Nov 9 13:46:57 2009 @@ -466,7 +466,7 @@ loc2 = cl self.mc.SHR(loc, loc2) - def genop_guard_oononnull(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) @@ -475,29 +475,11 @@ else: return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): - guard_opnum = guard_op.opnum - loc = arglocs[0] - self.mc.TEST(loc, loc) - if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) - else: - return self.implement_guard(addr, self.mc.JZ) - - genop_guard_int_is_true = genop_guard_oononnull - - def genop_oononnull(self, op, arglocs, resloc): + def genop_int_is_true(self, op, arglocs, resloc): self.mc.CMP(arglocs[0], imm8(0)) self.mc.MOV(resloc, imm8(0)) self.mc.SETNE(lower_byte(resloc)) - genop_int_is_true = genop_oononnull - - def genop_ooisnull(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) - self.mc.MOV(resloc, imm8(0)) - self.mc.SETE(lower_byte(resloc)) - def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as @@ -672,6 +654,7 @@ loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JZ) + genop_guard_guard_nonnull = genop_guard_guard_true def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): @@ -703,6 +686,7 @@ loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JNZ) + genop_guard_guard_isnull = genop_guard_guard_false def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): if guard_op.args[0].type == FLOAT: Modified: pypy/branch/guard-nonnull/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/x86/regalloc.py Mon Nov 9 13:46:57 2009 @@ -305,8 +305,7 @@ self.assembler.regalloc_perform_discard(op, arglocs) def can_optimize_cmp_op(self, op, i, operations): - if not (op.is_comparison() or op.opnum == rop.OOISNULL or - op.opnum == rop.OONONNULL): + if not op.is_comparison(): return False if (operations[i + 1].opnum != rop.GUARD_TRUE and operations[i + 1].opnum != rop.GUARD_FALSE): @@ -385,6 +384,8 @@ consider_guard_true = _consider_guard consider_guard_false = _consider_guard + consider_guard_nonnull = _consider_guard + consider_guard_isnull = _consider_guard def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] @@ -831,8 +832,6 @@ self.Perform(op, [argloc], resloc) consider_int_is_true = _consider_nullity - consider_ooisnull = _consider_nullity - consider_oononnull = _consider_nullity def consider_same_as(self, op, ignored): argloc = self.loc(op.args[0]) Modified: pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_gc_integration.py Mon Nov 9 13:46:57 2009 @@ -132,11 +132,9 @@ guard_value(i2, 1) [i2, i3, i4, i5, i6, i7, i0, i1, i8] guard_class(i4, 138998336) [i4, i5, i6, i7, i0, i1, i8] i11 = getfield_gc(i4, descr=descr0) - i12 = ooisnull(i11) - guard_false(i12) [i4, i5, i6, i7, i0, i1, i11, i8] + guard_nonnull(i11) [i4, i5, i6, i7, i0, i1, i11, i8] i13 = getfield_gc(i11, descr=descr0) - i14 = ooisnull(i13) - guard_true(i14) [i4, i5, i6, i7, i0, i1, i11, i8] + guard_isnull(i13) [i4, i5, i6, i7, i0, i1, i11, i8] i15 = getfield_gc(i4, descr=descr0) i17 = int_lt(i15, 0) guard_false(i17) [i4, i5, i6, i7, i0, i1, i11, i15, i8] Modified: pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_regalloc.py Mon Nov 9 13:46:57 2009 @@ -437,20 +437,6 @@ self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] - def test_nullity(self): - ops = ''' - [i0, i1, i2, i3, i4, i5, i6] - i10 = oononnull(i0) - i11 = ooisnull(i1) - i12 = oononnull(i2) - i13 = oononnull(i3) - i14 = ooisnull(i6) - i15 = ooisnull(i5) - finish(i10, i11, i12, i13, i14, i15) - ''' - self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) - assert self.getints(6) == [0, 0, 1, 1, 0, 0] - def test_strsetitem(self): ops = ''' [p0, i] Modified: pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/backend/x86/test/test_runner.py Mon Nov 9 13:46:57 2009 @@ -229,7 +229,7 @@ assert c.value == 3 def test_nullity_with_guard(self): - allops = [rop.OONONNULL, rop.OOISNULL, rop.INT_IS_TRUE] + allops = [rop.INT_IS_TRUE] guards = [rop.GUARD_TRUE, rop.GUARD_FALSE] p = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(lltype.GcStruct('x'))) Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/executor.py Mon Nov 9 13:46:57 2009 @@ -103,26 +103,6 @@ def do_same_as(cpu, box1): return box1 -def do_oononnull(cpu, box1): - tp = box1.type - if tp == INT: - x = bool(box1.getint()) - elif tp == REF: - x = bool(box1.getref_base()) - else: - assert False - return ConstInt(x) - -def do_ooisnull(cpu, box1): - tp = box1.type - if tp == INT: - x = bool(box1.getint()) - elif tp == REF: - x = bool(box1.getref_base()) - else: - assert False - return ConstInt(not x) - def do_oois(cpu, box1, box2): tp = box1.type assert tp == box2.type Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/history.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/history.py Mon Nov 9 13:46:57 2009 @@ -106,6 +106,9 @@ def set_future_value(self, cpu, j): raise NotImplementedError + def nonnull(self): + raise NotImplementedError + def repr_rpython(self): return '%s' % self @@ -177,9 +180,6 @@ def same_constant(self, other): raise NotImplementedError - def nonnull_constant(self): - raise NotImplementedError - def __repr__(self): return 'Const(%s)' % self._getrepr_() @@ -239,7 +239,7 @@ assert isinstance(other, Const) return self.value == other.getint() - def nonnull_constant(self): + def nonnull(self): return self.value != 0 def _getrepr_(self): @@ -286,7 +286,7 @@ assert isinstance(other, Const) return self.value == other.getaddr(self.cpu) - def nonnull_constant(self): + def nonnull(self): return bool(self.value) def _getrepr_(self): @@ -322,7 +322,7 @@ assert isinstance(other, ConstFloat) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return self.value != 0.0 def _getrepr_(self): @@ -368,7 +368,7 @@ assert isinstance(other, ConstPtr) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return bool(self.value) _getrepr_ = repr_pointer @@ -421,7 +421,7 @@ assert isinstance(other, ConstObj) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return bool(self.value) _getrepr_ = repr_object @@ -512,6 +512,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) + def nonnull(self): + return self.value != 0 + def _getrepr_(self): return self.value @@ -541,6 +544,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_float(j, self.value) + def nonnull(self): + return self.value != 0.0 + def _getrepr_(self): return self.value @@ -580,6 +586,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_ref(j, self.value) + def nonnull(self): + return bool(self.value) + def repr_rpython(self): return repr_rpython(self, 'bp') @@ -615,6 +624,9 @@ else: return 0 + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_ref(j, self.value) Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizefindnode.py Mon Nov 9 13:46:57 2009 @@ -168,11 +168,11 @@ def find_nodes_no_escape(self, op): pass # for operations that don't escape their arguments - find_nodes_OONONNULL = find_nodes_no_escape - find_nodes_OOISNULL = find_nodes_no_escape - find_nodes_OOIS = find_nodes_no_escape - find_nodes_OOISNOT = find_nodes_no_escape - find_nodes_INSTANCEOF = find_nodes_no_escape + find_nodes_OOIS = find_nodes_no_escape + find_nodes_OOISNOT = find_nodes_no_escape + find_nodes_INSTANCEOF = find_nodes_no_escape + find_nodes_GUARD_NONNULL = find_nodes_no_escape + find_nodes_GUARD_ISNULL = find_nodes_no_escape def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode() Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/optimizeopt.py Mon Nov 9 13:46:57 2009 @@ -69,7 +69,7 @@ if self.is_constant(): box = self.box assert isinstance(box, Const) - return not box.nonnull_constant() + return not box.nonnull() return False def make_constant(self, constbox): @@ -100,7 +100,7 @@ elif level == LEVEL_CONSTANT: box = self.box assert isinstance(box, Const) - return box.nonnull_constant() + return box.nonnull() else: return False @@ -114,25 +114,6 @@ # meaning it has been forced. return self.box is None -class BoolValue(OptValue): - - def __init__(self, box, fromvalue, reversed, nullconstbox): - OptValue.__init__(self, box) - # If later 'box' is turned into a constant False - # (resp. True), then 'fromvalue' will be known to - # be null (resp. non-null). If 'reversed', then - # this logic is reversed. - self.fromvalue = fromvalue - self.reversed = reversed - self.nullconstbox = nullconstbox # of the correct type - - def make_constant(self, constbox): - OptValue.make_constant(self, constbox) - if constbox.nonnull_constant() ^ self.reversed: - self.fromvalue.make_nonnull() - else: - self.fromvalue.make_constant(self.nullconstbox) - class ConstantValue(OptValue): level = LEVEL_CONSTANT @@ -434,11 +415,6 @@ self.make_equal_to(box, vvalue) return vvalue - def make_bool(self, box, fromvalue, reversed, nullconstbox): - value = BoolValue(box, fromvalue, reversed, nullconstbox) - self.make_equal_to(box, value) - return value - def new_ptr_box(self): return self.cpu.ts.BoxRef() @@ -585,6 +561,24 @@ self.emit_operation(op) value.make_constant(constbox) + def optimize_GUARD_ISNULL(self, op): + value = self.getvalue(op.args[0]) + if value.is_null(): + return + elif value.is_nonnull(): + raise InvalidLoop + self.emit_operation(op) + value.make_constant(self.cpu.ts.CONST_NULL) + + def optimize_GUARD_NONNULL(self, op): + value = self.getvalue(op.args[0]) + if value.is_nonnull(): + return + elif value.is_null(): + raise InvalidLoop + self.emit_operation(op) + value.make_nonnull() + def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.args[0]) emit_operation = True @@ -631,26 +625,19 @@ self.emit_operation(op) - def _optimize_nullness(self, op, expect_nonnull, nullconstbox): - value = self.getvalue(op.args[0]) + def _optimize_nullness(self, op, box, expect_nonnull): + value = self.getvalue(box) if value.is_nonnull(): self.make_constant_int(op.result, expect_nonnull) elif value.is_null(): self.make_constant_int(op.result, not expect_nonnull) else: - self.make_bool(op.result, value, not expect_nonnull, nullconstbox) self.emit_operation(op) - def optimize_OONONNULL(self, op): - self._optimize_nullness(op, True, self.cpu.ts.CONST_NULL) - - def optimize_OOISNULL(self, op): - self._optimize_nullness(op, False, self.cpu.ts.CONST_NULL) - def optimize_INT_IS_TRUE(self, op): - self._optimize_nullness(op, True, CONST_0) + self._optimize_nullness(op, op.args[0], True) - def _optimize_oois_ooisnot(self, op, expect_isnot, unary_opnum): + def _optimize_oois_ooisnot(self, op, expect_isnot): value0 = self.getvalue(op.args[0]) value1 = self.getvalue(op.args[1]) if value0.is_virtual(): @@ -662,19 +649,17 @@ elif value1.is_virtual(): self.make_constant_int(op.result, expect_isnot) elif value1.is_null(): - op = ResOperation(unary_opnum, [op.args[0]], op.result) - self._optimize_nullness(op, expect_isnot, self.cpu.ts.CONST_NULL) + self._optimize_nullness(op, op.args[0], expect_isnot) elif value0.is_null(): - op = ResOperation(unary_opnum, [op.args[1]], op.result) - self._optimize_nullness(op, expect_isnot, self.cpu.ts.CONST_NULL) + self._optimize_nullness(op, op.args[1], expect_isnot) else: self.optimize_default(op) def optimize_OOISNOT(self, op): - self._optimize_oois_ooisnot(op, True, rop.OONONNULL) + self._optimize_oois_ooisnot(op, True) def optimize_OOIS(self, op): - self._optimize_oois_ooisnot(op, False, rop.OOISNULL) + self._optimize_oois_ooisnot(op, False) def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py Mon Nov 9 13:46:57 2009 @@ -281,6 +281,10 @@ target = self.load_3byte() # load the 'target' argument self.pc = target # jump + def ignore_next_guard_nullness(self): + self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero + self.load_int() # past the 'box' argument + def dont_follow_jump(self): _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not assert ord(self.bytecode[self.pc]) == _op_goto_if_not @@ -445,13 +449,29 @@ else: self.execute(rop.INT_NEG, box) - @arguments("box") - def opimpl_ptr_nonzero(self, box): - self.execute(rop.OONONNULL, box) + @arguments("orgpc", "box") + def opimpl_ptr_nonzero(self, pc, box): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + res = ConstInt(1) + else: + opnum = rop.GUARD_ISNULL + res = ConstInt(0) + self.generate_guard(pc, opnum, box, []) + self.make_result_box(res) - @arguments("box") - def opimpl_ptr_iszero(self, box): - self.execute(rop.OOISNULL, box) + @arguments("orgpc", "box") + def opimpl_ptr_iszero(self, pc, box): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + res = ConstInt(0) + else: + opnum = rop.GUARD_ISNULL + res = ConstInt(1) + self.generate_guard(pc, opnum, box, []) + self.make_result_box(res) opimpl_oononnull = opimpl_ptr_nonzero opimpl_ooisnull = opimpl_ptr_iszero @@ -1527,6 +1547,8 @@ self.handle_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.raise_overflow_error() + elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: + self.framestack[-1].ignore_next_guard_nullness() def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.staticdata.num_green_args Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/resoperation.py Mon Nov 9 13:46:57 2009 @@ -1,7 +1,6 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized - class ResOperation(object): """The central ResOperation class, representing one operation.""" @@ -118,6 +117,8 @@ 'GUARD_FALSE', 'GUARD_VALUE', 'GUARD_CLASS', + 'GUARD_NONNULL', + 'GUARD_ISNULL', '_GUARD_FOLDABLE_LAST', 'GUARD_NO_EXCEPTION', 'GUARD_EXCEPTION', @@ -178,8 +179,6 @@ # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'OONONNULL/1b', - 'OOISNULL/1b', 'OOIS/2b', 'OOISNOT/2b', # Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py Mon Nov 9 13:46:57 2009 @@ -341,6 +341,28 @@ assert 0, "missing instantiate_*_%s in:\n%r" % (expected, names) + def test_oois_constant_null(self): + from pypy.rpython.lltypesystem import lltype + + S = lltype.GcStruct('S') + s = lltype.malloc(S) + NULL = lltype.nullptr(S) + def f(p, i): + if i % 2: + return p == NULL + elif i % 3: + return NULL == p + elif i % 4: + return p != NULL + else: + return NULL != p + graphs = self.make_graphs(f, [s, 5]) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = graphs + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert 'ptr_eq' not in jitcode._source + assert 'ptr_ne' not in jitcode._source class ImmutableFieldsTests: Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Nov 9 13:46:57 2009 @@ -474,10 +474,7 @@ [p3, p4, p2] p0 = new_with_vtable(ConstClass(node_vtable)) p1 = new_with_vtable(ConstClass(node_vtable)) - i1 = oononnull(p0) - guard_true(i1) [] - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] i3 = ooisnot(p0, NULL) guard_true(i3) [] i4 = oois(p0, NULL) @@ -827,14 +824,12 @@ def test_find_nodes_bug_1(self): ops = """ [p12] - i16 = ooisnull(p12) - guard_false(i16) [] + guard_nonnull(p12) [] guard_class(p12, ConstClass(node_vtable)) [] guard_class(p12, ConstClass(node_vtable)) [] i22 = getfield_gc_pure(p12, descr=valuedescr) escape(i22) - i25 = ooisnull(p12) - guard_false(i25) [] + guard_nonnull(p12) [] guard_class(p12, ConstClass(node_vtable)) [] guard_class(p12, ConstClass(node_vtable)) [] i29 = getfield_gc_pure(p12, descr=valuedescr) @@ -847,10 +842,8 @@ setarrayitem_gc(p35, 0, p33, descr=arraydescr3) p38 = new_with_vtable(ConstClass(u_vtable)) # U setfield_gc(p38, p35, descr=onedescr) - i39 = ooisnull(p38) - guard_false(i39) [] - i40 = oononnull(p38) - guard_true(i40) [] + guard_nonnull(p38) [] + guard_nonnull(p38) [] guard_class(p38, ConstClass(u_vtable)) [] p42 = getfield_gc(p38, descr=onedescr) # Array(NODE) i43 = arraylen_gc(p42, descr=arraydescr3) @@ -871,8 +864,7 @@ guard_false(i55) [] p56 = getfield_gc(p46, descr=ddescr) # Array(NODE) p58 = getarrayitem_gc(p56, 0, descr=arraydescr3) # NODE - i59 = ooisnull(p38) - guard_false(i59) [] + guard_nonnull(p38) [] jump(p58) """ self.find_nodes(ops, 'Virtual(node_vtable, valuedescr=Not)') Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 9 13:46:57 2009 @@ -338,10 +338,7 @@ ops = """ [p0] guard_class(p0, ConstClass(node_vtable)) [] - i0 = oononnull(p0) - guard_true(i0) [] - i1 = ooisnull(p0) - guard_false(i1) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ @@ -352,6 +349,7 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): + py.test.skip("too bad") ops = """ [i0] i1 = int_is_true(i0) @@ -371,31 +369,13 @@ def test_ooisnull_oononnull_2(self): ops = """ [p0] - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] - i1 = ooisnull(p0) - guard_false(i1) [] - jump(p0) - """ - expected = """ - [p0] - i0 = oononnull(p0) - guard_true(i0) [] - jump(p0) - """ - self.optimize_loop(ops, 'Not', expected) - ops = """ - [p0] - i1 = ooisnull(p0) - guard_false(i1) [] - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] + guard_nonnull(p0) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ [p0] - i1 = ooisnull(p0) - guard_false(i1) [] + guard_nonnull(p0) [] jump(p0) """ self.optimize_loop(ops, 'Not', expected) @@ -404,34 +384,14 @@ ops = """ [] p0 = escape() - i0 = ooisnull(p0) - guard_true(i0) [] - i1 = oononnull(p0) - guard_false(i1) [] - jump() - """ - expected = """ - [] - p0 = escape() - i0 = ooisnull(p0) - guard_true(i0) [] - jump() - """ - self.optimize_loop(ops, '', expected) - ops = """ - [] - p0 = escape() - i0 = oononnull(p0) - guard_false(i0) [] - i1 = ooisnull(p0) - guard_true(i1) [] + guard_isnull(p0) [] + guard_isnull(p0) [] jump() """ expected = """ [] p0 = escape() - i0 = oononnull(p0) - guard_false(i0) [] + guard_isnull(p0) [] jump() """ self.optimize_loop(ops, '', expected) @@ -441,17 +401,14 @@ [p0] pv = new_with_vtable(ConstClass(node_vtable)) setfield_gc(pv, p0, descr=valuedescr) - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] + guard_nonnull(p0) [] p1 = getfield_gc(pv, descr=valuedescr) - i1 = ooisnull(p1) - guard_false(i1) [] + guard_nonnull(p1) [] jump(p0) """ expected = """ [p0] - i0 = oononnull(p0) - guard_true(i0) [] + guard_nonnull(p0) [] jump(p0) """ self.optimize_loop(ops, 'Not', expected) @@ -489,10 +446,7 @@ guard_true(i0) [] i3 = oois(NULL, p0) guard_false(i1) [] - i4 = oononnull(p0) - guard_true(i4) [] - i5 = ooisnull(p0) - guard_false(i5) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ @@ -541,16 +495,16 @@ def test_guard_value_to_guard_false(self): ops = """ - [p] - i1 = ooisnull(p) - guard_value(i1, 0) [p] - jump(p) + [i] + i1 = int_is_true(i) + guard_value(i1, 0) [i] + jump(i) """ expected = """ - [p] - i1 = ooisnull(p) - guard_false(i1) [p] - jump(p) + [i] + i1 = int_is_true(i) + guard_false(i1) [i] + jump(i) """ self.optimize_loop(ops, 'Not', expected) @@ -686,10 +640,7 @@ def test_virtual_oois(self): ops = """ [p0, p1, p2] - i1 = oononnull(p0) - guard_true(i1) [] - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] i3 = ooisnot(p0, NULL) guard_true(i3) [] i4 = oois(p0, NULL) @@ -727,8 +678,7 @@ # the details of the algorithm... expected2 = """ [p0, p1, p2] - i1 = oononnull(p0) - guard_true(i1) [] + guard_nonnull(p0) [] i7 = ooisnot(p0, p1) guard_true(i7) [] i8 = oois(p0, p1) @@ -833,7 +783,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = ooisnull(p2) + i1 = oois(p2, NULL) jump(i1) """ expected = """ @@ -849,7 +799,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = ooisnull(p2) + i1 = oois(p2, NULL) jump(i1) """ expected = """ @@ -1003,8 +953,7 @@ ops = """ [i1, p0] setarrayitem_gc(p0, 0, i1, descr=arraydescr) - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] p1 = new_array(i1, descr=arraydescr) jump(i1, p1) """ @@ -1426,8 +1375,7 @@ ops = """ [i0, p1] p4 = getfield_gc(p1, descr=nextdescr) - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p2 = new_with_vtable(ConstClass(node_vtable)) @@ -1437,8 +1385,7 @@ """ expected = """ [i0, p4] - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p3 = escape() @@ -1451,8 +1398,7 @@ ops = """ [i0, p1] p4 = getarrayitem_gc(p1, 0, descr=arraydescr2) - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p2 = new_array(1, descr=arraydescr2) @@ -1462,8 +1408,7 @@ """ expected = """ [i0, p4] - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p3 = escape() @@ -1475,8 +1420,7 @@ def test_invalid_loop_1(self): ops = """ [p1] - i1 = ooisnull(p1) - guard_true(i1) [] + guard_isnull(p1) [] # p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) @@ -1501,8 +1445,7 @@ ops = """ [p1] p2 = getfield_gc(p1, descr=nextdescr) - i1 = ooisnull(p2) - guard_true(i1) [] + guard_isnull(p2) [] # p3 = new_with_vtable(ConstClass(node_vtable)) p4 = new_with_vtable(ConstClass(node_vtable)) From fijal at codespeak.net Mon Nov 9 13:57:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 13:57:21 +0100 (CET) Subject: [pypy-svn] r69089 - in pypy/branch/guard-nonnull/pypy/jit/metainterp: . test Message-ID: <20091109125721.B29A0168008@codespeak.net> Author: fijal Date: Mon Nov 9 13:57:21 2009 New Revision: 69089 Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py Log: (arigo, fijal) Optimize away ptr_eq(stuff, NULL) on codewriter level Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py Mon Nov 9 13:57:21 2009 @@ -808,6 +808,29 @@ serialize_op_uint_is_true = serialize_op_int_is_true + def _serialize_op_ptr_eq(self, op, opname): + arg0, arg1 = op.args + if isinstance(arg0, Constant) and not arg0.value: + self.emit(opname, self.var_position(arg1)) + self.register_var(op.result) + elif isinstance(arg1, Constant) and not arg1.value: + self.emit(opname, self.var_position(arg0)) + self.register_var(op.result) + else: + self.default_serialize_op(op) + + def serialize_op_ptr_eq(self, op): + self._serialize_op_ptr_eq(op, 'ptr_iszero') + + def serialize_op_ptr_ne(self, op): + self._serialize_op_ptr_eq(op, 'ptr_nonzero') + + def serialize_op_oois(self, op): + self._serialize_op_ptr_eq(op, 'ooisnull') + + def serialize_op_ooisnot(self, op): + self._serialize_op_ptr_eq(op, 'oononnull') + def serialize_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} STRUCT = op.args[0].value Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_codewriter.py Mon Nov 9 13:57:21 2009 @@ -363,6 +363,8 @@ jitcode = cw.make_one_bytecode((graphs[0], None), False) assert 'ptr_eq' not in jitcode._source assert 'ptr_ne' not in jitcode._source + assert jitcode._source.count('ptr_iszero') == 2 + assert jitcode._source.count('ptr_nonzero') == 2 class ImmutableFieldsTests: From arigo at codespeak.net Mon Nov 9 14:07:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Nov 2009 14:07:53 +0100 (CET) Subject: [pypy-svn] r69090 - pypy/trunk/pypy/tool Message-ID: <20091109130753.F3AED16804F@codespeak.net> Author: arigo Date: Mon Nov 9 14:07:53 2009 New Revision: 69090 Modified: pypy/trunk/pypy/tool/logparser.py Log: Only complain that "The time decreases!" if we are loading a log produced with PYPYLOG=filename, as opposed to PYPYLOG=:filename. Modified: pypy/trunk/pypy/tool/logparser.py ============================================================================== --- pypy/trunk/pypy/tool/logparser.py (original) +++ pypy/trunk/pypy/tool/logparser.py Mon Nov 9 14:07:53 2009 @@ -16,27 +16,34 @@ r_stop = re.compile(r"\[([0-9a-f]+)\] ([\w-]+)\}$") lasttime = 0 log = DebugLog() + time_decrase = False + performance_log = True + nested = 0 f = open(filename, 'r') for line in f: line = line.rstrip() match = r_start.match(line) if match: record = log.debug_start + nested += 1 else: match = r_stop.match(line) if match: record = log.debug_stop + nested -= 1 else: log.debug_print(line) + performance_log = performance_log and nested == 0 continue time = int(int(match.group(1), 16)) - if time < lasttime: - raise Exception("The time decreases! The log file may have been" - " produced on a multi-CPU machine and the process" - " moved between CPUs.") + time_decrase = time_decrase or time < lasttime lasttime = time record(match.group(2), time=int(match.group(1), 16)) f.close() + if performance_log and time_decrase: + raise Exception("The time decreases! The log file may have been" + " produced on a multi-CPU machine and the process" + " moved between CPUs.") return log def extract_category(log, catprefix='', toplevel=False): From cfbolz at codespeak.net Mon Nov 9 14:24:51 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 14:24:51 +0100 (CET) Subject: [pypy-svn] r69091 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091109132451.0C1FD16804F@codespeak.net> Author: cfbolz Date: Mon Nov 9 14:24:50 2009 New Revision: 69091 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: (pedronis, cfbolz): baby steps Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Mon Nov 9 14:24:50 2009 @@ -89,7 +89,7 @@ TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-1, TAGBOX) +UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) NULLREF = tag(-1, TAGCONST) From fijal at codespeak.net Mon Nov 9 14:36:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 14:36:31 +0100 (CET) Subject: [pypy-svn] r69092 - in pypy/branch/gc-dump-heap/pypy: rpython rpython/lltypesystem rpython/memory/gc rpython/memory/gctransform rpython/memory/test translator/c/test Message-ID: <20091109133631.966D816804F@codespeak.net> Author: fijal Date: Mon Nov 9 14:36:30 2009 New Revision: 69092 Modified: pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/transform.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py pypy/branch/gc-dump-heap/pypy/translator/c/test/test_boehm.py pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Log: (arigo, fijal) * Rename dump_heap to heap_stats * Implement something for the basic gctransform class (so it does not explode horribly on ie boehm) Modified: pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/llinterp.py Mon Nov 9 14:36:30 2009 @@ -824,7 +824,7 @@ def op_gc_assume_young_pointers(self, addr): raise NotImplementedError - def op_gc_dump_heap(self): + def op_gc_heap_stats(self): raise NotImplementedError def op_gc_obtain_free_space(self, size): Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/lloperation.py Mon Nov 9 14:36:30 2009 @@ -460,7 +460,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), - 'gc_dump_heap' : LLOp(canunwindgc=True), + 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/generation.py Mon Nov 9 14:36:30 2009 @@ -517,7 +517,7 @@ def _id_grow_older(self, obj, id, ignored): self.objects_with_id.setitem(obj, id) - def dump_heap_walk_roots(self): + def heap_stats_walk_roots(self): self.last_generation_root_objects.foreach( self._track_heap_ext, None) self.root_walker.walk_roots( Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gc/semispace.py Mon Nov 9 14:36:30 2009 @@ -658,13 +658,13 @@ def _track_heap_root(self, root): self.track_heap(root.address[0]) - def dump_heap_walk_roots(self): + def heap_stats_walk_roots(self): self.root_walker.walk_roots( SemiSpaceGC._track_heap_root, SemiSpaceGC._track_heap_root, SemiSpaceGC._track_heap_root) - def dump_heap(self): + def heap_stats(self): self._tracked_dict = self.AddressDict() max_tid = self.root_walker.gcdata.max_type_id ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, zero=True) @@ -676,7 +676,7 @@ while i < max_tid: self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map[i])) i += 1 - self.dump_heap_walk_roots() + self.heap_stats_walk_roots() self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) self._tracked_dict.delete() return ll_typeid_map Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/framework.py Mon Nov 9 14:36:30 2009 @@ -271,8 +271,8 @@ [s_gc, annmodel.SomeAddress()], annmodel.s_None) - if hasattr(GCClass, 'dump_heap'): - self.dump_heap_ptr = getfn(GCClass.dump_heap.im_func, + if hasattr(GCClass, 'heap_stats'): + self.heap_stats_ptr = getfn(GCClass.heap_stats.im_func, [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)), minimal_transform=False) self.get_member_index_ptr = getfn( @@ -648,10 +648,12 @@ hop.genop("direct_call", [self.assume_young_pointers_ptr, self.c_const_gc, v_addr]) - def gct_gc_dump_heap(self, hop): + def gct_gc_heap_stats(self, hop): + if not hasattr(self, 'heap_stats_ptr'): + return GCTransformer.gct_gc_heap_stats(self, hop) op = hop.spaceop livevars = self.push_roots(hop) - hop.genop("direct_call", [self.dump_heap_ptr, self.c_const_gc], + hop.genop("direct_call", [self.heap_stats_ptr, self.c_const_gc], resultvar=op.result) self.pop_roots(hop, livevars) Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/gctransform/transform.py Mon Nov 9 14:36:30 2009 @@ -388,6 +388,11 @@ # this assumes a non-moving GC. Moving GCs need to override this hop.rename('cast_ptr_to_int') + def gct_gc_heap_stats(self, hop): + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + + return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP), + lltype.nullptr(ARRAY_TYPEID_MAP))) class MinimalGCTransformer(BaseGCTransformer): def __init__(self, parenttransformer): Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_gc.py Mon Nov 9 14:36:30 2009 @@ -551,28 +551,6 @@ res = self.interpret(fn, [-1000], taggedpointers=True) assert res == 111 - - def test_gc_dump_heap(self): - if getattr(self.GCClass, 'dump_heap', None) is None: - py.test.skip("unsupported gc") - S = lltype.GcStruct('S', ('x', lltype.Signed)) - - def fun(fd): - l = [] - for i in range(10): - l.append(lltype.malloc(S)) - rgc.dump_heap(fd) - keepalive_until_here(l) - return 0 - - from pypy.tool.udir import udir - f = udir.join("gcdump_direct.log") - handle = open(str(f), "w") - run = self.interpret(fun, [handle.fileno()]) - handle.close() - assert f.read() == 'xxx' - - from pypy.rlib.objectmodel import UnboxedValue class TaggedBase(object): Modified: pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/memory/test/test_transformed_gc.py Mon Nov 9 14:36:30 2009 @@ -797,7 +797,7 @@ run = self.runner("do_malloc_operations_in_call") run([]) - def define_gc_dump_heap(cls): + def define_gc_heap_stats(cls): S = lltype.GcStruct('S', ('x', lltype.Signed)) l1 = [] l2 = [] @@ -815,7 +815,7 @@ # We cheat here and only read the table which we later on # process ourselves, otherwise this test takes ages llop.gc__collect(lltype.Void) - tb = rgc._dump_heap() + tb = rgc._heap_stats() a = 0 nr = 0 b = 0 @@ -836,8 +836,8 @@ return d * 1000 + c * 100 + b * 10 + a return f - def test_gc_dump_heap(self): - run = self.runner("gc_dump_heap") + def test_gc_heap_stats(self): + run = self.runner("gc_heap_stats") res = run([]) assert res % 10000 == 2611 totsize = (res / 10000) Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/translator/c/test/test_boehm.py (original) +++ pypy/branch/gc-dump-heap/pypy/translator/c/test/test_boehm.py Mon Nov 9 14:36:30 2009 @@ -380,6 +380,15 @@ c_fn = self.getcompiled(fn, []) assert c_fn() == False + def test_heap_stats(self): + from pypy.rlib import rgc + + def fn(): + return bool(rgc._heap_stats()) + + c_fn = self.getcompiled(fn, []) + assert not c_fn() + def test_malloc_nonmovable(self): TP = lltype.GcArray(lltype.Char) def func(): Modified: pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/translator/c/test/test_newgc.py Mon Nov 9 14:36:30 2009 @@ -900,7 +900,7 @@ res = self.run('gc_set_max_heap_size') assert res == 2 - def define_gc_dump_heap(cls): + def define_gc_heap_stats(cls): S = lltype.GcStruct('S', ('x', lltype.Signed)) l1 = [] l2 = [] @@ -912,7 +912,7 @@ l1.append(s) l2.append(s) l3.append(s) - tb = rgc._dump_heap() + tb = rgc._heap_stats() a = 0 nr = 0 b = 0 @@ -930,8 +930,8 @@ return c * 100 + b * 10 + a return f - def test_gc_dump_heap(self): - res = self.run("gc_dump_heap") + def test_gc_heap_stats(self): + res = self.run("gc_heap_stats") assert res == 3011 def definestr_string_builder(cls): From cfbolz at codespeak.net Mon Nov 9 15:46:16 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 15:46:16 +0100 (CET) Subject: [pypy-svn] r69093 - pypy/branch/compress-virtuals-resumedata/pypy/jit/tl/tla Message-ID: <20091109144616.B958E16804F@codespeak.net> Author: cfbolz Date: Mon Nov 9 15:46:15 2009 New Revision: 69093 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/tl/tla/tla.py Log: add a get_printable_location Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/tl/tla/tla.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/tl/tla/tla.py Mon Nov 9 15:46:15 2009 @@ -71,9 +71,13 @@ # ____________________________________________________________ +def get_printable_location(bytecode, pc): + return str(pc) + jitdriver = JitDriver(greens=['bytecode', 'pc'], reds=['self'], - virtualizables=['self']) + virtualizables=['self'], + get_printable_location=get_printable_location) class Frame(object): _virtualizable2_ = ['stackpos', 'stack[*]'] From cfbolz at codespeak.net Mon Nov 9 15:53:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 15:53:06 +0100 (CET) Subject: [pypy-svn] r69094 - pypy/trunk/pypy/jit/tl/tla Message-ID: <20091109145306.938AF16804F@codespeak.net> Author: cfbolz Date: Mon Nov 9 15:53:05 2009 New Revision: 69094 Modified: pypy/trunk/pypy/jit/tl/tla/tla.py Log: oops, this should have gone to trunk Modified: pypy/trunk/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/tla.py (original) +++ pypy/trunk/pypy/jit/tl/tla/tla.py Mon Nov 9 15:53:05 2009 @@ -71,9 +71,13 @@ # ____________________________________________________________ +def get_printable_location(bytecode, pc): + return str(pc) + jitdriver = JitDriver(greens=['bytecode', 'pc'], reds=['self'], - virtualizables=['self']) + virtualizables=['self'], + get_printable_location=get_printable_location) class Frame(object): _virtualizable2_ = ['stackpos', 'stack[*]'] From arigo at codespeak.net Mon Nov 9 15:54:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Nov 2009 15:54:49 +0100 (CET) Subject: [pypy-svn] r69095 - pypy/trunk/pypy/jit/tl/tla Message-ID: <20091109145449.57C7B16804F@codespeak.net> Author: arigo Date: Mon Nov 9 15:54:48 2009 New Revision: 69095 Added: pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py (contents, props changed) Log: An example that actually loops. Added: pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py Mon Nov 9 15:54:48 2009 @@ -0,0 +1,9 @@ +from pypy.jit.tl.tla import tla + +code = [ + tla.CONST_INT, 1, + tla.SUB, + tla.DUP, + tla.JUMP_IF, 0, + tla.RETURN + ] From cfbolz at codespeak.net Mon Nov 9 16:06:03 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 16:06:03 +0100 (CET) Subject: [pypy-svn] r69096 - pypy/trunk/pypy/jit/tl/tla Message-ID: <20091109150603.62A7416804F@codespeak.net> Author: cfbolz Date: Mon Nov 9 16:06:02 2009 New Revision: 69096 Modified: pypy/trunk/pypy/jit/tl/tla/tla.py Log: put None into stack Modified: pypy/trunk/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/tla.py (original) +++ pypy/trunk/pypy/jit/tl/tla/tla.py Mon Nov 9 16:06:02 2009 @@ -95,7 +95,9 @@ stackpos = self.stackpos - 1 assert stackpos >= 0 self.stackpos = stackpos - return self.stack[stackpos] + res = self.stack[stackpos] + self.stack[stackpos] = None + return res def interp(self): bytecode = self.bytecode From antocuni at codespeak.net Mon Nov 9 16:15:22 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 9 Nov 2009 16:15:22 +0100 (CET) Subject: [pypy-svn] r69097 - in pypy/trunk/pypy/translator/cli: . src test Message-ID: <20091109151522.1498E16804F@codespeak.net> Author: antocuni Date: Mon Nov 9 16:15:21 2009 New Revision: 69097 Added: pypy/trunk/pypy/translator/cli/src/debug.cs pypy/trunk/pypy/translator/cli/test/test_standalone.py (contents, props changed) Modified: pypy/trunk/pypy/translator/cli/entrypoint.py pypy/trunk/pypy/translator/cli/metavm.py pypy/trunk/pypy/translator/cli/opcodes.py pypy/trunk/pypy/translator/cli/rte.py pypy/trunk/pypy/translator/cli/src/pypylib.cs Log: (antocuni, arigo around) implement debug_{print,start,stop} & co. for CLI Modified: pypy/trunk/pypy/translator/cli/entrypoint.py ============================================================================== --- pypy/trunk/pypy/translator/cli/entrypoint.py (original) +++ pypy/trunk/pypy/translator/cli/entrypoint.py Mon Nov 9 16:15:21 2009 @@ -65,6 +65,8 @@ ilasm.call(self.cts.graph_to_signature(self.graph)) ilasm.opcode('pop') # XXX: return this value, if it's an int32 + + ilasm.call('void [pypylib]pypy.runtime.DebugPrint::close_file()') ilasm.opcode('ret') ilasm.end_function() self.db.pending_function(self.graph) Modified: pypy/trunk/pypy/translator/cli/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/cli/metavm.py (original) +++ pypy/trunk/pypy/translator/cli/metavm.py Mon Nov 9 16:15:21 2009 @@ -253,7 +253,7 @@ class _DebugPrint(MicroInstruction): def render(self, generator, op): - MAXARGS = 4 + MAXARGS = 8 if len(op.args) > MAXARGS: generator.db.genoo.log.WARNING('debug_print supported only up to ' '%d arguments (got %d)' % (MAXARGS, len(op.args))) @@ -268,11 +268,7 @@ boxtype = generator.cts.lltype_to_cts(TYPE) generator.ilasm.opcode('box', boxtype) - generator.ilasm.call('void [pypylib]pypy.runtime.Utils::debug_print(%s)' % signature) - -class _HaveDebugPrints(MicroInstruction): - def render(self, generator, op): - generator.ilasm.load_const(ootype.Bool, True) + generator.ilasm.call('void [pypylib]pypy.runtime.DebugPrint::DEBUG_PRINT(%s)' % signature) OOTYPE_TO_MNEMONIC = { @@ -310,4 +306,4 @@ SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() DebugPrint = _DebugPrint() -HaveDebugPrints = _HaveDebugPrints() + Modified: pypy/trunk/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/trunk/pypy/translator/cli/opcodes.py (original) +++ pypy/trunk/pypy/translator/cli/opcodes.py Mon Nov 9 16:15:21 2009 @@ -2,7 +2,7 @@ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, \ - DebugPrint, HaveDebugPrints + DebugPrint from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -78,10 +78,10 @@ 'resume_point': Ignore, 'debug_assert': Ignore, 'debug_print': [DebugPrint], - 'debug_start': Ignore, - 'debug_stop': Ignore, - 'have_debug_prints': [HaveDebugPrints], - 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], + 'debug_start': [PushAllArgs, 'call void [pypylib]pypy.runtime.DebugPrint::DEBUG_START(string)'], + 'debug_stop': [PushAllArgs, 'call void [pypylib]pypy.runtime.DebugPrint::DEBUG_STOP(string)'], + 'have_debug_prints': [PushAllArgs, 'call bool [pypylib]pypy.runtime.DebugPrint::HAVE_DEBUG_PRINTS()'], + 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Debug::DEBUG_FATALERROR(string)'], 'keepalive': Ignore, 'jit_marker': Ignore, 'promote_virtualizable': Ignore, Modified: pypy/trunk/pypy/translator/cli/rte.py ============================================================================== --- pypy/trunk/pypy/translator/cli/rte.py (original) +++ pypy/trunk/pypy/translator/cli/rte.py Mon Nov 9 16:15:21 2009 @@ -77,7 +77,8 @@ get_COMPILER = classmethod(get_COMPILER) class PyPyLibDLL(Target): - SOURCES = ['pypylib.cs', 'll_os.cs', 'll_os_path.cs', 'errno.cs', 'll_math.cs'] + SOURCES = ['pypylib.cs', 'll_os.cs', 'll_os_path.cs', 'errno.cs', 'll_math.cs', + 'debug.cs'] OUTPUT = 'pypylib.dll' FLAGS = ['/t:library', '/unsafe', '/r:main.exe'] DEPENDENCIES = [MainStub] Added: pypy/trunk/pypy/translator/cli/src/debug.cs ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/cli/src/debug.cs Mon Nov 9 16:15:21 2009 @@ -0,0 +1,179 @@ +using System; +using System.IO; +using System.Diagnostics; + +// this code is modeled after translator/c/src/debug.h +namespace pypy.runtime +{ + + public class Debug + { + public static void DEBUG_FATALERROR(string msg) + { + throw new Exception("debug_fatalerror: " + msg); + } + } + + public class DebugPrint + { + static Stopwatch watch = null; + static TextWriter debug_file = null; + static int have_debug_prints = -1; + static bool debug_ready = false; + static bool debug_profile = false; + static string debug_prefix = null; + + public static void close_file() + { + if (debug_file != null) + debug_file.Close(); + } + + public static bool HAVE_DEBUG_PRINTS() + { + if ((have_debug_prints & 1) != 0) { + debug_ensure_opened(); + return true; + } + return false; + } + + public static void DEBUG_START(string category) + { + debug_ensure_opened(); + /* Enter a nesting level. Nested debug_prints are disabled by + default because the following left shift introduces a 0 in the + last bit. Note that this logic assumes that we are never going + to nest debug_starts more than 31 levels (63 on 64-bits). */ + have_debug_prints <<= 1; + if (!debug_profile) { + /* non-profiling version */ + if (debug_prefix == null || !category.StartsWith(debug_prefix)) { + /* wrong section name, or no PYPYLOG at all, skip it */ + return; + } + /* else make this subsection active */ + have_debug_prints |= 1; + } + display_startstop("{", "", category); + } + + public static void DEBUG_STOP(string category) + { + if (debug_profile || (have_debug_prints & 1) != 0) + display_startstop("", "}", category); + have_debug_prints >>= 1; + } + + + static void setup_profiling() + { + watch = new Stopwatch(); + watch.Start(); + } + + static void debug_open() + { + string filename = Environment.GetEnvironmentVariable("PYPYLOG"); + if (filename != null && filename.Length > 0){ + int colon = filename.IndexOf(':'); + if (colon == -1) { + /* PYPYLOG=filename --- profiling version */ + debug_profile = true; + } + else { + /* PYPYLOG=prefix:filename --- conditional logging */ + debug_prefix = filename.Substring(0, colon); + filename = filename.Substring(colon+1); + } + if (filename != "-") + debug_file = File.CreateText(filename); + } + if (debug_file == null) + debug_file = System.Console.Error; + debug_ready = true; + setup_profiling(); + } + + static void debug_ensure_opened() { + if (!debug_ready) + debug_open(); + } + + static long read_timestamp() { + return watch.ElapsedMilliseconds; + } + + static void display_startstop(string prefix, + string postfix, + string category) + { + long timestamp = read_timestamp(); + debug_file.WriteLine("[{0:X}] {1}{2}{3}", + timestamp, + prefix, + category, + postfix); + } + + // ************************************************** + // debug_print family + // ************************************************** + public static void DEBUG_PRINT(object a0) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0}", a0); + } + + public static void DEBUG_PRINT(object a0, object a1) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1}", a0, a1); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2}", a0, a1, a2); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2, object a3) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2} {3}", + a0, a1, a2, a3); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2, object a3, + object a4) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2} {3} {4}", + a0, a1, a2, a3, a4); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2, object a3, + object a4, object a5) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2} {3} {4} {5}", + a0, a1, a2, a3, a4, a5); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2, object a3, + object a4, object a5, object a6) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2} {3} {4} {5} {6}", + a0, a1, a2, a3, a4, a5, a6); + } + + public static void DEBUG_PRINT(object a0, object a1, object a2, object a3, + object a4, object a5, object a6, object a7) + { + if (HAVE_DEBUG_PRINTS()) + debug_file.WriteLine("{0} {1} {2} {3} {4} {5} {6} {7}", + a0, a1, a2, a3, a4, a5, a6, a7); + } + } +} Modified: pypy/trunk/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/trunk/pypy/translator/cli/src/pypylib.cs Mon Nov 9 16:15:21 2009 @@ -382,31 +382,6 @@ public class Utils { - public static void debug_fatalerror(string msg) - { - throw new Exception("debug_fatalerror: " + msg); - } - - public static void debug_print(object a) - { - Console.Error.WriteLine(a); - } - - public static void debug_print(object a, object b) - { - Console.Error.WriteLine("{0} {1}", a, b); - } - - public static void debug_print(object a, object b, object c) - { - Console.Error.WriteLine("{0} {1} {2}", a, b, c); - } - - public static void debug_print(object a, object b, object c, object d) - { - Console.Error.WriteLine("{0} {1} {2} {3}", a, b, c, d); - } - public static DynamicMethod CreateDynamicMethod(string name, Type res, Type[] args) { return new DynamicMethod(name, res, args, typeof(Utils).Module); Added: pypy/trunk/pypy/translator/cli/test/test_standalone.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/cli/test/test_standalone.py Mon Nov 9 16:15:21 2009 @@ -0,0 +1,54 @@ +import subprocess +from pypy.translator.c.test.test_standalone import TestStandalone as CTestStandalone +from pypy.annotation.listdef import s_list_of_strings +from pypy.translator.translator import TranslationContext +from pypy.translator.cli.sdk import SDK + +class CliStandaloneBuilder(object): + + def __init__(self, translator, entry_point, config): + self.translator = translator + self.entry_point = entry_point + self.config = config + self.exe_name = None + + def compile(self): + from pypy.translator.cli.test.runtest import _build_gen_from_graph + graph = self.translator.graphs[0] + gen = _build_gen_from_graph(graph, self.translator, standalone=True) + gen.generate_source() + self.exe_name = gen.build_exe() + + def cmdexec(self, args='', env=None, err=False): + assert self.exe_name + stdout, stderr, retval = self.run(args, env=env) + if retval != 0: + raise Exception("Returned %d" % (retval,)) + if err: + return stdout, stderr + return stdout + + def run(self, args, env): + arglist = SDK.runtime() + [self.exe_name] + map(str, args) + env = env.copy() + env['LANG'] = 'C' + mono = subprocess.Popen(arglist, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + stdout, stderr = mono.communicate() + retval = mono.wait() + return stdout, stderr, retval + + +class TestStandalone(object): + config = None + + def compile(self, entry_point): + t = TranslationContext(self.config) + t.buildannotator().build_types(entry_point, [s_list_of_strings]) + t.buildrtyper(type_system='ootype').specialize() + + cbuilder = CliStandaloneBuilder(t, entry_point, t.config) + cbuilder.compile() + return t, cbuilder + + test_debug_print_start_stop = CTestStandalone.test_debug_print_start_stop.im_func From cfbolz at codespeak.net Mon Nov 9 16:16:45 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 9 Nov 2009 16:16:45 +0100 (CET) Subject: [pypy-svn] r69098 - pypy/trunk/pypy/jit/tl/tla Message-ID: <20091109151645.7B6C216804F@codespeak.net> Author: cfbolz Date: Mon Nov 9 16:16:44 2009 New Revision: 69098 Modified: pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py pypy/trunk/pypy/jit/tl/tla/test_tla.py Log: make loopabit have a nested loop Modified: pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py (original) +++ pypy/trunk/pypy/jit/tl/tla/loopabit.tla.py Mon Nov 9 16:16:44 2009 @@ -1,6 +1,12 @@ from pypy.jit.tl.tla import tla code = [ + tla.DUP, + tla.CONST_INT, 1, + tla.SUB, + tla.DUP, + tla.JUMP_IF, 1, + tla.POP, tla.CONST_INT, 1, tla.SUB, tla.DUP, Modified: pypy/trunk/pypy/jit/tl/tla/test_tla.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/test_tla.py (original) +++ pypy/trunk/pypy/jit/tl/tla/test_tla.py Mon Nov 9 16:16:44 2009 @@ -161,11 +161,17 @@ class TestLLtype(LLJitMixin): def test_loop(self): code = [ - tla.CONST_INT, 1, - tla.SUB, - tla.DUP, - tla.JUMP_IF, 0, - tla.RETURN + tla.DUP, + tla.CONST_INT, 1, + tla.SUB, + tla.DUP, + tla.JUMP_IF, 1, + tla.POP, + tla.CONST_INT, 1, + tla.SUB, + tla.DUP, + tla.JUMP_IF, 0, + tla.RETURN ] def interp_w(intvalue): w_result = interp(code, tla.W_IntObject(intvalue)) From fijal at codespeak.net Mon Nov 9 16:33:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 16:33:55 +0100 (CET) Subject: [pypy-svn] r69099 - in pypy/branch/guard-nonnull/pypy/jit/metainterp: . test Message-ID: <20091109153355.27C5B16804C@codespeak.net> Author: fijal Date: Mon Nov 9 16:33:54 2009 New Revision: 69099 Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_basic.py Log: (arigo, fijal) A test and a fix for bridge around pointer comparison Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/codewriter.py Mon Nov 9 16:33:54 2009 @@ -820,16 +820,18 @@ self.default_serialize_op(op) def serialize_op_ptr_eq(self, op): - self._serialize_op_ptr_eq(op, 'ptr_iszero') + self._serialize_op_ptr_eq(op, 'ooisnull') + serialize_op_oois = serialize_op_ptr_eq def serialize_op_ptr_ne(self, op): - self._serialize_op_ptr_eq(op, 'ptr_nonzero') + self._serialize_op_ptr_eq(op, 'oononnull') + serialize_op_ooisnot = serialize_op_ptr_ne - def serialize_op_oois(self, op): - self._serialize_op_ptr_eq(op, 'ooisnull') + def serialize_op_ptr_iszero(self, op): + self.default_serialize_op(op, 'ooisnull') - def serialize_op_ooisnot(self, op): - self._serialize_op_ptr_eq(op, 'oononnull') + def serialize_op_ptr_nonzero(self, op): + self.default_serialize_op(op, 'oononnull') def serialize_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/pyjitpl.py Mon Nov 9 16:33:54 2009 @@ -281,9 +281,24 @@ target = self.load_3byte() # load the 'target' argument self.pc = target # jump - def ignore_next_guard_nullness(self): + def ignore_next_guard_nullness(self, opnum): + _op_ooisnull = self.metainterp.staticdata._op_ooisnull + _op_oononnull = self.metainterp.staticdata._op_oononnull + bc = ord(self.bytecode[self.pc]) + if bc == _op_ooisnull: + if opnum == rop.GUARD_ISNULL: + res = ConstInt(0) + else: + res = ConstInt(1) + else: + assert bc == _op_oononnull + if opnum == rop.GUARD_ISNULL: + res = ConstInt(1) + else: + res = ConstInt(0) self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero self.load_int() # past the 'box' argument + self.make_result_box(res) def dont_follow_jump(self): _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not @@ -450,7 +465,7 @@ self.execute(rop.INT_NEG, box) @arguments("orgpc", "box") - def opimpl_ptr_nonzero(self, pc, box): + def opimpl_oononnull(self, pc, box): value = box.nonnull() if value: opnum = rop.GUARD_NONNULL @@ -462,7 +477,7 @@ self.make_result_box(res) @arguments("orgpc", "box") - def opimpl_ptr_iszero(self, pc, box): + def opimpl_ooisnull(self, pc, box): value = box.nonnull() if value: opnum = rop.GUARD_NONNULL @@ -473,9 +488,6 @@ self.generate_guard(pc, opnum, box, []) self.make_result_box(res) - opimpl_oononnull = opimpl_ptr_nonzero - opimpl_ooisnull = opimpl_ptr_iszero - @arguments("box", "box") def opimpl_ptr_eq(self, box1, box2): self.execute(rop.OOIS, box1, box2) @@ -1001,6 +1013,8 @@ self.warmrunnerdesc = warmrunnerdesc self._op_goto_if_not = self.find_opcode('goto_if_not') + self._op_ooisnull = self.find_opcode('ooisnull') + self._op_oononnull = self.find_opcode('oononnull') backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] @@ -1548,7 +1562,7 @@ elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.raise_overflow_error() elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: - self.framestack[-1].ignore_next_guard_nullness() + self.framestack[-1].ignore_next_guard_nullness(opnum) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.staticdata.num_green_args Modified: pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/guard-nonnull/pypy/jit/metainterp/test/test_basic.py Mon Nov 9 16:33:54 2009 @@ -1180,8 +1180,24 @@ history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value assert res == expected + def test_collapsing_ptr_eq(self): + S = lltype.GcStruct('S') + p = lltype.malloc(S) + driver = JitDriver(greens = [], reds = ['n', 'x']) + def f(n, x): + while n > 0: + driver.can_enter_jit(n=n, x=x) + driver.jit_merge_point(n=n, x=x) + if x: + n -= 1 + n -= 1 + def main(): + f(10, p) + f(10, lltype.nullptr(S)) + + self.meta_interp(main, []) class TestLLtype(BaseLLtypeTests, LLJitMixin): pass From fijal at codespeak.net Mon Nov 9 16:36:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 9 Nov 2009 16:36:36 +0100 (CET) Subject: [pypy-svn] r69100 - pypy/branch/gc-dump-heap/pypy/rlib Message-ID: <20091109153636.BB99116804C@codespeak.net> Author: fijal Date: Mon Nov 9 16:36:35 2009 New Revision: 69100 Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Log: Rename dump_heap to heap_stats here as well Modified: pypy/branch/gc-dump-heap/pypy/rlib/rgc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rlib/rgc.py (original) +++ pypy/branch/gc-dump-heap/pypy/rlib/rgc.py Mon Nov 9 16:36:35 2009 @@ -184,11 +184,11 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) -def _dump_heap(): +def _heap_stats(): raise NotImplementedError # can't be run directly class DumpHeapEntry(ExtRegistryEntry): - _about_ = _dump_heap + _about_ = _heap_stats def compute_result_annotation(self): from pypy.annotation import model as annmodel @@ -201,7 +201,7 @@ from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP from pypy.rpython.lltypesystem import lltype hop.exception_is_here() - return hop.genop('gc_dump_heap', [], resulttype=hop.r_result) + return hop.genop('gc_heap_stats', [], resulttype=hop.r_result) def malloc_nonmovable(TP, n=None, zero=False): """ Allocate a non-moving buffer or return nullptr. From arigo at codespeak.net Mon Nov 9 16:59:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Nov 2009 16:59:17 +0100 (CET) Subject: [pypy-svn] r69101 - pypy/branch/guard-nonnull/pypy/module/pypyjit/test Message-ID: <20091109155917.BBDE316804C@codespeak.net> Author: arigo Date: Mon Nov 9 16:59:16 2009 New Revision: 69101 Modified: pypy/branch/guard-nonnull/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix that test too. Modified: pypy/branch/guard-nonnull/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/guard-nonnull/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/guard-nonnull/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 9 16:59:16 2009 @@ -158,8 +158,7 @@ ops = self.get_by_bytecode("LOAD_GLOBAL") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "getfield_gc", "ooisnull", - "guard_false"] + "getfield_gc", "guard_nonnull"] assert not ops[1] # second LOAD_GLOBAL folded away ops = self.get_by_bytecode("CALL_FUNCTION") assert len(ops) == 2 @@ -203,7 +202,7 @@ ops = self.get_by_bytecode("LOAD_ATTR") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "ooisnull", "guard_false"] + "guard_nonnull"] assert not ops[1] # second LOAD_ATTR folded away def test_default_and_kw(self): From arigo at codespeak.net Mon Nov 9 18:20:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Nov 2009 18:20:05 +0100 (CET) Subject: [pypy-svn] r69102 - in pypy/trunk/pypy: jit/backend/llgraph jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20091109172005.B34C916804D@codespeak.net> Author: arigo Date: Mon Nov 9 18:20:03 2009 New Revision: 69102 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/test/test_random.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/executor.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Merge the branch 'guard-nonnull': Remove oononnull and ooisnull operations, replace with guard_nonnull/isnull. Kills a bit of stuff here and there. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Mon Nov 9 18:20:03 2009 @@ -104,8 +104,6 @@ 'new_with_vtable' : (('ref',), 'ref'), 'new' : ((), 'ref'), 'new_array' : (('int',), 'ref'), - 'oononnull' : (('ref',), 'bool'), - 'ooisnull' : (('ref',), 'bool'), 'oois' : (('ref', 'ref'), 'bool'), 'ooisnot' : (('ref', 'ref'), 'bool'), 'instanceof' : (('ref',), 'bool'), @@ -134,6 +132,8 @@ 'guard_exception' : (('ref',), 'ref'), 'guard_no_overflow' : ((), None), 'guard_overflow' : ((), None), + 'guard_nonnull' : (('ref',), None), + 'guard_isnull' : (('ref',), None), 'newstr' : (('int',), 'ref'), 'strlen' : (('ref',), 'int'), 'strgetitem' : (('ref', 'int'), 'int'), @@ -551,6 +551,9 @@ if value: raise GuardFailed + op_guard_nonnull = op_guard_true + op_guard_isnull = op_guard_false + def op_guard_class(self, _, value, expected_class): value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) expected_class = llmemory.cast_adr_to_ptr( Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Mon Nov 9 18:20:03 2009 @@ -441,9 +441,13 @@ def test_passing_guards(self): + t_box, T_box = self.alloc_instance(self.T) + nullbox = self.null_instance() all = [(rop.GUARD_TRUE, [BoxInt(1)]), (rop.GUARD_FALSE, [BoxInt(0)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)])] + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), + (rop.GUARD_NONNULL, [t_box]), + (rop.GUARD_ISNULL, [nullbox])] if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) for (opname, args) in all: @@ -460,9 +464,13 @@ # 'void') def test_failing_guards(self): + t_box, T_box = self.alloc_instance(self.T) + nullbox = self.null_instance() all = [(rop.GUARD_TRUE, [BoxInt(0)]), (rop.GUARD_FALSE, [BoxInt(1)]), - (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)])] + (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), + (rop.GUARD_NONNULL, [nullbox]), + (rop.GUARD_ISNULL, [t_box])] if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) for opname, args in all: @@ -493,10 +501,6 @@ assert r.value == 0 r = self.execute_operation(rop.OOISNOT, [u2_box, u1_box], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNULL, [u1_box], 'int') - assert r.value == 0 - r = self.execute_operation(rop.OONONNULL, [u2_box], 'int') - assert r.value == 1 # null_box = self.null_instance() r = self.execute_operation(rop.OOIS, [null_box, @@ -513,10 +517,6 @@ assert r.value == 1 r = self.execute_operation(rop.OOISNOT, [null_box, u1_box], 'int') assert r.value == 1 - r = self.execute_operation(rop.OOISNULL, [null_box], 'int') - assert r.value == 1 - r = self.execute_operation(rop.OONONNULL, [null_box], 'int') - assert r.value == 0 def test_array_basic(self): a_box, A = self.alloc_array_of(lltype.Signed, 342) @@ -962,10 +962,6 @@ assert r.value == 1 r = self.execute_operation(rop.OOISNOT, [BoxInt(v), BoxInt(v)], 'int') assert r.value == 0 - r = self.execute_operation(rop.OOISNULL, [BoxInt(v)], 'int') - assert r.value == 0 - r = self.execute_operation(rop.OONONNULL, [BoxInt(v)], 'int') - assert r.value == 1 lltype.free(x, flavor='raw') def test_new_plain_struct(self): @@ -1335,6 +1331,22 @@ [value, chr1, chr2]) assert len(dict.fromkeys([value, chr1, chr2]).keys()) == 3 + def test_guards_nongc(self): + x = lltype.malloc(lltype.Struct('x'), flavor='raw') + v = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x)) + vbox = BoxInt(v) + ops = [ + (rop.GUARD_NONNULL, vbox, False), + (rop.GUARD_ISNULL, vbox, True), + (rop.GUARD_NONNULL, BoxInt(0), True), + (rop.GUARD_ISNULL, BoxInt(0), False), + ] + for opname, arg, res in ops: + self.execute_operation(opname, [arg], 'void') + assert self.guard_failed == res + + lltype.free(x, flavor='raw') + class OOtypeBackendTest(BaseBackendTest): type_system = 'ootype' Modified: pypy/trunk/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_random.py Mon Nov 9 18:20:03 2009 @@ -57,27 +57,18 @@ def get_bool_var(self, r): if self.boolvars and r.random() < 0.8: - v = r.choice(self.boolvars) + return r.choice(self.boolvars) elif self.ptrvars and r.random() < 0.4: v, S = r.choice(self.ptrvars + self.prebuilt_ptr_consts)[:2] v2, S2 = r.choice(self.ptrvars + self.prebuilt_ptr_consts)[:2] if S == S2 and not (isinstance(v, ConstPtr) and isinstance(v2, ConstPtr)): if r.random() < 0.5: - v = self.do(rop.OOIS, [v, v2]) + return self.do(rop.OOIS, [v, v2]) else: - v = self.do(rop.OOISNOT, [v, v2]) - else: - if isinstance(v, ConstPtr): - v, S = r.choice(self.ptrvars) - if r.random() < 0.5: - v = self.do(rop.OONONNULL, [v]) - else: - v = self.do(rop.OOISNULL, [v]) - else: - v = r.choice(self.intvars) - v = self.do(rop.INT_IS_TRUE, [v]) - return v + return self.do(rop.OOISNOT, [v, v2]) + v = r.choice(self.intvars) + return self.do(rop.INT_IS_TRUE, [v]) def subset_of_intvars(self, r): subset = [] @@ -358,6 +349,16 @@ builder.should_fail_by = op builder.guard_op = op +class GuardPtrOperation(GuardOperation): + def gen_guard(self, builder, r): + if not builder.ptrvars: + raise CannotProduceOperation + box = r.choice(builder.ptrvars)[0] + op = ResOperation(self.opnum, [box], None) + passing = ((self.opnum == rop.GUARD_NONNULL and box.value) or + (self.opnum == rop.GUARD_ISNULL and not box.value)) + return op, passing + class GuardValueOperation(GuardOperation): def gen_guard(self, builder, r): v = r.choice(builder.intvars) @@ -410,6 +411,8 @@ OPERATIONS.append(GuardOperation(rop.GUARD_TRUE)) OPERATIONS.append(GuardOperation(rop.GUARD_FALSE)) OPERATIONS.append(GuardOperation(rop.GUARD_FALSE)) +OPERATIONS.append(GuardPtrOperation(rop.GUARD_NONNULL)) +OPERATIONS.append(GuardPtrOperation(rop.GUARD_ISNULL)) OPERATIONS.append(GuardValueOperation(rop.GUARD_VALUE)) for _op in [rop.INT_NEG, Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Nov 9 18:20:03 2009 @@ -466,7 +466,7 @@ loc2 = cl self.mc.SHR(loc, loc2) - def genop_guard_oononnull(self, op, guard_op, addr, arglocs, resloc): + def genop_guard_int_is_true(self, op, guard_op, addr, arglocs, resloc): guard_opnum = guard_op.opnum loc = arglocs[0] self.mc.TEST(loc, loc) @@ -475,29 +475,11 @@ else: return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_ooisnull(self, op, guard_op, addr, arglocs, resloc): - guard_opnum = guard_op.opnum - loc = arglocs[0] - self.mc.TEST(loc, loc) - if guard_opnum == rop.GUARD_TRUE: - return self.implement_guard(addr, self.mc.JNZ) - else: - return self.implement_guard(addr, self.mc.JZ) - - genop_guard_int_is_true = genop_guard_oononnull - - def genop_oononnull(self, op, arglocs, resloc): + def genop_int_is_true(self, op, arglocs, resloc): self.mc.CMP(arglocs[0], imm8(0)) self.mc.MOV(resloc, imm8(0)) self.mc.SETNE(lower_byte(resloc)) - genop_int_is_true = genop_oononnull - - def genop_ooisnull(self, op, arglocs, resloc): - self.mc.CMP(arglocs[0], imm8(0)) - self.mc.MOV(resloc, imm8(0)) - self.mc.SETE(lower_byte(resloc)) - def genop_same_as(self, op, arglocs, resloc): self.mov(arglocs[0], resloc) genop_cast_ptr_to_int = genop_same_as @@ -672,6 +654,7 @@ loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JZ) + genop_guard_guard_nonnull = genop_guard_guard_true def genop_guard_guard_no_exception(self, ign_1, guard_op, addr, locs, ign_2): @@ -703,6 +686,7 @@ loc = locs[0] self.mc.TEST(loc, loc) return self.implement_guard(addr, self.mc.JNZ) + genop_guard_guard_isnull = genop_guard_guard_false def genop_guard_guard_value(self, ign_1, guard_op, addr, locs, ign_2): if guard_op.args[0].type == FLOAT: Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon Nov 9 18:20:03 2009 @@ -305,8 +305,7 @@ self.assembler.regalloc_perform_discard(op, arglocs) def can_optimize_cmp_op(self, op, i, operations): - if not (op.is_comparison() or op.opnum == rop.OOISNULL or - op.opnum == rop.OONONNULL): + if not op.is_comparison(): return False if (operations[i + 1].opnum != rop.GUARD_TRUE and operations[i + 1].opnum != rop.GUARD_FALSE): @@ -385,6 +384,8 @@ consider_guard_true = _consider_guard consider_guard_false = _consider_guard + consider_guard_nonnull = _consider_guard + consider_guard_isnull = _consider_guard def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] @@ -831,8 +832,6 @@ self.Perform(op, [argloc], resloc) consider_int_is_true = _consider_nullity - consider_ooisnull = _consider_nullity - consider_oononnull = _consider_nullity def consider_same_as(self, op, ignored): argloc = self.loc(op.args[0]) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Mon Nov 9 18:20:03 2009 @@ -132,11 +132,9 @@ guard_value(i2, 1) [i2, i3, i4, i5, i6, i7, i0, i1, i8] guard_class(i4, 138998336) [i4, i5, i6, i7, i0, i1, i8] i11 = getfield_gc(i4, descr=descr0) - i12 = ooisnull(i11) - guard_false(i12) [i4, i5, i6, i7, i0, i1, i11, i8] + guard_nonnull(i11) [i4, i5, i6, i7, i0, i1, i11, i8] i13 = getfield_gc(i11, descr=descr0) - i14 = ooisnull(i13) - guard_true(i14) [i4, i5, i6, i7, i0, i1, i11, i8] + guard_isnull(i13) [i4, i5, i6, i7, i0, i1, i11, i8] i15 = getfield_gc(i4, descr=descr0) i17 = int_lt(i15, 0) guard_false(i17) [i4, i5, i6, i7, i0, i1, i11, i15, i8] Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Mon Nov 9 18:20:03 2009 @@ -437,20 +437,6 @@ self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] - def test_nullity(self): - ops = ''' - [i0, i1, i2, i3, i4, i5, i6] - i10 = oononnull(i0) - i11 = ooisnull(i1) - i12 = oononnull(i2) - i13 = oononnull(i3) - i14 = ooisnull(i6) - i15 = ooisnull(i5) - finish(i10, i11, i12, i13, i14, i15) - ''' - self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) - assert self.getints(6) == [0, 0, 1, 1, 0, 0] - def test_strsetitem(self): ops = ''' [p0, i] Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Mon Nov 9 18:20:03 2009 @@ -229,7 +229,7 @@ assert c.value == 3 def test_nullity_with_guard(self): - allops = [rop.OONONNULL, rop.OOISNULL, rop.INT_IS_TRUE] + allops = [rop.INT_IS_TRUE] guards = [rop.GUARD_TRUE, rop.GUARD_FALSE] p = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(lltype.GcStruct('x'))) Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Mon Nov 9 18:20:03 2009 @@ -808,6 +808,31 @@ serialize_op_uint_is_true = serialize_op_int_is_true + def _serialize_op_ptr_eq(self, op, opname): + arg0, arg1 = op.args + if isinstance(arg0, Constant) and not arg0.value: + self.emit(opname, self.var_position(arg1)) + self.register_var(op.result) + elif isinstance(arg1, Constant) and not arg1.value: + self.emit(opname, self.var_position(arg0)) + self.register_var(op.result) + else: + self.default_serialize_op(op) + + def serialize_op_ptr_eq(self, op): + self._serialize_op_ptr_eq(op, 'ooisnull') + serialize_op_oois = serialize_op_ptr_eq + + def serialize_op_ptr_ne(self, op): + self._serialize_op_ptr_eq(op, 'oononnull') + serialize_op_ooisnot = serialize_op_ptr_ne + + def serialize_op_ptr_iszero(self, op): + self.default_serialize_op(op, 'ooisnull') + + def serialize_op_ptr_nonzero(self, op): + self.default_serialize_op(op, 'oononnull') + def serialize_op_malloc(self, op): assert op.args[1].value == {'flavor': 'gc'} STRUCT = op.args[0].value Modified: pypy/trunk/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/executor.py (original) +++ pypy/trunk/pypy/jit/metainterp/executor.py Mon Nov 9 18:20:03 2009 @@ -103,26 +103,6 @@ def do_same_as(cpu, box1): return box1 -def do_oononnull(cpu, box1): - tp = box1.type - if tp == INT: - x = bool(box1.getint()) - elif tp == REF: - x = bool(box1.getref_base()) - else: - assert False - return ConstInt(x) - -def do_ooisnull(cpu, box1): - tp = box1.type - if tp == INT: - x = bool(box1.getint()) - elif tp == REF: - x = bool(box1.getref_base()) - else: - assert False - return ConstInt(not x) - def do_oois(cpu, box1, box2): tp = box1.type assert tp == box2.type Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Mon Nov 9 18:20:03 2009 @@ -106,6 +106,9 @@ def set_future_value(self, cpu, j): raise NotImplementedError + def nonnull(self): + raise NotImplementedError + def repr_rpython(self): return '%s' % self @@ -177,9 +180,6 @@ def same_constant(self, other): raise NotImplementedError - def nonnull_constant(self): - raise NotImplementedError - def __repr__(self): return 'Const(%s)' % self._getrepr_() @@ -239,7 +239,7 @@ assert isinstance(other, Const) return self.value == other.getint() - def nonnull_constant(self): + def nonnull(self): return self.value != 0 def _getrepr_(self): @@ -286,7 +286,7 @@ assert isinstance(other, Const) return self.value == other.getaddr(self.cpu) - def nonnull_constant(self): + def nonnull(self): return bool(self.value) def _getrepr_(self): @@ -322,7 +322,7 @@ assert isinstance(other, ConstFloat) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return self.value != 0.0 def _getrepr_(self): @@ -368,7 +368,7 @@ assert isinstance(other, ConstPtr) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return bool(self.value) _getrepr_ = repr_pointer @@ -421,7 +421,7 @@ assert isinstance(other, ConstObj) return self.value == other.value - def nonnull_constant(self): + def nonnull(self): return bool(self.value) _getrepr_ = repr_object @@ -512,6 +512,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) + def nonnull(self): + return self.value != 0 + def _getrepr_(self): return self.value @@ -541,6 +544,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_float(j, self.value) + def nonnull(self): + return self.value != 0.0 + def _getrepr_(self): return self.value @@ -580,6 +586,9 @@ def set_future_value(self, cpu, j): cpu.set_future_value_ref(j, self.value) + def nonnull(self): + return bool(self.value) + def repr_rpython(self): return repr_rpython(self, 'bp') @@ -615,6 +624,9 @@ else: return 0 + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_ref(j, self.value) Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Mon Nov 9 18:20:03 2009 @@ -168,11 +168,11 @@ def find_nodes_no_escape(self, op): pass # for operations that don't escape their arguments - find_nodes_OONONNULL = find_nodes_no_escape - find_nodes_OOISNULL = find_nodes_no_escape - find_nodes_OOIS = find_nodes_no_escape - find_nodes_OOISNOT = find_nodes_no_escape - find_nodes_INSTANCEOF = find_nodes_no_escape + find_nodes_OOIS = find_nodes_no_escape + find_nodes_OOISNOT = find_nodes_no_escape + find_nodes_INSTANCEOF = find_nodes_no_escape + find_nodes_GUARD_NONNULL = find_nodes_no_escape + find_nodes_GUARD_ISNULL = find_nodes_no_escape def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode() Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon Nov 9 18:20:03 2009 @@ -69,7 +69,7 @@ if self.is_constant(): box = self.box assert isinstance(box, Const) - return not box.nonnull_constant() + return not box.nonnull() return False def make_constant(self, constbox): @@ -100,7 +100,7 @@ elif level == LEVEL_CONSTANT: box = self.box assert isinstance(box, Const) - return box.nonnull_constant() + return box.nonnull() else: return False @@ -114,25 +114,6 @@ # meaning it has been forced. return self.box is None -class BoolValue(OptValue): - - def __init__(self, box, fromvalue, reversed, nullconstbox): - OptValue.__init__(self, box) - # If later 'box' is turned into a constant False - # (resp. True), then 'fromvalue' will be known to - # be null (resp. non-null). If 'reversed', then - # this logic is reversed. - self.fromvalue = fromvalue - self.reversed = reversed - self.nullconstbox = nullconstbox # of the correct type - - def make_constant(self, constbox): - OptValue.make_constant(self, constbox) - if constbox.nonnull_constant() ^ self.reversed: - self.fromvalue.make_nonnull() - else: - self.fromvalue.make_constant(self.nullconstbox) - class ConstantValue(OptValue): level = LEVEL_CONSTANT @@ -434,11 +415,6 @@ self.make_equal_to(box, vvalue) return vvalue - def make_bool(self, box, fromvalue, reversed, nullconstbox): - value = BoolValue(box, fromvalue, reversed, nullconstbox) - self.make_equal_to(box, value) - return value - def new_ptr_box(self): return self.cpu.ts.BoxRef() @@ -585,6 +561,24 @@ self.emit_operation(op) value.make_constant(constbox) + def optimize_GUARD_ISNULL(self, op): + value = self.getvalue(op.args[0]) + if value.is_null(): + return + elif value.is_nonnull(): + raise InvalidLoop + self.emit_operation(op) + value.make_constant(self.cpu.ts.CONST_NULL) + + def optimize_GUARD_NONNULL(self, op): + value = self.getvalue(op.args[0]) + if value.is_nonnull(): + return + elif value.is_null(): + raise InvalidLoop + self.emit_operation(op) + value.make_nonnull() + def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.args[0]) emit_operation = True @@ -631,26 +625,19 @@ self.emit_operation(op) - def _optimize_nullness(self, op, expect_nonnull, nullconstbox): - value = self.getvalue(op.args[0]) + def _optimize_nullness(self, op, box, expect_nonnull): + value = self.getvalue(box) if value.is_nonnull(): self.make_constant_int(op.result, expect_nonnull) elif value.is_null(): self.make_constant_int(op.result, not expect_nonnull) else: - self.make_bool(op.result, value, not expect_nonnull, nullconstbox) self.emit_operation(op) - def optimize_OONONNULL(self, op): - self._optimize_nullness(op, True, self.cpu.ts.CONST_NULL) - - def optimize_OOISNULL(self, op): - self._optimize_nullness(op, False, self.cpu.ts.CONST_NULL) - def optimize_INT_IS_TRUE(self, op): - self._optimize_nullness(op, True, CONST_0) + self._optimize_nullness(op, op.args[0], True) - def _optimize_oois_ooisnot(self, op, expect_isnot, unary_opnum): + def _optimize_oois_ooisnot(self, op, expect_isnot): value0 = self.getvalue(op.args[0]) value1 = self.getvalue(op.args[1]) if value0.is_virtual(): @@ -662,19 +649,17 @@ elif value1.is_virtual(): self.make_constant_int(op.result, expect_isnot) elif value1.is_null(): - op = ResOperation(unary_opnum, [op.args[0]], op.result) - self._optimize_nullness(op, expect_isnot, self.cpu.ts.CONST_NULL) + self._optimize_nullness(op, op.args[0], expect_isnot) elif value0.is_null(): - op = ResOperation(unary_opnum, [op.args[1]], op.result) - self._optimize_nullness(op, expect_isnot, self.cpu.ts.CONST_NULL) + self._optimize_nullness(op, op.args[1], expect_isnot) else: self.optimize_default(op) def optimize_OOISNOT(self, op): - self._optimize_oois_ooisnot(op, True, rop.OONONNULL) + self._optimize_oois_ooisnot(op, True) def optimize_OOIS(self, op): - self._optimize_oois_ooisnot(op, False, rop.OOISNULL) + self._optimize_oois_ooisnot(op, False) def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Mon Nov 9 18:20:03 2009 @@ -281,6 +281,25 @@ target = self.load_3byte() # load the 'target' argument self.pc = target # jump + def ignore_next_guard_nullness(self, opnum): + _op_ooisnull = self.metainterp.staticdata._op_ooisnull + _op_oononnull = self.metainterp.staticdata._op_oononnull + bc = ord(self.bytecode[self.pc]) + if bc == _op_ooisnull: + if opnum == rop.GUARD_ISNULL: + res = ConstInt(0) + else: + res = ConstInt(1) + else: + assert bc == _op_oononnull + if opnum == rop.GUARD_ISNULL: + res = ConstInt(1) + else: + res = ConstInt(0) + self.pc += 1 # past the bytecode for ptr_iszero/ptr_nonzero + self.load_int() # past the 'box' argument + self.make_result_box(res) + def dont_follow_jump(self): _op_goto_if_not = self.metainterp.staticdata._op_goto_if_not assert ord(self.bytecode[self.pc]) == _op_goto_if_not @@ -445,16 +464,29 @@ else: self.execute(rop.INT_NEG, box) - @arguments("box") - def opimpl_ptr_nonzero(self, box): - self.execute(rop.OONONNULL, box) - - @arguments("box") - def opimpl_ptr_iszero(self, box): - self.execute(rop.OOISNULL, box) + @arguments("orgpc", "box") + def opimpl_oononnull(self, pc, box): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + res = ConstInt(1) + else: + opnum = rop.GUARD_ISNULL + res = ConstInt(0) + self.generate_guard(pc, opnum, box, []) + self.make_result_box(res) - opimpl_oononnull = opimpl_ptr_nonzero - opimpl_ooisnull = opimpl_ptr_iszero + @arguments("orgpc", "box") + def opimpl_ooisnull(self, pc, box): + value = box.nonnull() + if value: + opnum = rop.GUARD_NONNULL + res = ConstInt(0) + else: + opnum = rop.GUARD_ISNULL + res = ConstInt(1) + self.generate_guard(pc, opnum, box, []) + self.make_result_box(res) @arguments("box", "box") def opimpl_ptr_eq(self, box1, box2): @@ -981,6 +1013,8 @@ self.warmrunnerdesc = warmrunnerdesc self._op_goto_if_not = self.find_opcode('goto_if_not') + self._op_ooisnull = self.find_opcode('ooisnull') + self._op_oononnull = self.find_opcode('oononnull') backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] @@ -1527,6 +1561,8 @@ self.handle_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.raise_overflow_error() + elif opnum == rop.GUARD_NONNULL or opnum == rop.GUARD_ISNULL: + self.framestack[-1].ignore_next_guard_nullness(opnum) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.staticdata.num_green_args Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Mon Nov 9 18:20:03 2009 @@ -1,7 +1,6 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized - class ResOperation(object): """The central ResOperation class, representing one operation.""" @@ -118,6 +117,8 @@ 'GUARD_FALSE', 'GUARD_VALUE', 'GUARD_CLASS', + 'GUARD_NONNULL', + 'GUARD_ISNULL', '_GUARD_FOLDABLE_LAST', 'GUARD_NO_EXCEPTION', 'GUARD_EXCEPTION', @@ -178,8 +179,6 @@ # 'SAME_AS/1', # gets a Const or a Box, turns it into another Box # - 'OONONNULL/1b', - 'OOISNULL/1b', 'OOIS/2b', 'OOISNOT/2b', # Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Mon Nov 9 18:20:03 2009 @@ -1180,8 +1180,24 @@ history.BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, x))).value assert res == expected + def test_collapsing_ptr_eq(self): + S = lltype.GcStruct('S') + p = lltype.malloc(S) + driver = JitDriver(greens = [], reds = ['n', 'x']) + def f(n, x): + while n > 0: + driver.can_enter_jit(n=n, x=x) + driver.jit_merge_point(n=n, x=x) + if x: + n -= 1 + n -= 1 + def main(): + f(10, p) + f(10, lltype.nullptr(S)) + + self.meta_interp(main, []) class TestLLtype(BaseLLtypeTests, LLJitMixin): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Mon Nov 9 18:20:03 2009 @@ -341,6 +341,30 @@ assert 0, "missing instantiate_*_%s in:\n%r" % (expected, names) + def test_oois_constant_null(self): + from pypy.rpython.lltypesystem import lltype + + S = lltype.GcStruct('S') + s = lltype.malloc(S) + NULL = lltype.nullptr(S) + def f(p, i): + if i % 2: + return p == NULL + elif i % 3: + return NULL == p + elif i % 4: + return p != NULL + else: + return NULL != p + graphs = self.make_graphs(f, [s, 5]) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = graphs + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert 'ptr_eq' not in jitcode._source + assert 'ptr_ne' not in jitcode._source + assert jitcode._source.count('ptr_iszero') == 2 + assert jitcode._source.count('ptr_nonzero') == 2 class ImmutableFieldsTests: Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Nov 9 18:20:03 2009 @@ -474,10 +474,7 @@ [p3, p4, p2] p0 = new_with_vtable(ConstClass(node_vtable)) p1 = new_with_vtable(ConstClass(node_vtable)) - i1 = oononnull(p0) - guard_true(i1) [] - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] i3 = ooisnot(p0, NULL) guard_true(i3) [] i4 = oois(p0, NULL) @@ -827,14 +824,12 @@ def test_find_nodes_bug_1(self): ops = """ [p12] - i16 = ooisnull(p12) - guard_false(i16) [] + guard_nonnull(p12) [] guard_class(p12, ConstClass(node_vtable)) [] guard_class(p12, ConstClass(node_vtable)) [] i22 = getfield_gc_pure(p12, descr=valuedescr) escape(i22) - i25 = ooisnull(p12) - guard_false(i25) [] + guard_nonnull(p12) [] guard_class(p12, ConstClass(node_vtable)) [] guard_class(p12, ConstClass(node_vtable)) [] i29 = getfield_gc_pure(p12, descr=valuedescr) @@ -847,10 +842,8 @@ setarrayitem_gc(p35, 0, p33, descr=arraydescr3) p38 = new_with_vtable(ConstClass(u_vtable)) # U setfield_gc(p38, p35, descr=onedescr) - i39 = ooisnull(p38) - guard_false(i39) [] - i40 = oononnull(p38) - guard_true(i40) [] + guard_nonnull(p38) [] + guard_nonnull(p38) [] guard_class(p38, ConstClass(u_vtable)) [] p42 = getfield_gc(p38, descr=onedescr) # Array(NODE) i43 = arraylen_gc(p42, descr=arraydescr3) @@ -871,8 +864,7 @@ guard_false(i55) [] p56 = getfield_gc(p46, descr=ddescr) # Array(NODE) p58 = getarrayitem_gc(p56, 0, descr=arraydescr3) # NODE - i59 = ooisnull(p38) - guard_false(i59) [] + guard_nonnull(p38) [] jump(p58) """ self.find_nodes(ops, 'Virtual(node_vtable, valuedescr=Not)') Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 9 18:20:03 2009 @@ -338,10 +338,7 @@ ops = """ [p0] guard_class(p0, ConstClass(node_vtable)) [] - i0 = oononnull(p0) - guard_true(i0) [] - i1 = ooisnull(p0) - guard_false(i1) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ @@ -352,6 +349,7 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_1(self): + py.test.skip("too bad") ops = """ [i0] i1 = int_is_true(i0) @@ -371,31 +369,13 @@ def test_ooisnull_oononnull_2(self): ops = """ [p0] - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] - i1 = ooisnull(p0) - guard_false(i1) [] - jump(p0) - """ - expected = """ - [p0] - i0 = oononnull(p0) - guard_true(i0) [] - jump(p0) - """ - self.optimize_loop(ops, 'Not', expected) - ops = """ - [p0] - i1 = ooisnull(p0) - guard_false(i1) [] - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] + guard_nonnull(p0) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ [p0] - i1 = ooisnull(p0) - guard_false(i1) [] + guard_nonnull(p0) [] jump(p0) """ self.optimize_loop(ops, 'Not', expected) @@ -404,34 +384,14 @@ ops = """ [] p0 = escape() - i0 = ooisnull(p0) - guard_true(i0) [] - i1 = oononnull(p0) - guard_false(i1) [] - jump() - """ - expected = """ - [] - p0 = escape() - i0 = ooisnull(p0) - guard_true(i0) [] - jump() - """ - self.optimize_loop(ops, '', expected) - ops = """ - [] - p0 = escape() - i0 = oononnull(p0) - guard_false(i0) [] - i1 = ooisnull(p0) - guard_true(i1) [] + guard_isnull(p0) [] + guard_isnull(p0) [] jump() """ expected = """ [] p0 = escape() - i0 = oononnull(p0) - guard_false(i0) [] + guard_isnull(p0) [] jump() """ self.optimize_loop(ops, '', expected) @@ -441,17 +401,14 @@ [p0] pv = new_with_vtable(ConstClass(node_vtable)) setfield_gc(pv, p0, descr=valuedescr) - i0 = oononnull(p0) # p0 != NULL - guard_true(i0) [] + guard_nonnull(p0) [] p1 = getfield_gc(pv, descr=valuedescr) - i1 = ooisnull(p1) - guard_false(i1) [] + guard_nonnull(p1) [] jump(p0) """ expected = """ [p0] - i0 = oononnull(p0) - guard_true(i0) [] + guard_nonnull(p0) [] jump(p0) """ self.optimize_loop(ops, 'Not', expected) @@ -489,10 +446,7 @@ guard_true(i0) [] i3 = oois(NULL, p0) guard_false(i1) [] - i4 = oononnull(p0) - guard_true(i4) [] - i5 = ooisnull(p0) - guard_false(i5) [] + guard_nonnull(p0) [] jump(p0) """ expected = """ @@ -541,16 +495,16 @@ def test_guard_value_to_guard_false(self): ops = """ - [p] - i1 = ooisnull(p) - guard_value(i1, 0) [p] - jump(p) + [i] + i1 = int_is_true(i) + guard_value(i1, 0) [i] + jump(i) """ expected = """ - [p] - i1 = ooisnull(p) - guard_false(i1) [p] - jump(p) + [i] + i1 = int_is_true(i) + guard_false(i1) [i] + jump(i) """ self.optimize_loop(ops, 'Not', expected) @@ -686,10 +640,7 @@ def test_virtual_oois(self): ops = """ [p0, p1, p2] - i1 = oononnull(p0) - guard_true(i1) [] - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] i3 = ooisnot(p0, NULL) guard_true(i3) [] i4 = oois(p0, NULL) @@ -727,8 +678,7 @@ # the details of the algorithm... expected2 = """ [p0, p1, p2] - i1 = oononnull(p0) - guard_true(i1) [] + guard_nonnull(p0) [] i7 = ooisnot(p0, p1) guard_true(i7) [] i8 = oois(p0, p1) @@ -833,7 +783,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = ooisnull(p2) + i1 = oois(p2, NULL) jump(i1) """ expected = """ @@ -849,7 +799,7 @@ p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc(p0, descr=nextdescr) - i1 = ooisnull(p2) + i1 = oois(p2, NULL) jump(i1) """ expected = """ @@ -1003,8 +953,7 @@ ops = """ [i1, p0] setarrayitem_gc(p0, 0, i1, descr=arraydescr) - i2 = ooisnull(p0) - guard_false(i2) [] + guard_nonnull(p0) [] p1 = new_array(i1, descr=arraydescr) jump(i1, p1) """ @@ -1426,8 +1375,7 @@ ops = """ [i0, p1] p4 = getfield_gc(p1, descr=nextdescr) - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p2 = new_with_vtable(ConstClass(node_vtable)) @@ -1437,8 +1385,7 @@ """ expected = """ [i0, p4] - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p3 = escape() @@ -1451,8 +1398,7 @@ ops = """ [i0, p1] p4 = getarrayitem_gc(p1, 0, descr=arraydescr2) - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p2 = new_array(1, descr=arraydescr2) @@ -1462,8 +1408,7 @@ """ expected = """ [i0, p4] - i2 = ooisnull(p4) - guard_false(i2) [] + guard_nonnull(p4) [] escape(p4) # p3 = escape() @@ -1475,8 +1420,7 @@ def test_invalid_loop_1(self): ops = """ [p1] - i1 = ooisnull(p1) - guard_true(i1) [] + guard_isnull(p1) [] # p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) @@ -1501,8 +1445,7 @@ ops = """ [p1] p2 = getfield_gc(p1, descr=nextdescr) - i1 = ooisnull(p2) - guard_true(i1) [] + guard_isnull(p2) [] # p3 = new_with_vtable(ConstClass(node_vtable)) p4 = new_with_vtable(ConstClass(node_vtable)) Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 9 18:20:03 2009 @@ -158,8 +158,7 @@ ops = self.get_by_bytecode("LOAD_GLOBAL") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "getfield_gc", "ooisnull", - "guard_false"] + "getfield_gc", "guard_nonnull"] assert not ops[1] # second LOAD_GLOBAL folded away ops = self.get_by_bytecode("CALL_FUNCTION") assert len(ops) == 2 @@ -203,7 +202,7 @@ ops = self.get_by_bytecode("LOAD_ATTR") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "ooisnull", "guard_false"] + "guard_nonnull"] assert not ops[1] # second LOAD_ATTR folded away def test_default_and_kw(self): From arigo at codespeak.net Mon Nov 9 18:21:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 9 Nov 2009 18:21:23 +0100 (CET) Subject: [pypy-svn] r69103 - pypy/branch/guard-nonnull Message-ID: <20091109172123.A35CD16804D@codespeak.net> Author: arigo Date: Mon Nov 9 18:21:23 2009 New Revision: 69103 Removed: pypy/branch/guard-nonnull/ Log: Remove merged branch. From david at codespeak.net Mon Nov 9 21:25:51 2009 From: david at codespeak.net (david at codespeak.net) Date: Mon, 9 Nov 2009 21:25:51 +0100 (CET) Subject: [pypy-svn] r69104 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20091109202551.68467168049@codespeak.net> Author: david Date: Mon Nov 9 21:25:48 2009 New Revision: 69104 Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.py Log: fix parser hack to use new io binary Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parserhack.py (original) +++ pypy/branch/io-lang/pypy/lang/io/parserhack.py Mon Nov 9 21:25:48 2009 @@ -62,7 +62,7 @@ io_file = py.magic.autopath().dirpath().join("parserhack.io") def parse(input, space=None): - child_in, child_out_err = os.popen4("osxvm %s" % io_file) + child_in, child_out_err = os.popen4("io %s" % io_file) child_in.write(input) child_in.close() s = child_out_err.read().strip() From david at codespeak.net Mon Nov 9 21:28:41 2009 From: david at codespeak.net (david at codespeak.net) Date: Mon, 9 Nov 2009 21:28:41 +0100 (CET) Subject: [pypy-svn] r69105 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20091109202841.4F233168049@codespeak.net> Author: david Date: Mon Nov 9 21:28:40 2009 New Revision: 69105 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_number.py Log: fix number tests to work on python 2.5 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_number.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_number.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_number.py Mon Nov 9 21:28:40 2009 @@ -1,6 +1,15 @@ from pypy.lang.io.parserhack import interpret from pypy.lang.io.model import W_Number -from math import isnan, isinf +import sys +if sys.version_info < (2, 6): + def isnan(num): + return num != num + def isinf(num): + # not sure if this works with long + from sys import maxint + return num > maxint +else: + from math import isnan, isinf import py def test_even_simpler(): From david at codespeak.net Mon Nov 9 21:32:51 2009 From: david at codespeak.net (david at codespeak.net) Date: Mon, 9 Nov 2009 21:32:51 +0100 (CET) Subject: [pypy-svn] r69106 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20091109203251.F3D8C16804C@codespeak.net> Author: david Date: Mon Nov 9 21:32:51 2009 New Revision: 69106 Modified: pypy/branch/io-lang/pypy/lang/io/object.py Log: Fix Object setSlot to use correct target and context in generated methods Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Mon Nov 9 21:32:51 2009 @@ -181,9 +181,13 @@ def object_new_slot(space, w_target, name, w_value): from pypy.lang.io.model import W_CFunction w_target.slots[name] = w_value - def setSlot(space, w_target, w_message, w_context): - w_target.slots[name] = w_message.arguments[0].eval(space, w_context, w_context) - return w_target + + def setSlot(my_space, w_w_target, w_w_message, w_w_context): + w_w_target.slots[name] = w_w_message.arguments[0].eval(my_space, + w_w_context, + w_w_target) + return w_w_target + w_target.slots['set%s' % (name[0].capitalize() + name[1:])] = W_CFunction(space, setSlot) @register_method('Object', 'updateSlot', unwrap_spec=[object, str, object]) From david at codespeak.net Mon Nov 9 21:35:26 2009 From: david at codespeak.net (david at codespeak.net) Date: Mon, 9 Nov 2009 21:35:26 +0100 (CET) Subject: [pypy-svn] r69107 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20091109203526.91B8F168049@codespeak.net> Author: david Date: Mon Nov 9 21:35:25 2009 New Revision: 69107 Modified: pypy/branch/io-lang/pypy/lang/io/coroutine.py pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/test/test_coro.py pypy/branch/io-lang/pypy/lang/io/test/test_io_extensions.py Log: Fix some lookup related coroutine issues Modified: pypy/branch/io-lang/pypy/lang/io/coroutine.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/coroutine.py (original) +++ pypy/branch/io-lang/pypy/lang/io/coroutine.py Mon Nov 9 21:35:25 2009 @@ -9,24 +9,13 @@ @register_method('Coroutine', 'isCurrent') def coroutine_is_current(space, w_target, w_message, w_context): return space.newbool(w_target is W_Coroutine.w_getcurrent(space)) - - at register_method('Coroutine', 'setRunMessage', unwrap_spec=[object, object]) -def coroutine_setRunMessage(space, w_coroutine, w_message): - w_coroutine.slots['runMessage'] = w_message - return w_coroutine - + @register_method('Coroutine', 'run') def coroutine_run(space, w_target, w_message, w_context): - w_target.run(space, w_target, w_context) + # XXX check this, because w_target.run(space, w_context, w_context) also works, maybe missing some scenarios + w_target.run(space, w_target, w_target) return w_target -# @register_method('Coroutine', 'setResult', unwrap_spec=[object, object]) -# def coroutine_set_result(space, w_coro, w_result): -# print w_result -# w_coro.slots['result'] = w_result -# return w_coro -# - @register_method('Coroutine', 'resume') def coroutine_switch(space, w_target, w_message, w_context): w_target.switch() \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py (original) +++ pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py Mon Nov 9 21:35:25 2009 @@ -38,13 +38,13 @@ def run(self, space, w_receiver, w_context): if self.thunk is None: - t = IoThunk(space, self.slots['runMessage'], self, w_context) + t = IoThunk(space, self.slots['runMessage'], w_receiver, w_context) self.bind(t) p = self.lookup('parentCoroutine') if not space.isnil(p): self.parent = p - self.switch() + class AppCoState(BaseCoState): def __init__(self, space): @@ -53,7 +53,7 @@ def post_install(self): self.main = W_Coroutine(self.space, self, [self.space.w_object]) - self.current = self.main.clone() + self.current = self.main # self.current.slots['parentCoroutine'] = self.main class IoThunk(AbstractThunk): @@ -65,4 +65,4 @@ def call(self): t = self.w_message.eval(self.space, self.w_receiver, self.w_context) - self.w_receiver.slots['result'] = thunk \ No newline at end of file + self.w_receiver.slots['result'] = t \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Mon Nov 9 21:35:25 2009 @@ -44,6 +44,9 @@ def clone(self): return W_Object(self.space, [self]) + def __repr__(self): + return "" % (self.slots.keys(),) + class W_Number(W_Object): """Number""" def __init__(self, space, value, protos = None): Modified: pypy/branch/io-lang/pypy/lang/io/test/test_coro.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_coro.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_coro.py Mon Nov 9 21:35:25 2009 @@ -66,7 +66,6 @@ assert res.slots['result'].value == 99 def test_coro_parent_resume_switch(): - # py.test.skip() inp = """ back := currentCoro p := Coroutine currentCoroutine clone do( @@ -131,4 +130,74 @@ assert isinstance(res, W_Coroutine) assert res.slots['runTarget'] is space.w_lobby assert res.slots['runLocals'] is space.w_lobby - \ No newline at end of file + +def test_xxx0(): + py.test.skip() + inp = """ + Lobby p := Coroutine currentCoroutine clone do ( + name := "Coro" + ) + p setRunMessage(message(99)) + Lobby a := Coroutine currentCoroutine clone do ( + name := "Coro-a" + ) + a setParentCoroutine(p) + a setRunMessage(message(23)) + a run; + """ + res,space = interpret(inp) + assert space.w_lobby.slots['a'].slots['result'].value == 23 + assert space.w_lobby.slots['p'].slots['result'].value == 99 + + +def test_lookup_problem1(): + inp = """ + p := 4 + result ::= 99 + Object do ( + foobar ::= nil + ) + a := Object clone + a setFoobar(p) + Lobby setResult(a foobar) + result + """ + res,space = interpret(inp) + print res + assert res.value == 4 + +def test_lookup_problem2(): + inp = """ + p := 4 + result ::= 99 + try( + Object do ( + foobar ::= nil + ) + a := Object clone + a setFoobar(p) + Lobby setResult(a foobar) + ) + result + """ + res,space = interpret(inp) + assert res.value == 4 + + +def test_lookup_problem3(): + inp = """ + result ::= 99 + try( + p := 4 + Object do ( + foobar ::= nil + ) + a := Object clone + a setFoobar(p) + Lobby setResult(a foobar) + ) + result + """ + res,space = interpret(inp) + print res + assert res.value == 4 \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_io_extensions.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_io_extensions.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_io_extensions.py Mon Nov 9 21:35:25 2009 @@ -16,7 +16,7 @@ assert isinstance(res, W_Object) assert res.protos == [space.w_object] -def test_map_as_lit(): +def test_map_as_list(): py.test.skip("Depends on ==") inp = 'Map clone atPut("1", 12345) atPut("2", 99) atPut("3", 3) atPut("4", 234) asList' res, space = interpret(inp) @@ -43,4 +43,44 @@ a = space.w_lobby.slots['a'] assert 'timers' in a.slots assert 'setTimers' in a.slots - # assert isinstance(a.slots['setTimers'], W_Block) \ No newline at end of file + # assert isinstance(a.slots['setTimers'], W_Block) + + +def test_new_slot_with_lookup(): + inp = """ + q := 99 + a := Object clone do ( + timer ::= 1 + ) + a setTimer(q) + a timer + """ + res, space = interpret(inp) + assert res.value == 99 + a = space.w_lobby.slots['a'] + assert 'timer' in a.slots + assert 'setTimer' in a.slots + +def test_new_slot_with_var(): + inp = """ + p := 4 + a := Coroutine currentCoroutine do ( + foobar ::= p + ) + a foobar + """ + res,space = interpret(inp) + assert res.value == 4 + +def test_new_slot_with_var2(): + inp = """ + p := 4 + q := 23 + a := Coroutine currentCoroutine do ( + foobar ::= p + ) + a setFoobar(q) + a foobar + """ + res,space = interpret(inp) + assert res.value == 23 \ No newline at end of file From fijal at codespeak.net Tue Nov 10 11:51:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Nov 2009 11:51:18 +0100 (CET) Subject: [pypy-svn] r69110 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091110105118.CB673168075@codespeak.net> Author: fijal Date: Tue Nov 10 11:51:17 2009 New Revision: 69110 Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Log: Fix this test Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Tue Nov 10 11:51:17 2009 @@ -363,8 +363,8 @@ jitcode = cw.make_one_bytecode((graphs[0], None), False) assert 'ptr_eq' not in jitcode._source assert 'ptr_ne' not in jitcode._source - assert jitcode._source.count('ptr_iszero') == 2 - assert jitcode._source.count('ptr_nonzero') == 2 + assert jitcode._source.count('oononnull') == 2 + assert jitcode._source.count('ooisnull') == 2 class ImmutableFieldsTests: From fijal at codespeak.net Tue Nov 10 13:31:43 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Nov 2009 13:31:43 +0100 (CET) Subject: [pypy-svn] r69111 - in pypy/branch/gc-dump-heap/pypy/module/gc: . test Message-ID: <20091110123143.D48EF168075@codespeak.net> Author: fijal Date: Tue Nov 10 13:31:42 2009 New Revision: 69111 Modified: pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py Log: Rename operation here as well (missing commit) Modified: pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/__init__.py Tue Nov 10 13:31:42 2009 @@ -12,5 +12,5 @@ 'disable_finalizers': 'interp_gc.disable_finalizers', 'estimate_heap_size': 'interp_gc.estimate_heap_size', 'garbage' : 'space.newlist([])', - 'dump_heap': 'interp_gc.dump_heap', + 'dump_heap_stats': 'interp_gc.dump_heap_stats', } Modified: pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/interp_gc.py Tue Nov 10 13:31:42 2009 @@ -54,12 +54,14 @@ space.wrap("can't estimate the heap size")) estimate_heap_size.unwrap_spec = [ObjSpace] -def dump_heap(space, filename): - tb = rgc._dump_heap() +def dump_heap_stats(space, filename): + tb = rgc._heap_stats() + if not tb: + raise OperationError(space.w_RuntimeError, + space.wrap("Wrong GC")) f = open_file_as_stream(filename, mode="w") for i in range(len(tb)): f.write("%d %d " % (tb[i].count, tb[i].size)) f.write(",".join([str(tb[i].links[j]) for j in range(len(tb))]) + "\n") f.close() - -dump_heap.unwrap_spec = [ObjSpace, str] +dump_heap_stats.unwrap_spec = [ObjSpace, str] Modified: pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py (original) +++ pypy/branch/gc-dump-heap/pypy/module/gc/test/test_gc.py Tue Nov 10 13:31:42 2009 @@ -85,11 +85,11 @@ self.size = size self.links = links - def fake_dump_heap(): + def fake_heap_stats(): return [X(1, 12, [0, 0]), X(2, 10, [10, 0])] - cls._dump_heap = rgc._dump_heap - rgc._dump_heap = fake_dump_heap + cls._heap_stats = rgc._heap_stats + rgc._heap_stats = fake_heap_stats fname = udir.join('gcdump.log') cls.space = gettestobjspace() cls.w_fname = cls.space.wrap(str(fname)) @@ -99,10 +99,10 @@ import py from pypy.rlib import rgc - rgc._dump_heap = cls._dump_heap + rgc._heap_stats = cls._heap_stats assert py.path.local(cls._fname).read() == '1 12 0,0\n2 10 10,0\n' - def test_gc_dump_heap(self): + def test_gc_heap_stats(self): import gc - gc.dump_heap(self.fname) + gc.dump_heap_stats(self.fname) From fijal at codespeak.net Tue Nov 10 13:35:43 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 10 Nov 2009 13:35:43 +0100 (CET) Subject: [pypy-svn] r69112 - pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem Message-ID: <20091110123543.7435B168077@codespeak.net> Author: fijal Date: Tue Nov 10 13:35:42 2009 New Revision: 69112 Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py Log: Fix test_lloperation Modified: pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/gc-dump-heap/pypy/rpython/lltypesystem/opimpl.py Tue Nov 10 13:35:42 2009 @@ -466,6 +466,9 @@ return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset) op_gc_gettypeptr_group.need_result_type = True +def op_get_member_index(memberoffset): + raise NotImplementedError + # ____________________________________________________________ def get_op_impl(opname): From arigo at codespeak.net Tue Nov 10 13:46:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Nov 2009 13:46:25 +0100 (CET) Subject: [pypy-svn] r69113 - in pypy/trunk/pypy: doc/jit module/gc module/gc/test rlib rpython rpython/lltypesystem rpython/memory rpython/memory/gc rpython/memory/gctransform rpython/memory/test tool translator/c/test Message-ID: <20091110124625.52F7C1680FA@codespeak.net> Author: arigo Date: Tue Nov 10 13:46:24 2009 New Revision: 69113 Added: pypy/trunk/pypy/tool/gcdump.py - copied unchanged from r69112, pypy/branch/gc-dump-heap/pypy/tool/gcdump.py Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt pypy/trunk/pypy/module/gc/__init__.py pypy/trunk/pypy/module/gc/interp_gc.py pypy/trunk/pypy/module/gc/test/test_gc.py pypy/trunk/pypy/rlib/rgc.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/rpython/lltypesystem/opimpl.py pypy/trunk/pypy/rpython/memory/gc/base.py pypy/trunk/pypy/rpython/memory/gc/generation.py pypy/trunk/pypy/rpython/memory/gc/semispace.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/transform.py pypy/trunk/pypy/rpython/memory/gctypelayout.py pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py pypy/trunk/pypy/translator/c/test/test_boehm.py pypy/trunk/pypy/translator/c/test/test_newgc.py Log: Merge the branch 'gc-dump-heap', adding rgc._heap_stats() that forces a dump of stats about object usage, classified per type id. Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Tue Nov 10 13:46:24 2009 @@ -32,6 +32,7 @@ longer suitable. can_enter_jit goes at the end of a application level loop. In the Python interpreter, this is the JUMP_ABSOLUTE bytecode. The Python interpreter defines its hints in pypy/module/pypyjit/interp_jit.py. +XXX by overloading default frame behavior from pyframe The interpreter wishing to use the PyPy's JIT must define a list of *green* variables and a list of *red* variables. The *green* variables are loop @@ -98,6 +99,8 @@ contain values that may change during the running of a loop. There are three kinds of normal boxes: BoxInt, BoxPtr, and BoxFloat, and four kinds of constant boxes: ConstInt, ConstPtr, ConstFloat, and ConstAddr. +XXX ConstAddr is only a hack for translation not to translate pointers, +XXX nothing more The meta-interpreter starts interpreting the JIT bytecode. Each operation is executed and then recorded in a list of operations, called the trace. Modified: pypy/trunk/pypy/module/gc/__init__.py ============================================================================== --- pypy/trunk/pypy/module/gc/__init__.py (original) +++ pypy/trunk/pypy/module/gc/__init__.py Tue Nov 10 13:46:24 2009 @@ -12,4 +12,5 @@ 'disable_finalizers': 'interp_gc.disable_finalizers', 'estimate_heap_size': 'interp_gc.estimate_heap_size', 'garbage' : 'space.newlist([])', + 'dump_heap_stats': 'interp_gc.dump_heap_stats', } Modified: pypy/trunk/pypy/module/gc/interp_gc.py ============================================================================== --- pypy/trunk/pypy/module/gc/interp_gc.py (original) +++ pypy/trunk/pypy/module/gc/interp_gc.py Tue Nov 10 13:46:24 2009 @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import ObjSpace from pypy.interpreter.error import OperationError from pypy.rlib import rgc +from pypy.rlib.streamio import open_file_as_stream def collect(space): "Run a full collection." @@ -52,3 +53,15 @@ raise OperationError(space.w_RuntimeError, space.wrap("can't estimate the heap size")) estimate_heap_size.unwrap_spec = [ObjSpace] + +def dump_heap_stats(space, filename): + tb = rgc._heap_stats() + if not tb: + raise OperationError(space.w_RuntimeError, + space.wrap("Wrong GC")) + f = open_file_as_stream(filename, mode="w") + for i in range(len(tb)): + f.write("%d %d " % (tb[i].count, tb[i].size)) + f.write(",".join([str(tb[i].links[j]) for j in range(len(tb))]) + "\n") + f.close() +dump_heap_stats.unwrap_spec = [ObjSpace, str] Modified: pypy/trunk/pypy/module/gc/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/module/gc/test/test_gc.py (original) +++ pypy/trunk/pypy/module/gc/test/test_gc.py Tue Nov 10 13:46:24 2009 @@ -1,3 +1,5 @@ +from pypy.conftest import gettestobjspace + class AppTestGC(object): def test_collect(self): import gc @@ -72,4 +74,35 @@ assert gc.isenabled() gc.enable() assert gc.isenabled() + +class AppTestGcDumpHeap(object): + def setup_class(cls): + from pypy.tool.udir import udir + from pypy.rlib import rgc + class X(object): + def __init__(self, count, size, links): + self.count = count + self.size = size + self.links = links + + def fake_heap_stats(): + return [X(1, 12, [0, 0]), X(2, 10, [10, 0])] + cls._heap_stats = rgc._heap_stats + rgc._heap_stats = fake_heap_stats + fname = udir.join('gcdump.log') + cls.space = gettestobjspace() + cls.w_fname = cls.space.wrap(str(fname)) + cls._fname = fname + + def teardown_class(cls): + import py + from pypy.rlib import rgc + + rgc._heap_stats = cls._heap_stats + assert py.path.local(cls._fname).read() == '1 12 0,0\n2 10 10,0\n' + + def test_gc_heap_stats(self): + import gc + gc.dump_heap_stats(self.fname) + Modified: pypy/trunk/pypy/rlib/rgc.py ============================================================================== --- pypy/trunk/pypy/rlib/rgc.py (original) +++ pypy/trunk/pypy/rlib/rgc.py Tue Nov 10 13:46:24 2009 @@ -184,6 +184,25 @@ hop.exception_cannot_occur() return hop.genop('gc_can_move', hop.args_v, resulttype=hop.r_result) +def _heap_stats(): + raise NotImplementedError # can't be run directly + +class DumpHeapEntry(ExtRegistryEntry): + _about_ = _heap_stats + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + from pypy.rpython.lltypesystem import lltype + return annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + from pypy.rpython.lltypesystem import lltype + hop.exception_is_here() + return hop.genop('gc_heap_stats', [], resulttype=hop.r_result) + def malloc_nonmovable(TP, n=None, zero=False): """ Allocate a non-moving buffer or return nullptr. When running directly, will pretend that gc is always Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Tue Nov 10 13:46:24 2009 @@ -810,6 +810,9 @@ def op_gc_assume_young_pointers(self, addr): raise NotImplementedError + def op_gc_heap_stats(self): + raise NotImplementedError + def op_gc_obtain_free_space(self, size): raise NotImplementedError Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Tue Nov 10 13:46:24 2009 @@ -422,6 +422,7 @@ 'extract_ushort': LLOp(canfold=True), 'combine_ushort': LLOp(canfold=True), 'gc_gettypeptr_group': LLOp(canfold=True), + 'get_member_index': LLOp(canfold=True), # __________ used by the JIT ________ @@ -460,6 +461,7 @@ 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), 'gc_assume_young_pointers': LLOp(), + 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Tue Nov 10 13:46:24 2009 @@ -483,6 +483,9 @@ return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset) op_gc_gettypeptr_group.need_result_type = True +def op_get_member_index(memberoffset): + raise NotImplementedError + # ____________________________________________________________ def get_op_impl(opname): Modified: pypy/trunk/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/base.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/base.py Tue Nov 10 13:46:24 2009 @@ -6,6 +6,11 @@ from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), + ('size', lltype.Signed), + ('links', lltype.Array(lltype.Signed))) +ARRAY_TYPEID_MAP = lltype.GcArray(lltype.Ptr(TYPEID_MAP)) + class GCBase(object): _alloc_flavor_ = "raw" moving_gc = False @@ -48,7 +53,8 @@ varsize_offset_to_variable_part, varsize_offset_to_length, varsize_offsets_to_gcpointers_in_var_part, - weakpointer_offset): + weakpointer_offset, + member_index): self.getfinalizer = getfinalizer self.is_varsize = is_varsize self.has_gcptr_in_varsize = has_gcptr_in_varsize @@ -60,6 +66,10 @@ self.varsize_offset_to_length = varsize_offset_to_length self.varsize_offsets_to_gcpointers_in_var_part = varsize_offsets_to_gcpointers_in_var_part self.weakpointer_offset = weakpointer_offset + self.member_index = member_index + + def get_member_index(self, type_id): + return self.member_index(type_id) def set_root_walker(self, root_walker): self.root_walker = root_walker Modified: pypy/trunk/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/generation.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/generation.py Tue Nov 10 13:46:24 2009 @@ -511,6 +511,17 @@ def _id_grow_older(self, obj, id, ignored): self.objects_with_id.setitem(obj, id) + def heap_stats_walk_roots(self): + self.last_generation_root_objects.foreach( + self._track_heap_ext, None) + self.root_walker.walk_roots( + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root) + + def _track_heap_ext(self, adr, ignored): + self.trace(adr, self.track_heap_parent, adr) + def debug_check_object(self, obj): """Check the invariants about 'obj' that should be true between collections.""" @@ -564,6 +575,7 @@ else: SemiSpaceGC.debug_check_can_copy(self, obj) + # ____________________________________________________________ import os Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Tue Nov 10 13:46:24 2009 @@ -10,7 +10,8 @@ from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck -from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc.base import MovingGCBase, ARRAY_TYPEID_MAP,\ + TYPEID_MAP import sys, os @@ -25,7 +26,6 @@ memoryError = MemoryError() - class SemiSpaceGC(MovingGCBase): _alloc_flavor_ = "raw" inline_simple_malloc = True @@ -623,3 +623,47 @@ hdr.tid |= GCFLAG_HASHTAKEN # return llmemory.cast_adr_to_int(obj) # direct case + + def track_heap_parent(self, obj, parent): + addr = obj.address[0] + parent_idx = llop.get_member_index(lltype.Signed, + self.get_type_id(parent)) + idx = llop.get_member_index(lltype.Signed, self.get_type_id(addr)) + self._ll_typeid_map[parent_idx].links[idx] += 1 + self.track_heap(addr) + + def track_heap(self, adr): + if self._tracked_dict.contains(adr): + return + self._tracked_dict.add(adr) + idx = llop.get_member_index(lltype.Signed, self.get_type_id(adr)) + self._ll_typeid_map[idx].count += 1 + totsize = self.get_size(adr) + self.size_gc_header() + self._ll_typeid_map[idx].size += llmemory.raw_malloc_usage(totsize) + self.trace(adr, self.track_heap_parent, adr) + + def _track_heap_root(self, root): + self.track_heap(root.address[0]) + + def heap_stats_walk_roots(self): + self.root_walker.walk_roots( + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root, + SemiSpaceGC._track_heap_root) + + def heap_stats(self): + self._tracked_dict = self.AddressDict() + max_tid = self.root_walker.gcdata.max_type_id + ll_typeid_map = lltype.malloc(ARRAY_TYPEID_MAP, max_tid, zero=True) + for i in range(max_tid): + ll_typeid_map[i] = lltype.malloc(TYPEID_MAP, max_tid, zero=True) + self._ll_typeid_map = ll_typeid_map + self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map)) + i = 0 + while i < max_tid: + self._tracked_dict.add(llmemory.cast_ptr_to_adr(ll_typeid_map[i])) + i += 1 + self.heap_stats_walk_roots() + self._ll_typeid_map = lltype.nullptr(ARRAY_TYPEID_MAP) + self._tracked_dict.delete() + return ll_typeid_map Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Nov 10 13:46:24 2009 @@ -24,7 +24,6 @@ TYPE_ID = rffi.USHORT - class CollectAnalyzer(graphanalyze.BoolGraphAnalyzer): def analyze_direct_call(self, graph, seen=None): @@ -117,6 +116,7 @@ def __init__(self, translator): from pypy.rpython.memory.gc.base import choose_gc_from_config + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP super(FrameworkGCTransformer, self).__init__(translator, inline=True) if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes @@ -147,6 +147,7 @@ gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcend = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() + gcdata.max_type_id = 13 # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} @@ -183,6 +184,9 @@ data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) + data_classdef.generalize_attr( + 'max_type_id', + annmodel.SomeInteger()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) @@ -262,6 +266,15 @@ [s_gc, annmodel.SomeAddress()], annmodel.s_None) + if hasattr(GCClass, 'heap_stats'): + self.heap_stats_ptr = getfn(GCClass.heap_stats.im_func, + [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)), + minimal_transform=False) + self.get_member_index_ptr = getfn( + GCClass.get_member_index.im_func, + [s_gc, annmodel.SomeInteger(knowntype=rffi.r_ushort)], + annmodel.SomeInteger()) + # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) if getattr(GCClass, 'inline_simple_malloc', False): @@ -492,14 +505,15 @@ ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address), len(addresses_of_static_ptrs), immortal=True) + for i in range(len(addresses_of_static_ptrs)): ll_static_roots_inside[i] = addresses_of_static_ptrs[i] ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)) ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc) ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs) - newgcdependencies = [] newgcdependencies.append(ll_static_roots_inside) + ll_instance.inst_max_type_id = len(group.members) self.write_typeid_list() return newgcdependencies @@ -634,6 +648,21 @@ hop.genop("direct_call", [self.assume_young_pointers_ptr, self.c_const_gc, v_addr]) + def gct_gc_heap_stats(self, hop): + if not hasattr(self, 'heap_stats_ptr'): + return GCTransformer.gct_gc_heap_stats(self, hop) + op = hop.spaceop + livevars = self.push_roots(hop) + hop.genop("direct_call", [self.heap_stats_ptr, self.c_const_gc], + resultvar=op.result) + self.pop_roots(hop, livevars) + + def gct_get_member_index(self, hop): + op = hop.spaceop + v_typeid = op.args[0] + hop.genop("direct_call", [self.get_member_index_ptr, self.c_const_gc, + v_typeid], resultvar=op.result) + def gct_gc_adr_of_nursery_free(self, hop): if getattr(self.gcdata.gc, 'nursery_free', None) is None: raise NotImplementedError("gc_adr_of_nursery_free only for generational gcs") Modified: pypy/trunk/pypy/rpython/memory/gctransform/transform.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/transform.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/transform.py Tue Nov 10 13:46:24 2009 @@ -388,6 +388,11 @@ # this assumes a non-moving GC. Moving GCs need to override this hop.rename('cast_ptr_to_int') + def gct_gc_heap_stats(self, hop): + from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP + + return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP), + lltype.nullptr(ARRAY_TYPEID_MAP))) class MinimalGCTransformer(BaseGCTransformer): def __init__(self, parenttransformer): Modified: pypy/trunk/pypy/rpython/memory/gctypelayout.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctypelayout.py (original) +++ pypy/trunk/pypy/rpython/memory/gctypelayout.py Tue Nov 10 13:46:24 2009 @@ -93,6 +93,10 @@ return weakptr_offset return -1 + def q_member_index(self, typeid): + infobits = self.get(typeid).infobits + return infobits & T_MEMBER_INDEX + def set_query_functions(self, gc): gc.set_query_functions( self.q_is_varsize, @@ -105,23 +109,26 @@ self.q_varsize_offset_to_variable_part, self.q_varsize_offset_to_length, self.q_varsize_offsets_to_gcpointers_in_var_part, - self.q_weakpointer_offset) + self.q_weakpointer_offset, + self.q_member_index) -T_IS_VARSIZE = 0x01 -T_HAS_GCPTR_IN_VARSIZE = 0x02 -T_IS_GCARRAY_OF_GCPTR = 0x04 -T_IS_WEAKREF = 0x08 +# the lowest 16bits are used to store group member index +T_MEMBER_INDEX = 0xffff +T_IS_VARSIZE = 0x10000 +T_HAS_GCPTR_IN_VARSIZE = 0x20000 +T_IS_GCARRAY_OF_GCPTR = 0x40000 +T_IS_WEAKREF = 0x80000 def _check_typeid(typeid): ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid), "invalid type_id") -def encode_type_shape(builder, info, TYPE): +def encode_type_shape(builder, info, TYPE, index): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) - infobits = 0 + infobits = index info.ofstoptrs = builder.offsets2table(offsets, TYPE) info.finalizer = builder.make_finalizer_funcptr_for_type(TYPE) if not TYPE._is_varsize(): @@ -212,12 +219,12 @@ fullinfo = lltype.malloc(GCData.VARSIZE_TYPE_INFO, immortal=True, zero=True) info = fullinfo.header + type_id = self.type_info_group.add_member(fullinfo) if self.can_encode_type_shape: - encode_type_shape(self, info, TYPE) + encode_type_shape(self, info, TYPE, type_id.index) else: - self._pending_type_shapes.append((info, TYPE)) + self._pending_type_shapes.append((info, TYPE, type_id.index)) # store it - type_id = self.type_info_group.add_member(fullinfo) self.id_of_type[TYPE] = type_id self.add_vtable_after_typeinfo(TYPE) return type_id @@ -257,8 +264,8 @@ def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True - for info, TYPE in self._pending_type_shapes: - encode_type_shape(self, info, TYPE) + for info, TYPE, index in self._pending_type_shapes: + encode_type_shape(self, info, TYPE, index) del self._pending_type_shapes def delay_encoding(self): Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Tue Nov 10 13:46:24 2009 @@ -4,7 +4,7 @@ from pypy.translator.c import gc from pypy.annotation import model as annmodel from pypy.annotation import policy as annpolicy -from pypy.rpython.lltypesystem import lltype, llmemory, llarena +from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi from pypy.rpython.memory.gctransform import framework from pypy.rpython.lltypesystem.lloperation import llop, void from pypy.rpython.memory.gc.marksweep import X_CLONE, X_POOL, X_POOL_PTR @@ -13,6 +13,7 @@ from pypy.rlib import rgc from pypy import conftest from pypy.rlib.rstring import StringBuilder +from pypy.rlib.objectmodel import keepalive_until_here INT_SIZE = struct.calcsize("i") # only for estimates @@ -794,6 +795,55 @@ run = self.runner("do_malloc_operations_in_call") run([]) + def define_gc_heap_stats(cls): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + l1 = [] + l2 = [] + l3 = [] + l4 = [] + + def f(): + for i in range(10): + s = lltype.malloc(S) + l1.append(s) + l2.append(s) + if i < 3: + l3.append(s) + l4.append(s) + # We cheat here and only read the table which we later on + # process ourselves, otherwise this test takes ages + llop.gc__collect(lltype.Void) + tb = rgc._heap_stats() + a = 0 + nr = 0 + b = 0 + c = 0 + d = 0 + e = 0 + for i in range(len(tb)): + if tb[i].count == 10: + a += 1 + nr = i + if tb[i].count > 50: + d += 1 + for i in range(len(tb)): + if tb[i].count == 4: + b += 1 + c += tb[i].links[nr] + e += tb[i].size + return d * 1000 + c * 100 + b * 10 + a + return f + + def test_gc_heap_stats(self): + run = self.runner("gc_heap_stats") + res = run([]) + assert res % 10000 == 2611 + totsize = (res / 10000) + size_of_int = rffi.sizeof(lltype.Signed) + assert (totsize - 26 * size_of_int) % 4 == 0 + # ^^^ a crude assumption that totsize - varsize would be dividable by 4 + # (and give fixedsize) + # ________________________________________________________________ class TestMarkSweepGC(GenericGCTests): Modified: pypy/trunk/pypy/translator/c/test/test_boehm.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_boehm.py (original) +++ pypy/trunk/pypy/translator/c/test/test_boehm.py Tue Nov 10 13:46:24 2009 @@ -380,6 +380,15 @@ c_fn = self.getcompiled(fn, []) assert c_fn() == False + def test_heap_stats(self): + from pypy.rlib import rgc + + def fn(): + return bool(rgc._heap_stats()) + + c_fn = self.getcompiled(fn, []) + assert not c_fn() + def test_malloc_nonmovable(self): TP = lltype.GcArray(lltype.Char) def func(): Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Tue Nov 10 13:46:24 2009 @@ -898,7 +898,41 @@ def test_gc_set_max_heap_size(self): res = self.run('gc_set_max_heap_size') assert res == 2 + + def define_gc_heap_stats(cls): + S = lltype.GcStruct('S', ('x', lltype.Signed)) + l1 = [] + l2 = [] + l3 = [] + def f(): + for i in range(10): + s = lltype.malloc(S) + l1.append(s) + l2.append(s) + l3.append(s) + tb = rgc._heap_stats() + a = 0 + nr = 0 + b = 0 + c = 0 + for i in range(len(tb)): + if tb[i].count == 10: + a += 1 + nr = i + for i in range(len(tb)): + if tb[i].count == 3: + b += 1 + c += tb[i].links[nr] + # we don't count b here since there can be more singletons, + # important one is c, a is for check + return c * 100 + b * 10 + a + return f + + def test_gc_heap_stats(self): + res = self.run("gc_heap_stats") + assert res == 3011 + def definestr_string_builder(cls): def fn(_): s = StringBuilder() From arigo at codespeak.net Tue Nov 10 13:46:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Nov 2009 13:46:46 +0100 (CET) Subject: [pypy-svn] r69114 - pypy/branch/gc-dump-heap Message-ID: <20091110124646.6AC111680FE@codespeak.net> Author: arigo Date: Tue Nov 10 13:46:45 2009 New Revision: 69114 Removed: pypy/branch/gc-dump-heap/ Log: Remove merged branch. From hpk at codespeak.net Tue Nov 10 15:11:23 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 15:11:23 +0100 (CET) Subject: [pypy-svn] r69115 - pypy/branch/py11 Message-ID: <20091110141123.24BC1168104@codespeak.net> Author: hpk Date: Tue Nov 10 15:11:22 2009 New Revision: 69115 Added: pypy/branch/py11/ - copied from r69112, pypy/trunk/ Log: opening a py-1.1.0 integration branch From arigo at codespeak.net Tue Nov 10 15:19:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Nov 2009 15:19:27 +0100 (CET) Subject: [pypy-svn] r69117 - pypy/branch/merge-guards-2 Message-ID: <20091110141927.E3D66168104@codespeak.net> Author: arigo Date: Tue Nov 10 15:19:27 2009 New Revision: 69117 Added: pypy/branch/merge-guards-2/ - copied from r69116, pypy/trunk/ Log: A branch in which to merge guard_nonnull/guard_class/guard_value. From hpk at codespeak.net Tue Nov 10 15:38:04 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 15:38:04 +0100 (CET) Subject: [pypy-svn] r69118 - pypy/branch/py11 Message-ID: <20091110143804.76533168104@codespeak.net> Author: hpk Date: Tue Nov 10 15:38:03 2009 New Revision: 69118 Modified: pypy/branch/py11/ (props changed) Log: strike py-lib external dependency From hpk at codespeak.net Tue Nov 10 16:01:03 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 16:01:03 +0100 (CET) Subject: [pypy-svn] r69120 - in pypy/branch/py11/pypy: . annotation/test bin config config/test doc doc/config doc/statistic jit/backend jit/backend/x86 jit/tl jit/tl/spli jit/tl/test jit/tool lang/gameboy/tool lang/js lang/js/test/ecma lang/prolog/interpreter lang/scheme lang/smalltalk/tool lib/distributed lib/test2 module/__builtin__/test module/_codecs/test module/_sre/test module/sys/test rlib/parsing/test rlib/rsdl/test rpython/microbench tool tool/algo/test tool/pytest tool/rest tool/test translator translator/benchmark translator/c translator/c/test translator/cli/test translator/goal translator/goal/test2 translator/microbench/pybench translator/platform translator/sandbox translator/sandbox/test translator/test translator/tool Message-ID: <20091110150103.B2407168105@codespeak.net> Author: hpk Date: Tue Nov 10 16:01:00 2009 New Revision: 69120 Added: pypy/branch/py11/pypy/tool/difftime.py (contents, props changed) pypy/branch/py11/pypy/tool/rest/ pypy/branch/py11/pypy/tool/rest/__init__.py (contents, props changed) pypy/branch/py11/pypy/tool/rest/convert.py (contents, props changed) pypy/branch/py11/pypy/tool/rest/directive.py (contents, props changed) pypy/branch/py11/pypy/tool/rest/rest.py (contents, props changed) pypy/branch/py11/pypy/tool/rest/rst.py (contents, props changed) Modified: pypy/branch/py11/pypy/annotation/test/autopath.py pypy/branch/py11/pypy/bin/autopath.py pypy/branch/py11/pypy/config/autopath.py pypy/branch/py11/pypy/config/makerestdoc.py pypy/branch/py11/pypy/config/test/test_makerestdoc.py pypy/branch/py11/pypy/conftest.py pypy/branch/py11/pypy/doc/config/autopath.py pypy/branch/py11/pypy/doc/confrest_oldpy.py pypy/branch/py11/pypy/doc/statistic/confrest.py pypy/branch/py11/pypy/jit/backend/autopath.py pypy/branch/py11/pypy/jit/backend/x86/autopath.py pypy/branch/py11/pypy/jit/tl/autopath.py pypy/branch/py11/pypy/jit/tl/spli/autopath.py pypy/branch/py11/pypy/jit/tl/test/test_pypyjit.py pypy/branch/py11/pypy/jit/tool/autopath.py pypy/branch/py11/pypy/lang/gameboy/tool/autopath.py pypy/branch/py11/pypy/lang/js/autopath.py pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py pypy/branch/py11/pypy/lang/prolog/interpreter/autopath.py pypy/branch/py11/pypy/lang/scheme/autopath.py pypy/branch/py11/pypy/lang/smalltalk/tool/autopath.py pypy/branch/py11/pypy/lib/distributed/socklayer.py pypy/branch/py11/pypy/lib/test2/autopath.py pypy/branch/py11/pypy/module/__builtin__/test/autopath.py pypy/branch/py11/pypy/module/_codecs/test/autopath.py pypy/branch/py11/pypy/module/_sre/test/autopath.py pypy/branch/py11/pypy/module/_sre/test/test_app_sre.py pypy/branch/py11/pypy/module/sys/test/autopath.py pypy/branch/py11/pypy/rlib/parsing/test/autopath.py pypy/branch/py11/pypy/rlib/rsdl/test/autopath.py pypy/branch/py11/pypy/rpython/microbench/autopath.py pypy/branch/py11/pypy/test_all.py pypy/branch/py11/pypy/tool/algo/test/autopath.py pypy/branch/py11/pypy/tool/ansi_mandelbrot.py pypy/branch/py11/pypy/tool/ansi_print.py pypy/branch/py11/pypy/tool/autopath.py pypy/branch/py11/pypy/tool/genstatistic.py pypy/branch/py11/pypy/tool/pytest/appsupport.py pypy/branch/py11/pypy/tool/pytest/autopath.py pypy/branch/py11/pypy/tool/statistic_over_time.py pypy/branch/py11/pypy/tool/test/autopath.py pypy/branch/py11/pypy/tool/test/test_pytestsupport.py pypy/branch/py11/pypy/translator/autopath.py pypy/branch/py11/pypy/translator/benchmark/autopath.py pypy/branch/py11/pypy/translator/c/autopath.py pypy/branch/py11/pypy/translator/c/test/autopath.py pypy/branch/py11/pypy/translator/cli/test/autopath.py pypy/branch/py11/pypy/translator/goal/autopath.py pypy/branch/py11/pypy/translator/goal/test2/autopath.py pypy/branch/py11/pypy/translator/microbench/pybench/autopath.py pypy/branch/py11/pypy/translator/platform/__init__.py pypy/branch/py11/pypy/translator/sandbox/autopath.py pypy/branch/py11/pypy/translator/sandbox/test/autopath.py pypy/branch/py11/pypy/translator/test/autopath.py pypy/branch/py11/pypy/translator/tool/autopath.py Log: * put some internal rest-related and doc-generation files that pypy relied on into pypy/tool/rest * use pytest-1.1' py.impl.* instead of py.__ magic * have autopath look for "pypy" for import tweaks Modified: pypy/branch/py11/pypy/annotation/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/annotation/test/autopath.py (original) +++ pypy/branch/py11/pypy/annotation/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/bin/autopath.py ============================================================================== --- pypy/branch/py11/pypy/bin/autopath.py (original) +++ pypy/branch/py11/pypy/bin/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/config/autopath.py ============================================================================== --- pypy/branch/py11/pypy/config/autopath.py (original) +++ pypy/branch/py11/pypy/config/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/config/makerestdoc.py ============================================================================== --- pypy/branch/py11/pypy/config/makerestdoc.py (original) +++ pypy/branch/py11/pypy/config/makerestdoc.py Tue Nov 10 16:01:00 2009 @@ -1,6 +1,6 @@ import py -from py.__.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link -from py.__.rest.rst import Directive, Em, Quote, Text +from pypy.tool.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link +from pypy.tool.rest.rst import Directive, Em, Quote, Text from pypy.config.config import ChoiceOption, BoolOption, StrOption, IntOption from pypy.config.config import FloatOption, OptionDescription, Option, Config @@ -212,7 +212,7 @@ """ register a :config: ReST link role for use in documentation. """ try: from docutils.parsers.rst import directives, states, roles - from py.__.rest.directive import register_linkrole + from py.impl.rest.directive import register_linkrole except ImportError: return # enable :config: link role Modified: pypy/branch/py11/pypy/config/test/test_makerestdoc.py ============================================================================== --- pypy/branch/py11/pypy/config/test/test_makerestdoc.py (original) +++ pypy/branch/py11/pypy/config/test/test_makerestdoc.py Tue Nov 10 16:01:00 2009 @@ -1,7 +1,7 @@ from pypy.config.config import * from pypy.config.makerestdoc import make_cmdline_overview -from py.__.misc.rest import process as restcheck +from pypy.tool.rest.rest import process as restcheck tempdir = py.test.ensuretemp('config') Modified: pypy/branch/py11/pypy/conftest.py ============================================================================== --- pypy/branch/py11/pypy/conftest.py (original) +++ pypy/branch/py11/pypy/conftest.py Tue Nov 10 16:01:00 2009 @@ -1,5 +1,5 @@ import py, sys, os -from py.__.test.outcome import Failed +from py.impl.test.outcome import Failed from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport Modified: pypy/branch/py11/pypy/doc/config/autopath.py ============================================================================== --- pypy/branch/py11/pypy/doc/config/autopath.py (original) +++ pypy/branch/py11/pypy/doc/config/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/doc/confrest_oldpy.py ============================================================================== --- pypy/branch/py11/pypy/doc/confrest_oldpy.py (original) +++ pypy/branch/py11/pypy/doc/confrest_oldpy.py Tue Nov 10 16:01:00 2009 @@ -1,6 +1,6 @@ import py -from py.__.misc.rest import convert_rest_html, strip_html_header -from py.__.misc.difftime import worded_time +from pypy.tool.rest.rest import convert_rest_html, strip_html_header +from pypy.tool.difftime import worded_time html = py.xml.html Modified: pypy/branch/py11/pypy/doc/statistic/confrest.py ============================================================================== --- pypy/branch/py11/pypy/doc/statistic/confrest.py (original) +++ pypy/branch/py11/pypy/doc/statistic/confrest.py Tue Nov 10 16:01:00 2009 @@ -1,5 +1,5 @@ import py -from py.__.doc.confrest import * +from py.impl.doc.confrest import * class PyPyPage(Page): def fill_menubar(self): Modified: pypy/branch/py11/pypy/jit/backend/autopath.py ============================================================================== --- pypy/branch/py11/pypy/jit/backend/autopath.py (original) +++ pypy/branch/py11/pypy/jit/backend/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/jit/backend/x86/autopath.py ============================================================================== --- pypy/branch/py11/pypy/jit/backend/x86/autopath.py (original) +++ pypy/branch/py11/pypy/jit/backend/x86/autopath.py Tue Nov 10 16:01:00 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/branch/py11/pypy/jit/tl/autopath.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/autopath.py (original) +++ pypy/branch/py11/pypy/jit/tl/autopath.py Tue Nov 10 16:01:00 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/branch/py11/pypy/jit/tl/spli/autopath.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/spli/autopath.py (original) +++ pypy/branch/py11/pypy/jit/tl/spli/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/jit/tl/test/test_pypyjit.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/test/test_pypyjit.py (original) +++ pypy/branch/py11/pypy/jit/tl/test/test_pypyjit.py Tue Nov 10 16:01:00 2009 @@ -21,7 +21,7 @@ def check_crasher(func_name): try: JIT_EXECUTABLE.sysexec(CRASH_FILE, func_name) - except py.__.process.cmdexec.ExecutionFailed, e: + except py.impl.process.cmdexec.ExecutionFailed, e: print "stderr" print "------" print e.err Modified: pypy/branch/py11/pypy/jit/tool/autopath.py ============================================================================== --- pypy/branch/py11/pypy/jit/tool/autopath.py (original) +++ pypy/branch/py11/pypy/jit/tool/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lang/gameboy/tool/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/tool/autopath.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/tool/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lang/js/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/autopath.py (original) +++ pypy/branch/py11/pypy/lang/js/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py Tue Nov 10 16:01:00 2009 @@ -2,7 +2,7 @@ from pypy.lang.js.interpreter import * from pypy.lang.js.jsobj import W_Array, JsBaseExcept from pypy.rlib.parsing.parsing import ParseError -from py.__.test.outcome import Failed, ExceptionFailure +from py.impl.test.outcome import Failed, ExceptionFailure import pypy.lang.js as js from pypy.lang.js import interpreter Modified: pypy/branch/py11/pypy/lang/prolog/interpreter/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lang/prolog/interpreter/autopath.py (original) +++ pypy/branch/py11/pypy/lang/prolog/interpreter/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lang/scheme/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lang/scheme/autopath.py (original) +++ pypy/branch/py11/pypy/lang/scheme/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lang/smalltalk/tool/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lang/smalltalk/tool/autopath.py (original) +++ pypy/branch/py11/pypy/lang/smalltalk/tool/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/branch/py11/pypy/lib/distributed/socklayer.py (original) +++ pypy/branch/py11/pypy/lib/distributed/socklayer.py Tue Nov 10 16:01:00 2009 @@ -1,7 +1,7 @@ import py from socket import socket -from py.__.green.msgstruct import decodemessage, message +from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal import sys Modified: pypy/branch/py11/pypy/lib/test2/autopath.py ============================================================================== --- pypy/branch/py11/pypy/lib/test2/autopath.py (original) +++ pypy/branch/py11/pypy/lib/test2/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/module/__builtin__/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/module/__builtin__/test/autopath.py (original) +++ pypy/branch/py11/pypy/module/__builtin__/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/module/_codecs/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/module/_codecs/test/autopath.py (original) +++ pypy/branch/py11/pypy/module/_codecs/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/module/_sre/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/module/_sre/test/autopath.py (original) +++ pypy/branch/py11/pypy/module/_sre/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/py11/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/py11/pypy/module/_sre/test/test_app_sre.py Tue Nov 10 16:01:00 2009 @@ -3,7 +3,7 @@ from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option -from py.__.test.outcome import Skipped +from py.impl.test.outcome import Skipped def init_globals_hack(space): space.appexec([space.wrap(autopath.this_dir)], """(this_dir): Modified: pypy/branch/py11/pypy/module/sys/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/module/sys/test/autopath.py (original) +++ pypy/branch/py11/pypy/module/sys/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/rlib/parsing/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/test/autopath.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/rlib/rsdl/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/rlib/rsdl/test/autopath.py (original) +++ pypy/branch/py11/pypy/rlib/rsdl/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/rpython/microbench/autopath.py ============================================================================== --- pypy/branch/py11/pypy/rpython/microbench/autopath.py (original) +++ pypy/branch/py11/pypy/rpython/microbench/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/test_all.py ============================================================================== --- pypy/branch/py11/pypy/test_all.py (original) +++ pypy/branch/py11/pypy/test_all.py Tue Nov 10 16:01:00 2009 @@ -1,6 +1,6 @@ #! /usr/bin/env python -if __name__ == '__main__': +if __name__ == '__main__': import tool.autopath - import py - py.test.cmdline.main() + import py + py.cmdline.pytest() Modified: pypy/branch/py11/pypy/tool/algo/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/tool/algo/test/autopath.py (original) +++ pypy/branch/py11/pypy/tool/algo/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/branch/py11/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/branch/py11/pypy/tool/ansi_mandelbrot.py Tue Nov 10 16:01:00 2009 @@ -1,6 +1,6 @@ import sys -from py.__.io.terminalwriter import ansi_print, get_terminal_width +from py.impl.io.terminalwriter import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/branch/py11/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/py11/pypy/tool/ansi_print.py (original) +++ pypy/branch/py11/pypy/tool/ansi_print.py Tue Nov 10 16:01:00 2009 @@ -4,7 +4,7 @@ import sys -from py.__.io.terminalwriter import ansi_print +from py.impl.io.terminalwriter import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: Modified: pypy/branch/py11/pypy/tool/autopath.py ============================================================================== --- pypy/branch/py11/pypy/tool/autopath.py (original) +++ pypy/branch/py11/pypy/tool/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Added: pypy/branch/py11/pypy/tool/difftime.py ============================================================================== --- (empty file) +++ pypy/branch/py11/pypy/tool/difftime.py Tue Nov 10 16:01:00 2009 @@ -0,0 +1,32 @@ +import py + +_time_desc = { + 1 : 'second', 60 : 'minute', 3600 : 'hour', 86400 : 'day', + 2628000 : 'month', 31536000 : 'year', } + +def worded_diff_time(ctime): + difftime = py.std.time.time() - ctime + keys = _time_desc.keys() + keys.sort() + for i, key in py.builtin.enumerate(keys): + if key >=difftime: + break + l = [] + keylist = keys[:i] + + keylist.reverse() + for key in keylist[:1]: + div = int(difftime / key) + if div==0: + break + difftime -= div * key + plural = div > 1 and 's' or '' + l.append('%d %s%s' %(div, _time_desc[key], plural)) + return ", ".join(l) + " ago " + +_months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +def worded_time(ctime): + tm = py.std.time.gmtime(ctime) + return "%s %d, %d" % (_months[tm.tm_mon-1], tm.tm_mday, tm.tm_year) Modified: pypy/branch/py11/pypy/tool/genstatistic.py ============================================================================== --- pypy/branch/py11/pypy/tool/genstatistic.py (original) +++ pypy/branch/py11/pypy/tool/genstatistic.py Tue Nov 10 16:01:00 2009 @@ -1,7 +1,7 @@ import autopath import py -from py.__.misc.cmdline import countloc +from py.impl.misc.cmdline import countloc from py.xml import raw pypydir = py.path.local(autopath.pypydir) Modified: pypy/branch/py11/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/py11/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/py11/pypy/tool/pytest/appsupport.py Tue Nov 10 16:01:00 2009 @@ -1,9 +1,10 @@ import autopath import py -from py.__.magic import exprinfo +import py.impl.code.assertion +from py.impl.code import _assertionold as exprinfo from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from py.__.test.outcome import ExceptionFailure +from py.impl.test.outcome import ExceptionFailure # ____________________________________________________________ Modified: pypy/branch/py11/pypy/tool/pytest/autopath.py ============================================================================== --- pypy/branch/py11/pypy/tool/pytest/autopath.py (original) +++ pypy/branch/py11/pypy/tool/pytest/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Added: pypy/branch/py11/pypy/tool/rest/__init__.py ============================================================================== Added: pypy/branch/py11/pypy/tool/rest/convert.py ============================================================================== --- (empty file) +++ pypy/branch/py11/pypy/tool/rest/convert.py Tue Nov 10 16:01:00 2009 @@ -0,0 +1,163 @@ +import py + +ExecutionFailed = py.process.cmdexec.Error +# utility functions to convert between various formats + +format_to_dotargument = {"png": "png", + "eps": "ps", + "ps": "ps", + "pdf": "ps", + } + +def ps2eps(ps): + # XXX write a pure python version + if not py.path.local.sysfind("ps2epsi") and \ + not py.path.local.sysfind("ps2eps"): + raise SystemExit("neither ps2eps nor ps2epsi found") + try: + eps = ps.new(ext=".eps") + py.process.cmdexec('ps2epsi "%s" "%s"' % (ps, eps)) + except ExecutionFailed: + py.process.cmdexec('ps2eps -l -f "%s"' % ps) + +def ps2pdf(ps, compat_level="1.2"): + if not py.path.local.sysfind("gs"): + raise SystemExit("ERROR: gs not found") + pdf = ps.new(ext=".pdf") + options = dict(OPTIONS="-dSAFER -dCompatibilityLevel=%s" % compat_level, + infile=ps, outfile=pdf) + cmd = ('gs %(OPTIONS)s -q -dNOPAUSE -dBATCH -sDEVICE=pdfwrite ' + '"-sOutputFile=%(outfile)s" %(OPTIONS)s -c .setpdfwrite ' + '-f "%(infile)s"') % options + py.process.cmdexec(cmd) + return pdf + +def eps2pdf(eps): + # XXX write a pure python version + if not py.path.local.sysfind("epstopdf"): + raise SystemExit("ERROR: epstopdf not found") + py.process.cmdexec('epstopdf "%s"' % eps) + +def dvi2eps(dvi, dest=None): + if dest is None: + dest = eps.new(ext=".eps") + command = 'dvips -q -E -n 1 -D 600 -p 1 -o "%s" "%s"' % (dest, dvi) + if not py.path.local.sysfind("dvips"): + raise SystemExit("ERROR: dvips not found") + py.process.cmdexec(command) + +def convert_dot(fn, new_extension): + if not py.path.local.sysfind("dot"): + raise SystemExit("ERROR: dot not found") + result = fn.new(ext=new_extension) + print result + arg = "-T%s" % (format_to_dotargument[new_extension], ) + py.std.os.system('dot "%s" "%s" > "%s"' % (arg, fn, result)) + if new_extension == "eps": + ps = result.new(ext="ps") + result.move(ps) + ps2eps(ps) + ps.remove() + elif new_extension == "pdf": + # convert to eps file first, to get the bounding box right + eps = result.new(ext="eps") + ps = result.new(ext="ps") + result.move(ps) + ps2eps(ps) + eps2pdf(eps) + ps.remove() + eps.remove() + return result + + +class latexformula2png(object): + def __init__(self, formula, dest, temp=None): + self.formula = formula + try: + import Image + self.Image = Image + self.scale = 2 # create a larger image + self.upscale = 5 # create the image upscale times larger, then scale it down + except ImportError: + self.scale = 2 + self.upscale = 1 + self.Image = None + self.output_format = ('pngmono', 'pnggray', 'pngalpha')[2] + if temp is None: + temp = py.test.ensuretemp("latexformula") + self.temp = temp + self.latex = self.temp.join('formula.tex') + self.dvi = self.temp.join('formula.dvi') + self.eps = self.temp.join('formula.eps') + self.png = self.temp.join('formula.png') + self.saveas(dest) + + def saveas(self, dest): + self.gen_latex() + self.gen_dvi() + dvi2eps(self.dvi, self.eps) + self.gen_png() + self.scale_image() + self.png.copy(dest) + + def gen_latex(self): + self.latex.write (""" + \\documentclass{article} + \\pagestyle{empty} + \\begin{document} + + %s + \\pagebreak + + \\end{document} + """ % (self.formula)) + + def gen_dvi(self): + origdir = py.path.local() + self.temp.chdir() + py.process.cmdexec('latex "%s"' % (self.latex)) + origdir.chdir() + + def gen_png(self): + tempdir = py.path.local.mkdtemp() + + re_bbox = py.std.re.compile('%%BoundingBox:\s*(\d+) (\d+) (\d+) (\d+)') + eps = self.eps.read() + x1, y1, x2, y2 = [int(i) for i in re_bbox.search(eps).groups()] + X = x2 - x1 + 2 + Y = y2 - y1 + 2 + mx = -x1 + my = -y1 + ps = self.temp.join('temp.ps') + source = self.eps + ps.write(""" + 1 1 1 setrgbcolor + newpath + -1 -1 moveto + %(X)d -1 lineto + %(X)d %(Y)d lineto + -1 %(Y)d lineto + closepath + fill + %(mx)d %(my)d translate + 0 0 0 setrgbcolor + (%(source)s) run + + """ % locals()) + + sx = int((x2 - x1) * self.scale * self.upscale) + sy = int((y2 - y1) * self.scale * self.upscale) + res = 72 * self.scale * self.upscale + command = ('gs -q -g%dx%d -r%dx%d -sDEVICE=%s -sOutputFile="%s" ' + '-dNOPAUSE -dBATCH "%s"') % ( + sx, sy, res, res, self.output_format, self.png, ps) + py.process.cmdexec(command) + + def scale_image(self): + if self.Image is None: + return + image = self.Image.open(str(self.png)) + image.resize((image.size[0] / self.upscale, + image.size[1] / self.upscale), + self.Image.ANTIALIAS).save(str(self.png)) + Added: pypy/branch/py11/pypy/tool/rest/directive.py ============================================================================== --- (empty file) +++ pypy/branch/py11/pypy/tool/rest/directive.py Tue Nov 10 16:01:00 2009 @@ -0,0 +1,115 @@ +# XXX this file is messy since it tries to deal with several docutils versions +import py + +from pypy.tool.rest.convert import convert_dot, latexformula2png + +import sys +import docutils +from docutils import nodes +from docutils.parsers.rst import directives, states, roles +from docutils.parsers.rst.directives import images + +if hasattr(images, "image"): + directives_are_functions = True +else: + directives_are_functions = False + +try: + from docutils.utils import unescape # docutils version > 0.3.5 +except ImportError: + from docutils.parsers.rst.states import unescape # docutils 0.3.5 + +if not directives_are_functions: + ImageClass = images.Image + +else: + class ImageClass(object): + option_spec = images.image.options + def run(self): + return images.image(u'image', + self.arguments, + self.options, + self.content, + self.lineno, + self.content_offset, + self.block_text, + self.state, + self.state_machine) + + +backend_to_image_format = {"html": "png", "latex": "pdf"} + +class GraphvizDirective(ImageClass): + def convert(self, fn, path): + path = py.path.local(path).dirpath() + dot = path.join(fn) + result = convert_dot(dot, backend_to_image_format[_backend]) + return result.relto(path) + + def run(self): + newname = self.convert(self.arguments[0], + self.state.document.settings._source) + text = self.block_text.replace("graphviz", "image", 1) + self.block_text = text.replace(self.arguments[0], newname, 1) + self.name = u'image' + self.arguments = [newname] + return ImageClass.run(self) + + def old_interface(self): + def f(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + for arg in "name arguments options content lineno " \ + "content_offset block_text state state_machine".split(): + setattr(self, arg, locals()[arg]) + return self.run() + f.arguments = (1, 0, 1) + f.options = self.option_spec + return f + + +_backend = None +def set_backend_and_register_directives(backend): + #XXX this is only used to work around the inflexibility of docutils: + # a directive does not know the target format + global _backend + _backend = backend + if not directives_are_functions: + directives.register_directive("graphviz", GraphvizDirective) + else: + directives.register_directive("graphviz", + GraphvizDirective().old_interface()) + roles.register_canonical_role("latexformula", latexformula_role) + +def latexformula_role(name, rawtext, text, lineno, inliner, + options={}, content=[]): + if _backend == 'latex': + options['format'] = 'latex' + return roles.raw_role(name, rawtext, text, lineno, inliner, + options, content) + else: + # XXX: make the place of the image directory configurable + sourcedir = py.path.local(inliner.document.settings._source).dirpath() + imagedir = sourcedir.join("img") + if not imagedir.check(): + imagedir.mkdir() + # create halfway senseful imagename: + # use hash of formula + alphanumeric characters of it + # could + imagename = "%s_%s.png" % ( + hash(text), "".join([c for c in text if c.isalnum()])) + image = imagedir.join(imagename) + latexformula2png(unescape(text, True), image) + imagenode = nodes.image(image.relto(sourcedir), uri=image.relto(sourcedir)) + return [imagenode], [] +latexformula_role.content = True +latexformula_role.options = {} + +def register_linkrole(role_name, callback): + def source_role(name, rawtext, text, lineno, inliner, options={}, + content=[]): + text, target = callback(name, text) + reference_node = nodes.reference(rawtext, text, name=text, refuri=target) + return [reference_node], [] + source_role.content = True + source_role.options = {} + roles.register_canonical_role(role_name, source_role) Added: pypy/branch/py11/pypy/tool/rest/rest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/pypy/tool/rest/rest.py Tue Nov 10 16:01:00 2009 @@ -0,0 +1,81 @@ +import py +import sys, os, traceback +import re + +if hasattr(sys.stdout, 'fileno') and os.isatty(sys.stdout.fileno()): + def log(msg): + print msg +else: + def log(msg): + pass + +def convert_rest_html(source, source_path, stylesheet=None, encoding='latin1'): + from pypy.tool.rest import directive + """ return html latin1-encoded document for the given input. + source a ReST-string + sourcepath where to look for includes (basically) + stylesheet path (to be used if any) + """ + from docutils.core import publish_string + directive.set_backend_and_register_directives("html") + kwargs = { + 'stylesheet' : stylesheet, + 'stylesheet_path': None, + 'traceback' : 1, + 'embed_stylesheet': 0, + 'output_encoding' : encoding, + #'halt' : 0, # 'info', + 'halt_level' : 2, + } + # docutils uses os.getcwd() :-( + source_path = os.path.abspath(str(source_path)) + prevdir = os.getcwd() + try: + #os.chdir(os.path.dirname(source_path)) + return publish_string(source, source_path, writer_name='html', + settings_overrides=kwargs) + finally: + os.chdir(prevdir) + +def process(txtpath, encoding='latin1'): + """ process a textfile """ + log("processing %s" % txtpath) + assert txtpath.check(ext='.txt') + if isinstance(txtpath, py.path.svnwc): + txtpath = txtpath.localpath + htmlpath = txtpath.new(ext='.html') + #svninfopath = txtpath.localpath.new(ext='.svninfo') + + style = txtpath.dirpath('style.css') + if style.check(): + stylesheet = style.basename + else: + stylesheet = None + content = unicode(txtpath.read(), encoding) + doc = convert_rest_html(content, txtpath, stylesheet=stylesheet, encoding=encoding) + htmlpath.write(doc) + #log("wrote %r" % htmlpath) + #if txtpath.check(svnwc=1, versioned=1): + # info = txtpath.info() + # svninfopath.dump(info) + +rex1 = re.compile(ur'.*(.*).*', re.MULTILINE | re.DOTALL) +rex2 = re.compile(ur'.*
(.*)
.*', re.MULTILINE | re.DOTALL) + +def strip_html_header(string, encoding='utf8'): + """ return the content of the body-tag """ + uni = unicode(string, encoding) + for rex in rex1,rex2: + match = rex.search(uni) + if not match: + break + uni = match.group(1) + return uni + +class Project: # used for confrest.py files + def __init__(self, sourcepath): + self.sourcepath = sourcepath + def process(self, path): + return process(path) + def get_htmloutputpath(self, path): + return path.new(ext='html') Added: pypy/branch/py11/pypy/tool/rest/rst.py ============================================================================== --- (empty file) +++ pypy/branch/py11/pypy/tool/rest/rst.py Tue Nov 10 16:01:00 2009 @@ -0,0 +1,417 @@ + +""" reStructuredText generation tools + + provides an api to build a tree from nodes, which can be converted to + ReStructuredText on demand + + note that not all of ReST is supported, a usable subset is offered, but + certain features aren't supported, and also certain details (like how links + are generated, or how escaping is done) can not be controlled +""" + +from __future__ import generators + +import py + +def escape(txt): + """escape ReST markup""" + if not isinstance(txt, str) and not isinstance(txt, unicode): + txt = str(txt) + # XXX this takes a very naive approach to escaping, but it seems to be + # sufficient... + for c in '\\*`|:_': + txt = txt.replace(c, '\\%s' % (c,)) + return txt + +class RestError(Exception): + """ raised on containment errors (wrong parent) """ + +class AbstractMetaclass(type): + def __new__(cls, *args): + obj = super(AbstractMetaclass, cls).__new__(cls, *args) + parent_cls = obj.parentclass + if parent_cls is None: + return obj + if not isinstance(parent_cls, list): + class_list = [parent_cls] + else: + class_list = parent_cls + if obj.allow_nesting: + class_list.append(obj) + + for _class in class_list: + if not _class.allowed_child: + _class.allowed_child = {obj:True} + else: + _class.allowed_child[obj] = True + return obj + +class AbstractNode(object): + """ Base class implementing rest generation + """ + sep = '' + __metaclass__ = AbstractMetaclass + parentclass = None # this exists to allow parent to know what + # children can exist + allow_nesting = False + allowed_child = {} + defaults = {} + + _reg_whitespace = py.std.re.compile('\s+') + + def __init__(self, *args, **kwargs): + self.parent = None + self.children = [] + for child in args: + self._add(child) + for arg in kwargs: + setattr(self, arg, kwargs[arg]) + + def join(self, *children): + """ add child nodes + + returns a reference to self + """ + for child in children: + self._add(child) + return self + + def add(self, child): + """ adds a child node + + returns a reference to the child + """ + self._add(child) + return child + + def _add(self, child): + if child.__class__ not in self.allowed_child: + raise RestError("%r cannot be child of %r" % \ + (child.__class__, self.__class__)) + self.children.append(child) + child.parent = self + + def __getitem__(self, item): + return self.children[item] + + def __setitem__(self, item, value): + self.children[item] = value + + def text(self): + """ return a ReST string representation of the node """ + return self.sep.join([child.text() for child in self.children]) + + def wordlist(self): + """ return a list of ReST strings for this node and its children """ + return [self.text()] + +class Rest(AbstractNode): + """ Root node of a document """ + + sep = "\n\n" + def __init__(self, *args, **kwargs): + AbstractNode.__init__(self, *args, **kwargs) + self.links = {} + + def render_links(self, check=False): + """render the link attachments of the document""" + assert not check, "Link checking not implemented" + if not self.links: + return "" + link_texts = [] + # XXX this could check for duplicates and remove them... + for link, target in self.links.iteritems(): + link_texts.append(".. _`%s`: %s" % (escape(link), target)) + return "\n" + "\n".join(link_texts) + "\n\n" + + def text(self): + outcome = [] + if (isinstance(self.children[0], Transition) or + isinstance(self.children[-1], Transition)): + raise ValueError, ('document must not begin or end with a ' + 'transition') + for child in self.children: + outcome.append(child.text()) + + # always a trailing newline + text = self.sep.join([i for i in outcome if i]) + "\n" + return text + self.render_links() + +class Transition(AbstractNode): + """ a horizontal line """ + parentclass = Rest + + def __init__(self, char='-', width=80, *args, **kwargs): + self.char = char + self.width = width + super(Transition, self).__init__(*args, **kwargs) + + def text(self): + return (self.width - 1) * self.char + +class Paragraph(AbstractNode): + """ simple paragraph """ + + parentclass = Rest + sep = " " + indent = "" + width = 80 + + def __init__(self, *args, **kwargs): + # make shortcut + args = list(args) + for num, arg in py.builtin.enumerate(args): + if isinstance(arg, str): + args[num] = Text(arg) + super(Paragraph, self).__init__(*args, **kwargs) + + def text(self): + texts = [] + for child in self.children: + texts += child.wordlist() + + buf = [] + outcome = [] + lgt = len(self.indent) + + def grab(buf): + outcome.append(self.indent + self.sep.join(buf)) + + texts.reverse() + while texts: + next = texts[-1] + if not next: + texts.pop() + continue + if lgt + len(self.sep) + len(next) <= self.width or not buf: + buf.append(next) + lgt += len(next) + len(self.sep) + texts.pop() + else: + grab(buf) + lgt = len(self.indent) + buf = [] + grab(buf) + return "\n".join(outcome) + +class SubParagraph(Paragraph): + """ indented sub paragraph """ + + indent = " " + +class Title(Paragraph): + """ title element """ + + parentclass = Rest + belowchar = "=" + abovechar = "" + + def text(self): + txt = self._get_text() + lines = [] + if self.abovechar: + lines.append(self.abovechar * len(txt)) + lines.append(txt) + if self.belowchar: + lines.append(self.belowchar * len(txt)) + return "\n".join(lines) + + def _get_text(self): + txt = [] + for node in self.children: + txt += node.wordlist() + return ' '.join(txt) + +class AbstractText(AbstractNode): + parentclass = [Paragraph, Title] + start = "" + end = "" + def __init__(self, _text): + self._text = _text + + def text(self): + text = self.escape(self._text) + return self.start + text + self.end + + def escape(self, text): + if not isinstance(text, str) and not isinstance(text, unicode): + text = str(text) + if self.start: + text = text.replace(self.start, '\\%s' % (self.start,)) + if self.end and self.end != self.start: + text = text.replace(self.end, '\\%s' % (self.end,)) + return text + +class Text(AbstractText): + def wordlist(self): + text = escape(self._text) + return self._reg_whitespace.split(text) + +class LiteralBlock(AbstractText): + parentclass = Rest + start = '::\n\n' + + def text(self): + if not self._text.strip(): + return '' + text = self.escape(self._text).split('\n') + for i, line in py.builtin.enumerate(text): + if line.strip(): + text[i] = ' %s' % (line,) + return self.start + '\n'.join(text) + +class Em(AbstractText): + start = "*" + end = "*" + +class Strong(AbstractText): + start = "**" + end = "**" + +class Quote(AbstractText): + start = '``' + end = '``' + +class Anchor(AbstractText): + start = '_`' + end = '`' + +class Footnote(AbstractText): + def __init__(self, note, symbol=False): + raise NotImplemented('XXX') + +class Citation(AbstractText): + def __init__(self, text, cite): + raise NotImplemented('XXX') + +class ListItem(Paragraph): + allow_nesting = True + item_chars = '*+-' + + def text(self): + idepth = self.get_indent_depth() + indent = self.indent + (idepth + 1) * ' ' + txt = '\n\n'.join(self.render_children(indent)) + ret = [] + item_char = self.item_chars[idepth] + ret += [indent[len(item_char)+1:], item_char, ' ', txt[len(indent):]] + return ''.join(ret) + + def render_children(self, indent): + txt = [] + buffer = [] + def render_buffer(fro, to): + if not fro: + return + p = Paragraph(indent=indent, *fro) + p.parent = self.parent + to.append(p.text()) + for child in self.children: + if isinstance(child, AbstractText): + buffer.append(child) + else: + if buffer: + render_buffer(buffer, txt) + buffer = [] + txt.append(child.text()) + + render_buffer(buffer, txt) + return txt + + def get_indent_depth(self): + depth = 0 + current = self + while (current.parent is not None and + isinstance(current.parent, ListItem)): + depth += 1 + current = current.parent + return depth + +class OrderedListItem(ListItem): + item_chars = ["#."] * 5 + +class DListItem(ListItem): + item_chars = None + def __init__(self, term, definition, *args, **kwargs): + self.term = term + super(DListItem, self).__init__(definition, *args, **kwargs) + + def text(self): + idepth = self.get_indent_depth() + indent = self.indent + (idepth + 1) * ' ' + txt = '\n\n'.join(self.render_children(indent)) + ret = [] + ret += [indent[2:], self.term, '\n', txt] + return ''.join(ret) + +class Link(AbstractText): + start = '`' + end = '`_' + + def __init__(self, _text, target): + self._text = _text + self.target = target + self.rest = None + + def text(self): + if self.rest is None: + self.rest = self.find_rest() + if self.rest.links.get(self._text, self.target) != self.target: + raise ValueError('link name %r already in use for a different ' + 'target' % (self.target,)) + self.rest.links[self._text] = self.target + return AbstractText.text(self) + + def find_rest(self): + # XXX little overkill, but who cares... + next = self + while next.parent is not None: + next = next.parent + return next + +class InternalLink(AbstractText): + start = '`' + end = '`_' + +class LinkTarget(Paragraph): + def __init__(self, name, target): + self.name = name + self.target = target + + def text(self): + return ".. _`%s`:%s\n" % (self.name, self.target) + +class Substitution(AbstractText): + def __init__(self, text, **kwargs): + raise NotImplemented('XXX') + +class Directive(Paragraph): + indent = ' ' + def __init__(self, name, *args, **options): + self.name = name + self.content = options.pop('content', []) + children = list(args) + super(Directive, self).__init__(*children) + self.options = options + + def text(self): + # XXX not very pretty... + namechunksize = len(self.name) + 2 + self.children.insert(0, Text('X' * namechunksize)) + txt = super(Directive, self).text() + txt = '.. %s::%s' % (self.name, txt[namechunksize + 3:],) + options = '\n'.join([' :%s: %s' % (k, v) for (k, v) in + self.options.iteritems()]) + if options: + txt += '\n%s' % (options,) + + if self.content: + txt += '\n' + for item in self.content: + assert item.parentclass == Rest, 'only top-level items allowed' + assert not item.indent + item.indent = ' ' + txt += '\n' + item.text() + + return txt + Modified: pypy/branch/py11/pypy/tool/statistic_over_time.py ============================================================================== --- pypy/branch/py11/pypy/tool/statistic_over_time.py (original) +++ pypy/branch/py11/pypy/tool/statistic_over_time.py Tue Nov 10 16:01:00 2009 @@ -1,5 +1,5 @@ import py -from py.__.misc.cmdline.countloc import get_loccount +from py.impl.misc.cmdline.countloc import get_loccount import datetime import time Modified: pypy/branch/py11/pypy/tool/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/tool/test/autopath.py (original) +++ pypy/branch/py11/pypy/tool/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/branch/py11/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/py11/pypy/tool/test/test_pytestsupport.py Tue Nov 10 16:01:00 2009 @@ -1,5 +1,5 @@ import autopath -from py.__.magic import exprinfo +from py.impl.code import _assertionold as exprinfo from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.argument import Arguments Modified: pypy/branch/py11/pypy/translator/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/autopath.py (original) +++ pypy/branch/py11/pypy/translator/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/benchmark/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/benchmark/autopath.py (original) +++ pypy/branch/py11/pypy/translator/benchmark/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/c/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/c/autopath.py (original) +++ pypy/branch/py11/pypy/translator/c/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/c/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/c/test/autopath.py (original) +++ pypy/branch/py11/pypy/translator/c/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/cli/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/cli/test/autopath.py (original) +++ pypy/branch/py11/pypy/translator/cli/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/goal/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/autopath.py (original) +++ pypy/branch/py11/pypy/translator/goal/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/goal/test2/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/test2/autopath.py (original) +++ pypy/branch/py11/pypy/translator/goal/test2/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/microbench/pybench/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/microbench/pybench/autopath.py (original) +++ pypy/branch/py11/pypy/translator/microbench/pybench/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/py11/pypy/translator/platform/__init__.py (original) +++ pypy/branch/py11/pypy/translator/platform/__init__.py Tue Nov 10 16:01:00 2009 @@ -6,7 +6,7 @@ import sys, py, os from pypy.tool.ansi_print import ansi_log -from py.__.code import safe_repr +from py.impl.code.code import safe_repr log = py.log.Producer("platform") py.log.setconsumer("platform", ansi_log) @@ -33,9 +33,9 @@ def __repr__(self): if self.err: - return "" % safe_repr._repr(self.err) + return "" % safe_repr(self.err) else: - return "" % safe_repr._repr(self.out) + return "" % safe_repr(self.out) __str__ = __repr__ Modified: pypy/branch/py11/pypy/translator/sandbox/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/sandbox/autopath.py (original) +++ pypy/branch/py11/pypy/translator/sandbox/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/sandbox/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/sandbox/test/autopath.py (original) +++ pypy/branch/py11/pypy/translator/sandbox/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/test/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/test/autopath.py (original) +++ pypy/branch/py11/pypy/translator/test/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/py11/pypy/translator/tool/autopath.py ============================================================================== --- pypy/branch/py11/pypy/translator/tool/autopath.py (original) +++ pypy/branch/py11/pypy/translator/tool/autopath.py Tue Nov 10 16:01:00 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break From hpk at codespeak.net Tue Nov 10 16:26:47 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 16:26:47 +0100 (CET) Subject: [pypy-svn] r69121 - in pypy/branch/py11: . lib-python pypy pypy/bin pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/doc/tool pypy/jit pypy/jit/backend/test pypy/jit/tl pypy/jit/tl/tla pypy/lang/gameboy/debug pypy/lang/gameboy/profiling pypy/lang/gameboy/profiling/evaluation pypy/lang/gameboy/test pypy/lang/js pypy/lang/js/test pypy/lang/js/test/ecma pypy/lang/prolog/interpreter pypy/lang/scheme pypy/lang/scheme/test pypy/lang/smalltalk/test pypy/lang/smalltalk/tool pypy/lib/app_test/ctypes_tests pypy/module/bz2/test pypy/module/pypyjit/test pypy/objspace/std/test pypy/rlib/parsing pypy/rlib/parsing/test pypy/rlib/rsdl pypy/rlib/test pypy/rpython/module/test pypy/rpython/test pypy/tool pypy/tool/bench pypy/tool/pytest pypy/tool/pytest/test pypy/tool/test pypy/translator pypy/translator/benchmark pypy/translator/c/test pypy/translator/cli pypy/translator/goal pypy/translator/jvm pypy/translator/platform/test pypy/translator/test Message-ID: <20091110152647.3CFA5168102@codespeak.net> Author: hpk Date: Tue Nov 10 16:26:43 2009 New Revision: 69121 Modified: pypy/branch/py11/lib-python/conftest.py pypy/branch/py11/pypy/bin/py.py pypy/branch/py11/pypy/config/config.py pypy/branch/py11/pypy/config/makerestdoc.py pypy/branch/py11/pypy/config/pypyoption.py pypy/branch/py11/pypy/config/test/test_pypyoption.py pypy/branch/py11/pypy/conftest.py pypy/branch/py11/pypy/doc/config/generate.py pypy/branch/py11/pypy/doc/config/makemodules.py pypy/branch/py11/pypy/doc/confrest.py pypy/branch/py11/pypy/doc/confrest_oldpy.py pypy/branch/py11/pypy/doc/conftest.py pypy/branch/py11/pypy/doc/statistic/confrest.py pypy/branch/py11/pypy/doc/test_redirections.py pypy/branch/py11/pypy/doc/tool/makeref.py pypy/branch/py11/pypy/doc/tool/mydot.py pypy/branch/py11/pypy/jit/backend/test/conftest.py pypy/branch/py11/pypy/jit/conftest.py pypy/branch/py11/pypy/jit/tl/conftest.py pypy/branch/py11/pypy/jit/tl/targettlc.py pypy/branch/py11/pypy/jit/tl/targettlr.py pypy/branch/py11/pypy/jit/tl/tla/targettla.py pypy/branch/py11/pypy/jit/tl/tla/tla_assembler.py pypy/branch/py11/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py pypy/branch/py11/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py pypy/branch/py11/pypy/lang/gameboy/profiling/gameboyTest.py pypy/branch/py11/pypy/lang/gameboy/test/test_cartridge.py pypy/branch/py11/pypy/lang/gameboy/test/test_rom.py pypy/branch/py11/pypy/lang/js/jsparser.py pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py pypy/branch/py11/pypy/lang/js/test/test_interactive.py pypy/branch/py11/pypy/lang/js/test/test_parser.py pypy/branch/py11/pypy/lang/prolog/interpreter/conftest.py pypy/branch/py11/pypy/lang/prolog/interpreter/interactive.py pypy/branch/py11/pypy/lang/prolog/interpreter/parsing.py pypy/branch/py11/pypy/lang/scheme/execution.py pypy/branch/py11/pypy/lang/scheme/test/test_interactive.py pypy/branch/py11/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/py11/pypy/lang/smalltalk/tool/analyseimage.py pypy/branch/py11/pypy/lib/app_test/ctypes_tests/conftest.py pypy/branch/py11/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/branch/py11/pypy/module/pypyjit/test/conftest.py pypy/branch/py11/pypy/objspace/std/test/test_complexobject.py pypy/branch/py11/pypy/rlib/parsing/ebnfparse.py pypy/branch/py11/pypy/rlib/parsing/makepackrat.py pypy/branch/py11/pypy/rlib/parsing/regexparse.py pypy/branch/py11/pypy/rlib/parsing/test/test_pythonlexer.py pypy/branch/py11/pypy/rlib/parsing/test/test_pythonparse.py pypy/branch/py11/pypy/rlib/rsdl/eci.py pypy/branch/py11/pypy/rlib/test/test_listsort.py pypy/branch/py11/pypy/rpython/module/test/test_ll_os_path.py pypy/branch/py11/pypy/rpython/test/test_rbuiltin.py pypy/branch/py11/pypy/tool/bench/pypyresult.py pypy/branch/py11/pypy/tool/option.py pypy/branch/py11/pypy/tool/pytest/genreportdata.py pypy/branch/py11/pypy/tool/pytest/htmlreport.py pypy/branch/py11/pypy/tool/pytest/test/test_new_count.py pypy/branch/py11/pypy/tool/test/test_conftest1.py pypy/branch/py11/pypy/tool/udir.py pypy/branch/py11/pypy/translator/benchmark/benchmarks.py pypy/branch/py11/pypy/translator/benchmark/jitbench.py pypy/branch/py11/pypy/translator/c/test/test_extfunc.py pypy/branch/py11/pypy/translator/cli/conftest.py pypy/branch/py11/pypy/translator/driver.py pypy/branch/py11/pypy/translator/goal/targetgbfullprofiling.py pypy/branch/py11/pypy/translator/goal/targetgbimplementation.py pypy/branch/py11/pypy/translator/goal/targetgbrom4.py pypy/branch/py11/pypy/translator/goal/targetpreimportedpypy.py pypy/branch/py11/pypy/translator/goal/targetpypystandalone.py pypy/branch/py11/pypy/translator/goal/translate.py pypy/branch/py11/pypy/translator/interactive.py pypy/branch/py11/pypy/translator/jvm/conftest.py pypy/branch/py11/pypy/translator/jvm/genjvm.py pypy/branch/py11/pypy/translator/platform/test/test_darwin.py pypy/branch/py11/pypy/translator/platform/test/test_maemo.py pypy/branch/py11/pypy/translator/test/test_driver.py pypy/branch/py11/pytest_resultlog.py Log: replacing addgroup/getgroup and py.magic.autopath as advertised Modified: pypy/branch/py11/lib-python/conftest.py ============================================================================== --- pypy/branch/py11/lib-python/conftest.py (original) +++ pypy/branch/py11/lib-python/conftest.py Tue Nov 10 16:26:43 2009 @@ -28,7 +28,7 @@ # def pytest_addoption(parser): - group = parser.addgroup("complicance testing options") + group = parser.getgroup("complicance testing options") group.addoption('-T', '--timeout', action="store", type="string", default="1000", dest="timeout", help="fail a test module after the given timeout. " Modified: pypy/branch/py11/pypy/bin/py.py ============================================================================== --- pypy/branch/py11/pypy/bin/py.py (original) +++ pypy/branch/py11/pypy/bin/py.py Tue Nov 10 16:26:43 2009 @@ -12,7 +12,7 @@ pass from pypy.tool import option -from py.compat.optparse import make_option +from optparse import make_option from pypy.interpreter import main, interactive, error, gateway from pypy.config.config import OptionDescription, BoolOption, StrOption from pypy.config.config import Config, to_optparse Modified: pypy/branch/py11/pypy/config/config.py ============================================================================== --- pypy/branch/py11/pypy/config/config.py (original) +++ pypy/branch/py11/pypy/config/config.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ import py -from py.compat import optparse +import optparse from pypy.tool.pairtype import extendabletype SUPPRESS_USAGE = optparse.SUPPRESS_USAGE Modified: pypy/branch/py11/pypy/config/makerestdoc.py ============================================================================== --- pypy/branch/py11/pypy/config/makerestdoc.py (original) +++ pypy/branch/py11/pypy/config/makerestdoc.py Tue Nov 10 16:26:43 2009 @@ -7,7 +7,7 @@ from pypy.config.config import ArbitraryOption, DEFAULT_OPTION_NAME from pypy.config.config import _getnegation -configdocdir = py.magic.autopath().dirpath().dirpath().join("doc", "config") +configdocdir = py.path.local(__file__).dirpath().dirpath().join("doc", "config") def get_fullpath(opt, path): if path: Modified: pypy/branch/py11/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/py11/pypy/config/pypyoption.py (original) +++ pypy/branch/py11/pypy/config/pypyoption.py Tue Nov 10 16:26:43 2009 @@ -5,7 +5,7 @@ from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConflictConfigError -modulepath = py.magic.autopath().dirpath().dirpath().join("module") +modulepath = py.path.local(__file__).dirpath().dirpath().join("module") all_modules = [p.basename for p in modulepath.listdir() if p.check(dir=True, dotfile=False) and p.join('__init__.py').check()] Modified: pypy/branch/py11/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/branch/py11/pypy/config/test/test_pypyoption.py (original) +++ pypy/branch/py11/pypy/config/test/test_pypyoption.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ from pypy.config.config import Config, ConfigError from pypy.config.translationoption import set_opt_level -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() def test_required(): conf = get_pypy_config() Modified: pypy/branch/py11/pypy/conftest.py ============================================================================== --- pypy/branch/py11/pypy/conftest.py (original) +++ pypy/branch/py11/pypy/conftest.py Tue Nov 10 16:26:43 2009 @@ -9,7 +9,7 @@ from pypy.tool.udir import udir from pypy.tool.autopath import pypydir -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() # pytest settings pytest_plugins = "resultlog", @@ -36,7 +36,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("pypy options") + group = parser.getgroup("pypy options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") group.addoption('-A', '--runappdirect', action="store_true", Modified: pypy/branch/py11/pypy/doc/config/generate.py ============================================================================== --- pypy/branch/py11/pypy/doc/config/generate.py (original) +++ pypy/branch/py11/pypy/doc/config/generate.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ from pypy.config import pypyoption, translationoption, config from pypy.doc.config.confrest import all_optiondescrs -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() for descr in all_optiondescrs: prefix = descr._name Modified: pypy/branch/py11/pypy/doc/config/makemodules.py ============================================================================== --- pypy/branch/py11/pypy/doc/config/makemodules.py (original) +++ pypy/branch/py11/pypy/doc/config/makemodules.py Tue Nov 10 16:26:43 2009 @@ -2,7 +2,7 @@ import py from pypy.config import pypyoption, translationoption, config -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() if __name__ == '__main__': c = config.Config(pypyoption.pypy_optiondescription).usemodules Modified: pypy/branch/py11/pypy/doc/confrest.py ============================================================================== --- pypy/branch/py11/pypy/doc/confrest.py (original) +++ pypy/branch/py11/pypy/doc/confrest.py Tue Nov 10 16:26:43 2009 @@ -34,7 +34,7 @@ class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' Modified: pypy/branch/py11/pypy/doc/confrest_oldpy.py ============================================================================== --- pypy/branch/py11/pypy/doc/confrest_oldpy.py (original) +++ pypy/branch/py11/pypy/doc/confrest_oldpy.py Tue Nov 10 16:26:43 2009 @@ -104,7 +104,7 @@ class Project: - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "py lib" prefix_title = "" # we have a logo already containing "py lib" encoding = 'latin1' Modified: pypy/branch/py11/pypy/doc/conftest.py ============================================================================== --- pypy/branch/py11/pypy/doc/conftest.py (original) +++ pypy/branch/py11/pypy/doc/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,12 +1,12 @@ import py from pypy.config.makerestdoc import register_config_role -docdir = py.magic.autopath().dirpath() +docdir = py.path.local(__file__).dirpath() pytest_plugins = "pytest_restdoc" def pytest_addoption(parser): - group = parser.addgroup("pypy-doc options") + group = parser.getgroup("pypy-doc options") group.addoption('--pypy-doctests', action="store_true", dest="pypy_doctests", default=False, help="enable doctests in .txt files") Modified: pypy/branch/py11/pypy/doc/statistic/confrest.py ============================================================================== --- pypy/branch/py11/pypy/doc/statistic/confrest.py (original) +++ pypy/branch/py11/pypy/doc/statistic/confrest.py Tue Nov 10 16:26:43 2009 @@ -17,7 +17,7 @@ " ", id="menubar") class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' encoding = 'latin1' Modified: pypy/branch/py11/pypy/doc/test_redirections.py ============================================================================== --- pypy/branch/py11/pypy/doc/test_redirections.py (original) +++ pypy/branch/py11/pypy/doc/test_redirections.py Tue Nov 10 16:26:43 2009 @@ -1,6 +1,6 @@ import py -redir = py.magic.autopath().dirpath('redirections') +redir = py.path.local(__file__).dirpath('redirections') def checkexist(path): print "checking", path Modified: pypy/branch/py11/pypy/doc/tool/makeref.py ============================================================================== --- pypy/branch/py11/pypy/doc/tool/makeref.py (original) +++ pypy/branch/py11/pypy/doc/tool/makeref.py Tue Nov 10 16:26:43 2009 @@ -1,6 +1,6 @@ import py -py.magic.autopath() +py.path.local(__file__) import pypy pypydir = py.path.local(pypy.__file__).dirpath() distdir = pypydir.dirpath() Modified: pypy/branch/py11/pypy/doc/tool/mydot.py ============================================================================== --- pypy/branch/py11/pypy/doc/tool/mydot.py (original) +++ pypy/branch/py11/pypy/doc/tool/mydot.py Tue Nov 10 16:26:43 2009 @@ -62,7 +62,7 @@ if __name__ == '__main__': - from py.compat import optparse + import optparse parser = optparse.OptionParser() parser.add_option("-T", dest="format", help="output format") Modified: pypy/branch/py11/pypy/jit/backend/test/conftest.py ============================================================================== --- pypy/branch/py11/pypy/jit/backend/test/conftest.py (original) +++ pypy/branch/py11/pypy/jit/backend/test/conftest.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup('random test options') + group = parser.getgroup('random test options') group.addoption('--random-seed', action="store", type="int", default=random.randrange(0, 10000), dest="randomseed", Modified: pypy/branch/py11/pypy/jit/conftest.py ============================================================================== --- pypy/branch/py11/pypy/jit/conftest.py (original) +++ pypy/branch/py11/pypy/jit/conftest.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("JIT options") + group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", default=False, dest="run_slow_tests", help="run all the compiled tests (instead of just a few)") Modified: pypy/branch/py11/pypy/jit/tl/conftest.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/conftest.py (original) +++ pypy/branch/py11/pypy/jit/tl/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit.py options") + group = parser.getgroup("pypyjit.py options") group.addoption('--ootype', action="store_true", dest="ootype", default=False, help="use ootype") Modified: pypy/branch/py11/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/py11/pypy/jit/tl/targettlc.py Tue Nov 10 16:26:43 2009 @@ -1,6 +1,6 @@ import time import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/py11/pypy/jit/tl/targettlr.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/targettlr.py (original) +++ pypy/branch/py11/pypy/jit/tl/targettlr.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlr import interpret from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/py11/pypy/jit/tl/tla/targettla.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/tla/targettla.py (original) +++ pypy/branch/py11/pypy/jit/tl/tla/targettla.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla import tla Modified: pypy/branch/py11/pypy/jit/tl/tla/tla_assembler.py ============================================================================== --- pypy/branch/py11/pypy/jit/tl/tla/tla_assembler.py (original) +++ pypy/branch/py11/pypy/jit/tl/tla/tla_assembler.py Tue Nov 10 16:26:43 2009 @@ -2,7 +2,7 @@ import sys import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla.test_tla import assemble def usage(): Modified: pypy/branch/py11/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py Tue Nov 10 16:26:43 2009 @@ -11,7 +11,7 @@ # ------------------------------------------------------------------------------ -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" # filename = ROM_PATH + "/rom9/rom9.gb" filename = "/home/tverwaes/roms/SuperMarioLand.gb" SOCKET_PORT = 55682 @@ -83,7 +83,7 @@ # ------------------------------------------------------------------------------ -MARIO_DIR = str(py.magic.autopath().dirpath().dirpath()\ +MARIO_DIR = str(py.path.local(__file__).dirpath().dirpath()\ .dirpath().dirpath()\ .dirpath().dirpath()) + "/mario" Modified: pypy/branch/py11/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Tue Nov 10 16:26:43 2009 @@ -8,7 +8,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/branch/py11/pypy/lang/gameboy/profiling/gameboyTest.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/profiling/gameboyTest.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/profiling/gameboyTest.py Tue Nov 10 16:26:43 2009 @@ -7,7 +7,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/branch/py11/pypy/lang/gameboy/test/test_cartridge.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/test/test_cartridge.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/test/test_cartridge.py Tue Nov 10 16:26:43 2009 @@ -8,7 +8,7 @@ def mapToByte(value): return ord(value) & 0xFF -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" CONTENT = "abcdefghijklmnopqrstuvwxyz1234567890" MAPPED_CONTENT = map_to_byte(CONTENT) Modified: pypy/branch/py11/pypy/lang/gameboy/test/test_rom.py ============================================================================== --- pypy/branch/py11/pypy/lang/gameboy/test/test_rom.py (original) +++ pypy/branch/py11/pypy/lang/gameboy/test/test_rom.py Tue Nov 10 16:26:43 2009 @@ -6,7 +6,7 @@ from pypy.lang.gameboy.gameboy import * -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" EMULATION_CYCLES = 64 # ------------------------------------------------------------------------------ Modified: pypy/branch/py11/pypy/lang/js/jsparser.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/jsparser.py (original) +++ pypy/branch/py11/pypy/lang/js/jsparser.py Tue Nov 10 16:26:43 2009 @@ -2,7 +2,7 @@ from pypy.rlib.parsing.parsing import ParseError, Rule import py -GFILE = py.magic.autopath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/branch/py11/pypy/lang/js/test/ecma/conftest.py Tue Nov 10 16:26:43 2009 @@ -8,7 +8,7 @@ interpreter.TEST = True -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() exclusionlist = ['shell.js', 'browser.js'] def pytest_addoption(parser): Modified: pypy/branch/py11/pypy/lang/js/test/test_interactive.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/test/test_interactive.py (original) +++ pypy/branch/py11/pypy/lang/js/test/test_interactive.py Tue Nov 10 16:26:43 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv): - return self._spawn(str(py.magic.autopath().dirpath().dirpath().join('js_interactive.py')), argv) + return self._spawn(str(py.path.local(__file__).dirpath().dirpath().join('js_interactive.py')), argv) def prompt_send(self, message): self.child.expect('js>') Modified: pypy/branch/py11/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/branch/py11/pypy/lang/js/test/test_parser.py (original) +++ pypy/branch/py11/pypy/lang/js/test/test_parser.py Tue Nov 10 16:26:43 2009 @@ -9,7 +9,7 @@ from pypy import conftest import sys -GFILE = py.magic.autopath().dirpath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/branch/py11/pypy/lang/prolog/interpreter/conftest.py ============================================================================== --- pypy/branch/py11/pypy/lang/prolog/interpreter/conftest.py (original) +++ pypy/branch/py11/pypy/lang/prolog/interpreter/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,6 +1,6 @@ import py, sys -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() Option = py.test.config.Option Modified: pypy/branch/py11/pypy/lang/prolog/interpreter/interactive.py ============================================================================== --- pypy/branch/py11/pypy/lang/prolog/interpreter/interactive.py (original) +++ pypy/branch/py11/pypy/lang/prolog/interpreter/interactive.py Tue Nov 10 16:26:43 2009 @@ -7,7 +7,7 @@ import py import sys -#sys.path.append(str(py.magic.autopath().dirpath().dirpath())) +#sys.path.append(str(py.path.local(__file__).dirpath().dirpath())) from pypy.rlib.parsing.parsing import ParseError from pypy.rlib.parsing.deterministic import LexerError Modified: pypy/branch/py11/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/branch/py11/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/branch/py11/pypy/lang/prolog/interpreter/parsing.py Tue Nov 10 16:26:43 2009 @@ -3403,7 +3403,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/py11/pypy/lang/scheme/execution.py ============================================================================== --- pypy/branch/py11/pypy/lang/scheme/execution.py (original) +++ pypy/branch/py11/pypy/lang/scheme/execution.py Tue Nov 10 16:26:43 2009 @@ -19,7 +19,7 @@ except (TypeError, AttributeError): pass -de_file = py.magic.autopath().dirpath().join("r5rs_derived_expr.ss") +de_file = py.path.local(__file__).dirpath().join("r5rs_derived_expr.ss") de_code = de_file.read() de_expr_lst = parse(de_code) Modified: pypy/branch/py11/pypy/lang/scheme/test/test_interactive.py ============================================================================== --- pypy/branch/py11/pypy/lang/scheme/test/test_interactive.py (original) +++ pypy/branch/py11/pypy/lang/scheme/test/test_interactive.py Tue Nov 10 16:26:43 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv=[]): - path = py.magic.autopath()/".."/".."/"interactive.py" + path = py.path.local(__file__)/".."/".."/"interactive.py" return self._spawn(str(path), argv) def test_interactive(self): Modified: pypy/branch/py11/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/py11/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/py11/pypy/lang/smalltalk/test/test_miniimage.py Tue Nov 10 16:26:43 2009 @@ -12,7 +12,7 @@ def setup_module(module, filename='mini.image'): space = objspace.ObjSpace() - module.mini_image = py.magic.autopath().dirpath().dirpath().join(filename) + module.mini_image = py.path.local(__file__).dirpath().dirpath().join(filename) module.reader = open_miniimage(space) reader.initialize() module.image = squeakimage.SqueakImage() Modified: pypy/branch/py11/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/branch/py11/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/branch/py11/pypy/lang/smalltalk/tool/analyseimage.py Tue Nov 10 16:26:43 2009 @@ -6,7 +6,7 @@ from pypy.lang.smalltalk import interpreter import sys -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') +mini_image = py.path.local(__file__).dirpath().dirpath().join('mini.image') def get_miniimage(space): return squeakimage.ImageReader(space, squeakimage.Stream(mini_image.open())) Modified: pypy/branch/py11/pypy/lib/app_test/ctypes_tests/conftest.py ============================================================================== --- pypy/branch/py11/pypy/lib/app_test/ctypes_tests/conftest.py (original) +++ pypy/branch/py11/pypy/lib/app_test/ctypes_tests/conftest.py Tue Nov 10 16:26:43 2009 @@ -11,7 +11,7 @@ from pypy.translator.platform import platform from pypy.translator.tool.cbuild import ExternalCompilationInfo udir = py.test.ensuretemp('_ctypes_test') - cfile = py.magic.autopath().dirpath().join("_ctypes_test.c") + cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': libraries = ['oleaut32'] Modified: pypy/branch/py11/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/branch/py11/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/branch/py11/pypy/module/bz2/test/test_bz2_compdecomp.py Tue Nov 10 16:26:43 2009 @@ -25,7 +25,7 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.BUGGY_DATA = py.magic.autopath().dirpath().join('data.bz2').read() + mod.BUGGY_DATA = py.path.local(__file__).dirpath().join('data.bz2').read() mod.decompress = decompress class AppTestBZ2Compressor(CheckAllocation): Modified: pypy/branch/py11/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/branch/py11/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/branch/py11/pypy/module/pypyjit/test/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit options") + group = parser.getgroup("pypyjit options") group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/branch/py11/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/py11/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/py11/pypy/objspace/std/test/test_complexobject.py Tue Nov 10 16:26:43 2009 @@ -67,7 +67,7 @@ sys.path.append(%r) import helper return helper - """ % (str(py.magic.autopath().dirpath()))) + """ % (str(py.path.local(__file__).dirpath()))) def test_div(self): h = self.helper Modified: pypy/branch/py11/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/ebnfparse.py Tue Nov 10 16:26:43 2009 @@ -2125,7 +2125,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/py11/pypy/rlib/parsing/makepackrat.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/makepackrat.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/makepackrat.py Tue Nov 10 16:26:43 2009 @@ -718,7 +718,7 @@ def test_generate(): - f = py.magic.autopath().dirpath().join("pypackrat.py") + f = py.path.local(__file__).dirpath().join("pypackrat.py") from pypackrat import PyPackratSyntaxParser p = PyPackratSyntaxParser(syntax) t = p.file() Modified: pypy/branch/py11/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/regexparse.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/regexparse.py Tue Nov 10 16:26:43 2009 @@ -1966,7 +1966,7 @@ def test_generate(): - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/py11/pypy/rlib/parsing/test/test_pythonlexer.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/test/test_pythonlexer.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/test/test_pythonlexer.py Tue Nov 10 16:26:43 2009 @@ -230,7 +230,7 @@ assert tokens[i * 3].name == 'String' def test_self(): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlexer.tokenize(s) print tokens @@ -263,7 +263,7 @@ "op": "operator", } import tokenize, token - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlex(s) print [t.name for t in tokens][:20] tokens2 = list(tokenize.generate_tokens(iter(s.splitlines(True)).next)) Modified: pypy/branch/py11/pypy/rlib/parsing/test/test_pythonparse.py ============================================================================== --- pypy/branch/py11/pypy/rlib/parsing/test/test_pythonparse.py (original) +++ pypy/branch/py11/pypy/rlib/parsing/test/test_pythonparse.py Tue Nov 10 16:26:43 2009 @@ -7,7 +7,7 @@ from pypy.rlib.parsing.parsing import PackratParser, Symbol, ParseError, Rule from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function -grammar = py.magic.autopath().dirpath().join("pygrammar.txt").read(mode='rt') +grammar = py.path.local(__file__).dirpath().join("pygrammar.txt").read(mode='rt') def test_parse_grammar(): @@ -240,12 +240,12 @@ t = self.ToAST.transform(t) def test_parse_this(self): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() t = self.parse(s) t = self.ToAST.transform(t) def test_parsing(self): - s = py.magic.autopath().dirpath().dirpath().join("parsing.py").read() + s = py.path.local(__file__).dirpath().dirpath().join("parsing.py").read() t = self.parse(s) t = self.ToAST.transform(t) Modified: pypy/branch/py11/pypy/rlib/rsdl/eci.py ============================================================================== --- pypy/branch/py11/pypy/rlib/rsdl/eci.py (original) +++ pypy/branch/py11/pypy/rlib/rsdl/eci.py Tue Nov 10 16:26:43 2009 @@ -9,7 +9,7 @@ includes = ['SDL.h'], include_dirs = ['/Library/Frameworks/SDL.framework/Headers'], link_files = [ - str(py.magic.autopath().dirpath().join('macosx-sdl-main/SDLMain.m')), + str(py.path.local(__file__).dirpath().join('macosx-sdl-main/SDLMain.m')), ], frameworks = ['SDL', 'Cocoa'] ) Modified: pypy/branch/py11/pypy/rlib/test/test_listsort.py ============================================================================== --- pypy/branch/py11/pypy/rlib/test/test_listsort.py (original) +++ pypy/branch/py11/pypy/rlib/test/test_listsort.py Tue Nov 10 16:26:43 2009 @@ -35,7 +35,7 @@ sorttest(lst1) def test_file(): - for fn in py.magic.autopath().dirpath().listdir(): + for fn in py.path.local(__file__).dirpath().listdir(): if fn.ext == '.py': lines1 = fn.readlines() sorttest(lines1) Modified: pypy/branch/py11/pypy/rpython/module/test/test_ll_os_path.py ============================================================================== --- pypy/branch/py11/pypy/rpython/module/test/test_ll_os_path.py (original) +++ pypy/branch/py11/pypy/rpython/module/test/test_ll_os_path.py Tue Nov 10 16:26:43 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir def test_exists(): - filename = impl.to_rstr(str(py.magic.autopath())) + filename = impl.to_rstr(str(py.path.local(__file__))) assert impl.ll_os_path_exists(filename) == True assert not impl.ll_os_path_exists(impl.to_rstr( "strange_filename_that_looks_improbable.sde")) Modified: pypy/branch/py11/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/branch/py11/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/branch/py11/pypy/rpython/test/test_rbuiltin.py Tue Nov 10 16:26:43 2009 @@ -294,7 +294,7 @@ def f(fn): fn = hlstr(fn) return os.path.exists(fn) - filename = self.string_to_ll(str(py.magic.autopath())) + filename = self.string_to_ll(str(py.path.local(__file__))) assert self.interpret(f, [filename]) == True #assert self.interpret(f, [ # self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False @@ -308,7 +308,7 @@ fn = hlstr(fn) return os.path.isdir(fn) assert self.interpret(f, [self.string_to_ll("/")]) == True - assert self.interpret(f, [self.string_to_ll(str(py.magic.autopath()))]) == False + assert self.interpret(f, [self.string_to_ll(str(py.path.local(__file__)))]) == False assert self.interpret(f, [self.string_to_ll("another/unlikely/directory/name")]) == False def test_pbc_isTrue(self): Modified: pypy/branch/py11/pypy/tool/bench/pypyresult.py ============================================================================== --- pypy/branch/py11/pypy/tool/bench/pypyresult.py (original) +++ pypy/branch/py11/pypy/tool/bench/pypyresult.py Tue Nov 10 16:26:43 2009 @@ -52,7 +52,7 @@ if __name__ == "__main__": - x = py.magic.autopath().dirpath("bench-unix.benchmark_result") + x = py.path.local(__file__).dirpath("bench-unix.benchmark_result") db = ResultDB() db.parsepickle(x) Modified: pypy/branch/py11/pypy/tool/option.py ============================================================================== --- pypy/branch/py11/pypy/tool/option.py (original) +++ pypy/branch/py11/pypy/tool/option.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ import os from pypy.config.pypyoption import get_pypy_config from pypy.config.config import Config, OptionDescription, to_optparse -from py.compat import optparse +import optparse extra_useage = """For detailed descriptions of all the options see http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" Modified: pypy/branch/py11/pypy/tool/pytest/genreportdata.py ============================================================================== --- pypy/branch/py11/pypy/tool/pytest/genreportdata.py (original) +++ pypy/branch/py11/pypy/tool/pytest/genreportdata.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ import py import sys -mydir = py.magic.autopath().dirpath().realpath() +mydir = py.path.local(__file__).dirpath().realpath() from pypy.tool.pytest import htmlreport from pypy.tool.pytest import confpath Modified: pypy/branch/py11/pypy/tool/pytest/htmlreport.py ============================================================================== --- pypy/branch/py11/pypy/tool/pytest/htmlreport.py (original) +++ pypy/branch/py11/pypy/tool/pytest/htmlreport.py Tue Nov 10 16:26:43 2009 @@ -232,7 +232,7 @@ t = self.rep.render_latest_table(self.rep.results) assert unicode(t) -mydir = py.magic.autopath().dirpath() +mydir = py.path.local(__file__).dirpath() def getpicklepath(): return mydir.join('.htmlreport.pickle') Modified: pypy/branch/py11/pypy/tool/pytest/test/test_new_count.py ============================================================================== --- pypy/branch/py11/pypy/tool/pytest/test/test_new_count.py (original) +++ pypy/branch/py11/pypy/tool/pytest/test/test_new_count.py Tue Nov 10 16:26:43 2009 @@ -2,7 +2,7 @@ import py #from pypy.tool.pytest.confpath import testresultdir from pypy.tool.pytest.result import ResultFromMime -testpath = py.magic.autopath().dirpath('data') +testpath = py.path.local(__file__).dirpath('data') class TestResultCache: Modified: pypy/branch/py11/pypy/tool/test/test_conftest1.py ============================================================================== --- pypy/branch/py11/pypy/tool/test/test_conftest1.py (original) +++ pypy/branch/py11/pypy/tool/test/test_conftest1.py Tue Nov 10 16:26:43 2009 @@ -1,7 +1,7 @@ import py -innertest = py.magic.autopath().dirpath('conftest1_innertest.py') +innertest = py.path.local(__file__).dirpath('conftest1_innertest.py') pytest_plugins = "pytest_pytester" class TestPyPyTests: Modified: pypy/branch/py11/pypy/tool/udir.py ============================================================================== --- pypy/branch/py11/pypy/tool/udir.py (original) +++ pypy/branch/py11/pypy/tool/udir.py Tue Nov 10 16:26:43 2009 @@ -35,7 +35,7 @@ dir = local(dir) if basename is None: try: - p = py.magic.autopath().dirpath() + p = py.path.local(__file__).dirpath() basename = svn_info(py.path.svnwc(p).info().url) except: basename = '' Modified: pypy/branch/py11/pypy/translator/benchmark/benchmarks.py ============================================================================== --- pypy/branch/py11/pypy/translator/benchmark/benchmarks.py (original) +++ pypy/branch/py11/pypy/translator/benchmark/benchmarks.py Tue Nov 10 16:26:43 2009 @@ -34,7 +34,7 @@ def external_dependency(dirname, svnurl, revision): """Check out (if necessary) a given fixed revision of a svn url.""" - dirpath = py.magic.autopath().dirpath().join(dirname) + dirpath = py.path.local(__file__).dirpath().join(dirname) revtag = dirpath.join('-svn-rev-') if dirpath.check(): if not revtag.check() or int(revtag.read()) != revision: @@ -70,13 +70,13 @@ return get_result(txt, PYSTONE_PATTERN) def run_richards(executable='/usr/local/bin/python', n=5): - richards = py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py') + richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') txt = run_cmd('"%s" %s %s' % (executable, richards, n)) return get_result(txt, RICHARDS_PATTERN) def run_translate(executable='/usr/local/bin/python'): - translate = py.magic.autopath().dirpath().dirpath().join('goal').join('translate.py') - target = py.magic.autopath().dirpath().dirpath().join('goal').join('targetrpystonedalone.py') + translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') + target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' T = time.time() status = os.system(argstr%(executable, translate, target)) @@ -87,7 +87,7 @@ def run_docutils(executable='/usr/local/bin/python'): docutilssvnpath = 'docutils' # subdir of the local dir - translatetxt = py.magic.autopath().dirpath().dirpath().dirpath().join('doc').join('translation.txt') + translatetxt = py.path.local(__file__).dirpath().dirpath().dirpath().join('doc').join('translation.txt') command = """import sys sys.modules['unicodedata'] = sys # docutils need 'import unicodedata' to work, but no more... sys.path[0:0] = ['%s', '%s/extras'] @@ -123,7 +123,7 @@ templess is some simple templating language, to check out use 'svn co -r100 http://johnnydebris.net/templess/trunk templess' """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() pypath = os.path.dirname(os.path.dirname(py.__file__)) templessdir = here.join('templess') testscript = templessdir.join('test/oneshot.py') @@ -143,7 +143,7 @@ def run_gadfly(executable='/usr/local/bin/python'): """ run some tests in the gadfly pure Python database """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() gadfly = here.join('gadfly') testscript = gadfly.join('test', 'testsubset.py') command = 'PYTHONPATH="%s" "%s" "%s"' % (gadfly, executable, testscript) @@ -167,7 +167,7 @@ def run_mako(executable='/usr/local/bin/python'): """ run some tests in the mako templating system """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() mako = here.join('mako') testscript = mako.join('examples', 'bench', 'basic.py') command = 'PYTHONPATH="%s" "%s" "%s" mako' % (mako.join('lib'), Modified: pypy/branch/py11/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/branch/py11/pypy/translator/benchmark/jitbench.py (original) +++ pypy/branch/py11/pypy/translator/benchmark/jitbench.py Tue Nov 10 16:26:43 2009 @@ -14,7 +14,7 @@ response.read() def run_richards(executable='python'): - richards = str(py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py')) + richards = str(py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py')) pipe = subprocess.Popen([executable, richards], stdout=subprocess.PIPE, stderr=subprocess.PIPE) return pipe.communicate() Modified: pypy/branch/py11/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/py11/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/py11/pypy/translator/c/test/test_extfunc.py Tue Nov 10 16:26:43 2009 @@ -141,7 +141,7 @@ os.unlink(filename) def test_os_access(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_access(path, mode): return os.access(path, mode) f = compile(call_access, [str, int]) @@ -150,7 +150,7 @@ def test_os_stat(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) has_blksize = hasattr(os.stat_result, 'st_blksize') has_blocks = hasattr(os.stat_result, 'st_blocks') def call_stat(): @@ -189,7 +189,7 @@ def test_os_fstat(): if os.environ.get('PYPY_CC', '').startswith('tcc'): py.test.skip("segfault with tcc :-(") - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_fstat(): fd = os.open(filename, os.O_RDONLY, 0777) st = os.fstat(fd) Modified: pypy/branch/py11/pypy/translator/cli/conftest.py ============================================================================== --- pypy/branch/py11/pypy/translator/cli/conftest.py (original) +++ pypy/branch/py11/pypy/translator/cli/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-cli options") + group = parser.getgroup("pypy-cli options") group.addoption('--source', action="store_true", dest="source", default=False, help="only generate IL source, don't compile") group.addoption('--wd', action="store_true", dest="wd", default=False, Modified: pypy/branch/py11/pypy/translator/driver.py ============================================================================== --- pypy/branch/py11/pypy/translator/driver.py (original) +++ pypy/branch/py11/pypy/translator/driver.py Tue Nov 10 16:26:43 2009 @@ -9,7 +9,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings from pypy.annotation import policy as annpolicy -from py.compat import optparse +import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS Modified: pypy/branch/py11/pypy/translator/goal/targetgbfullprofiling.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/targetgbfullprofiling.py (original) +++ pypy/branch/py11/pypy/translator/goal/targetgbfullprofiling.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import GameBoyProfiler -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" def entry_point(argv=None): if argv is not None and len(argv) > 1: Modified: pypy/branch/py11/pypy/translator/goal/targetgbimplementation.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/targetgbimplementation.py (original) +++ pypy/branch/py11/pypy/translator/goal/targetgbimplementation.py Tue Nov 10 16:26:43 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.gameboy_implementation import GameBoyImplementation -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" print ROM_PATH Modified: pypy/branch/py11/pypy/translator/goal/targetgbrom4.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/targetgbrom4.py (original) +++ pypy/branch/py11/pypy/translator/goal/targetgbrom4.py Tue Nov 10 16:26:43 2009 @@ -4,7 +4,7 @@ from pypy.lang.gameboy.gameboy import GameBoy -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" EMULATION_CYCLES = 1<<24 Modified: pypy/branch/py11/pypy/translator/goal/targetpreimportedpypy.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/targetpreimportedpypy.py (original) +++ pypy/branch/py11/pypy/translator/goal/targetpreimportedpypy.py Tue Nov 10 16:26:43 2009 @@ -23,7 +23,7 @@ "random", ] -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/branch/py11/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/py11/pypy/translator/goal/targetpypystandalone.py Tue Nov 10 16:26:43 2009 @@ -11,7 +11,7 @@ from pypy.tool.option import make_objspace from pypy.translator.goal.nanos import setup_nanos -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/branch/py11/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/py11/pypy/translator/goal/translate.py (original) +++ pypy/branch/py11/pypy/translator/goal/translate.py Tue Nov 10 16:26:43 2009 @@ -84,7 +84,7 @@ import py # we want 2.4 expand_default functionality -optparse = py.compat.optparse +import optparse from pypy.tool.ansi_print import ansi_log log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) Modified: pypy/branch/py11/pypy/translator/interactive.py ============================================================================== --- pypy/branch/py11/pypy/translator/interactive.py (original) +++ pypy/branch/py11/pypy/translator/interactive.py Tue Nov 10 16:26:43 2009 @@ -1,4 +1,4 @@ -from py.compat import optparse +import optparse import autopath from pypy.translator.translator import TranslationContext Modified: pypy/branch/py11/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/branch/py11/pypy/translator/jvm/conftest.py (original) +++ pypy/branch/py11/pypy/translator/jvm/conftest.py Tue Nov 10 16:26:43 2009 @@ -1,6 +1,6 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-jvm options") + group = parser.getgroup("pypy-jvm options") group.addoption('--java', action='store', dest='java', default='java', help='Define the java executable to use') group.addoption('--javac', action='store', dest='javac', Modified: pypy/branch/py11/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/py11/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/py11/pypy/translator/jvm/genjvm.py Tue Nov 10 16:26:43 2009 @@ -83,7 +83,7 @@ self.jasmin_files = None # Determine various paths: - self.thisdir = py.magic.autopath().dirpath() + self.thisdir = py.path.local(__file__).dirpath() self.rootdir = self.thisdir.join('src') self.srcdir = self.rootdir.join('pypy') self.jnajar = self.rootdir.join('jna.jar') Modified: pypy/branch/py11/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/py11/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/py11/pypy/translator/platform/test/test_darwin.py Tue Nov 10 16:26:43 2009 @@ -32,7 +32,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(frameworks=('Cocoa',), include_dirs=(includedir,)) executable = self.platform.compile([objcfile], eci) Modified: pypy/branch/py11/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/branch/py11/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/branch/py11/pypy/translator/platform/test/test_maemo.py Tue Nov 10 16:26:43 2009 @@ -26,7 +26,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(include_dirs=(includedir,)) executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) Modified: pypy/branch/py11/pypy/translator/test/test_driver.py ============================================================================== --- pypy/branch/py11/pypy/translator/test/test_driver.py (original) +++ pypy/branch/py11/pypy/translator/test/test_driver.py Tue Nov 10 16:26:43 2009 @@ -1,7 +1,7 @@ import py from pypy.translator.driver import TranslationDriver -from py.compat import optparse +import optparse def test_ctr(): td = TranslationDriver() Modified: pypy/branch/py11/pytest_resultlog.py ============================================================================== --- pypy/branch/py11/pytest_resultlog.py (original) +++ pypy/branch/py11/pytest_resultlog.py Tue Nov 10 16:26:43 2009 @@ -6,7 +6,7 @@ import py def pytest_addoption(parser): - group = parser.addgroup("resultlog", "resultlog plugin options") + group = parser.getgroup("resultlog", "resultlog plugin options") group.addoption('--resultlog', action="store", dest="resultlog", metavar="path", default=None, help="path for machine-readable result log.") From pedronis at codespeak.net Tue Nov 10 16:35:12 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 10 Nov 2009 16:35:12 +0100 (CET) Subject: [pypy-svn] r69122 - pypy/trunk/pypy/translator/c/src Message-ID: <20091110153512.995C4168102@codespeak.net> Author: pedronis Date: Tue Nov 10 16:35:12 2009 New Revision: 69122 Modified: pypy/trunk/pypy/translator/c/src/debug.h Log: detect linux Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Tue Nov 10 16:35:12 2009 @@ -45,7 +45,7 @@ #ifndef PYPY_NOT_MAIN_FILE #include -#if defined(__GNUC__) && !defined(WIN32) +#if defined(__GNUC__) && defined(__linux__) # include static void pypy_setup_profiling() { From arigo at codespeak.net Tue Nov 10 16:38:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Nov 2009 16:38:56 +0100 (CET) Subject: [pypy-svn] r69123 - in pypy/branch/merge-guards-2/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091110153856.08192168102@codespeak.net> Author: arigo Date: Tue Nov 10 16:38:56 2009 New Revision: 69123 Modified: pypy/branch/merge-guards-2/pypy/jit/backend/llgraph/llimpl.py pypy/branch/merge-guards-2/pypy/jit/metainterp/optimizeopt.py pypy/branch/merge-guards-2/pypy/jit/metainterp/resoperation.py pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_basic.py pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_optimizeopt.py Log: Merge guard_nonnull followed by guard_class and/or guard_value. Modified: pypy/branch/merge-guards-2/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/backend/llgraph/llimpl.py Tue Nov 10 16:38:56 2009 @@ -134,6 +134,7 @@ 'guard_overflow' : ((), None), 'guard_nonnull' : (('ref',), None), 'guard_isnull' : (('ref',), None), + 'guard_nonnull_class' : (('ref', 'ref'), None), 'newstr' : (('int',), 'ref'), 'strlen' : (('ref',), 'int'), 'strgetitem' : (('ref', 'int'), 'int'), @@ -562,6 +563,11 @@ if value.typeptr != expected_class: raise GuardFailed + def op_guard_nonnull_class(self, _, value, expected_class): + if not value: + raise GuardFailed + self.op_guard_class(_, value, expected_class) + def op_guard_value(self, _, value, expected_value): if value != expected_value: raise GuardFailed Modified: pypy/branch/merge-guards-2/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/metainterp/optimizeopt.py Tue Nov 10 16:38:56 2009 @@ -42,8 +42,8 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'guard_class_index', 'level') - guard_class_index = -1 + _attrs_ = ('box', 'known_class', 'last_guard_index', 'level') + last_guard_index = -1 level = LEVEL_UNKNOWN @@ -88,10 +88,15 @@ return None def make_constant_class(self, classbox, opindex): - if self.level < LEVEL_KNOWNCLASS: - self.known_class = classbox - self.level = LEVEL_KNOWNCLASS - self.guard_class_index = opindex + assert self.level < LEVEL_KNOWNCLASS + self.known_class = classbox + self.level = LEVEL_KNOWNCLASS + self.last_guard_index = opindex + + def make_nonnull(self, opindex): + assert self.level < LEVEL_NONNULL + self.level = LEVEL_NONNULL + self.last_guard_index = opindex def is_nonnull(self): level = self.level @@ -104,7 +109,7 @@ else: return False - def make_nonnull(self): + def ensure_nonnull(self): if self.level < LEVEL_NONNULL: self.level = LEVEL_NONNULL @@ -577,17 +582,26 @@ elif value.is_null(): raise InvalidLoop self.emit_operation(op) - value.make_nonnull() + value.make_nonnull(len(self.newoperations) - 1) def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.args[0]) emit_operation = True - if value.guard_class_index != -1: - # there already has been a guard_class on this value, which is - # rather silly. replace the original guard_class with a guard_value - guard_class_op = self.newoperations[value.guard_class_index] - guard_class_op.opnum = op.opnum - guard_class_op.args[1] = op.args[1] + if value.last_guard_index != -1: + # there already has been a guard_nonnull or guard_class or + # guard_nonnull_class on this value, which is rather silly. + # replace the original guard with a guard_value + old_guard_op = self.newoperations[value.last_guard_index] + old_opnum = old_guard_op.opnum + old_guard_op.opnum = op.opnum + old_guard_op.args = [old_guard_op.args[0], op.args[1]] + if old_opnum == rop.GUARD_NONNULL: + # hack hack hack. Change the guard_opnum on + # old_guard_op.descr so that when resuming, + # the operation is not skipped by pyjitpl.py. + descr = old_guard_op.descr + assert isinstance(descr, compile.ResumeGuardDescr) + descr.guard_opnum = rop.GUARD_NONNULL_CLASS emit_operation = False constbox = op.args[1] assert isinstance(constbox, Const) @@ -610,8 +624,29 @@ # earlier, in optimizefindnode.py. assert realclassbox.same_constant(expectedclassbox) return - self.emit_operation(op) - value.make_constant_class(expectedclassbox, len(self.newoperations) - 1) + emit_operation = True + if value.last_guard_index != -1: + # there already has been a guard_nonnull or guard_class or + # guard_nonnull_class on this value. + old_guard_op = self.newoperations[value.last_guard_index] + if old_guard_op.opnum == rop.GUARD_NONNULL: + # it was a guard_nonnull, which we replace with a + # guard_nonnull_class. + old_guard_op.opnum = rop.GUARD_NONNULL_CLASS + old_guard_op.args = [old_guard_op.args[0], op.args[1]] + # hack hack hack. Change the guard_opnum on + # old_guard_op.descr so that when resuming, + # the operation is not skipped by pyjitpl.py. + descr = old_guard_op.descr + assert isinstance(descr, compile.ResumeGuardDescr) + descr.guard_opnum = rop.GUARD_NONNULL_CLASS + emit_operation = False + if emit_operation: + self.emit_operation(op) + last_guard_index = len(self.newoperations) - 1 + else: + last_guard_index = value.last_guard_index + value.make_constant_class(expectedclassbox, last_guard_index) def optimize_GUARD_NO_EXCEPTION(self, op): if not self.exception_might_have_happened: @@ -669,7 +704,7 @@ assert fieldvalue is not None self.make_equal_to(op.result, fieldvalue) else: - value.make_nonnull() + value.ensure_nonnull() self.heap_op_optimizer.optimize_GETFIELD_GC(op, value) # note: the following line does not mean that the two operations are @@ -681,7 +716,7 @@ if value.is_virtual(): value.setfield(op.descr, self.getvalue(op.args[1])) else: - value.make_nonnull() + value.ensure_nonnull() fieldvalue = self.getvalue(op.args[1]) self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue) @@ -708,7 +743,7 @@ if value.is_virtual(): self.make_constant_int(op.result, value.getlength()) else: - value.make_nonnull() + value.ensure_nonnull() self.optimize_default(op) def optimize_GETARRAYITEM_GC(self, op): @@ -719,7 +754,7 @@ itemvalue = value.getitem(indexbox.getint()) self.make_equal_to(op.result, itemvalue) return - value.make_nonnull() + value.ensure_nonnull() self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value) # note: the following line does not mean that the two operations are @@ -733,7 +768,7 @@ if indexbox is not None: value.setitem(indexbox.getint(), self.getvalue(op.args[2])) return - value.make_nonnull() + value.ensure_nonnull() fieldvalue = self.getvalue(op.args[2]) self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue) @@ -873,7 +908,7 @@ self.optimizer.make_equal_to(op.result, fieldvalue) return # default case: produce the operation - value.make_nonnull() + value.ensure_nonnull() self.optimizer.optimize_default(op) # then remember the result of reading the field fieldvalue = self.optimizer.getvalue(op.result) Modified: pypy/branch/merge-guards-2/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/metainterp/resoperation.py Tue Nov 10 16:38:56 2009 @@ -119,6 +119,7 @@ 'GUARD_CLASS', 'GUARD_NONNULL', 'GUARD_ISNULL', + 'GUARD_NONNULL_CLASS', '_GUARD_FOLDABLE_LAST', 'GUARD_NO_EXCEPTION', 'GUARD_EXCEPTION', Modified: pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_basic.py Tue Nov 10 16:38:56 2009 @@ -1003,10 +1003,10 @@ class A(object): def g(self, x): - return x - 1 + return x - 5 class B(A): def g(self, y): - return y - 2 + return y - 3 a1 = A() a2 = A() @@ -1021,8 +1021,125 @@ hint(a, promote=True) return x res = self.meta_interp(f, [299], listops=True) + assert res == f(299) self.check_loops(guard_class=0, guard_value=3) + def test_merge_guardnonnull_guardclass(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 3 + class B(A): + def g(self, y): + return y - 5 + + a1 = A() + b1 = B() + def f(x): + l = [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x = a.g(x) + else: + x -= 7 + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, + guard_nonnull_class=2, guard_isnull=1) + + def test_merge_guardnonnull_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + pass + class B(A): + pass + + a1 = A() + b1 = B() + def f(x): + l = [b1] * 100 + [None] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x -= 5 + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + + def test_merge_guardnonnull_guardvalue_2(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + pass + class B(A): + pass + + a1 = A() + b1 = B() + def f(x): + l = [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x -= 5 + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + + def test_merge_guardnonnull_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 3 + class B(A): + def g(self, y): + return y - 5 + + a1 = A() + a2 = A() + b1 = B() + def f(x): + l = [a2] * 100 + [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x = a.g(x) + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [399], listops=True) + assert res == f(399) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=3, + guard_nonnull_class=0, guard_isnull=1) + def test_residual_call_doesnt_lose_info(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) Modified: pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/metainterp/test/test_optimizeopt.py Tue Nov 10 16:38:56 2009 @@ -1472,6 +1472,56 @@ """ self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + def test_merge_guard_nonnull_guard_class(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_class(p1, ConstClass(node_vtable)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_nonnull_class(p1, ConstClass(node_vtable)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + def test_merge_guard_nonnull_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + def test_merge_guard_nonnull_guard_class_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_class(p1, ConstClass(node_vtable)) [i2] + i4 = int_sub(i3, 1) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i4, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + i4 = int_sub(i3, 1) + jump(p2, i0, i1, i4, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) # ---------- From afa at codespeak.net Tue Nov 10 17:20:54 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 10 Nov 2009 17:20:54 +0100 (CET) Subject: [pypy-svn] r69125 - in pypy/branch/msvc-asmgcroot/pypy/translator: c c/gcc c/gcc/test platform Message-ID: <20091110162054.80445168104@codespeak.net> Author: afa Date: Tue Nov 10 17:20:54 2009 New Revision: 69125 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Log: More fixes in trackgcroot to enable -O2 optimization on Windows. "pypy-c-jit.exe richards.py" runs in 3.4s, similar to linux. Next step: fix callbacks Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/test_trackgcroot.py Tue Nov 10 17:20:54 2009 @@ -8,7 +8,6 @@ from pypy.translator.c.gcc.trackgcroot import DarwinAssemblerParser from pypy.translator.c.gcc.trackgcroot import compress_callshape from pypy.translator.c.gcc.trackgcroot import decompress_callshape -from pypy.translator.c.gcc.trackgcroot import OFFSET_LABELS from pypy.translator.c.gcc.trackgcroot import PARSERS from StringIO import StringIO @@ -159,7 +158,8 @@ expectedlines.insert(i-1, '%s::\n' % (label,)) else: expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) - expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + expectedlines.insert(i-1, '%s=.+%d\n' % (label, + tracker.OFFSET_LABELS)) if format == 'msvc' and r_gcroot_constant.match(line): expectedlines[i] = ';' + expectedlines[i] expectedlines[i+1] = (expectedlines[i+1] Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Tue Nov 10 17:20:54 2009 @@ -13,8 +13,6 @@ from pypy.translator.c.gcc.instruction import LOC_REG, LOC_NOWHERE, LOC_MASK from pypy.translator.c.gcc.instruction import LOC_EBP_BASED, LOC_ESP_BASED -OFFSET_LABELS = 2**30 - class FunctionGcRootTracker(object): skip = 0 @@ -22,7 +20,7 @@ def init_regexp(cls): cls.r_label = re.compile(cls.LABEL+"[:]\s*$") cls.r_globl = re.compile(r"\t[.]globl\t"+cls.LABEL+"\s*$") - cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) + cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) cls.r_insn = re.compile(r"\t([a-z]\w*)\s") cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") @@ -323,7 +321,7 @@ # value a big constant, which is subtracted again when we need # the original value for gcmaptable.s. That's a hack. self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, - OFFSET_LABELS)) + self.OFFSET_LABELS)) self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label @@ -558,6 +556,11 @@ if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 while not self.r_jmptable_end.match(self.lines[tablelin]): + # skip empty lines + if (not self.lines[tablelin].strip() + or self.lines[tablelin].startswith(';')): + tablelin += 1 + continue match = self.r_jmptable_item.match(self.lines[tablelin]) if not match: raise NoPatternMatch(repr(self.lines[tablelin])) @@ -624,32 +627,19 @@ def visit_call(self, line): match = self.r_unaryinsn.match(line) + if match is None: assert self.r_unaryinsn_star.match(line) # indirect call - else: - target = match.group(1) - if target in self.FUNCTIONS_NOT_RETURNING: - return [InsnStop(), InsnCannotFollowEsp()] - if self.format == 'mingw32' and target == '__alloca': - # in functions with large stack requirements, windows - # needs a call to _alloca(), to turn reserved pages - # into committed memory. - # With mingw32 gcc at least, %esp is not used before - # this call. So we don't bother to compute the exact - # stack effect. - return [InsnCannotFollowEsp()] - if target in self.labels: - lineoffset = self.labels[target].lineno - self.currentlineno - if lineoffset >= 0: - assert lineoffset in (1,2) - return [InsnStackAdjust(-4)] - insns = [InsnCall(self.currentlineno), - InsnSetLocal(self.EAX)] # the result is there - if self.format in ('mingw32', 'msvc'): + return [InsnCall(self.currentlineno), + InsnSetLocal(self.EAX)] # the result is there + + target = match.group(1) + + if self.format in ('msvc'): # On win32, the address of a foreign function must be # computed, the optimizer may store it in a register. We - # could ignore this, except when the function has a - # __stdcall calling convention... + # could ignore this, except when the function need special + # processing (not returning, __stdcall...) def find_register(target): reg = [] def walker(insn, locs): @@ -658,8 +648,9 @@ for s in insn.all_sources_of(loc): sources.append(s) for source in sources: - if re.match("DWORD PTR .+@", source): - reg.append(source) + m = re.match("DWORD PTR (.+)", source) + if m: + reg.append(m.group(1)) if reg: return yield tuple(sources) @@ -673,14 +664,33 @@ if sources: target, = sources + if target in self.FUNCTIONS_NOT_RETURNING: + return [InsnStop(), InsnCannotFollowEsp()] + if self.format == 'mingw32' and target == '__alloca': + # in functions with large stack requirements, windows + # needs a call to _alloca(), to turn reserved pages + # into committed memory. + # With mingw32 gcc at least, %esp is not used before + # this call. So we don't bother to compute the exact + # stack effect. + return [InsnCannotFollowEsp()] + if target in self.labels: + lineoffset = self.labels[target].lineno - self.currentlineno + if lineoffset >= 0: + assert lineoffset in (1,2) + return [InsnStackAdjust(-4)] + + insns = [InsnCall(self.currentlineno), + InsnSetLocal(self.EAX)] # the result is there + if self.format in ('mingw32', 'msvc'): # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size - if match and '@' in target and not target.startswith('@'): + if '@' in target and not target.startswith('@'): insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) # Some (intrinsic?) functions use the "fastcall" calling convention # XXX without any declaration, how can we guess the stack effect? - if match and target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: + if target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: insns.append(InsnStackAdjust(16)) return insns @@ -696,6 +706,7 @@ for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 2**30 TOP_OF_STACK = '0(%esp)' r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") @@ -742,6 +753,7 @@ format = 'darwin' r_functionstart = re.compile(r"_(\w+):\s*$") + OFFSET_LABELS = 0 def __init__(self, lines, filetag=0): match = self.r_functionstart.match(lines[0]) @@ -753,7 +765,6 @@ class MsvcFunctionGcRootTracker(FunctionGcRootTracker): format = 'msvc' - ESP = 'esp' EBP = 'ebp' EAX = 'eax' @@ -764,6 +775,7 @@ OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 0 r_functionstart = re.compile(r"; Function compile flags: ") r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") @@ -783,7 +795,7 @@ r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") - r_jmptable_end = re.compile(r"[^\t]") + r_jmptable_end = re.compile(r"[^\t\n;]") FUNCTIONS_NOT_RETURNING = { '_abort': None, @@ -1346,7 +1358,9 @@ print >> output, '\tDD\t%s' % (label,) print >> output, '\tDD\t%d' % (n,) else: - print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) + print >> output, '\t.long\t%s-%d' % ( + label, + PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) print >> output, '\t.long\t%d' % (n,) _globl('__gcmapend') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Tue Nov 10 17:20:54 2009 @@ -519,7 +519,7 @@ mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory # even in debug builds - mk.definition('ASM_CFLAGS', '$(CFLAGS) /Od /Oi /Ob1') + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Oi /Ob1') mk.rule('.SUFFIXES', '.s', []) mk.rule('.s.obj', '', 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') Modified: pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/platform/windows.py Tue Nov 10 17:20:54 2009 @@ -247,7 +247,7 @@ rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('.c.obj', '', '$(CC) $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), + ('.c.obj', '', '$(CC) /nologo $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: From antocuni at codespeak.net Tue Nov 10 17:50:33 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 10 Nov 2009 17:50:33 +0100 (CET) Subject: [pypy-svn] r69126 - in pypy/trunk/pypy/jit: backend/cli metainterp/test Message-ID: <20091110165033.3C7B8168104@codespeak.net> Author: antocuni Date: Tue Nov 10 17:50:32 2009 New Revision: 69126 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py Log: add a test for guard_nonnull and guard_isnull, and implement them in the cli backend Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Tue Nov 10 17:50:32 2009 @@ -471,6 +471,12 @@ def emit_op_guard_false(self, op): self.emit_guard_bool(op, OpCodes.Brtrue) + def emit_op_guard_nonnull(self, op): + self.emit_guard_bool(op, OpCodes.Brfalse) + + def emit_op_guard_isnull(self, op): + self.emit_guard_bool(op, OpCodes.Brtrue) + def emit_op_guard_value(self, op): assert len(op.args) == 2 il_label = self.newbranch(op) Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Tue Nov 10 17:50:32 2009 @@ -1049,6 +1049,35 @@ res = self.meta_interp(f, [20], listops=True) self.check_loops(getfield_gc=1, getarrayitem_gc=0) + def test_guard_isnull_nonnull(self): + myjitdriver = JitDriver(greens = [], reds = ['x', 'res']) + class A(object): + pass + + @dont_look_inside + def create(x): + if x >= -40: + return A() + return None + + def f(x): + res = 0 + while x > 0: + myjitdriver.can_enter_jit(x=x, res=res) + myjitdriver.jit_merge_point(x=x, res=res) + obj = create(x-1) + if obj is not None: + res += 1 + obj2 = create(x-1000) + if obj2 is None: + res += 1 + x -= 1 + return res + res = self.meta_interp(f, [21]) + assert res == 42 + self.check_loops(guard_nonnull=1, guard_isnull=1) + + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): From antocuni at codespeak.net Tue Nov 10 17:52:10 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 10 Nov 2009 17:52:10 +0100 (CET) Subject: [pypy-svn] r69127 - in pypy/trunk/pypy: jit/backend/cli/test jit/backend/test translator/cli/src Message-ID: <20091110165210.3B821168104@codespeak.net> Author: antocuni Date: Tue Nov 10 17:52:09 2009 New Revision: 69127 Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py pypy/trunk/pypy/jit/backend/cli/test/test_runner.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/translator/cli/src/pypylib.cs Log: make test_basic and test_runner fully passing with py.test.net Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Tue Nov 10 17:52:09 2009 @@ -30,6 +30,7 @@ test_instantiate_classes = skip test_zerodivisionerror = skip test_isinstance = skip + test_isinstance_2 = skip test_oois = skip test_oostring_instance = skip test_long_long = skip @@ -44,3 +45,10 @@ test_instantiate_does_not_call = skip test_listcomp = skip test_tuple_immutable = skip + test_oosend_look_inside_only_one = skip + test_residual_external_call = skip + test_merge_guardclass_guardvalue = skip + test_residual_call_doesnt_lose_info = skip + test_oohash = skip + test_identityhash = skip + test_guard_isnull_nonnull = skip Modified: pypy/trunk/pypy/jit/backend/cli/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_runner.py Tue Nov 10 17:52:09 2009 @@ -20,6 +20,8 @@ class TestRunner(CliJitMixin, OOtypeBackendTest): + avoid_instances = True + def skip(self): py.test.skip("not supported in non-translated version") @@ -29,6 +31,7 @@ test_field = skip test_field_basic = skip test_ooops = skip + test_jump = skip def test_unused_result_float(self): py.test.skip('fixme! max 32 inputargs so far') Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Nov 10 17:52:09 2009 @@ -86,6 +86,8 @@ class BaseBackendTest(Runner): + avoid_instances = False + def test_compile_linear_loop(self): i0 = BoxInt() i1 = BoxInt() @@ -446,8 +448,12 @@ all = [(rop.GUARD_TRUE, [BoxInt(1)]), (rop.GUARD_FALSE, [BoxInt(0)]), (rop.GUARD_VALUE, [BoxInt(42), BoxInt(42)]), + ] + if not self.avoid_instances: + all.extend([ (rop.GUARD_NONNULL, [t_box]), - (rop.GUARD_ISNULL, [nullbox])] + (rop.GUARD_ISNULL, [nullbox]) + ]) if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(3.5), BoxFloat(3.5)])) for (opname, args) in all: @@ -469,8 +475,11 @@ all = [(rop.GUARD_TRUE, [BoxInt(0)]), (rop.GUARD_FALSE, [BoxInt(1)]), (rop.GUARD_VALUE, [BoxInt(42), BoxInt(41)]), + ] + if not self.avoid_instances: + all.extend([ (rop.GUARD_NONNULL, [nullbox]), - (rop.GUARD_ISNULL, [t_box])] + (rop.GUARD_ISNULL, [t_box])]) if self.cpu.supports_floats: all.append((rop.GUARD_VALUE, [BoxFloat(-1.0), BoxFloat(1.0)])) for opname, args in all: Modified: pypy/trunk/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/trunk/pypy/translator/cli/src/pypylib.cs Tue Nov 10 17:52:09 2009 @@ -113,9 +113,9 @@ public delegate void LoopDelegate(InputArgs args); public class InputArgs { - public int[] ints = new int[32]; - public double[] floats = new double[32]; - public object[] objs = new object[32]; + public int[] ints = new int[256]; + public double[] floats = new double[256]; + public object[] objs = new object[256]; public object exc_value = null; public int failed_op = -1; From pedronis at codespeak.net Tue Nov 10 18:31:23 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 10 Nov 2009 18:31:23 +0100 (CET) Subject: [pypy-svn] r69128 - in pypy/trunk/pypy/translator/c/gcc: . test/darwin Message-ID: <20091110173123.EBA75168104@codespeak.net> Author: pedronis Date: Tue Nov 10 18:31:23 2009 New Revision: 69128 Added: pypy/trunk/pypy/translator/c/gcc/test/darwin/track_0f.s (contents, props changed) Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: fix asmgcroot on Darwin Added: pypy/trunk/pypy/translator/c/gcc/test/darwin/track_0f.s ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/c/gcc/test/darwin/track_0f.s Tue Nov 10 18:31:23 2009 @@ -0,0 +1,105 @@ +_pypy_debug_open: + subl $92, %esp + movl %ebx, 76(%esp) + call L161 +"L00000000014$pb": +L161: + popl %ebx + movl %ebp, 88(%esp) + movl %esi, 80(%esp) + movl %edi, 84(%esp) + leal LC18-"L00000000014$pb"(%ebx), %eax + movl %eax, (%esp) + call L_getenv$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + testl %eax, %eax + movl %eax, %ebp + je L147 + cmpb $0, (%eax) + jne L158 +L147: + movl _pypy_debug_file-"L00000000014$pb"(%ebx), %esi + testl %esi, %esi + je L159 +L154: + movb $1, _debug_ready-"L00000000014$pb"(%ebx) + movl 80(%esp), %esi + movl 76(%esp), %ebx + movl 84(%esp), %edi + movl 88(%esp), %ebp + addl $92, %esp + ret + .align 4,0x90 +L158: + movl $58, 4(%esp) + movl %eax, (%esp) + call L_strchr$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + testl %eax, %eax + movl %eax, %edi + je L160 + movl %eax, %esi + subl %ebp, %esi + leal 1(%esi), %eax + movl %eax, (%esp) + call L_malloc$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + movl %ebp, 4(%esp) + leal 1(%edi), %ebp + movl %esi, 8(%esp) + movl %eax, _debug_prefix-"L00000000014$pb"(%ebx) + movl %eax, (%esp) + call L_memcpy$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + movl _debug_prefix-"L00000000014$pb"(%ebx), %eax + movb $0, (%eax,%esi) +L152: + leal LC19-"L00000000014$pb"(%ebx), %eax + movl $2, %ecx + cld + movl %ebp, %esi + movl %eax, %edi + repz + cmpsb + mov $0, %eax + je 0f + movzbl -1(%esi), %eax + movzbl -1(%edi), %ecx + subl %ecx,%eax +0: + testl %eax, %eax + je L147 + leal LC20-"L00000000014$pb"(%ebx), %eax + movl %eax, 4(%esp) + movl %ebp, (%esp) + call L_fopen$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + movl %eax, _pypy_debug_file-"L00000000014$pb"(%ebx) + jmp L147 +L160: + movb $1, _debug_profile-"L00000000014$pb"(%ebx) + jmp L152 + .align 4,0x90 +L159: + movl L___sF$non_lazy_ptr-"L00000000014$pb"(%ebx), %eax + movl $2, (%esp) + addl $176, %eax + movl %eax, _pypy_debug_file-"L00000000014$pb"(%ebx) + call L_isatty$stub + ;; expected {92(%esp) | 76(%esp), 80(%esp), 84(%esp), 88(%esp) | } + testl %eax, %eax + je L154 + leal LC21-"L00000000014$pb"(%ebx), %eax + movl 80(%esp), %esi + movl %eax, _debug_start_colors_1-"L00000000014$pb"(%ebx) + leal LC22-"L00000000014$pb"(%ebx), %eax + movl 84(%esp), %edi + movl %eax, _debug_start_colors_2-"L00000000014$pb"(%ebx) + leal LC23-"L00000000014$pb"(%ebx), %eax + movl 88(%esp), %ebp + movl %eax, _debug_stop_colors-"L00000000014$pb"(%ebx) + movb $1, _debug_ready-"L00000000014$pb"(%ebx) + movl 76(%esp), %ebx + addl $92, %esp + ret + .align 4,0x90 \ No newline at end of file Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Tue Nov 10 18:31:23 2009 @@ -23,14 +23,19 @@ r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") r_functionstart_darwin = re.compile(r"_(\w+):\s*$") -OFFSET_LABELS = 2**30 +if sys.platform != 'darwin': + OFFSET_LABELS = 2**30 +else: + OFFSET_LABELS = 0 # inside functions r_label = re.compile(LABEL+"[:]\s*$") +r_rel_label = re.compile(r"(\d+):\s*$") r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) r_insn = re.compile(r"\t([a-z]\w*)\s") r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") +r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") @@ -358,9 +363,16 @@ self.labels = {} # {name: Label()} for lineno, line in enumerate(self.lines): match = r_label.match(line) + label = None if match: label = match.group(1) - assert label not in self.labels, "duplicate label" + else: + # labels used by: j* NNNf + match = r_rel_label.match(line) + if match: + label = "rel %d" % lineno + if label: + assert label not in self.labels, "duplicate label: %s" % label self.labels[label] = Label(label, lineno) def append_instruction(self, insn): @@ -721,6 +733,8 @@ # gcc -fno-unit-at-a-time. return self.insns_for_copy(source, target) + visit_mov = visit_movl + def visit_pushl(self, line): match = r_unaryinsn.match(line) source = match.group(1) @@ -825,8 +839,20 @@ def conditional_jump(self, line): match = r_jump.match(line) if not match: - raise UnrecognizedOperation(line) - label = match.group(1) + match = r_jump_rel_label.match(line) + if not match: + raise UnrecognizedOperation(line) + # j* NNNf + label = match.group(1) + label += ":" + i = self.currentlineno + 1 + while True: + if self.lines[i].startswith(label): + label = "rel %d" % i + break + i += 1 + else: + label = match.group(1) self.register_jump_to(label) return [] From antocuni at codespeak.net Tue Nov 10 18:57:26 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 10 Nov 2009 18:57:26 +0100 (CET) Subject: [pypy-svn] r69130 - in pypy/trunk/pypy/translator: cli cli/test oosupport Message-ID: <20091110175726.683A5168104@codespeak.net> Author: antocuni Date: Tue Nov 10 18:57:25 2009 New Revision: 69130 Modified: pypy/trunk/pypy/translator/cli/function.py pypy/trunk/pypy/translator/cli/test/test_dotnet.py pypy/trunk/pypy/translator/oosupport/function.py Log: ugly workaround to make the exceptiontranformer work with .NET valuetypes Modified: pypy/trunk/pypy/translator/cli/function.py ============================================================================== --- pypy/trunk/pypy/translator/cli/function.py (original) +++ pypy/trunk/pypy/translator/cli/function.py Tue Nov 10 18:57:25 2009 @@ -139,6 +139,21 @@ self.store(link.last_exc_value) self._setup_link(link) + def _dont_store(self, to_load, to_store): + # ugly workaround to make the exceptiontransformer work with + # valuetypes: when exceptiontransforming a function whose result is a + # .NET valuetype, it tries to store a null into the return variable. + # Since it is not possible to store a null into a valuetype, and that + # in that case the value is not used anyway, we simply ignore it. + from pypy.translator.cli.dotnet import NativeInstance + if isinstance(to_load, flowmodel.Constant): + value = to_load.value + is_null = not value + T = ootype.typeOf(to_load.value) + if isinstance(T, NativeInstance) and T._is_value_type and is_null: + return True + return OOFunction._dont_store(self, to_load, to_store) + def render_numeric_switch(self, block): if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): # TODO: it could be faster to check is the values fit in Modified: pypy/trunk/pypy/translator/cli/test/test_dotnet.py ============================================================================== --- pypy/trunk/pypy/translator/cli/test/test_dotnet.py (original) +++ pypy/trunk/pypy/translator/cli/test/test_dotnet.py Tue Nov 10 18:57:25 2009 @@ -632,6 +632,20 @@ return d[OpCodes.Add] assert self.interpret(fn, []) == 42 + def test_valuetype_exceptiontransform(self): + def foo(x): + if x: + return OpCodes.Add + raise ValueError + def fn(x): + try: + foo(x) + return False + except ValueError: + return True + res = self.interpret(fn, [0], exctrans=True) + assert res + def test_classof(self): int32_class = classof(System.Int32) def fn(): @@ -694,7 +708,7 @@ class TestPythonnet(TestDotnetRtyping): # don't interpreter functions but execute them directly through pythonnet - def interpret(self, f, args, backendopt='ignored'): + def interpret(self, f, args, backendopt='ignored', exctrans='ignored'): return f(*args) def _skip_pythonnet(self, msg): Modified: pypy/trunk/pypy/translator/oosupport/function.py ============================================================================== --- pypy/trunk/pypy/translator/oosupport/function.py (original) +++ pypy/trunk/pypy/translator/oosupport/function.py Tue Nov 10 18:57:25 2009 @@ -321,6 +321,9 @@ self._setup_link(link) self.generator.branch_unconditionally(target_label) + def _dont_store(self, to_load, to_store): + return False + def _setup_link(self, link): target = link.target linkvars = [] @@ -329,6 +332,8 @@ continue if to_load.concretetype is ootype.Void: continue + if self._dont_store(to_load, to_store): + continue linkvars.append((to_load, to_store)) # after SSI_to_SSA it can happen to have to_load = [a, b] and From arigo at codespeak.net Tue Nov 10 19:00:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 10 Nov 2009 19:00:21 +0100 (CET) Subject: [pypy-svn] r69131 - in pypy/branch/merge-guards-2/pypy/jit/backend: test x86 Message-ID: <20091110180021.25CE0168102@codespeak.net> Author: arigo Date: Tue Nov 10 19:00:21 2009 New Revision: 69131 Modified: pypy/branch/merge-guards-2/pypy/jit/backend/test/runner_test.py pypy/branch/merge-guards-2/pypy/jit/backend/test/test_ll_random.py pypy/branch/merge-guards-2/pypy/jit/backend/x86/assembler.py pypy/branch/merge-guards-2/pypy/jit/backend/x86/regalloc.py Log: Implement guard_nonnull_class in the x86 backend. Modified: pypy/branch/merge-guards-2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/backend/test/runner_test.py Tue Nov 10 19:00:21 2009 @@ -460,8 +460,8 @@ #null_box = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(T))) self.execute_operation(rop.GUARD_CLASS, [t_box, T_box], 'void') assert not self.guard_failed - #self.execute_operation(rop.GUARD_CLASS_INVERSE, [t_box, null_box], - # 'void') + self.execute_operation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], 'void') + assert not self.guard_failed def test_failing_guards(self): t_box, T_box = self.alloc_instance(self.T) @@ -480,10 +480,12 @@ def test_failing_guard_class(self): t_box, T_box = self.alloc_instance(self.T) u_box, U_box = self.alloc_instance(self.U) - #null_box = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(T))) + null_box = self.null_instance() for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]), (rop.GUARD_CLASS, [u_box, T_box]), - #(rop.GUARD_VALUE_INVERSE, [BoxInt(10), BoxInt(10)]), + (rop.GUARD_NONNULL_CLASS, [t_box, U_box]), + (rop.GUARD_NONNULL_CLASS, [u_box, T_box]), + (rop.GUARD_NONNULL_CLASS, [null_box, T_box]), ]: assert self.execute_operation(opname, args, 'void') == None assert self.guard_failed Modified: pypy/branch/merge-guards-2/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/backend/test/test_ll_random.py Tue Nov 10 19:00:21 2009 @@ -203,6 +203,21 @@ op = ResOperation(self.opnum, [v, c_vtable2], None) return op, (vtable == vtable2) +class GuardNonNullClassOperation(GuardClassOperation): + def gen_guard(self, builder, r): + if r.random() < 0.5: + return GuardClassOperation.gen_guard(self, builder, r) + else: + v = BoxPtr(lltype.nullptr(llmemory.GCREF.TO)) + op = ResOperation(rop.SAME_AS, [ConstPtr(v.value)], v) + builder.loop.operations.append(op) + v2, S2 = builder.get_structptr_var(r, must_have_vtable=True) + vtable2 = S2._hints['vtable']._as_ptr() + c_vtable2 = ConstAddr(llmemory.cast_ptr_to_adr(vtable2), + builder.cpu) + op = ResOperation(self.opnum, [v, c_vtable2], None) + return op, False + class GetFieldOperation(test_random.AbstractOperation): def field_descr(self, builder, r): v, S = builder.get_structptr_var(r) @@ -577,6 +592,7 @@ OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL)) OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL)) OPERATIONS.append(CallOperationException(rop.CALL)) +OPERATIONS.append(GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) LLtypeOperationBuilder.OPERATIONS = OPERATIONS Modified: pypy/branch/merge-guards-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/backend/x86/assembler.py Tue Nov 10 19:00:21 2009 @@ -696,10 +696,10 @@ self.mc.CMP(locs[0], locs[1]) return self.implement_guard(addr, self.mc.JNE) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): + def _cmp_guard_class(self, mc, locs): offset = self.cpu.vtable_offset if offset is not None: - self.mc.CMP(mem(locs[0], offset), locs[1]) + mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -714,8 +714,24 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - type_info_group) >> 2 - self.mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) - # + mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + + def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): + self._cmp_guard_class(self.mc._mc, locs) + return self.implement_guard(addr, self.mc.JNE) + + def genop_guard_guard_nonnull_class(self, ign_1, guard_op, + addr, locs, ign_2): + mc = self.mc._mc + mc.CMP(locs[0], imm8(1)) + mc.write(constlistofchars('\x72\x00')) # JB later + jb_location = mc.get_relative_pos() + self._cmp_guard_class(mc, locs) + # patch the JB above + offset = mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + mc.overwrite(jb_location-1, [chr(offset)]) + # return self.implement_guard(addr, self.mc.JNE) def _no_const_locs(self, args): @@ -880,8 +896,9 @@ print msg raise NotImplementedError(msg) - def not_implemented_op_guard(self, op, regalloc, arglocs, resloc, descr): - msg = "not implemented operation (guard): %s" % op.getopname() + def not_implemented_op_guard(self, op, guard_op, + failaddr, arglocs, resloc): + msg = "not implemented operation (guard): %s" % guard_op.getopname() print msg raise NotImplementedError(msg) Modified: pypy/branch/merge-guards-2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/merge-guards-2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/merge-guards-2/pypy/jit/backend/x86/regalloc.py Tue Nov 10 19:00:21 2009 @@ -424,7 +424,9 @@ y = self.loc(op.args[1]) self.perform_guard(op, [x, y], None) self.rm.possibly_free_vars(op.args) - + + consider_guard_nonnull_class = consider_guard_class + def _consider_binop_part(self, op, ignored): x = op.args[0] argloc = self.loc(op.args[1]) From hpk at codespeak.net Tue Nov 10 19:02:22 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 19:02:22 +0100 (CET) Subject: [pypy-svn] r69132 - pypy/build/testrunner/test Message-ID: <20091110180222.DD186168102@codespeak.net> Author: hpk Date: Tue Nov 10 19:02:22 2009 New Revision: 69132 Modified: pypy/build/testrunner/test/test_runner.py Log: fix a better location for pytest script Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Tue Nov 10 19:02:22 2009 @@ -1,7 +1,9 @@ import py, sys, os, signal, cStringIO, tempfile import runner +import pypy +pytest_script = py.path.local(pypy.__file__).dirpath('test_all.py') class TestRunHelper(object): @@ -158,7 +160,7 @@ runner.invoke_in_thread = cls.real_invoke_in_thread[0] def test_one_dir(self): - test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + test_driver = [pytest_script] log = cStringIO.StringIO() out = cStringIO.StringIO() @@ -194,7 +196,7 @@ assert nfailures == 6 def test_one_dir_dry_run(self): - test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + test_driver = [pytest_script] log = cStringIO.StringIO() out = cStringIO.StringIO() @@ -220,7 +222,7 @@ assert "test_normal" in line def test_many_dirs(self): - test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + test_driver = [pytest_script] log = cStringIO.StringIO() out = cStringIO.StringIO() @@ -260,7 +262,7 @@ assert set(cleanedup) == set(alltestdirs) def test_timeout(self): - test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + test_driver = [pytest_script] log = cStringIO.StringIO() out = cStringIO.StringIO() @@ -295,7 +297,7 @@ assert log_lines[1] == ' Failed to run interp' def test_run_bad_get_test_driver(self): - test_driver = [py.path.local(py.__file__).dirpath('bin', 'py.test')] + test_driver = [pytest_script] log = cStringIO.StringIO() out = cStringIO.StringIO() From cfbolz at codespeak.net Tue Nov 10 19:17:36 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 10 Nov 2009 19:17:36 +0100 (CET) Subject: [pypy-svn] r69135 - pypy/trunk/pypy/interpreter/test Message-ID: <20091110181736.C3A8A168105@codespeak.net> Author: cfbolz Date: Tue Nov 10 19:17:35 2009 New Revision: 69135 Modified: pypy/trunk/pypy/interpreter/test/test_code.py Log: this (obscurely enough) should make the failing appdirect-test pass Modified: pypy/trunk/pypy/interpreter/test/test_code.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_code.py (original) +++ pypy/trunk/pypy/interpreter/test/test_code.py Tue Nov 10 19:17:35 2009 @@ -195,8 +195,9 @@ exec """if 1: + r = range def f(): - return [l for l in range(100)] + return [l for l in r(100)] def g(): return [l for l in [1, 2, 3, 4]] """ From hpk at codespeak.net Tue Nov 10 19:17:38 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Tue, 10 Nov 2009 19:17:38 +0100 (CET) Subject: [pypy-svn] r69134 - in pypy/branch/py11: dotviewer py py/impl py/impl/cmdline py/impl/code py/impl/compat py/impl/io py/impl/log py/impl/path py/impl/path/gateway py/impl/process py/impl/test py/impl/test/dist py/impl/test/looponfail py/plugin Message-ID: <20091110181738.F23A8168105@codespeak.net> Author: hpk Date: Tue Nov 10 19:17:34 2009 New Revision: 69134 Added: pypy/branch/py11/py/ pypy/branch/py11/py/__init__.py (contents, props changed) pypy/branch/py11/py/apipkg.py (contents, props changed) pypy/branch/py11/py/impl/ pypy/branch/py11/py/impl/__init__.py (contents, props changed) pypy/branch/py11/py/impl/_com.py (contents, props changed) pypy/branch/py11/py/impl/_metainfo.py (contents, props changed) pypy/branch/py11/py/impl/builtin.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/ pypy/branch/py11/py/impl/cmdline/__init__.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pycleanup.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pyconvert_unittest.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pycountloc.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pylookup.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pysvnwcrevert.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pytest.py (contents, props changed) pypy/branch/py11/py/impl/cmdline/pywhich.py (contents, props changed) pypy/branch/py11/py/impl/code/ pypy/branch/py11/py/impl/code/__init__.py (contents, props changed) pypy/branch/py11/py/impl/code/_assertionnew.py (contents, props changed) pypy/branch/py11/py/impl/code/_assertionold.py (contents, props changed) pypy/branch/py11/py/impl/code/assertion.py (contents, props changed) pypy/branch/py11/py/impl/code/code.py (contents, props changed) pypy/branch/py11/py/impl/code/oldmagic.py (contents, props changed) pypy/branch/py11/py/impl/code/oldmagic2.py (contents, props changed) pypy/branch/py11/py/impl/code/source.py (contents, props changed) pypy/branch/py11/py/impl/compat/ pypy/branch/py11/py/impl/compat/__init__.py (contents, props changed) pypy/branch/py11/py/impl/compat/dep_doctest.py (contents, props changed) pypy/branch/py11/py/impl/compat/dep_optparse.py (contents, props changed) pypy/branch/py11/py/impl/compat/dep_subprocess.py (contents, props changed) pypy/branch/py11/py/impl/compat/dep_textwrap.py (contents, props changed) pypy/branch/py11/py/impl/error.py (contents, props changed) pypy/branch/py11/py/impl/io/ pypy/branch/py11/py/impl/io/__init__.py (contents, props changed) pypy/branch/py11/py/impl/io/capture.py (contents, props changed) pypy/branch/py11/py/impl/io/terminalwriter.py (contents, props changed) pypy/branch/py11/py/impl/log/ pypy/branch/py11/py/impl/log/__init__.py (contents, props changed) pypy/branch/py11/py/impl/log/log.py (contents, props changed) pypy/branch/py11/py/impl/log/warning.py (contents, props changed) pypy/branch/py11/py/impl/path/ pypy/branch/py11/py/impl/path/__init__.py (contents, props changed) pypy/branch/py11/py/impl/path/cacheutil.py (contents, props changed) pypy/branch/py11/py/impl/path/common.py (contents, props changed) pypy/branch/py11/py/impl/path/gateway/ pypy/branch/py11/py/impl/path/gateway/__init__.py (contents, props changed) pypy/branch/py11/py/impl/path/gateway/channeltest.py (contents, props changed) pypy/branch/py11/py/impl/path/gateway/channeltest2.py (contents, props changed) pypy/branch/py11/py/impl/path/gateway/remotepath.py (contents, props changed) pypy/branch/py11/py/impl/path/local.py (contents, props changed) pypy/branch/py11/py/impl/path/local.py.orig pypy/branch/py11/py/impl/path/svnurl.py (contents, props changed) pypy/branch/py11/py/impl/path/svnwc.py (contents, props changed) pypy/branch/py11/py/impl/process/ pypy/branch/py11/py/impl/process/__init__.py (contents, props changed) pypy/branch/py11/py/impl/process/cmdexec.py (contents, props changed) pypy/branch/py11/py/impl/process/forkedfunc.py (contents, props changed) pypy/branch/py11/py/impl/process/killproc.py (contents, props changed) pypy/branch/py11/py/impl/std.py (contents, props changed) pypy/branch/py11/py/impl/test/ pypy/branch/py11/py/impl/test/__init__.py (contents, props changed) pypy/branch/py11/py/impl/test/cmdline.py (contents, props changed) pypy/branch/py11/py/impl/test/collect.py (contents, props changed) pypy/branch/py11/py/impl/test/compat.py (contents, props changed) pypy/branch/py11/py/impl/test/config.py (contents, props changed) pypy/branch/py11/py/impl/test/conftesthandle.py (contents, props changed) pypy/branch/py11/py/impl/test/defaultconftest.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/ pypy/branch/py11/py/impl/test/dist/__init__.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/dsession.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/gwmanage.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/mypickle.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/nodemanage.py (contents, props changed) pypy/branch/py11/py/impl/test/dist/txnode.py (contents, props changed) pypy/branch/py11/py/impl/test/funcargs.py (contents, props changed) pypy/branch/py11/py/impl/test/looponfail/ pypy/branch/py11/py/impl/test/looponfail/__init__.py (contents, props changed) pypy/branch/py11/py/impl/test/looponfail/remote.py (contents, props changed) pypy/branch/py11/py/impl/test/looponfail/util.py (contents, props changed) pypy/branch/py11/py/impl/test/outcome.py (contents, props changed) pypy/branch/py11/py/impl/test/parseopt.py (contents, props changed) pypy/branch/py11/py/impl/test/pluginmanager.py (contents, props changed) pypy/branch/py11/py/impl/test/pycollect.py (contents, props changed) pypy/branch/py11/py/impl/test/session.py (contents, props changed) pypy/branch/py11/py/impl/xmlgen.py (contents, props changed) pypy/branch/py11/py/plugin/ pypy/branch/py11/py/plugin/__init__.py (contents, props changed) pypy/branch/py11/py/plugin/hookspec.py (contents, props changed) pypy/branch/py11/py/plugin/pytest__pytest.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_assertion.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_capture.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_default.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_doctest.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_figleaf.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_helpconfig.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_hooklog.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_mark.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_monkeypatch.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_nose.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_pastebin.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_pdb.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_pylint.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_pytester.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_recwarn.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_restdoc.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_resultlog.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_runner.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_skipping.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_terminal.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_tmpdir.py (contents, props changed) pypy/branch/py11/py/plugin/pytest_unittest.py (contents, props changed) Modified: pypy/branch/py11/dotviewer/conftest.py Log: adding a copy of the py-1.1.0 py lib directly to the pypy root tree (at some point it should just be installed globally) Modified: pypy/branch/py11/dotviewer/conftest.py ============================================================================== --- pypy/branch/py11/dotviewer/conftest.py (original) +++ pypy/branch/py11/dotviewer/conftest.py Tue Nov 10 19:17:34 2009 @@ -1,7 +1,7 @@ import py def pytest_addoption(parser): - group = parser.addgroup("dotviever") + group = parser.getgroup("dotviever") group.addoption('--pygame', action="store_true", dest="pygame", default=False, help="allow interactive tests using Pygame") Added: pypy/branch/py11/py/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +""" +py.test and pylib: rapid testing and development utils + +this module uses apipkg.py for lazy-loading sub modules +and classes. The initpkg-dictionary below specifies +name->value mappings where value can be another namespace +dictionary or an import path. + +(c) Holger Krekel and others, 2009 +""" +version = "1.1.0" + +__version__ = version = version or "1.1.x" +import py.apipkg + +py.apipkg.initpkg(__name__, dict( + # access to all standard lib modules + std = '.impl.std:std', + # access to all posix errno's as classes + error = '.impl.error:error', + + _impldir = '.impl._metainfo:impldir', + _dir = '.impl._metainfo:pydir', + _pydirs = '.impl._metainfo:pydirs', + version = 'py:__version__', # backward compatibility + + _com = { + 'Registry': '.impl._com:Registry', + 'MultiCall': '.impl._com:MultiCall', + 'comregistry': '.impl._com:comregistry', + 'HookRelay': '.impl._com:HookRelay', + }, + cmdline = { + 'pytest': '.impl.cmdline.pytest:main', + 'pylookup': '.impl.cmdline.pylookup:main', + 'pycountloc': '.impl.cmdline.pycountlog:main', + 'pytest': '.impl.test.cmdline:main', + 'pylookup': '.impl.cmdline.pylookup:main', + 'pycountloc': '.impl.cmdline.pycountloc:main', + 'pycleanup': '.impl.cmdline.pycleanup:main', + 'pywhich' : '.impl.cmdline.pywhich:main', + 'pysvnwcrevert' : '.impl.cmdline.pysvnwcrevert:main', + 'pyconvert_unittest' : '.impl.cmdline.pyconvert_unittest:main', + }, + + test = { + # helpers for use from test functions or collectors + '__doc__' : '.impl.test:__doc__', + '_PluginManager' : '.impl.test.pluginmanager:PluginManager', + 'raises' : '.impl.test.outcome:raises', + 'skip' : '.impl.test.outcome:skip', + 'importorskip' : '.impl.test.outcome:importorskip', + 'fail' : '.impl.test.outcome:fail', + 'exit' : '.impl.test.outcome:exit', + # configuration/initialization related test api + 'config' : '.impl.test.config:config_per_process', + 'ensuretemp' : '.impl.test.config:ensuretemp', + 'collect': { + 'Collector' : '.impl.test.collect:Collector', + 'Directory' : '.impl.test.collect:Directory', + 'File' : '.impl.test.collect:File', + 'Item' : '.impl.test.collect:Item', + 'Module' : '.impl.test.pycollect:Module', + 'Class' : '.impl.test.pycollect:Class', + 'Instance' : '.impl.test.pycollect:Instance', + 'Generator' : '.impl.test.pycollect:Generator', + 'Function' : '.impl.test.pycollect:Function', + '_fillfuncargs' : '.impl.test.funcargs:fillfuncargs', + }, + }, + + # hook into the top-level standard library + process = { + '__doc__' : '.impl.process:__doc__', + 'cmdexec' : '.impl.process.cmdexec:cmdexec', + 'kill' : '.impl.process.killproc:kill', + 'ForkedFunc' : '.impl.process.forkedfunc:ForkedFunc', + }, + + path = { + '__doc__' : '.impl.path:__doc__', + 'svnwc' : '.impl.path.svnwc:SvnWCCommandPath', + 'svnurl' : '.impl.path.svnurl:SvnCommandPath', + 'local' : '.impl.path.local:LocalPath', + 'SvnAuth' : '.impl.path.svnwc:SvnAuth', + }, + + # some nice slightly magic APIs + magic = { + 'invoke' : '.impl.code.oldmagic:invoke', + 'revoke' : '.impl.code.oldmagic:revoke', + 'patch' : '.impl.code.oldmagic:patch', + 'revert' : '.impl.code.oldmagic:revert', + 'autopath' : '.impl.path.local:autopath', + 'AssertionError' : '.impl.code.oldmagic2:AssertionError', + }, + + # python inspection/code-generation API + code = { + '__doc__' : '.impl.code:__doc__', + 'compile' : '.impl.code.source:compile_', + 'Source' : '.impl.code.source:Source', + 'Code' : '.impl.code.code:Code', + 'Frame' : '.impl.code.code:Frame', + 'ExceptionInfo' : '.impl.code.code:ExceptionInfo', + 'Traceback' : '.impl.code.code:Traceback', + 'getfslineno' : '.impl.code.source:getfslineno', + 'getrawcode' : '.impl.code.code:getrawcode', + 'patch_builtins' : '.impl.code.code:patch_builtins', + 'unpatch_builtins' : '.impl.code.code:unpatch_builtins', + '_AssertionError' : '.impl.code.assertion:AssertionError', + }, + + # backports and additions of builtins + builtin = { + '__doc__' : '.impl.builtin:__doc__', + 'enumerate' : '.impl.builtin:enumerate', + 'reversed' : '.impl.builtin:reversed', + 'sorted' : '.impl.builtin:sorted', + 'set' : '.impl.builtin:set', + 'frozenset' : '.impl.builtin:frozenset', + 'BaseException' : '.impl.builtin:BaseException', + 'GeneratorExit' : '.impl.builtin:GeneratorExit', + 'print_' : '.impl.builtin:print_', + '_reraise' : '.impl.builtin:_reraise', + '_tryimport' : '.impl.builtin:_tryimport', + 'exec_' : '.impl.builtin:exec_', + '_basestring' : '.impl.builtin:_basestring', + '_totext' : '.impl.builtin:_totext', + '_isbytes' : '.impl.builtin:_isbytes', + '_istext' : '.impl.builtin:_istext', + '_getimself' : '.impl.builtin:_getimself', + '_getfuncdict' : '.impl.builtin:_getfuncdict', + 'builtins' : '.impl.builtin:builtins', + 'execfile' : '.impl.builtin:execfile', + 'callable' : '.impl.builtin:callable', + }, + + # input-output helping + io = { + '__doc__' : '.impl.io:__doc__', + 'dupfile' : '.impl.io.capture:dupfile', + 'TextIO' : '.impl.io.capture:TextIO', + 'BytesIO' : '.impl.io.capture:BytesIO', + 'FDCapture' : '.impl.io.capture:FDCapture', + 'StdCapture' : '.impl.io.capture:StdCapture', + 'StdCaptureFD' : '.impl.io.capture:StdCaptureFD', + 'TerminalWriter' : '.impl.io.terminalwriter:TerminalWriter', + }, + + # small and mean xml/html generation + xml = { + '__doc__' : '.impl.xmlgen:__doc__', + 'html' : '.impl.xmlgen:html', + 'Tag' : '.impl.xmlgen:Tag', + 'raw' : '.impl.xmlgen:raw', + 'Namespace' : '.impl.xmlgen:Namespace', + 'escape' : '.impl.xmlgen:escape', + }, + + log = { + # logging API ('producers' and 'consumers' connected via keywords) + '__doc__' : '.impl.log:__doc__', + '_apiwarn' : '.impl.log.warning:_apiwarn', + 'Producer' : '.impl.log.log:Producer', + 'setconsumer' : '.impl.log.log:setconsumer', + '_setstate' : '.impl.log.log:setstate', + '_getstate' : '.impl.log.log:getstate', + 'Path' : '.impl.log.log:Path', + 'STDOUT' : '.impl.log.log:STDOUT', + 'STDERR' : '.impl.log.log:STDERR', + 'Syslog' : '.impl.log.log:Syslog', + }, + + # compatibility modules (deprecated) + compat = { + '__doc__' : '.impl.compat:__doc__', + 'doctest' : '.impl.compat.dep_doctest:doctest', + 'optparse' : '.impl.compat.dep_optparse:optparse', + 'textwrap' : '.impl.compat.dep_textwrap:textwrap', + 'subprocess' : '.impl.compat.dep_subprocess:subprocess', + }, +)) Added: pypy/branch/py11/py/apipkg.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/apipkg.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,69 @@ +""" +apipkg: control the exported namespace of a python package. + +see http://pypi.python.org/pypi/apipkg + +(c) holger krekel, 2009 - MIT license +""" +import sys +from types import ModuleType + +__version__ = "1.0b2" + +def initpkg(pkgname, exportdefs): + """ initialize given package from the export definitions. """ + mod = ApiModule(pkgname, exportdefs, implprefix=pkgname) + oldmod = sys.modules[pkgname] + mod.__file__ = getattr(oldmod, '__file__', None) + mod.__version__ = getattr(oldmod, '__version__', None) + mod.__path__ = getattr(oldmod, '__path__', None) + sys.modules[pkgname] = mod + +def importobj(modpath, attrname): + module = __import__(modpath, None, None, ['__doc__']) + return getattr(module, attrname) + +class ApiModule(ModuleType): + def __init__(self, name, importspec, implprefix=None): + self.__name__ = name + self.__all__ = list(importspec) + self.__map__ = {} + self.__implprefix__ = implprefix or name + for name, importspec in importspec.items(): + if isinstance(importspec, dict): + subname = '%s.%s'%(self.__name__, name) + apimod = ApiModule(subname, importspec, implprefix) + sys.modules[subname] = apimod + setattr(self, name, apimod) + else: + modpath, attrname = importspec.split(':') + if modpath[0] == '.': + modpath = implprefix + modpath + if name == '__doc__': + self.__doc__ = importobj(modpath, attrname) + else: + self.__map__[name] = (modpath, attrname) + + def __repr__(self): + return '' % (self.__name__,) + + def __getattr__(self, name): + try: + modpath, attrname = self.__map__[name] + except KeyError: + raise AttributeError(name) + else: + result = importobj(modpath, attrname) + setattr(self, name, result) + del self.__map__[name] + return result + + def __dict__(self): + # force all the content of the module to be loaded when __dict__ is read + dictdescr = ModuleType.__dict__['__dict__'] + dict = dictdescr.__get__(self) + if dict is not None: + for name in self.__all__: + hasattr(self, name) # force attribute load, ignore errors + return dict + __dict__ = property(__dict__) Added: pypy/branch/py11/py/impl/__init__.py ============================================================================== Added: pypy/branch/py11/py/impl/_com.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/_com.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,125 @@ +""" +py lib plugins and plugin call management +""" + +import py +import inspect + +__all__ = ['Registry', 'MultiCall', 'comregistry', 'HookRelay'] + +class MultiCall: + """ execute a call into multiple python functions/methods. """ + + def __init__(self, methods, kwargs, firstresult=False): + self.methods = methods[:] + self.kwargs = kwargs.copy() + self.kwargs['__multicall__'] = self + self.results = [] + self.firstresult = firstresult + + def __repr__(self): + status = "%d results, %d meths" % (len(self.results), len(self.methods)) + return "" %(status, self.kwargs) + + def execute(self): + while self.methods: + method = self.methods.pop() + kwargs = self.getkwargs(method) + res = method(**kwargs) + if res is not None: + self.results.append(res) + if self.firstresult: + return res + if not self.firstresult: + return self.results + + def getkwargs(self, method): + kwargs = {} + for argname in varnames(method): + try: + kwargs[argname] = self.kwargs[argname] + except KeyError: + pass # might be optional param + return kwargs + +def varnames(func): + ismethod = inspect.ismethod(func) + rawcode = py.code.getrawcode(func) + try: + return rawcode.co_varnames[ismethod:] + except AttributeError: + return () + +class Registry: + """ + Manage Plugins: register/unregister call calls to plugins. + """ + def __init__(self, plugins=None): + if plugins is None: + plugins = [] + self._plugins = plugins + + def register(self, plugin): + assert not isinstance(plugin, str) + assert not plugin in self._plugins + self._plugins.append(plugin) + + def unregister(self, plugin): + self._plugins.remove(plugin) + + def isregistered(self, plugin): + return plugin in self._plugins + + def __iter__(self): + return iter(self._plugins) + + def listattr(self, attrname, plugins=None, extra=(), reverse=False): + l = [] + if plugins is None: + plugins = self._plugins + candidates = list(plugins) + list(extra) + for plugin in candidates: + try: + l.append(getattr(plugin, attrname)) + except AttributeError: + continue + if reverse: + l.reverse() + return l + +class HookRelay: + def __init__(self, hookspecs, registry): + self._hookspecs = hookspecs + self._registry = registry + for name, method in vars(hookspecs).items(): + if name[:1] != "_": + setattr(self, name, self._makecall(name)) + + def _makecall(self, name, extralookup=None): + hookspecmethod = getattr(self._hookspecs, name) + firstresult = getattr(hookspecmethod, 'firstresult', False) + return HookCaller(self, name, firstresult=firstresult, + extralookup=extralookup) + + def _getmethods(self, name, extralookup=()): + return self._registry.listattr(name, extra=extralookup) + + def _performcall(self, name, multicall): + return multicall.execute() + +class HookCaller: + def __init__(self, hookrelay, name, firstresult, extralookup=None): + self.hookrelay = hookrelay + self.name = name + self.firstresult = firstresult + self.extralookup = extralookup and [extralookup] or () + + def __repr__(self): + return "" %(self.name,) + + def __call__(self, **kwargs): + methods = self.hookrelay._getmethods(self.name, self.extralookup) + mc = MultiCall(methods, kwargs, firstresult=self.firstresult) + return self.hookrelay._performcall(self.name, mc) + +comregistry = Registry([]) Added: pypy/branch/py11/py/impl/_metainfo.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/_metainfo.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,9 @@ + +import py + +pydir = py.path.local(py.__file__).dirpath() +impldir = pydir.join("impl") + +# list of all directories beloging to py +assert impldir.relto(pydir) +pydirs = [pydir] Added: pypy/branch/py11/py/impl/builtin.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/builtin.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,203 @@ +import sys + +try: + reversed = reversed +except NameError: + def reversed(sequence): + """reversed(sequence) -> reverse iterator over values of the sequence + + Return a reverse iterator + """ + if hasattr(sequence, '__reversed__'): + return sequence.__reversed__() + if not hasattr(sequence, '__getitem__'): + raise TypeError("argument to reversed() must be a sequence") + return reversed_iterator(sequence) + + class reversed_iterator(object): + + def __init__(self, seq): + self.seq = seq + self.remaining = len(seq) + + def __iter__(self): + return self + + def next(self): + i = self.remaining + if i > 0: + i -= 1 + item = self.seq[i] + self.remaining = i + return item + raise StopIteration + + def __length_hint__(self): + return self.remaining + +try: + sorted = sorted +except NameError: + builtin_cmp = cmp # need to use cmp as keyword arg + + def sorted(iterable, cmp=None, key=None, reverse=0): + use_cmp = None + if key is not None: + if cmp is None: + def use_cmp(x, y): + return builtin_cmp(x[0], y[0]) + else: + def use_cmp(x, y): + return cmp(x[0], y[0]) + l = [(key(element), element) for element in iterable] + else: + if cmp is not None: + use_cmp = cmp + l = list(iterable) + if use_cmp is not None: + l.sort(use_cmp) + else: + l.sort() + if reverse: + l.reverse() + if key is not None: + return [element for (_, element) in l] + return l + +try: + set, frozenset = set, frozenset +except NameError: + from sets import set, frozenset + +# pass through +enumerate = enumerate + +try: + BaseException = BaseException +except NameError: + BaseException = Exception + +try: + GeneratorExit = GeneratorExit +except NameError: + class GeneratorExit(Exception): + """ This exception is never raised, it is there to make it possible to + write code compatible with CPython 2.5 even in lower CPython + versions.""" + pass + GeneratorExit.__module__ = 'exceptions' + +if sys.version_info >= (3, 0): + exec ("print_ = print ; exec_=exec") + import builtins + + # some backward compatibility helpers + _basestring = str + def _totext(obj, encoding): + if isinstance(obj, bytes): + obj = obj.decode(encoding) + elif not isinstance(obj, str): + obj = str(obj) + return obj + + def _isbytes(x): + return isinstance(x, bytes) + def _istext(x): + return isinstance(x, str) + + def _getimself(function): + return getattr(function, '__self__', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def execfile(fn, globs=None, locs=None): + if globs is None: + back = sys._getframe(1) + globs = back.f_globals + locs = back.f_locals + del back + elif locs is None: + locs = globs + fp = open(fn, "rb") + try: + source = fp.read() + finally: + fp.close() + co = compile(source, fn, "exec", dont_inherit=True) + exec_(co, globs, locs) + + def callable(obj): + return hasattr(obj, "__call__") + +else: + import __builtin__ as builtins + _totext = unicode + _basestring = basestring + execfile = execfile + callable = callable + def _isbytes(x): + return isinstance(x, str) + def _istext(x): + return isinstance(x, unicode) + + def _getimself(function): + return getattr(function, 'im_self', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def print_(*args, **kwargs): + """ minimal backport of py3k print statement. """ + sep = ' ' + if 'sep' in kwargs: + sep = kwargs.pop('sep') + end = '\n' + if 'end' in kwargs: + end = kwargs.pop('end') + file = 'file' in kwargs and kwargs.pop('file') or sys.stdout + if kwargs: + args = ", ".join([str(x) for x in kwargs]) + raise TypeError("invalid keyword arguments: %s" % args) + at_start = True + for x in args: + if not at_start: + file.write(sep) + file.write(str(x)) + at_start = False + file.write(end) + + def exec_(obj, globals=None, locals=None): + """ minimal backport of py3k exec statement. """ + if globals is None: + frame = sys._getframe(1) + globals = frame.f_globals + if locals is None: + locals = frame.f_locals + elif locals is None: + locals = globals + exec2(obj, globals, locals) + +if sys.version_info >= (3,0): + exec (""" +def _reraise(cls, val, tb): + assert hasattr(val, '__traceback__') + raise val +""") +else: + exec (""" +def _reraise(cls, val, tb): + raise cls, val, tb +def exec2(obj, globals, locals): + exec obj in globals, locals +""") + +def _tryimport(*names): + """ return the first successfully imported module. """ + assert names + for name in names: + try: + return __import__(name, None, None, '__doc__') + except ImportError: + excinfo = sys.exc_info() + _reraise(*excinfo) Added: pypy/branch/py11/py/impl/cmdline/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/py11/py/impl/cmdline/pycleanup.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pycleanup.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +"""\ +py.cleanup [PATH] + +Delete pyc file recursively, starting from PATH (which defaults to the current +working directory). Don't follow links and don't recurse into directories with +a ".". +""" +import py + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + parser.add_option("-e", "--remove", dest="ext", default=".pyc", action="store", + help="remove files with the given comma-separated list of extensions" + ) + parser.add_option("-n", "--dryrun", dest="dryrun", default=False, + action="store_true", + help="display would-be-removed filenames" + ) + parser.add_option("-d", action="store_true", dest="removedir", + help="remove empty directories") + (options, args) = parser.parse_args() + if not args: + args = ["."] + ext = options.ext.split(",") + def shouldremove(p): + return p.ext in ext + + for arg in args: + path = py.path.local(arg) + py.builtin.print_("cleaning path", path, "of extensions", ext) + for x in path.visit(shouldremove, lambda x: x.check(dotfile=0, link=0)): + remove(x, options) + if options.removedir: + for x in path.visit(lambda x: x.check(dir=1), + lambda x: x.check(dotfile=0, link=0)): + if not x.listdir(): + remove(x, options) + +def remove(path, options): + if options.dryrun: + py.builtin.print_("would remove", path) + else: + py.builtin.print_("removing", path) + path.remove() + Added: pypy/branch/py11/py/impl/cmdline/pyconvert_unittest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pyconvert_unittest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,249 @@ +import re +import sys +import parser + +d={} +# d is the dictionary of unittest changes, keyed to the old name +# used by unittest. +# d[old][0] is the new replacement function. +# d[old][1] is the operator you will substitute, or '' if there is none. +# d[old][2] is the possible number of arguments to the unittest +# function. + +# Old Unittest Name new name operator # of args +d['assertRaises'] = ('raises', '', ['Any']) +d['fail'] = ('raise AssertionError', '', [0,1]) +d['assert_'] = ('assert', '', [1,2]) +d['failIf'] = ('assert not', '', [1,2]) +d['assertEqual'] = ('assert', ' ==', [2,3]) +d['failIfEqual'] = ('assert not', ' ==', [2,3]) +d['assertIn'] = ('assert', ' in', [2,3]) +d['assertNotIn'] = ('assert', ' not in', [2,3]) +d['assertNotEqual'] = ('assert', ' !=', [2,3]) +d['failUnlessEqual'] = ('assert', ' ==', [2,3]) +d['assertAlmostEqual'] = ('assert round', ' ==', [2,3,4]) +d['failIfAlmostEqual'] = ('assert not round', ' ==', [2,3,4]) +d['assertNotAlmostEqual'] = ('assert round', ' !=', [2,3,4]) +d['failUnlessAlmostEquals'] = ('assert round', ' ==', [2,3,4]) + +# the list of synonyms +d['failUnlessRaises'] = d['assertRaises'] +d['failUnless'] = d['assert_'] +d['assertEquals'] = d['assertEqual'] +d['assertNotEquals'] = d['assertNotEqual'] +d['assertAlmostEquals'] = d['assertAlmostEqual'] +d['assertNotAlmostEquals'] = d['assertNotAlmostEqual'] + +# set up the regular expressions we will need +leading_spaces = re.compile(r'^(\s*)') # this never fails + +pat = '' +for k in d.keys(): # this complicated pattern to match all unittests + pat += '|' + r'^(\s*)' + 'self.' + k + r'\(' # \tself.whatever( + +old_names = re.compile(pat[1:]) +linesep='\n' # nobody will really try to convert files not read + # in text mode, will they? + + +def blocksplitter(fp): + '''split a file into blocks that are headed by functions to rename''' + + blocklist = [] + blockstring = '' + + for line in fp: + interesting = old_names.match(line) + if interesting : + if blockstring: + blocklist.append(blockstring) + blockstring = line # reset the block + else: + blockstring += line + + blocklist.append(blockstring) + return blocklist + +def rewrite_utest(block): + '''rewrite every block to use the new utest functions''' + + '''returns the rewritten unittest, unless it ran into problems, + in which case it just returns the block unchanged. + ''' + utest = old_names.match(block) + + if not utest: + return block + + old = utest.group(0).lstrip()[5:-1] # the name we want to replace + new = d[old][0] # the name of the replacement function + op = d[old][1] # the operator you will use , or '' if there is none. + possible_args = d[old][2] # a list of the number of arguments the + # unittest function could possibly take. + + if possible_args == ['Any']: # just rename assertRaises & friends + return re.sub('self.'+old, new, block) + + message_pos = possible_args[-1] + # the remaining unittests can have an optional message to print + # when they fail. It is always the last argument to the function. + + try: + indent, argl, trailer = decompose_unittest(old, block) + + except SyntaxError: # but we couldn't parse it! + return block + + argnum = len(argl) + if argnum not in possible_args: + # sanity check - this one isn't real either + return block + + elif argnum == message_pos: + message = argl[-1] + argl = argl[:-1] + else: + message = None + + if argnum is 0 or (argnum is 1 and argnum is message_pos): #unittest fail() + string = '' + if message: + message = ' ' + message + + elif message_pos is 4: # assertAlmostEqual & friends + try: + pos = argl[2].lstrip() + except IndexError: + pos = '7' # default if none is specified + string = '(%s -%s, %s)%s 0' % (argl[0], argl[1], pos, op ) + + else: # assert_, assertEquals and all the rest + string = ' ' + op.join(argl) + + if message: + string = string + ',' + message + + return indent + new + string + trailer + +def decompose_unittest(old, block): + '''decompose the block into its component parts''' + + ''' returns indent, arglist, trailer + indent -- the indentation + arglist -- the arguments to the unittest function + trailer -- any extra junk after the closing paren, such as #commment + ''' + + indent = re.match(r'(\s*)', block).group() + pat = re.search('self.' + old + r'\(', block) + + args, trailer = get_expr(block[pat.end():], ')') + arglist = break_args(args, []) + + if arglist == ['']: # there weren't any + return indent, [], trailer + + for i in range(len(arglist)): + try: + parser.expr(arglist[i].lstrip('\t ')) + except SyntaxError: + if i == 0: + arglist[i] = '(' + arglist[i] + ')' + else: + arglist[i] = ' (' + arglist[i] + ')' + + return indent, arglist, trailer + +def break_args(args, arglist): + '''recursively break a string into a list of arguments''' + try: + first, rest = get_expr(args, ',') + if not rest: + return arglist + [first] + else: + return [first] + break_args(rest, arglist) + except SyntaxError: + return arglist + [args] + +def get_expr(s, char): + '''split a string into an expression, and the rest of the string''' + + pos=[] + for i in range(len(s)): + if s[i] == char: + pos.append(i) + if pos == []: + raise SyntaxError # we didn't find the expected char. Ick. + + for p in pos: + # make the python parser do the hard work of deciding which comma + # splits the string into two expressions + try: + parser.expr('(' + s[:p] + ')') + return s[:p], s[p+1:] + except SyntaxError: # It's not an expression yet + pass + raise SyntaxError # We never found anything that worked. + + +def main(): + import sys + import py + + usage = "usage: %prog [-s [filename ...] | [-i | -c filename ...]]" + optparser = py.std.optparse.OptionParser(usage) + + def select_output (option, opt, value, optparser, **kw): + if hasattr(optparser, 'output'): + optparser.error( + 'Cannot combine -s -i and -c options. Use one only.') + else: + optparser.output = kw['output'] + + optparser.add_option("-s", "--stdout", action="callback", + callback=select_output, + callback_kwargs={'output':'stdout'}, + help="send your output to stdout") + + optparser.add_option("-i", "--inplace", action="callback", + callback=select_output, + callback_kwargs={'output':'inplace'}, + help="overwrite files in place") + + optparser.add_option("-c", "--copy", action="callback", + callback=select_output, + callback_kwargs={'output':'copy'}, + help="copy files ... fn.py --> fn_cp.py") + + options, args = optparser.parse_args() + + output = getattr(optparser, 'output', 'stdout') + + if output in ['inplace', 'copy'] and not args: + optparser.error( + '-i and -c option require at least one filename') + + if not args: + s = '' + for block in blocksplitter(sys.stdin): + s += rewrite_utest(block) + sys.stdout.write(s) + + else: + for infilename in args: # no error checking to see if we can open, etc. + infile = file(infilename) + s = '' + for block in blocksplitter(infile): + s += rewrite_utest(block) + if output == 'inplace': + outfile = file(infilename, 'w+') + elif output == 'copy': # yes, just go clobber any existing .cp + outfile = file (infilename[:-3]+ '_cp.py', 'w+') + else: + outfile = sys.stdout + + outfile.write(s) + + +if __name__ == '__main__': + main() Added: pypy/branch/py11/py/impl/cmdline/pycountloc.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pycountloc.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# hands on script to compute the non-empty Lines of Code +# for tests and non-test code + +"""\ +py.countloc [PATHS] + +Count (non-empty) lines of python code and number of python files recursively +starting from a list of paths given on the command line (starting from the +current working directory). Distinguish between test files and normal ones and +report them separately. +""" +import py + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + (options, args) = parser.parse_args() + countloc(args) + +def nodot(p): + return p.check(dotfile=0) + +class FileCounter(object): + def __init__(self): + self.file2numlines = {} + self.numlines = 0 + self.numfiles = 0 + + def addrecursive(self, directory, fil="*.py", rec=nodot): + for x in directory.visit(fil, rec): + self.addfile(x) + + def addfile(self, fn, emptylines=False): + if emptylines: + s = len(p.readlines()) + else: + s = 0 + for i in fn.readlines(): + if i.strip(): + s += 1 + self.file2numlines[fn] = s + self.numfiles += 1 + self.numlines += s + + def getnumlines(self, fil): + numlines = 0 + for path, value in self.file2numlines.items(): + if fil(path): + numlines += value + return numlines + + def getnumfiles(self, fil): + numfiles = 0 + for path in self.file2numlines: + if fil(path): + numfiles += 1 + return numfiles + +def get_loccount(locations=None): + if locations is None: + localtions = [py.path.local()] + counter = FileCounter() + for loc in locations: + counter.addrecursive(loc, '*.py', rec=nodot) + + def istestfile(p): + return p.check(fnmatch='test_*.py') + isnottestfile = lambda x: not istestfile(x) + + numfiles = counter.getnumfiles(isnottestfile) + numlines = counter.getnumlines(isnottestfile) + numtestfiles = counter.getnumfiles(istestfile) + numtestlines = counter.getnumlines(istestfile) + + return counter, numfiles, numlines, numtestfiles, numtestlines + +def countloc(paths=None): + if not paths: + paths = ['.'] + locations = [py.path.local(x) for x in paths] + (counter, numfiles, numlines, numtestfiles, + numtestlines) = get_loccount(locations) + + items = counter.file2numlines.items() + items.sort(lambda x,y: cmp(x[1], y[1])) + for x, y in items: + print("%3d %30s" % (y,x)) + + print("%30s %3d" %("number of testfiles", numtestfiles)) + print("%30s %3d" %("number of non-empty testlines", numtestlines)) + print("%30s %3d" %("number of files", numfiles)) + print("%30s %3d" %("number of non-empty lines", numlines)) + Added: pypy/branch/py11/py/impl/cmdline/pylookup.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pylookup.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +"""\ +py.lookup [search_directory] SEARCH_STRING [options] + +Looks recursively at Python files for a SEARCH_STRING, starting from the +present working directory. Prints the line, with the filename and line-number +prepended.""" + +import sys, os +import py +from py.impl.io.terminalwriter import ansi_print, terminal_width +import re + +def rec(p): + return p.check(dotfile=0) + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-i", "--ignore-case", action="store_true", dest="ignorecase", + help="ignore case distinctions") +parser.add_option("-C", "--context", action="store", type="int", dest="context", + default=0, help="How many lines of output to show") + +def find_indexes(search_line, string): + indexes = [] + before = 0 + while 1: + i = search_line.find(string, before) + if i == -1: + break + indexes.append(i) + before = i + len(string) + return indexes + +def main(): + (options, args) = parser.parse_args() + if len(args) == 2: + search_dir, string = args + search_dir = py.path.local(search_dir) + else: + search_dir = py.path.local() + string = args[0] + if options.ignorecase: + string = string.lower() + for x in search_dir.visit('*.py', rec): + # match filename directly + s = x.relto(search_dir) + if options.ignorecase: + s = s.lower() + if s.find(string) != -1: + sys.stdout.write("%s: filename matches %r" %(x, string) + "\n") + + try: + s = x.read() + except py.error.ENOENT: + pass # whatever, probably broken link (ie emacs lock) + searchs = s + if options.ignorecase: + searchs = s.lower() + if s.find(string) != -1: + lines = s.splitlines() + if options.ignorecase: + searchlines = s.lower().splitlines() + else: + searchlines = lines + for i, (line, searchline) in enumerate(zip(lines, searchlines)): + indexes = find_indexes(searchline, string) + if not indexes: + continue + if not options.context: + sys.stdout.write("%s:%d: " %(x.relto(search_dir), i+1)) + last_index = 0 + for index in indexes: + sys.stdout.write(line[last_index: index]) + ansi_print(line[index: index+len(string)], + file=sys.stdout, esc=31, newline=False) + last_index = index + len(string) + sys.stdout.write(line[last_index:] + "\n") + else: + context = (options.context)/2 + for count in range(max(0, i-context), min(len(lines) - 1, i+context+1)): + print("%s:%d: %s" %(x.relto(search_dir), count+1, lines[count].rstrip())) + print("-" * terminal_width) Added: pypy/branch/py11/py/impl/cmdline/pysvnwcrevert.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pysvnwcrevert.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,55 @@ +#! /usr/bin/env python +"""\ +py.svnwcrevert [options] WCPATH + +Running this script and then 'svn up' puts the working copy WCPATH in a state +as clean as a fresh check-out. + +WARNING: you'll loose all local changes, obviously! + +This script deletes all files that have been modified +or that svn doesn't explicitly know about, including svn:ignored files +(like .pyc files, hint hint). + +The goal of this script is to leave the working copy with some files and +directories possibly missing, but - most importantly - in a state where +the following 'svn up' won't just crash. +""" + +import sys, py + +def kill(p, root): + print('< %s' % (p.relto(root),)) + p.remove(rec=1) + +def svnwcrevert(path, root=None, precious=[]): + if root is None: + root = path + wcpath = py.path.svnwc(path) + try: + st = wcpath.status() + except ValueError: # typically, "bad char in wcpath" + kill(path, root) + return + for p in path.listdir(): + if p.basename == '.svn' or p.basename in precious: + continue + wcp = py.path.svnwc(p) + if wcp not in st.unchanged and wcp not in st.external: + kill(p, root) + elif p.check(dir=1): + svnwcrevert(p, root) + +# XXX add a functional test + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-p", "--precious", + action="append", dest="precious", default=[], + help="preserve files with this name") + +def main(): + opts, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + svnwcrevert(py.path.local(args[0]), precious=opts.precious) Added: pypy/branch/py11/py/impl/cmdline/pytest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pytest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,5 @@ +#!/usr/bin/env python +import py + +def main(): + py.test.cmdline.main() Added: pypy/branch/py11/py/impl/cmdline/pywhich.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/cmdline/pywhich.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +"""\ +py.which [name] + +print the location of the given python module or package name +""" + +import sys + +def main(): + name = sys.argv[1] + try: + mod = __import__(name) + except ImportError: + sys.stderr.write("could not import: " + name + "\n") + else: + try: + location = mod.__file__ + except AttributeError: + sys.stderr.write("module (has no __file__): " + str(mod)) + else: + print(location) Added: pypy/branch/py11/py/impl/code/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +""" python inspection/code generation API """ Added: pypy/branch/py11/py/impl/code/_assertionnew.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/_assertionnew.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,337 @@ +""" +Like _assertion.py but using builtin AST. It should replace _assertion.py +eventually. +""" + +import sys +import ast + +import py +from py.impl.code.assertion import _format_explanation, BuiltinAssertionError + + +if sys.platform.startswith("java") and sys.version_info < (2, 5, 2): + # See http://bugs.jython.org/issue1497 + _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict", + "ListComp", "GeneratorExp", "Yield", "Compare", "Call", + "Repr", "Num", "Str", "Attribute", "Subscript", "Name", + "List", "Tuple") + _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign", + "AugAssign", "Print", "For", "While", "If", "With", "Raise", + "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom", + "Exec", "Global", "Expr", "Pass", "Break", "Continue") + _expr_nodes = set(getattr(ast, name) for name in _exprs) + _stmt_nodes = set(getattr(ast, name) for name in _stmts) + def _is_ast_expr(node): + return node.__class__ in _expr_nodes + def _is_ast_stmt(node): + return node.__class__ in _stmt_nodes +else: + def _is_ast_expr(node): + return isinstance(node, ast.expr) + def _is_ast_stmt(node): + return isinstance(node, ast.stmt) + + +class Failure(Exception): + """Error found while interpreting AST.""" + + def __init__(self, explanation=""): + self.cause = sys.exc_info() + self.explanation = explanation + + +def interpret(source, frame, should_fail=False): + mod = ast.parse(source) + visitor = DebugInterpreter(frame) + try: + visitor.visit(mod) + except Failure: + failure = sys.exc_info()[1] + return getfailure(failure) + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --nomagic)") + +def run(offending_line, frame=None): + if frame is None: + frame = py.code.Frame(sys._getframe(1)) + return interpret(offending_line, frame) + +def getfailure(failure): + explanation = _format_explanation(failure.explanation) + value = failure.cause[1] + if str(value): + lines = explanation.splitlines() + if not lines: + lines.append("") + lines[0] += " << %s" % (value,) + explanation = "\n".join(lines) + text = "%s: %s" % (failure.cause[0].__name__, explanation) + if text.startswith("AssertionError: assert "): + text = text[16:] + return text + + +operator_map = { + ast.BitOr : "|", + ast.BitXor : "^", + ast.BitAnd : "&", + ast.LShift : "<<", + ast.RShift : ">>", + ast.Add : "+", + ast.Sub : "-", + ast.Mult : "*", + ast.Div : "/", + ast.FloorDiv : "//", + ast.Mod : "%", + ast.Eq : "==", + ast.NotEq : "!=", + ast.Lt : "<", + ast.LtE : "<=", + ast.Gt : ">", + ast.GtE : ">=", + ast.Is : "is", + ast.IsNot : "is not", + ast.In : "in", + ast.NotIn : "not in" +} + +unary_map = { + ast.Not : "not %s", + ast.Invert : "~%s", + ast.USub : "-%s", + ast.UAdd : "+%s" +} + + +class DebugInterpreter(ast.NodeVisitor): + """Interpret AST nodes to gleam useful debugging information.""" + + def __init__(self, frame): + self.frame = frame + + def generic_visit(self, node): + # Fallback when we don't have a special implementation. + if _is_ast_expr(node): + mod = ast.Expression(node) + co = self._compile(mod) + try: + result = self.frame.eval(co) + except Exception: + raise Failure() + explanation = self.frame.repr(result) + return explanation, result + elif _is_ast_stmt(node): + mod = ast.Module([node]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co) + except Exception: + raise Failure() + return None, None + else: + raise AssertionError("can't handle %s" %(node,)) + + def _compile(self, source, mode="eval"): + return compile(source, "", mode) + + def visit_Expr(self, expr): + return self.visit(expr.value) + + def visit_Module(self, mod): + for stmt in mod.body: + self.visit(stmt) + + def visit_Name(self, name): + explanation, result = self.generic_visit(name) + # See if the name is local. + source = "%r in locals() is not globals()" % (name.id,) + co = self._compile(source) + try: + local = self.frame.eval(co) + except Exception: + # have to assume it isn't + local = False + if not local: + return name.id, result + return explanation, result + + def visit_Compare(self, comp): + left = comp.left + left_explanation, left_result = self.visit(left) + got_result = False + for op, next_op in zip(comp.ops, comp.comparators): + if got_result and not result: + break + next_explanation, next_result = self.visit(next_op) + op_symbol = operator_map[op.__class__] + explanation = "%s %s %s" % (left_explanation, op_symbol, + next_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=next_result) + except Exception: + raise Failure(explanation) + else: + got_result = True + left_explanation, left_result = next_explanation, next_result + return explanation, result + + def visit_BoolOp(self, boolop): + is_or = isinstance(boolop.op, ast.Or) + explanations = [] + for operand in boolop.values: + explanation, result = self.visit(operand) + explanations.append(explanation) + if result == is_or: + break + name = is_or and " or " or " and " + explanation = "(" + name.join(explanations) + ")" + return explanation, result + + def visit_UnaryOp(self, unary): + pattern = unary_map[unary.op.__class__] + operand_explanation, operand_result = self.visit(unary.operand) + explanation = pattern % (operand_explanation,) + co = self._compile(pattern % ("__exprinfo_expr",)) + try: + result = self.frame.eval(co, __exprinfo_expr=operand_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_BinOp(self, binop): + left_explanation, left_result = self.visit(binop.left) + right_explanation, right_result = self.visit(binop.right) + symbol = operator_map[binop.op.__class__] + explanation = "(%s %s %s)" % (left_explanation, symbol, + right_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=right_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_Call(self, call): + func_explanation, func = self.visit(call.func) + arg_explanations = [] + ns = {"__exprinfo_func" : func} + arguments = [] + for arg in call.args: + arg_explanation, arg_result = self.visit(arg) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + arguments.append(arg_name) + arg_explanations.append(arg_explanation) + for keyword in call.keywords: + arg_explanation, arg_result = self.visit(keyword.value) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + keyword_source = "%s=%%s" % (keyword.id) + arguments.append(keyword_source % (arg_name,)) + arg_explanations.append(keyword_source % (arg_explanation,)) + if call.starargs: + arg_explanation, arg_result = self.visit(call.starargs) + arg_name = "__exprinfo_star" + ns[arg_name] = arg_result + arguments.append("*%s" % (arg_name,)) + arg_explanations.append("*%s" % (arg_explanation,)) + if call.kwargs: + arg_explanation, arg_result = self.visit(call.kwargs) + arg_name = "__exprinfo_kwds" + ns[arg_name] = arg_result + arguments.append("**%s" % (arg_name,)) + arg_explanations.append("**%s" % (arg_explanation,)) + args_explained = ", ".join(arg_explanations) + explanation = "%s(%s)" % (func_explanation, args_explained) + args = ", ".join(arguments) + source = "__exprinfo_func(%s)" % (args,) + co = self._compile(source) + try: + result = self.frame.eval(co, **ns) + except Exception: + raise Failure(explanation) + # Only show result explanation if it's not a builtin call or returns a + # bool. + if not isinstance(call.func, ast.Name) or \ + not self._is_builtin_name(call.func): + source = "isinstance(__exprinfo_value, bool)" + co = self._compile(source) + try: + is_bool = self.frame.eval(co, __exprinfo_value=result) + except Exception: + is_bool = False + if not is_bool: + pattern = "%s\n{%s = %s\n}" + rep = self.frame.repr(result) + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def _is_builtin_name(self, name): + pattern = "%r not in globals() and %r not in locals()" + source = pattern % (name.id, name.id) + co = self._compile(source) + try: + return self.frame.eval(co) + except Exception: + return False + + def visit_Attribute(self, attr): + if not isinstance(attr.ctx, ast.Load): + return self.generic_visit(attr) + source_explanation, source_result = self.visit(attr.value) + explanation = "%s.%s" % (source_explanation, attr.attr) + source = "__exprinfo_expr.%s" % (attr.attr,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + raise Failure(explanation) + # Check if the attr is from an instance. + source = "%r in getattr(__exprinfo_expr, '__dict__', {})" + source = source % (attr.attr,) + co = self._compile(source) + try: + from_instance = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + from_instance = True + if from_instance: + rep = self.frame.repr(result) + pattern = "%s\n{%s = %s\n}" + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def visit_Assert(self, assrt): + test_explanation, test_result = self.visit(assrt.test) + if test_explanation.startswith("False\n{False =") and \ + test_explanation.endswith("\n"): + test_explanation = test_explanation[15:-2] + explanation = "assert %s" % (test_explanation,) + if not test_result: + try: + raise BuiltinAssertionError + except Exception: + raise Failure(explanation) + return explanation, test_result + + def visit_Assign(self, assign): + value_explanation, value_result = self.visit(assign.value) + explanation = "... = %s" % (value_explanation,) + name = ast.Name("__exprinfo_expr", ast.Load(), assign.value.lineno, + assign.value.col_offset) + new_assign = ast.Assign(assign.targets, name, assign.lineno, + assign.col_offset) + mod = ast.Module([new_assign]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co, __exprinfo_expr=value_result) + except Exception: + raise Failure(explanation) + return explanation, value_result Added: pypy/branch/py11/py/impl/code/_assertionold.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/_assertionold.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,558 @@ +import py +import sys, inspect +from compiler import parse, ast, pycodegen +from py.impl.code.assertion import BuiltinAssertionError, _format_explanation + +passthroughex = (KeyboardInterrupt, SystemExit, MemoryError) + +class Failure: + def __init__(self, node): + self.exc, self.value, self.tb = sys.exc_info() + self.node = node + +class View(object): + """View base class. + + If C is a subclass of View, then C(x) creates a proxy object around + the object x. The actual class of the proxy is not C in general, + but a *subclass* of C determined by the rules below. To avoid confusion + we call view class the class of the proxy (a subclass of C, so of View) + and object class the class of x. + + Attributes and methods not found in the proxy are automatically read on x. + Other operations like setting attributes are performed on the proxy, as + determined by its view class. The object x is available from the proxy + as its __obj__ attribute. + + The view class selection is determined by the __view__ tuples and the + optional __viewkey__ method. By default, the selected view class is the + most specific subclass of C whose __view__ mentions the class of x. + If no such subclass is found, the search proceeds with the parent + object classes. For example, C(True) will first look for a subclass + of C with __view__ = (..., bool, ...) and only if it doesn't find any + look for one with __view__ = (..., int, ...), and then ..., object,... + If everything fails the class C itself is considered to be the default. + + Alternatively, the view class selection can be driven by another aspect + of the object x, instead of the class of x, by overriding __viewkey__. + See last example at the end of this module. + """ + + _viewcache = {} + __view__ = () + + def __new__(rootclass, obj, *args, **kwds): + self = object.__new__(rootclass) + self.__obj__ = obj + self.__rootclass__ = rootclass + key = self.__viewkey__() + try: + self.__class__ = self._viewcache[key] + except KeyError: + self.__class__ = self._selectsubclass(key) + return self + + def __getattr__(self, attr): + # attributes not found in the normal hierarchy rooted on View + # are looked up in the object's real class + return getattr(self.__obj__, attr) + + def __viewkey__(self): + return self.__obj__.__class__ + + def __matchkey__(self, key, subclasses): + if inspect.isclass(key): + keys = inspect.getmro(key) + else: + keys = [key] + for key in keys: + result = [C for C in subclasses if key in C.__view__] + if result: + return result + return [] + + def _selectsubclass(self, key): + subclasses = list(enumsubclasses(self.__rootclass__)) + for C in subclasses: + if not isinstance(C.__view__, tuple): + C.__view__ = (C.__view__,) + choices = self.__matchkey__(key, subclasses) + if not choices: + return self.__rootclass__ + elif len(choices) == 1: + return choices[0] + else: + # combine the multiple choices + return type('?', tuple(choices), {}) + + def __repr__(self): + return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__) + + +def enumsubclasses(cls): + for subcls in cls.__subclasses__(): + for subsubclass in enumsubclasses(subcls): + yield subsubclass + yield cls + + +class Interpretable(View): + """A parse tree node with a few extra methods.""" + explanation = None + + def is_builtin(self, frame): + return False + + def eval(self, frame): + # fall-back for unknown expression nodes + try: + expr = ast.Expression(self.__obj__) + expr.filename = '' + self.__obj__.filename = '' + co = pycodegen.ExpressionCodeGenerator(expr).getCode() + result = frame.eval(co) + except passthroughex: + raise + except: + raise Failure(self) + self.result = result + self.explanation = self.explanation or frame.repr(self.result) + + def run(self, frame): + # fall-back for unknown statement nodes + try: + expr = ast.Module(None, ast.Stmt([self.__obj__])) + expr.filename = '' + co = pycodegen.ModuleCodeGenerator(expr).getCode() + frame.exec_(co) + except passthroughex: + raise + except: + raise Failure(self) + + def nice_explanation(self): + return _format_explanation(self.explanation) + + +class Name(Interpretable): + __view__ = ast.Name + + def is_local(self, frame): + co = compile('%r in locals() is not globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_global(self, frame): + co = compile('%r in globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_builtin(self, frame): + co = compile('%r not in locals() and %r not in globals()' % ( + self.name, self.name), '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + super(Name, self).eval(frame) + if not self.is_local(frame): + self.explanation = self.name + +class Compare(Interpretable): + __view__ = ast.Compare + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + for operation, expr2 in self.ops: + if hasattr(self, 'result'): + # shortcutting in chained expressions + if not frame.is_true(self.result): + break + expr2 = Interpretable(expr2) + expr2.eval(frame) + self.explanation = "%s %s %s" % ( + expr.explanation, operation, expr2.explanation) + co = compile("__exprinfo_left %s __exprinfo_right" % operation, + '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_left=expr.result, + __exprinfo_right=expr2.result) + except passthroughex: + raise + except: + raise Failure(self) + expr = expr2 + +class And(Interpretable): + __view__ = ast.And + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if not frame.is_true(expr.result): + break + self.explanation = '(' + ' and '.join(explanations) + ')' + +class Or(Interpretable): + __view__ = ast.Or + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if frame.is_true(expr.result): + break + self.explanation = '(' + ' or '.join(explanations) + ')' + + +# == Unary operations == +keepalive = [] +for astclass, astpattern in { + ast.Not : 'not __exprinfo_expr', + ast.Invert : '(~__exprinfo_expr)', + }.items(): + + class UnaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + expr = Interpretable(self.expr) + expr.eval(frame) + self.explanation = astpattern.replace('__exprinfo_expr', + expr.explanation) + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(UnaryArith) + +# == Binary operations == +for astclass, astpattern in { + ast.Add : '(__exprinfo_left + __exprinfo_right)', + ast.Sub : '(__exprinfo_left - __exprinfo_right)', + ast.Mul : '(__exprinfo_left * __exprinfo_right)', + ast.Div : '(__exprinfo_left / __exprinfo_right)', + ast.Mod : '(__exprinfo_left % __exprinfo_right)', + ast.Power : '(__exprinfo_left ** __exprinfo_right)', + }.items(): + + class BinaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + left = Interpretable(self.left) + left.eval(frame) + right = Interpretable(self.right) + right.eval(frame) + self.explanation = (astpattern + .replace('__exprinfo_left', left .explanation) + .replace('__exprinfo_right', right.explanation)) + try: + self.result = frame.eval(co, __exprinfo_left=left.result, + __exprinfo_right=right.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(BinaryArith) + + +class CallFunc(Interpretable): + __view__ = ast.CallFunc + + def is_bool(self, frame): + co = compile('isinstance(__exprinfo_value, bool)', '?', 'eval') + try: + return frame.is_true(frame.eval(co, __exprinfo_value=self.result)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + node = Interpretable(self.node) + node.eval(frame) + explanations = [] + vars = {'__exprinfo_fn': node.result} + source = '__exprinfo_fn(' + for a in self.args: + if isinstance(a, ast.Keyword): + keyword = a.name + a = a.expr + else: + keyword = None + a = Interpretable(a) + a.eval(frame) + argname = '__exprinfo_%d' % len(vars) + vars[argname] = a.result + if keyword is None: + source += argname + ',' + explanations.append(a.explanation) + else: + source += '%s=%s,' % (keyword, argname) + explanations.append('%s=%s' % (keyword, a.explanation)) + if self.star_args: + star_args = Interpretable(self.star_args) + star_args.eval(frame) + argname = '__exprinfo_star' + vars[argname] = star_args.result + source += '*' + argname + ',' + explanations.append('*' + star_args.explanation) + if self.dstar_args: + dstar_args = Interpretable(self.dstar_args) + dstar_args.eval(frame) + argname = '__exprinfo_kwds' + vars[argname] = dstar_args.result + source += '**' + argname + ',' + explanations.append('**' + dstar_args.explanation) + self.explanation = "%s(%s)" % ( + node.explanation, ', '.join(explanations)) + if source.endswith(','): + source = source[:-1] + source += ')' + co = compile(source, '?', 'eval') + try: + self.result = frame.eval(co, **vars) + except passthroughex: + raise + except: + raise Failure(self) + if not node.is_builtin(frame) or not self.is_bool(frame): + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +class Getattr(Interpretable): + __view__ = ast.Getattr + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + co = compile('__exprinfo_expr.%s' % self.attrname, '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + self.explanation = '%s.%s' % (expr.explanation, self.attrname) + # if the attribute comes from the instance, its value is interesting + co = compile('hasattr(__exprinfo_expr, "__dict__") and ' + '%r in __exprinfo_expr.__dict__' % self.attrname, + '?', 'eval') + try: + from_instance = frame.is_true( + frame.eval(co, __exprinfo_expr=expr.result)) + except passthroughex: + raise + except: + from_instance = True + if from_instance: + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +# == Re-interpretation of full statements == + +class Assert(Interpretable): + __view__ = ast.Assert + + def run(self, frame): + test = Interpretable(self.test) + test.eval(frame) + # simplify 'assert False where False = ...' + if (test.explanation.startswith('False\n{False = ') and + test.explanation.endswith('\n}')): + test.explanation = test.explanation[15:-2] + # print the result as 'assert ' + self.result = test.result + self.explanation = 'assert ' + test.explanation + if not frame.is_true(test.result): + try: + raise BuiltinAssertionError + except passthroughex: + raise + except: + raise Failure(self) + +class Assign(Interpretable): + __view__ = ast.Assign + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = '... = ' + expr.explanation + # fall-back-run the rest of the assignment + ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr')) + mod = ast.Module(None, ast.Stmt([ass])) + mod.filename = '' + co = pycodegen.ModuleCodeGenerator(mod).getCode() + try: + frame.exec_(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + +class Discard(Interpretable): + __view__ = ast.Discard + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = expr.explanation + +class Stmt(Interpretable): + __view__ = ast.Stmt + + def run(self, frame): + for stmt in self.nodes: + stmt = Interpretable(stmt) + stmt.run(frame) + + +def report_failure(e): + explanation = e.node.nice_explanation() + if explanation: + explanation = ", in: " + explanation + else: + explanation = "" + sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation)) + +def check(s, frame=None): + if frame is None: + import sys + frame = sys._getframe(1) + frame = py.code.Frame(frame) + expr = parse(s, 'eval') + assert isinstance(expr, ast.Expression) + node = Interpretable(expr.node) + try: + node.eval(frame) + except passthroughex: + raise + except Failure: + e = sys.exc_info()[1] + report_failure(e) + else: + if not frame.is_true(node.result): + sys.stderr.write("assertion failed: %s\n" % node.nice_explanation()) + + +########################################################### +# API / Entry points +# ######################################################### + +def interpret(source, frame, should_fail=False): + module = Interpretable(parse(source, 'exec').node) + #print "got module", module + if isinstance(frame, py.std.types.FrameType): + frame = py.code.Frame(frame) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + return getfailure(e) + except passthroughex: + raise + except: + import traceback + traceback.print_exc() + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --nomagic)") + else: + return None + +def getmsg(excinfo): + if isinstance(excinfo, tuple): + excinfo = py.code.ExceptionInfo(excinfo) + #frame, line = gettbline(tb) + #frame = py.code.Frame(frame) + #return interpret(line, frame) + + tb = excinfo.traceback[-1] + source = str(tb.statement).strip() + x = interpret(source, tb.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + return x + +def getfailure(e): + explanation = e.node.nice_explanation() + if str(e.value): + lines = explanation.split('\n') + lines[0] += " << %s" % (e.value,) + explanation = '\n'.join(lines) + text = "%s: %s" % (e.exc.__name__, explanation) + if text.startswith('AssertionError: assert '): + text = text[16:] + return text + +def run(s, frame=None): + if frame is None: + import sys + frame = sys._getframe(1) + frame = py.code.Frame(frame) + module = Interpretable(parse(s, 'exec').node) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + report_failure(e) + + +if __name__ == '__main__': + # example: + def f(): + return 5 + def g(): + return 3 + def h(x): + return 'never' + check("f() * g() == 5") + check("not f()") + check("not (f() and g() or 0)") + check("f() == g()") + i = 4 + check("i == f()") + check("len(f()) == 0") + check("isinstance(2+3+4, float)") + + run("x = i") + check("x == 5") + + run("assert not f(), 'oops'") + run("a, b, c = 1, 2") + run("a, b, c = f()") + + check("max([f(),g()]) == 4") + check("'hello'[g()] == 'h'") + run("'guk%d' % h(f())") Added: pypy/branch/py11/py/impl/code/assertion.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/assertion.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,75 @@ +import sys +import py + +BuiltinAssertionError = py.builtin.builtins.AssertionError + + +def _format_explanation(explanation): + # uck! See CallFunc for where \n{ and \n} escape sequences are used + raw_lines = (explanation or '').split('\n') + # escape newlines not followed by { and } + lines = [raw_lines[0]] + for l in raw_lines[1:]: + if l.startswith('{') or l.startswith('}'): + lines.append(l) + else: + lines[-1] += '\\n' + l + + result = lines[:1] + stack = [0] + stackcnt = [0] + for line in lines[1:]: + if line.startswith('{'): + if stackcnt[-1]: + s = 'and ' + else: + s = 'where ' + stack.append(len(result)) + stackcnt[-1] += 1 + stackcnt.append(0) + result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) + else: + assert line.startswith('}') + stack.pop() + stackcnt.pop() + result[stack[-1]] += line[1:] + assert len(stack) == 1 + return '\n'.join(result) + + +if sys.version_info >= (2, 6) or (sys.platform.startswith("java")): + from py.impl.code._assertionnew import interpret +else: + from py.impl.code._assertionold import interpret + + +class AssertionError(BuiltinAssertionError): + + def __init__(self, *args): + BuiltinAssertionError.__init__(self, *args) + if args: + try: + self.msg = str(args[0]) + except (KeyboardInterrupt, SystemExit): + raise + except: + self.msg = "<[broken __repr__] %s at %0xd>" %( + args[0].__class__, id(args[0])) + else: + f = py.code.Frame(sys._getframe(1)) + try: + source = f.statement + source = str(source.deindent()).strip() + except py.error.ENOENT: + source = None + # this can also occur during reinterpretation, when the + # co_filename is set to "". + if source: + self.msg = interpret(source, f, should_fail=True) + if not self.args: + self.args = (self.msg,) + else: + self.msg = None + +if sys.version_info > (3, 0): + AssertionError.__module__ = "builtins" Added: pypy/branch/py11/py/impl/code/code.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/code.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,767 @@ +import py +import sys + +builtin_repr = repr + +repr = py.builtin._tryimport('repr', 'reprlib') + +class Code(object): + """ wrapper around Python code objects """ + def __init__(self, rawcode): + rawcode = py.code.getrawcode(rawcode) + self.raw = rawcode + try: + self.filename = rawcode.co_filename + self.firstlineno = rawcode.co_firstlineno - 1 + self.name = rawcode.co_name + except AttributeError: + raise TypeError("not a code object: %r" %(rawcode,)) + + def __eq__(self, other): + return self.raw == other.raw + + def __ne__(self, other): + return not self == other + + def new(self, rec=False, **kwargs): + """ return new code object with modified attributes. + if rec-cursive is true then dive into code + objects contained in co_consts. + """ + if sys.platform.startswith("java"): + # XXX jython does not support the below co_filename hack + return self.raw + names = [x for x in dir(self.raw) if x[:3] == 'co_'] + for name in kwargs: + if name not in names: + raise TypeError("unknown code attribute: %r" %(name, )) + if rec and hasattr(self.raw, 'co_consts'): # jython + newconstlist = [] + co = self.raw + cotype = type(co) + for c in co.co_consts: + if isinstance(c, cotype): + c = self.__class__(c).new(rec=True, **kwargs) + newconstlist.append(c) + return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) + for name in names: + if name not in kwargs: + kwargs[name] = getattr(self.raw, name) + arglist = [ + kwargs['co_argcount'], + kwargs['co_nlocals'], + kwargs.get('co_stacksize', 0), # jython + kwargs.get('co_flags', 0), # jython + kwargs.get('co_code', ''), # jython + kwargs.get('co_consts', ()), # jython + kwargs.get('co_names', []), # + kwargs['co_varnames'], + kwargs['co_filename'], + kwargs['co_name'], + kwargs['co_firstlineno'], + kwargs.get('co_lnotab', ''), #jython + kwargs.get('co_freevars', None), #jython + kwargs.get('co_cellvars', None), # jython + ] + if sys.version_info >= (3,0): + arglist.insert(1, kwargs['co_kwonlyargcount']) + return self.raw.__class__(*arglist) + else: + return py.std.new.code(*arglist) + + def path(self): + """ return a py.path.local object pointing to the source code """ + fn = self.raw.co_filename + try: + return fn.__path__ + except AttributeError: + p = py.path.local(self.raw.co_filename) + if not p.check(file=1): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p + + path = property(path, None, None, "path of this code object") + + def fullsource(self): + """ return a py.code.Source object for the full source file of the code + """ + from py.impl.code import source + full, _ = source.findsource(self.raw) + return full + fullsource = property(fullsource, None, None, + "full source containing this code object") + + def source(self): + """ return a py.code.Source object for the code object's source only + """ + # return source only for that part of code + return py.code.Source(self.raw) + + def getargs(self): + """ return a tuple with the argument names for the code object + """ + # handfull shortcut for getting args + raw = self.raw + return raw.co_varnames[:raw.co_argcount] + +class Frame(object): + """Wrapper around a Python frame holding f_locals and f_globals + in which expressions can be evaluated.""" + + def __init__(self, frame): + self.code = py.code.Code(frame.f_code) + self.lineno = frame.f_lineno - 1 + self.f_globals = frame.f_globals + self.f_locals = frame.f_locals + self.raw = frame + + def statement(self): + if self.code.fullsource is None: + return py.code.Source("") + return self.code.fullsource.getstatement(self.lineno) + statement = property(statement, None, None, + "statement this frame is at") + + def eval(self, code, **vars): + """ evaluate 'code' in the frame + + 'vars' are optional additional local variables + + returns the result of the evaluation + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + return eval(code, self.f_globals, f_locals) + + def exec_(self, code, **vars): + """ exec 'code' in the frame + + 'vars' are optiona; additional local variables + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + py.builtin.exec_(code, self.f_globals, f_locals ) + + def repr(self, object): + """ return a 'safe' (non-recursive, one-line) string repr for 'object' + """ + return safe_repr(object) + + def is_true(self, object): + return object + + def getargs(self): + """ return a list of tuples (name, value) for all arguments + """ + retval = [] + for arg in self.code.getargs(): + try: + retval.append((arg, self.f_locals[arg])) + except KeyError: + pass # this can occur when using Psyco + return retval + +class TracebackEntry(object): + """ a single entry in a traceback """ + + exprinfo = None + + def __init__(self, rawentry): + self._rawentry = rawentry + self.frame = py.code.Frame(rawentry.tb_frame) + # Ugh. 2.4 and 2.5 differs here when encountering + # multi-line statements. Not sure about the solution, but + # should be portable + self.lineno = rawentry.tb_lineno - 1 + self.relline = self.lineno - self.frame.code.firstlineno + + def __repr__(self): + return "" %(self.frame.code.path, self.lineno+1) + + def statement(self): + """ return a py.code.Source object for the current statement """ + source = self.frame.code.fullsource + return source.getstatement(self.lineno) + statement = property(statement, None, None, + "statement of this traceback entry.") + + def path(self): + return self.frame.code.path + path = property(path, None, None, "path to the full source code") + + def getlocals(self): + return self.frame.f_locals + locals = property(getlocals, None, None, "locals of underlaying frame") + + def reinterpret(self): + """Reinterpret the failing statement and returns a detailed information + about what operations are performed.""" + if self.exprinfo is None: + from py.impl.code import assertion + source = str(self.statement).strip() + x = assertion.interpret(source, self.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + self.exprinfo = x + return self.exprinfo + + def getfirstlinesource(self): + return self.frame.code.firstlineno + + def getsource(self): + """ return failing source code. """ + source = self.frame.code.fullsource + if source is None: + return None + start = self.getfirstlinesource() + end = self.lineno + try: + _, end = source.getstatementrange(end) + except IndexError: + end = self.lineno + 1 + # heuristic to stop displaying source on e.g. + # if something: # assume this causes a NameError + # # _this_ lines and the one + # below we don't want from entry.getsource() + for i in range(self.lineno, end): + if source[i].rstrip().endswith(':'): + end = i + 1 + break + return source[start:end] + source = property(getsource) + + def ishidden(self): + """ return True if the current frame has a var __tracebackhide__ + resolving to True + + mostly for internal use + """ + try: + return self.frame.eval("__tracebackhide__") + except (SystemExit, KeyboardInterrupt): + raise + except: + return False + + def __str__(self): + try: + fn = str(self.path) + except py.error.Error: + fn = '???' + name = self.frame.code.name + try: + line = str(self.statement).lstrip() + except KeyboardInterrupt: + raise + except: + line = "???" + return " File %r:%d in %s\n %s\n" %(fn, self.lineno+1, name, line) + + def name(self): + return self.frame.code.raw.co_name + name = property(name, None, None, "co_name of underlaying code") + +class Traceback(list): + """ Traceback objects encapsulate and offer higher level + access to Traceback entries. + """ + Entry = TracebackEntry + def __init__(self, tb): + """ initialize from given python traceback object. """ + if hasattr(tb, 'tb_next'): + def f(cur): + while cur is not None: + yield self.Entry(cur) + cur = cur.tb_next + list.__init__(self, f(tb)) + else: + list.__init__(self, tb) + + def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None): + """ return a Traceback instance wrapping part of this Traceback + + by provding any combination of path, lineno and firstlineno, the + first frame to start the to-be-returned traceback is determined + + this allows cutting the first part of a Traceback instance e.g. + for formatting reasons (removing some uninteresting bits that deal + with handling of the exception/traceback) + """ + for x in self: + code = x.frame.code + codepath = code.path + if ((path is None or codepath == path) and + (excludepath is None or (hasattr(codepath, 'relto') and + not codepath.relto(excludepath))) and + (lineno is None or x.lineno == lineno) and + (firstlineno is None or x.frame.code.firstlineno == firstlineno)): + return Traceback(x._rawentry) + return self + + def __getitem__(self, key): + val = super(Traceback, self).__getitem__(key) + if isinstance(key, type(slice(0))): + val = self.__class__(val) + return val + + def filter(self, fn=lambda x: not x.ishidden()): + """ return a Traceback instance with certain items removed + + fn is a function that gets a single argument, a TracebackItem + instance, and should return True when the item should be added + to the Traceback, False when not + + by default this removes all the TracebackItems which are hidden + (see ishidden() above) + """ + return Traceback(filter(fn, self)) + + def getcrashentry(self): + """ return last non-hidden traceback entry that lead + to the exception of a traceback. + """ + tb = self.filter() + if not tb: + tb = self + return tb[-1] + + def recursionindex(self): + """ return the index of the frame/TracebackItem where recursion + originates if appropriate, None if no recursion occurred + """ + cache = {} + for i, entry in enumerate(self): + key = entry.frame.code.path, entry.lineno + #print "checking for recursion at", key + l = cache.setdefault(key, []) + if l: + f = entry.frame + loc = f.f_locals + for otherloc in l: + if f.is_true(f.eval(co_equal, + __recursioncache_locals_1=loc, + __recursioncache_locals_2=otherloc)): + return i + l.append(entry.frame.f_locals) + return None + +co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2', + '?', 'eval') + +class ExceptionInfo(object): + """ wraps sys.exc_info() objects and offers + help for navigating the traceback. + """ + _striptext = '' + def __init__(self, tup=None, exprinfo=None): + # NB. all attributes are private! Subclasses or other + # ExceptionInfo-like classes may have different attributes. + if tup is None: + tup = sys.exc_info() + if exprinfo is None and isinstance(tup[1], py.code._AssertionError): + exprinfo = getattr(tup[1], 'msg', None) + if exprinfo is None: + exprinfo = str(tup[1]) + if exprinfo and exprinfo.startswith('assert '): + self._striptext = 'AssertionError: ' + self._excinfo = tup + self.type, self.value, tb = self._excinfo + self.typename = self.type.__name__ + self.traceback = py.code.Traceback(tb) + + def __repr__(self): + return "" % (self.typename, len(self.traceback)) + + def exconly(self, tryshort=False): + """ return the exception as a string + + when 'tryshort' resolves to True, and the exception is a + py.code._AssertionError, only the actual exception part of + the exception representation is returned (so 'AssertionError: ' is + removed from the beginning) + """ + lines = py.std.traceback.format_exception_only(self.type, self.value) + text = ''.join(lines) + text = text.rstrip() + if tryshort: + if text.startswith(self._striptext): + text = text[len(self._striptext):] + return text + + def errisinstance(self, exc): + """ return True if the exception is an instance of exc """ + return isinstance(self.value, exc) + + def _getreprcrash(self): + exconly = self.exconly(tryshort=True) + entry = self.traceback.getcrashentry() + path, lineno = entry.path, entry.lineno + reprcrash = ReprFileLocation(path, lineno+1, exconly) + return reprcrash + + def getrepr(self, showlocals=False, style="long", + abspath=False, tbfilter=True, funcargs=False): + """ return str()able representation of this exception info. + showlocals: show locals per traceback entry + style: long|short|no traceback style + tbfilter: hide entries (where __tracebackhide__ is true) + """ + fmt = FormattedExcinfo(showlocals=showlocals, style=style, + abspath=abspath, tbfilter=tbfilter, funcargs=funcargs) + return fmt.repr_excinfo(self) + + def __str__(self): + entry = self.traceback[-1] + loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly()) + return str(loc) + +class FormattedExcinfo(object): + """ presenting information about failing Functions and Generators. """ + # for traceback entries + flow_marker = ">" + fail_marker = "E" + + def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False): + self.showlocals = showlocals + self.style = style + self.tbfilter = tbfilter + self.funcargs = funcargs + self.abspath = abspath + + def _getindent(self, source): + # figure out indent for given source + try: + s = str(source.getstatement(len(source)-1)) + except KeyboardInterrupt: + raise + except: + try: + s = str(source[-1]) + except KeyboardInterrupt: + raise + except: + return 0 + return 4 + (len(s) - len(s.lstrip())) + + def _getentrysource(self, entry): + source = entry.getsource() + if source is not None: + source = source.deindent() + return source + + def _saferepr(self, obj): + return safe_repr(obj) + + def repr_args(self, entry): + if self.funcargs: + args = [] + for argname, argvalue in entry.frame.getargs(): + args.append((argname, self._saferepr(argvalue))) + return ReprFuncArgs(args) + + def get_source(self, source, line_index=-1, excinfo=None): + """ return formatted and marked up source lines. """ + lines = [] + if source is None: + source = py.code.Source("???") + line_index = 0 + if line_index < 0: + line_index += len(source) + for i in range(len(source)): + if i == line_index: + prefix = self.flow_marker + " " + else: + prefix = " " + line = prefix + source[i] + lines.append(line) + if excinfo is not None: + indent = self._getindent(source) + lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) + return lines + + def get_exconly(self, excinfo, indent=4, markall=False): + lines = [] + indent = " " * indent + # get the real exception information out + exlines = excinfo.exconly(tryshort=True).split('\n') + failindent = self.fail_marker + indent[1:] + for line in exlines: + lines.append(failindent + line) + if not markall: + failindent = indent + return lines + + def repr_locals(self, locals): + if self.showlocals: + lines = [] + keys = list(locals) + keys.sort() + for name in keys: + value = locals[name] + if name == '__builtins__': + lines.append("__builtins__ = ") + else: + # This formatting could all be handled by the + # _repr() function, which is only repr.Repr in + # disguise, so is very configurable. + str_repr = self._saferepr(value) + #if len(str_repr) < 70 or not isinstance(value, + # (list, tuple, dict)): + lines.append("%-10s = %s" %(name, str_repr)) + #else: + # self._line("%-10s =\\" % (name,)) + # # XXX + # py.std.pprint.pprint(value, stream=self.excinfowriter) + return ReprLocals(lines) + + def repr_traceback_entry(self, entry, excinfo=None): + # excinfo is not None if this is the last tb entry + source = self._getentrysource(entry) + if source is None: + source = py.code.Source("???") + line_index = 0 + else: + line_index = entry.lineno - entry.getfirstlinesource() + + lines = [] + if self.style == "long": + reprargs = self.repr_args(entry) + lines.extend(self.get_source(source, line_index, excinfo)) + message = excinfo and excinfo.typename or "" + path = self._makepath(entry.path) + filelocrepr = ReprFileLocation(path, entry.lineno+1, message) + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, filelocrepr) + else: + if self.style == "short": + line = source[line_index].lstrip() + lines.append(' File "%s", line %d, in %s' % ( + entry.path.basename, entry.lineno+1, entry.name)) + lines.append(" " + line) + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None) + + def _makepath(self, path): + if not self.abspath: + np = py.path.local().bestrelpath(path) + if len(np) < len(str(path)): + path = np + return path + + def repr_traceback(self, excinfo): + traceback = excinfo.traceback + if self.tbfilter: + traceback = traceback.filter() + recursionindex = None + if excinfo.errisinstance(RuntimeError): + recursionindex = traceback.recursionindex() + last = traceback[-1] + entries = [] + extraline = None + for index, entry in enumerate(traceback): + einfo = (last == entry) and excinfo or None + reprentry = self.repr_traceback_entry(entry, einfo) + entries.append(reprentry) + if index == recursionindex: + extraline = "!!! Recursion detected (same locals & position)" + break + return ReprTraceback(entries, extraline, style=self.style) + + def repr_excinfo(self, excinfo): + reprtraceback = self.repr_traceback(excinfo) + reprcrash = excinfo._getreprcrash() + return ReprExceptionInfo(reprtraceback, reprcrash) + +class TerminalRepr: + def __str__(self): + tw = py.io.TerminalWriter(stringio=True) + self.toterminal(tw) + return tw.stringio.getvalue().strip() + + def __repr__(self): + return "<%s instance at %0x>" %(self.__class__, id(self)) + +class ReprExceptionInfo(TerminalRepr): + def __init__(self, reprtraceback, reprcrash): + self.reprtraceback = reprtraceback + self.reprcrash = reprcrash + self.sections = [] + + def addsection(self, name, content, sep="-"): + self.sections.append((name, content, sep)) + + def toterminal(self, tw): + self.reprtraceback.toterminal(tw) + for name, content, sep in self.sections: + tw.sep(sep, name) + tw.line(content) + +class ReprTraceback(TerminalRepr): + entrysep = "_ " + + def __init__(self, reprentries, extraline, style): + self.reprentries = reprentries + self.extraline = extraline + self.style = style + + def toterminal(self, tw): + sepok = False + for entry in self.reprentries: + if self.style == "long": + if sepok: + tw.sep(self.entrysep) + tw.line("") + sepok = True + entry.toterminal(tw) + if self.extraline: + tw.line(self.extraline) + +class ReprEntry(TerminalRepr): + localssep = "_ " + + def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr): + self.lines = lines + self.reprfuncargs = reprfuncargs + self.reprlocals = reprlocals + self.reprfileloc = filelocrepr + + def toterminal(self, tw): + if self.reprfuncargs: + self.reprfuncargs.toterminal(tw) + for line in self.lines: + red = line.startswith("E ") + tw.line(line, bold=True, red=red) + if self.reprlocals: + #tw.sep(self.localssep, "Locals") + tw.line("") + self.reprlocals.toterminal(tw) + if self.reprfileloc: + tw.line("") + self.reprfileloc.toterminal(tw) + + def __str__(self): + return "%s\n%s\n%s" % ("\n".join(self.lines), + self.reprlocals, + self.reprfileloc) + +class ReprFileLocation(TerminalRepr): + def __init__(self, path, lineno, message): + self.path = str(path) + self.lineno = lineno + self.message = message + + def toterminal(self, tw): + # filename and lineno output for each entry, + # using an output format that most editors unterstand + msg = self.message + i = msg.find("\n") + if i != -1: + msg = msg[:i] + tw.line("%s:%s: %s" %(self.path, self.lineno, msg)) + +class ReprLocals(TerminalRepr): + def __init__(self, lines): + self.lines = lines + + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + +class ReprFuncArgs(TerminalRepr): + def __init__(self, args): + self.args = args + + def toterminal(self, tw): + if self.args: + linesofar = "" + for name, value in self.args: + ns = "%s = %s" %(name, value) + if len(ns) + len(linesofar) + 2 > tw.fullwidth: + if linesofar: + tw.line(linesofar) + linesofar = ns + else: + if linesofar: + linesofar += ", " + ns + else: + linesofar = ns + if linesofar: + tw.line(linesofar) + tw.line("") + + + +class SafeRepr(repr.Repr): + """ subclass of repr.Repr that limits the resulting size of repr() + and includes information on exceptions raised during the call. + """ + def __init__(self, *args, **kwargs): + repr.Repr.__init__(self, *args, **kwargs) + self.maxstring = 240 # 3 * 80 chars + self.maxother = 160 # 2 * 80 chars + + def repr(self, x): + return self._callhelper(repr.Repr.repr, self, x) + + def repr_instance(self, x, level): + return self._callhelper(builtin_repr, x) + + def _callhelper(self, call, x, *args): + try: + # Try the vanilla repr and make sure that the result is a string + s = call(x, *args) + except (KeyboardInterrupt, MemoryError, SystemExit): + raise + except: + cls, e, tb = sys.exc_info() + try: + exc_name = cls.__name__ + except: + exc_name = 'unknown' + try: + exc_info = str(e) + except: + exc_info = 'unknown' + return '<[%s("%s") raised in repr()] %s object at 0x%x>' % ( + exc_name, exc_info, x.__class__.__name__, id(x)) + else: + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + +safe_repr = SafeRepr().repr + +oldbuiltins = {} + +def patch_builtins(assertion=True, compile=True): + """ put compile and AssertionError builtins to Python's builtins. """ + if assertion: + from py.impl.code import assertion + l = oldbuiltins.setdefault('AssertionError', []) + l.append(py.builtin.builtins.AssertionError) + py.builtin.builtins.AssertionError = assertion.AssertionError + if compile: + l = oldbuiltins.setdefault('compile', []) + l.append(py.builtin.builtins.compile) + py.builtin.builtins.compile = py.code.compile + +def unpatch_builtins(assertion=True, compile=True): + """ remove compile and AssertionError builtins from Python builtins. """ + if assertion: + py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop() + if compile: + py.builtin.builtins.compile = oldbuiltins['compile'].pop() + +def getrawcode(obj): + """ return code object for given function. """ + obj = getattr(obj, 'im_func', obj) + obj = getattr(obj, 'func_code', obj) + obj = getattr(obj, 'f_code', obj) + obj = getattr(obj, '__code__', obj) + return obj + Added: pypy/branch/py11/py/impl/code/oldmagic.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/oldmagic.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,62 @@ +""" deprecated module for turning on/off some features. """ + +import py + +from py.builtin import builtins as cpy_builtin + +def invoke(assertion=False, compile=False): + """ (deprecated) invoke magic, currently you can specify: + + assertion patches the builtin AssertionError to try to give + more meaningful AssertionErrors, which by means + of deploying a mini-interpreter constructs + a useful error message. + """ + py.log._apiwarn("1.1", + "py.magic.invoke() is deprecated, use py.code.patch_builtins()", + stacklevel=2, + ) + py.code.patch_builtins(assertion=assertion, compile=compile) + +def revoke(assertion=False, compile=False): + """ (deprecated) revoke previously invoked magic (see invoke()).""" + py.log._apiwarn("1.1", + "py.magic.revoke() is deprecated, use py.code.unpatch_builtins()", + stacklevel=2, + ) + py.code.unpatch_builtins(assertion=assertion, compile=compile) + +patched = {} + +def patch(namespace, name, value): + """ (deprecated) rebind the 'name' on the 'namespace' to the 'value', + possibly and remember the original value. Multiple + invocations to the same namespace/name pair will + remember a list of old values. + """ + py.log._apiwarn("1.1", + "py.magic.patch() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + orig = getattr(namespace, name) + patched.setdefault(nref, []).append(orig) + setattr(namespace, name, value) + return orig + +def revert(namespace, name): + """ (deprecated) revert to the orginal value the last patch modified. + Raise ValueError if no such original value exists. + """ + py.log._apiwarn("1.1", + "py.magic.revert() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + if nref not in patched or not patched[nref]: + raise ValueError("No original value stored for %s.%s" % nref) + current = getattr(namespace, name) + orig = patched[nref].pop() + setattr(namespace, name, orig) + return current + Added: pypy/branch/py11/py/impl/code/oldmagic2.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/oldmagic2.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,6 @@ + +import py + +py.log._apiwarn("1.1", "py.magic.AssertionError is deprecated, use py.code._AssertionError", stacklevel=2) + +from py.code import _AssertionError as AssertionError Added: pypy/branch/py11/py/impl/code/source.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/code/source.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,347 @@ +from __future__ import generators +import sys +import inspect, tokenize +import py +cpy_compile = compile + +try: + import _ast + from _ast import PyCF_ONLY_AST as _AST_FLAG +except ImportError: + _AST_FLAG = 0 + _ast = None + + +class Source(object): + """ a immutable object holding a source code fragment, + possibly deindenting it. + """ + def __init__(self, *parts, **kwargs): + self.lines = lines = [] + de = kwargs.get('deindent', True) + rstrip = kwargs.get('rstrip', True) + for part in parts: + if not part: + partlines = [] + if isinstance(part, Source): + partlines = part.lines + elif isinstance(part, py.builtin._basestring): + partlines = part.split('\n') + if rstrip: + while partlines: + if partlines[-1].strip(): + break + partlines.pop() + else: + partlines = getsource(part, deindent=de).lines + if de: + partlines = deindent(partlines) + lines.extend(partlines) + + def __eq__(self, other): + try: + return self.lines == other.lines + except AttributeError: + if isinstance(other, str): + return str(self) == other + return False + + def __getitem__(self, key): + if isinstance(key, int): + return self.lines[key] + else: + if key.step not in (None, 1): + raise IndexError("cannot slice a Source with a step") + return self.__getslice__(key.start, key.stop) + + def __len__(self): + return len(self.lines) + + def __getslice__(self, start, end): + newsource = Source() + newsource.lines = self.lines[start:end] + return newsource + + def strip(self): + """ return new source object with trailing + and leading blank lines removed. + """ + start, end = 0, len(self) + while start < end and not self.lines[start].strip(): + start += 1 + while end > start and not self.lines[end-1].strip(): + end -= 1 + source = Source() + source.lines[:] = self.lines[start:end] + return source + + def putaround(self, before='', after='', indent=' ' * 4): + """ return a copy of the source object with + 'before' and 'after' wrapped around it. + """ + before = Source(before) + after = Source(after) + newsource = Source() + lines = [ (indent + line) for line in self.lines] + newsource.lines = before.lines + lines + after.lines + return newsource + + def indent(self, indent=' ' * 4): + """ return a copy of the source object with + all lines indented by the given indent-string. + """ + newsource = Source() + newsource.lines = [(indent+line) for line in self.lines] + return newsource + + def getstatement(self, lineno): + """ return Source statement which contains the + given linenumber (counted from 0). + """ + start, end = self.getstatementrange(lineno) + return self[start:end] + + def getstatementrange(self, lineno): + """ return (start, end) tuple which spans the minimal + statement region which containing the given lineno. + """ + # XXX there must be a better than these heuristic ways ... + # XXX there may even be better heuristics :-) + if not (0 <= lineno < len(self)): + raise IndexError("lineno out of range") + + # 1. find the start of the statement + from codeop import compile_command + for start in range(lineno, -1, -1): + trylines = self.lines[start:lineno+1] + # quick hack to indent the source and get it as a string in one go + trylines.insert(0, 'def xxx():') + trysource = '\n '.join(trylines) + # ^ space here + try: + compile_command(trysource) + except (SyntaxError, OverflowError, ValueError): + pass + else: + break # got a valid or incomplete statement + + # 2. find the end of the statement + for end in range(lineno+1, len(self)+1): + trysource = self[start:end] + if trysource.isparseable(): + break + + return start, end + + def getblockend(self, lineno): + # XXX + lines = [x + '\n' for x in self.lines[lineno:]] + blocklines = inspect.getblock(lines) + #print blocklines + return lineno + len(blocklines) - 1 + + def deindent(self, offset=None): + """ return a new source object deindented by offset. + If offset is None then guess an indentation offset from + the first non-blank line. Subsequent lines which have a + lower indentation offset will be copied verbatim as + they are assumed to be part of multilines. + """ + # XXX maybe use the tokenizer to properly handle multiline + # strings etc.pp? + newsource = Source() + newsource.lines[:] = deindent(self.lines, offset) + return newsource + + def isparseable(self, deindent=True): + """ return True if source is parseable, heuristically + deindenting it by default. + """ + try: + import parser + except ImportError: + syntax_checker = lambda x: compile(x, 'asd', 'exec') + else: + syntax_checker = parser.suite + + if deindent: + source = str(self.deindent()) + else: + source = str(self) + try: + #compile(source+'\n', "x", "exec") + syntax_checker(source+'\n') + except SyntaxError: + return False + else: + return True + + def __str__(self): + return "\n".join(self.lines) + + def compile(self, filename=None, mode='exec', + flag=generators.compiler_flag, + dont_inherit=0, _genframe=None): + """ return compiled code object. if filename is None + invent an artificial filename which displays + the source/line position of the caller frame. + """ + if not filename or py.path.local(filename).check(file=0): + if _genframe is None: + _genframe = sys._getframe(1) # the caller + fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno + if not filename: + filename = '' % (fn, lineno) + else: + filename = '' % (filename, fn, lineno) + source = "\n".join(self.lines) + '\n' + try: + co = cpy_compile(source, filename, mode, flag) + except SyntaxError: + ex = sys.exc_info()[1] + # re-represent syntax errors from parsing python strings + msglines = self.lines[:ex.lineno] + if ex.offset: + msglines.append(" "*ex.offset + '^') + msglines.append("syntax error probably generated here: %s" % filename) + newex = SyntaxError('\n'.join(msglines)) + newex.offset = ex.offset + newex.lineno = ex.lineno + newex.text = ex.text + raise newex + else: + if flag & _AST_FLAG: + return co + co_filename = MyStr(filename) + co_filename.__source__ = self + return py.code.Code(co).new(rec=1, co_filename=co_filename) + #return newcode_withfilename(co, co_filename) + +# +# public API shortcut functions +# + +def compile_(source, filename=None, mode='exec', flags= + generators.compiler_flag, dont_inherit=0): + """ compile the given source to a raw code object, + which points back to the source code through + "co_filename.__source__". All code objects + contained in the code object will recursively + also have this special subclass-of-string + filename. + """ + if _ast is not None and isinstance(source, _ast.AST): + # XXX should Source support having AST? + return cpy_compile(source, filename, mode, flags, dont_inherit) + _genframe = sys._getframe(1) # the caller + s = Source(source) + co = s.compile(filename, mode, flags, _genframe=_genframe) + return co + + +def getfslineno(obj): + try: + code = py.code.Code(obj) + except TypeError: + # fallback to + fn = (py.std.inspect.getsourcefile(obj) or + py.std.inspect.getfile(obj)) + fspath = fn and py.path.local(fn) or None + if fspath: + try: + _, lineno = findsource(obj) + except IOError: + lineno = None + else: + lineno = None + else: + fspath = code.path + lineno = code.firstlineno + return fspath, lineno + +# +# helper functions +# +class MyStr(str): + """ custom string which allows to add attributes. """ + +def findsource(obj): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno + else: + lineno = obj.co_firstlineno - 1 + return fullsource, lineno + + +def getsource(obj, **kwargs): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) + else: + lineno = obj.co_firstlineno - 1 + end = fullsource.getblockend(lineno) + return Source(fullsource[lineno:end+1], deident=True) + + +def deindent(lines, offset=None): + if offset is None: + for line in lines: + line = line.expandtabs() + s = line.lstrip() + if s: + offset = len(line)-len(s) + break + else: + offset = 0 + if offset == 0: + return list(lines) + newlines = [] + def readline_generator(lines): + for line in lines: + yield line + '\n' + while True: + yield '' + + r = readline_generator(lines) + try: + readline = r.next + except AttributeError: + readline = r.__next__ + + try: + for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(readline): + if sline > len(lines): + break # End of input reached + if sline > len(newlines): + line = lines[sline - 1].expandtabs() + if line.lstrip() and line[:offset].isspace(): + line = line[offset:] # Deindent + newlines.append(line) + + for i in range(sline, eline): + # Don't deindent continuing lines of + # multiline tokens (i.e. multiline strings) + newlines.append(lines[i]) + except (IndentationError, tokenize.TokenError): + pass + # Add any lines we didn't see. E.g. if an exception was raised. + newlines.extend(lines[len(newlines):]) + return newlines Added: pypy/branch/py11/py/impl/compat/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/compat/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,2 @@ +""" compatibility modules (taken from 2.4.4) """ + Added: pypy/branch/py11/py/impl/compat/dep_doctest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/compat/dep_doctest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,4 @@ +import py + +py.log._apiwarn("1.1", "py.compat.doctest deprecated, use standard library version.", stacklevel="initpkg") +doctest = py.std.doctest Added: pypy/branch/py11/py/impl/compat/dep_optparse.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/compat/dep_optparse.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,4 @@ +import py +py.log._apiwarn("1.1", "py.compat.optparse deprecated, use standard library version.", stacklevel="initpkg") + +optparse = py.std.optparse Added: pypy/branch/py11/py/impl/compat/dep_subprocess.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/compat/dep_subprocess.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,4 @@ + +import py +py.log._apiwarn("1.1", "py.compat.subprocess deprecated, use standard library version.", stacklevel="initpkg") +subprocess = py.std.subprocess Added: pypy/branch/py11/py/impl/compat/dep_textwrap.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/compat/dep_textwrap.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,4 @@ +import py + +py.log._apiwarn("1.1", "py.compat.textwrap deprecated, use standard library version.", stacklevel="initpkg") +textwrap = py.std.textwrap Added: pypy/branch/py11/py/impl/error.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/error.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,83 @@ +""" +create errno-specific classes for IO or os calls. + +""" +import sys, os, errno + +class Error(EnvironmentError): + def __repr__(self): + return "%s.%s %r: %s " %(self.__class__.__module__, + self.__class__.__name__, + self.__class__.__doc__, + " ".join(map(str, self.args)), + #repr(self.args) + ) + + def __str__(self): + s = "[%s]: %s" %(self.__class__.__doc__, + " ".join(map(str, self.args)), + ) + return s + +_winerrnomap = { + 2: errno.ENOENT, + 3: errno.ENOENT, + 17: errno.EEXIST, + 22: errno.ENOTDIR, + 267: errno.ENOTDIR, + 5: errno.EACCES, # anything better? +} + +class ErrorMaker(object): + """ lazily provides Exception classes for each possible POSIX errno + (as defined per the 'errno' module). All such instances + subclass EnvironmentError. + """ + Error = Error + _errno2class = {} + + def __getattr__(self, name): + eno = getattr(errno, name) + cls = self._geterrnoclass(eno) + setattr(self, name, cls) + return cls + + def _geterrnoclass(self, eno): + try: + return self._errno2class[eno] + except KeyError: + clsname = errno.errorcode.get(eno, "UnknownErrno%d" %(eno,)) + errorcls = type(Error)(clsname, (Error,), + {'__module__':'py.error', + '__doc__': os.strerror(eno)}) + self._errno2class[eno] = errorcls + return errorcls + + def checked_call(self, func, *args): + """ call a function and raise an errno-exception if applicable. """ + __tracebackhide__ = True + try: + return func(*args) + except self.Error: + raise + except EnvironmentError: + cls, value, tb = sys.exc_info() + if not hasattr(value, 'errno'): + raise + __tracebackhide__ = False + errno = value.errno + try: + if not isinstance(value, WindowsError): + raise NameError + except NameError: + # we are not on Windows, or we got a proper OSError + cls = self._geterrnoclass(errno) + else: + try: + cls = self._geterrnoclass(_winerrnomap[errno]) + except KeyError: + raise value + raise cls("%s%r" % (func.__name__, args)) + __tracebackhide__ = True + +error = ErrorMaker() Added: pypy/branch/py11/py/impl/io/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/io/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +""" input/output helping """ Added: pypy/branch/py11/py/impl/io/capture.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/io/capture.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,344 @@ +import os +import sys +import py +import tempfile + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +if sys.version_info < (3,0): + class TextIO(StringIO): + def write(self, data): + if not isinstance(data, unicode): + data = unicode(data, getattr(self, '_encoding', 'UTF-8')) + StringIO.write(self, data) +else: + TextIO = StringIO + +try: + from io import BytesIO +except ImportError: + class BytesIO(StringIO): + def write(self, data): + if isinstance(data, unicode): + raise TypeError("not a byte value: %r" %(data,)) + StringIO.write(self, data) + +class FDCapture: + """ Capture IO to/from a given os-level filedescriptor. """ + + def __init__(self, targetfd, tmpfile=None): + """ save targetfd descriptor, and open a new + temporary file there. If no tmpfile is + specified a tempfile.Tempfile() will be opened + in text mode. + """ + self.targetfd = targetfd + if tmpfile is None: + f = tempfile.TemporaryFile('wb+') + tmpfile = dupfile(f, encoding="UTF-8") + f.close() + self.tmpfile = tmpfile + self._savefd = os.dup(targetfd) + os.dup2(self.tmpfile.fileno(), targetfd) + self._patched = [] + + def setasfile(self, name, module=sys): + """ patch . to self.tmpfile + """ + key = (module, name) + self._patched.append((key, getattr(module, name))) + setattr(module, name, self.tmpfile) + + def unsetfiles(self): + """ unpatch all patched items + """ + while self._patched: + (module, name), value = self._patched.pop() + setattr(module, name, value) + + def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ + os.dup2(self._savefd, self.targetfd) + self.unsetfiles() + os.close(self._savefd) + self.tmpfile.seek(0) + return self.tmpfile + + def writeorg(self, data): + """ write a string to the original file descriptor + """ + tempfp = tempfile.TemporaryFile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(data) + finally: + tempfp.close() + + +def dupfile(f, mode=None, buffering=0, raising=False, encoding=None): + """ return a new open file object that's a duplicate of f + + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' + defines whether an exception is raised when an incompatible + file object is passed in (if raising is False, the file + object itself will be returned) + """ + try: + fd = f.fileno() + except AttributeError: + if raising: + raise + return f + newfd = os.dup(fd) + mode = mode and mode or f.mode + if sys.version_info >= (3,0): + if encoding is not None: + mode = mode.replace("b", "") + buffering = True + return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + else: + f = os.fdopen(newfd, mode, buffering) + if encoding is not None: + return EncodedFile(f, encoding) + return f + +class EncodedFile(object): + def __init__(self, _stream, encoding): + self._stream = _stream + self.encoding = encoding + + def write(self, obj): + if isinstance(obj, unicode): + obj = obj.encode(self.encoding) + elif isinstance(obj, str): + pass + else: + obj = str(obj) + self._stream.write(obj) + + def writelines(self, linelist): + data = ''.join(linelist) + self.write(data) + + def __getattr__(self, name): + return getattr(self._stream, name) + +class Capture(object): + def call(cls, func, *args, **kwargs): + """ return a (res, out, err) tuple where + out and err represent the output/error output + during function execution. + call the given function with args/kwargs + and capture output/error during its execution. + """ + so = cls() + try: + res = func(*args, **kwargs) + finally: + out, err = so.reset() + return res, out, err + call = classmethod(call) + + def reset(self): + """ reset sys.stdout/stderr and return captured output as strings. """ + if hasattr(self, '_suspended'): + outfile = self._kwargs['out'] + errfile = self._kwargs['err'] + del self._kwargs + else: + outfile, errfile = self.done() + out, err = "", "" + if outfile: + out = outfile.read() + outfile.close() + if errfile and errfile != outfile: + err = errfile.read() + errfile.close() + return out, err + + def suspend(self): + """ return current snapshot captures, memorize tempfiles. """ + assert not hasattr(self, '_suspended') + self._suspended = True + outerr = self.readouterr() + outfile, errfile = self.done() + self._kwargs['out'] = outfile + self._kwargs['err'] = errfile + return outerr + + def resume(self): + """ resume capturing with original temp files. """ + assert self._suspended + self._initialize(**self._kwargs) + del self._suspended + + +class StdCaptureFD(Capture): + """ This class allows to capture writes to FD1 and FD2 + and may connect a NULL file to FD0 (and prevent + reads from sys.stdin) + """ + def __init__(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + if in_: + self._oldin = (sys.stdin, os.dup(0)) + sys.stdin = DontReadFromInput() + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if out: + tmpfile = None + if hasattr(out, 'write'): + tmpfile = out + self.out = py.io.FDCapture(1, tmpfile=tmpfile) + if patchsys: + self.out.setasfile('stdout') + if err: + if mixed and out: + tmpfile = self.out.tmpfile + elif hasattr(err, 'write'): + tmpfile = err + else: + tmpfile = None + self.err = py.io.FDCapture(2, tmpfile=tmpfile) + if patchsys: + self.err.setasfile('stderr') + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + if hasattr(self, 'out'): + outfile = self.out.done() + else: + outfile = None + if hasattr(self, 'err'): + errfile = self.err.done() + else: + errfile = None + if hasattr(self, '_oldin'): + oldsys, oldfd = self._oldin + os.dup2(oldfd, 0) + os.close(oldfd) + sys.stdin = oldsys + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + l = [] + for name in ('out', 'err'): + res = "" + if hasattr(self, name): + f = getattr(self, name).tmpfile + f.seek(0) + res = f.read() + f.truncate(0) + f.seek(0) + l.append(res) + return l + +class StdCapture(Capture): + """ This class allows to capture writes to sys.stdout|stderr "in-memory" + and will raise errors on tries to read from sys.stdin. It only + modifies sys.stdout|stderr|stdin attributes and does not + touch underlying File Descriptors (use StdCaptureFD for that). + """ + def __init__(self, out=True, err=True, in_=True, mixed=False): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out, err, in_, mixed): + self._out = out + self._err = err + self._in = in_ + if out: + self._oldout = sys.stdout + if not hasattr(out, 'write'): + out = TextIO() + sys.stdout = self.out = out + if err: + self._olderr = sys.stderr + if out and mixed: + err = self.out + elif not hasattr(err, 'write'): + err = TextIO() + sys.stderr = self.err = err + if in_: + self._oldin = sys.stdin + sys.stdin = self.newin = DontReadFromInput() + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + o,e = sys.stdout, sys.stderr + if self._out: + try: + sys.stdout = self._oldout + except AttributeError: + raise IOError("stdout capturing already reset") + del self._oldout + outfile = self.out + outfile.seek(0) + else: + outfile = None + if self._err: + try: + sys.stderr = self._olderr + except AttributeError: + raise IOError("stderr capturing already reset") + del self._olderr + errfile = self.err + errfile.seek(0) + else: + errfile = None + if self._in: + sys.stdin = self._oldin + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + out = err = "" + if self._out: + out = sys.stdout.getvalue() + sys.stdout.truncate(0) + if self._err: + err = sys.stderr.getvalue() + sys.stderr.truncate(0) + return out, err + +class DontReadFromInput: + """Temporary stub class. Ideally when stdin is accessed, the + capturing should be turned off, with possibly all data captured + so far sent to the screen. This should be configurable, though, + because in automated test runs it is better to crash than + hang indefinitely. + """ + def read(self, *args): + raise IOError("reading from stdin while output is captured") + readline = read + readlines = read + __iter__ = read + + def fileno(self): + raise ValueError("redirected Stdin is pseudofile, has no fileno()") + def isatty(self): + return False + +try: + devnullpath = os.devnull +except AttributeError: + if os.name == 'nt': + devnullpath = 'NUL' + else: + devnullpath = '/dev/null' + + Added: pypy/branch/py11/py/impl/io/terminalwriter.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/io/terminalwriter.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,264 @@ +""" + +Helper functions for writing to terminals and files. + +""" + + +import sys, os +import py + +def _getdimensions(): + import termios,fcntl,struct + call = fcntl.ioctl(0,termios.TIOCGWINSZ,"\000"*8) + height,width = struct.unpack( "hhhh", call ) [:2] + return height, width + +if sys.platform == 'win32': + # ctypes access to the Windows console + + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + FOREGROUND_BLUE = 0x0001 # text color contains blue. + FOREGROUND_GREEN = 0x0002 # text color contains green. + FOREGROUND_RED = 0x0004 # text color contains red. + FOREGROUND_WHITE = 0x0007 + FOREGROUND_INTENSITY = 0x0008 # text color is intensified. + BACKGROUND_BLUE = 0x0010 # background color contains blue. + BACKGROUND_GREEN = 0x0020 # background color contains green. + BACKGROUND_RED = 0x0040 # background color contains red. + BACKGROUND_WHITE = 0x0070 + BACKGROUND_INTENSITY = 0x0080 # background color is intensified. + + def GetStdHandle(kind): + import ctypes + return ctypes.windll.kernel32.GetStdHandle(kind) + + def SetConsoleTextAttribute(handle, attr): + import ctypes + ctypes.windll.kernel32.SetConsoleTextAttribute( + handle, attr) + + def _getdimensions(): + import ctypes + from ctypes import wintypes + + SHORT = ctypes.c_short + class COORD(ctypes.Structure): + _fields_ = [('X', SHORT), + ('Y', SHORT)] + class SMALL_RECT(ctypes.Structure): + _fields_ = [('Left', SHORT), + ('Top', SHORT), + ('Right', SHORT), + ('Bottom', SHORT)] + class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): + _fields_ = [('dwSize', COORD), + ('dwCursorPosition', COORD), + ('wAttributes', wintypes.WORD), + ('srWindow', SMALL_RECT), + ('dwMaximumWindowSize', COORD)] + STD_OUTPUT_HANDLE = -11 + handle = GetStdHandle(STD_OUTPUT_HANDLE) + info = CONSOLE_SCREEN_BUFFER_INFO() + ctypes.windll.kernel32.GetConsoleScreenBufferInfo( + handle, ctypes.byref(info)) + # Substract one from the width, otherwise the cursor wraps + # and the ending \n causes an empty line to display. + return info.dwSize.Y, info.dwSize.X - 1 + +def get_terminal_width(): + try: + height, width = _getdimensions() + except (SystemExit, KeyboardInterrupt): + raise + except: + # FALLBACK + width = int(os.environ.get('COLUMNS', 80))-1 + # XXX the windows getdimensions may be bogus, let's sanify a bit + width = max(width, 40) # we alaways need 40 chars + return width + +terminal_width = get_terminal_width() + +# XXX unify with _escaped func below +def ansi_print(text, esc, file=None, newline=True, flush=False): + if file is None: + file = sys.stderr + text = text.rstrip() + if esc and not isinstance(esc, tuple): + esc = (esc,) + if esc and sys.platform != "win32" and file.isatty(): + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text + + '\x1b[0m') # ANSI color code "reset" + if newline: + text += '\n' + + if esc and sys.platform == "win32" and file.isatty(): + if 1 in esc: + bold = True + esc = tuple([x for x in esc if x != 1]) + else: + bold = False + esctable = {() : FOREGROUND_WHITE, # normal + (31,): FOREGROUND_RED, # red + (32,): FOREGROUND_GREEN, # green + (33,): FOREGROUND_GREEN|FOREGROUND_RED, # yellow + (34,): FOREGROUND_BLUE, # blue + (35,): FOREGROUND_BLUE|FOREGROUND_RED, # purple + (36,): FOREGROUND_BLUE|FOREGROUND_GREEN, # cyan + (37,): FOREGROUND_WHITE, # white + (39,): FOREGROUND_WHITE, # reset + } + attr = esctable.get(esc, FOREGROUND_WHITE) + if bold: + attr |= FOREGROUND_INTENSITY + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + if file is sys.stderr: + handle = GetStdHandle(STD_ERROR_HANDLE) + else: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + SetConsoleTextAttribute(handle, attr) + file.write(text) + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + else: + file.write(text) + + if flush: + file.flush() + +def should_do_markup(file): + return hasattr(file, 'isatty') and file.isatty() \ + and os.environ.get('TERM') != 'dumb' + +class TerminalWriter(object): + _esctable = dict(black=30, red=31, green=32, yellow=33, + blue=34, purple=35, cyan=36, white=37, + Black=40, Red=41, Green=42, Yellow=43, + Blue=44, Purple=45, Cyan=46, White=47, + bold=1, light=2, blink=5, invert=7) + + # XXX deprecate stringio argument + def __init__(self, file=None, stringio=False, encoding=None): + self.encoding = encoding + + if file is None: + if stringio: + self.stringio = file = py.io.TextIO() + else: + file = py.std.sys.stdout + elif hasattr(file, '__call__'): + file = WriteFile(file, encoding=encoding) + self._file = file + self.fullwidth = get_terminal_width() + self.hasmarkup = should_do_markup(file) + + def _escaped(self, text, esc): + if esc and self.hasmarkup: + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text +'\x1b[0m') + return text + + def markup(self, text, **kw): + esc = [] + for name in kw: + if name not in self._esctable: + raise ValueError("unknown markup: %r" %(name,)) + if kw[name]: + esc.append(self._esctable[name]) + return self._escaped(text, tuple(esc)) + + def sep(self, sepchar, title=None, fullwidth=None, **kw): + if fullwidth is None: + fullwidth = self.fullwidth + # the goal is to have the line be as long as possible + # under the condition that len(line) <= fullwidth + if title is not None: + # we want 2 + 2*len(fill) + len(title) <= fullwidth + # i.e. 2 + 2*len(sepchar)*N + len(title) <= fullwidth + # 2*len(sepchar)*N <= fullwidth - len(title) - 2 + # N <= (fullwidth - len(title) - 2) // (2*len(sepchar)) + N = (fullwidth - len(title) - 2) // (2*len(sepchar)) + fill = sepchar * N + line = "%s %s %s" % (fill, title, fill) + else: + # we want len(sepchar)*N <= fullwidth + # i.e. N <= fullwidth // len(sepchar) + line = sepchar * (fullwidth // len(sepchar)) + # in some situations there is room for an extra sepchar at the right, + # in particular if we consider that with a sepchar like "_ " the + # trailing space is not important at the end of the line + if len(line) + len(sepchar.rstrip()) <= fullwidth: + line += sepchar.rstrip() + + self.line(line, **kw) + + def write(self, s, **kw): + if s: + s = self._getbytestring(s) + if self.hasmarkup and kw: + s = self.markup(s, **kw) + self._file.write(s) + self._file.flush() + + def _getbytestring(self, s): + # XXX review this and the whole logic + if self.encoding and sys.version_info < (3,0) and isinstance(s, unicode): + return s.encode(self.encoding) + elif not isinstance(s, str): + return str(s) + return s + + def line(self, s='', **kw): + self.write(s, **kw) + self.write('\n') + +class Win32ConsoleWriter(TerminalWriter): + def write(self, s, **kw): + if s: + s = self._getbytestring(s) + if self.hasmarkup: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + + if self.hasmarkup and kw: + attr = 0 + if kw.pop('bold', False): + attr |= FOREGROUND_INTENSITY + + if kw.pop('red', False): + attr |= FOREGROUND_RED + elif kw.pop('blue', False): + attr |= FOREGROUND_BLUE + elif kw.pop('green', False): + attr |= FOREGROUND_GREEN + else: + attr |= FOREGROUND_WHITE + + SetConsoleTextAttribute(handle, attr) + self._file.write(s) + self._file.flush() + if self.hasmarkup: + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + + def line(self, s="", **kw): + self.write(s+"\n", **kw) + +if sys.platform == 'win32': + TerminalWriter = Win32ConsoleWriter + +class WriteFile(object): + def __init__(self, writemethod, encoding=None): + self.encoding = encoding + self._writemethod = writemethod + + def write(self, data): + if self.encoding: + data = data.encode(self.encoding) + self._writemethod(data) + + def flush(self): + return + + Added: pypy/branch/py11/py/impl/log/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/log/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,2 @@ +""" logging API ('producers' and 'consumers' connected via keywords) """ + Added: pypy/branch/py11/py/impl/log/log.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/log/log.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,184 @@ +""" +basic logging functionality based on a producer/consumer scheme. + +XXX implement this API: (maybe put it into slogger.py?) + + log = Logger( + info=py.log.STDOUT, + debug=py.log.STDOUT, + command=None) + log.info("hello", "world") + log.command("hello", "world") + + log = Logger(info=Logger(something=...), + debug=py.log.STDOUT, + command=None) +""" +import py, sys + +class Message(object): + def __init__(self, keywords, args): + self.keywords = keywords + self.args = args + + def content(self): + return " ".join(map(str, self.args)) + + def prefix(self): + return "[%s] " % (":".join(self.keywords)) + + def __str__(self): + return self.prefix() + self.content() + + +class Producer(object): + """ (deprecated) Log producer API which sends messages to be logged + to a 'consumer' object, which then prints them to stdout, + stderr, files, etc. Used extensively by PyPy-1.1. + """ + + Message = Message # to allow later customization + keywords2consumer = {} + + def __init__(self, keywords, keywordmapper=None, **kw): + if hasattr(keywords, 'split'): + keywords = tuple(keywords.split()) + self._keywords = keywords + if keywordmapper is None: + keywordmapper = default_keywordmapper + self._keywordmapper = keywordmapper + + def __repr__(self): + return "" % ":".join(self._keywords) + + def __getattr__(self, name): + if '_' in name: + raise AttributeError(name) + producer = self.__class__(self._keywords + (name,)) + setattr(self, name, producer) + return producer + + def __call__(self, *args): + """ write a message to the appropriate consumer(s) """ + func = self._keywordmapper.getconsumer(self._keywords) + if func is not None: + func(self.Message(self._keywords, args)) + +class KeywordMapper: + def __init__(self): + self.keywords2consumer = {} + + def getstate(self): + return self.keywords2consumer.copy() + def setstate(self, state): + self.keywords2consumer.clear() + self.keywords2consumer.update(state) + + def getconsumer(self, keywords): + """ return a consumer matching the given keywords. + + tries to find the most suitable consumer by walking, starting from + the back, the list of keywords, the first consumer matching a + keyword is returned (falling back to py.log.default) + """ + for i in range(len(keywords), 0, -1): + try: + return self.keywords2consumer[keywords[:i]] + except KeyError: + continue + return self.keywords2consumer.get('default', default_consumer) + + def setconsumer(self, keywords, consumer): + """ set a consumer for a set of keywords. """ + # normalize to tuples + if isinstance(keywords, str): + keywords = tuple(filter(None, keywords.split())) + elif hasattr(keywords, '_keywords'): + keywords = keywords._keywords + elif not isinstance(keywords, tuple): + raise TypeError("key %r is not a string or tuple" % (keywords,)) + if consumer is not None and not py.builtin.callable(consumer): + if not hasattr(consumer, 'write'): + raise TypeError( + "%r should be None, callable or file-like" % (consumer,)) + consumer = File(consumer) + self.keywords2consumer[keywords] = consumer + +def default_consumer(msg): + """ the default consumer, prints the message to stdout (using 'print') """ + sys.stderr.write(str(msg)+"\n") + +default_keywordmapper = KeywordMapper() + +def setconsumer(keywords, consumer): + default_keywordmapper.setconsumer(keywords, consumer) + +def setstate(state): + default_keywordmapper.setstate(state) +def getstate(): + return default_keywordmapper.getstate() + +# +# Consumers +# + +class File(object): + """ log consumer wrapping a file(-like) object """ + def __init__(self, f): + assert hasattr(f, 'write') + #assert isinstance(f, file) or not hasattr(f, 'open') + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + self._file.write(str(msg) + "\n") + +class Path(object): + """ log consumer that opens and writes to a Path """ + def __init__(self, filename, append=False, + delayed_create=False, buffering=False): + self._append = append + self._filename = str(filename) + self._buffering = buffering + if not delayed_create: + self._openfile() + + def _openfile(self): + mode = self._append and 'a' or 'w' + f = open(self._filename, mode) + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + if not hasattr(self, "_file"): + self._openfile() + self._file.write(str(msg) + "\n") + if not self._buffering: + self._file.flush() + +def STDOUT(msg): + """ consumer that writes to sys.stdout """ + sys.stdout.write(str(msg)+"\n") + +def STDERR(msg): + """ consumer that writes to sys.stderr """ + sys.stderr.write(str(msg)+"\n") + +class Syslog: + """ consumer that writes to the syslog daemon """ + + def __init__(self, priority = None): + if priority is None: + priority = self.LOG_INFO + self.priority = priority + + def __call__(self, msg): + """ write a message to the log """ + py.std.syslog.syslog(self.priority, str(msg)) + +for _prio in "EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG".split(): + _prio = "LOG_" + _prio + try: + setattr(Syslog, _prio, getattr(py.std.syslog, _prio)) + except AttributeError: + pass Added: pypy/branch/py11/py/impl/log/warning.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/log/warning.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,72 @@ +import py, sys + +class Warning(DeprecationWarning): + def __init__(self, msg, path, lineno): + self.msg = msg + self.path = path + self.lineno = lineno + def __repr__(self): + return "%s:%d: %s" %(self.path, self.lineno+1, self.msg) + def __str__(self): + return self.msg + +def _apiwarn(startversion, msg, stacklevel=2, function=None): + # below is mostly COPIED from python2.4/warnings.py's def warn() + # Get context information + if stacklevel == "initpkg": + frame = sys._getframe(stacklevel == "initpkg" and 1 or stacklevel) + level = 2 + while frame: + co = frame.f_code + if co.co_name == "__getattr__" and co.co_filename.find("initpkg") !=-1: + stacklevel = level + break + level += 1 + frame = frame.f_back + else: + stacklevel = 1 + msg = "%s (since version %s)" %(msg, startversion) + warn(msg, stacklevel=stacklevel+1, function=function) + +def warn(msg, stacklevel=1, function=None): + if function is not None: + filename = py.std.inspect.getfile(function) + lineno = py.code.getrawcode(function).co_firstlineno + else: + try: + caller = sys._getframe(stacklevel) + except ValueError: + globals = sys.__dict__ + lineno = 1 + else: + globals = caller.f_globals + lineno = caller.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + filename = globals.get('__file__') + if filename: + fnl = filename.lower() + if fnl.endswith(".pyc") or fnl.endswith(".pyo"): + filename = filename[:-1] + elif fnl.endswith("$py.class"): + filename = filename.replace('$py.class', '.py') + else: + if module == "__main__": + try: + filename = sys.argv[0] + except AttributeError: + # embedded interpreters don't have sys.argv, see bug #839151 + filename = '__main__' + if not filename: + filename = module + path = py.path.local(filename) + warning = Warning(msg, path, lineno) + py.std.warnings.warn_explicit(warning, category=Warning, + filename=str(warning.path), + lineno=warning.lineno, + registry=py.std.warnings.__dict__.setdefault( + "__warningsregistry__", {}) + ) + Added: pypy/branch/py11/py/impl/path/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +""" unified file system api """ Added: pypy/branch/py11/py/impl/path/cacheutil.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/cacheutil.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,111 @@ +""" +This module contains multithread-safe cache implementations. + +All Caches have + + getorbuild(key, builder) + delentry(key) + +methods and allow configuration when instantiating the cache class. +""" +from time import time as gettime + +class BasicCache(object): + def __init__(self, maxentries=128): + self.maxentries = maxentries + self.prunenum = int(maxentries - maxentries/8) + self._dict = {} + + def _getentry(self, key): + return self._dict[key] + + def _putentry(self, key, entry): + self._prunelowestweight() + self._dict[key] = entry + + def delentry(self, key, raising=False): + try: + del self._dict[key] + except KeyError: + if raising: + raise + + def getorbuild(self, key, builder): + try: + entry = self._getentry(key) + except KeyError: + entry = self._build(key, builder) + self._putentry(key, entry) + return entry.value + + def _prunelowestweight(self): + """ prune out entries with lowest weight. """ + numentries = len(self._dict) + if numentries >= self.maxentries: + # evict according to entry's weight + items = [(entry.weight, key) + for key, entry in self._dict.items()] + items.sort() + index = numentries - self.prunenum + if index > 0: + for weight, key in items[:index]: + # in MT situations the element might be gone + self.delentry(key, raising=False) + +class BuildcostAccessCache(BasicCache): + """ A BuildTime/Access-counting cache implementation. + the weight of a value is computed as the product of + + num-accesses-of-a-value * time-to-build-the-value + + The values with the least such weights are evicted + if the cache maxentries threshold is superceded. + For implementation flexibility more than one object + might be evicted at a time. + """ + # time function to use for measuring build-times + + def _build(self, key, builder): + start = gettime() + val = builder() + end = gettime() + return WeightedCountingEntry(val, end-start) + + +class WeightedCountingEntry(object): + def __init__(self, value, oneweight): + self._value = value + self.weight = self._oneweight = oneweight + + def value(self): + self.weight += self._oneweight + return self._value + value = property(value) + +class AgingCache(BasicCache): + """ This cache prunes out cache entries that are too old. + """ + def __init__(self, maxentries=128, maxseconds=10.0): + super(AgingCache, self).__init__(maxentries) + self.maxseconds = maxseconds + + def _getentry(self, key): + entry = self._dict[key] + if entry.isexpired(): + self.delentry(key) + raise KeyError(key) + return entry + + def _build(self, key, builder): + val = builder() + entry = AgingEntry(val, gettime() + self.maxseconds) + return entry + +class AgingEntry(object): + def __init__(self, value, expirationtime): + self.value = value + self.weight = expirationtime + + def isexpired(self): + t = gettime() + return t >= self.weight Added: pypy/branch/py11/py/impl/path/common.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/common.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,333 @@ +""" +""" +import os, sys +import py + +class Checkers: + _depend_on_existence = 'exists', 'link', 'dir', 'file' + + def __init__(self, path): + self.path = path + + def dir(self): + raise NotImplementedError + + def file(self): + raise NotImplementedError + + def dotfile(self): + return self.path.basename.startswith('.') + + def ext(self, arg): + if not arg.startswith('.'): + arg = '.' + arg + return self.path.ext == arg + + def exists(self): + raise NotImplementedError + + def basename(self, arg): + return self.path.basename == arg + + def basestarts(self, arg): + return self.path.basename.startswith(arg) + + def relto(self, arg): + return self.path.relto(arg) + + def fnmatch(self, arg): + return FNMatcher(arg)(self.path) + + def endswith(self, arg): + return str(self.path).endswith(arg) + + def _evaluate(self, kw): + for name, value in kw.items(): + invert = False + meth = None + try: + meth = getattr(self, name) + except AttributeError: + if name[:3] == 'not': + invert = True + try: + meth = getattr(self, name[3:]) + except AttributeError: + pass + if meth is None: + raise TypeError( + "no %r checker available for %r" % (name, self.path)) + try: + if py.code.getrawcode(meth).co_argcount > 1: + if (not meth(value)) ^ invert: + return False + else: + if bool(value) ^ bool(meth()) ^ invert: + return False + except (py.error.ENOENT, py.error.ENOTDIR): + for name in self._depend_on_existence: + if name in kw: + if kw.get(name): + return False + name = 'not' + name + if name in kw: + if not kw.get(name): + return False + return True + +class NeverRaised(Exception): + pass + +class PathBase(object): + """ shared implementation for filesystem path objects.""" + Checkers = Checkers + + def __div__(self, other): + return self.join(str(other)) + __truediv__ = __div__ # py3k + + def basename(self): + """ basename part of path. """ + return self._getbyspec('basename')[0] + basename = property(basename, None, None, basename.__doc__) + + def purebasename(self): + """ pure base name of the path.""" + return self._getbyspec('purebasename')[0] + purebasename = property(purebasename, None, None, purebasename.__doc__) + + def ext(self): + """ extension of the path (including the '.').""" + return self._getbyspec('ext')[0] + ext = property(ext, None, None, ext.__doc__) + + def dirpath(self, *args, **kwargs): + """ return the directory Path of the current Path joined + with any given path arguments. + """ + return self.new(basename='').join(*args, **kwargs) + + def read(self, mode='r'): + """ read and return a bytestring from reading the path. """ + if sys.version_info < (2,3): + for x in 'u', 'U': + if x in mode: + mode = mode.replace(x, '') + f = self.open(mode) + try: + return f.read() + finally: + f.close() + + def readlines(self, cr=1): + """ read and return a list of lines from the path. if cr is False, the +newline will be removed from the end of each line. """ + if not cr: + content = self.read('rU') + return content.split('\n') + else: + f = self.open('rU') + try: + return f.readlines() + finally: + f.close() + + def load(self): + """ (deprecated) return object unpickled from self.read() """ + f = self.open('rb') + try: + return py.error.checked_call(py.std.pickle.load, f) + finally: + f.close() + + def move(self, target): + """ move this path to target. """ + if target.relto(self): + raise py.error.EINVAL(target, + "cannot move path into a subdirectory of itself") + try: + self.rename(target) + except py.error.EXDEV: # invalid cross-device link + self.copy(target) + self.remove() + + def __repr__(self): + """ return a string representation of this path. """ + return repr(str(self)) + + def check(self, **kw): + """ check a path for existence, or query its properties + + without arguments, this returns True if the path exists (on the + filesystem), False if not + + with (keyword only) arguments, the object compares the value + of the argument with the value of a property with the same name + (if it has one, else it raises a TypeError) + + when for example the keyword argument 'ext' is '.py', this will + return True if self.ext == '.py', False otherwise + """ + if not kw: + kw = {'exists' : 1} + return self.Checkers(self)._evaluate(kw) + + def relto(self, relpath): + """ return a string which is the relative part of the path + to the given 'relpath'. + """ + if not isinstance(relpath, (str, PathBase)): + raise TypeError("%r: not a string or path object" %(relpath,)) + strrelpath = str(relpath) + if strrelpath and strrelpath[-1] != self.sep: + strrelpath += self.sep + #assert strrelpath[-1] == self.sep + #assert strrelpath[-2] != self.sep + strself = str(self) + if sys.platform == "win32": + if os.path.normcase(strself).startswith( + os.path.normcase(strrelpath)): + return strself[len(strrelpath):] + elif strself.startswith(strrelpath): + return strself[len(strrelpath):] + return "" + + def bestrelpath(self, dest): + """ return a string which is a relative path from self + to dest such that self.join(bestrelpath) == dest and + if not such path can be determined return dest. + """ + try: + base = self.common(dest) + if not base: # can be the case on windows + return str(dest) + self2base = self.relto(base) + reldest = dest.relto(base) + if self2base: + n = self2base.count(self.sep) + 1 + else: + n = 0 + l = ['..'] * n + if reldest: + l.append(reldest) + target = dest.sep.join(l) + return target + except AttributeError: + return str(dest) + + + def parts(self, reverse=False): + """ return a root-first list of all ancestor directories + plus the path itself. + """ + current = self + l = [self] + while 1: + last = current + current = current.dirpath() + if last == current: + break + l.insert(0, current) + if reverse: + l.reverse() + return l + + def common(self, other): + """ return the common part shared with the other path + or None if there is no common part. + """ + last = None + for x, y in zip(self.parts(), other.parts()): + if x != y: + return last + last = x + return last + + def __add__(self, other): + """ return new path object with 'other' added to the basename""" + return self.new(basename=self.basename+str(other)) + + def __cmp__(self, other): + """ return sort value (-1, 0, +1). """ + try: + return cmp(self.strpath, other.strpath) + except AttributeError: + return cmp(str(self), str(other)) # self.path, other.path) + + def __lt__(self, other): + try: + return self.strpath < other.strpath + except AttributeError: + return str(self) < str(other) + + def visit(self, fil=None, rec=None, ignore=NeverRaised): + """ yields all paths below the current one + + fil is a filter (glob pattern or callable), if not matching the + path will not be yielded, defaulting to None (everything is + returned) + + rec is a filter (glob pattern or callable) that controls whether + a node is descended, defaulting to None + + ignore is an Exception class that is ignoredwhen calling dirlist() + on any of the paths (by default, all exceptions are reported) + """ + if isinstance(fil, str): + fil = FNMatcher(fil) + if rec: + if isinstance(rec, str): + rec = fnmatch(fil) + elif not hasattr(rec, '__call__'): + rec = None + try: + entries = self.listdir() + except ignore: + return + dirs = [p for p in entries + if p.check(dir=1) and (rec is None or rec(p))] + for subdir in dirs: + for p in subdir.visit(fil=fil, rec=rec, ignore=ignore): + yield p + for p in entries: + if fil is None or fil(p): + yield p + + def _sortlist(self, res, sort): + if sort: + if hasattr(sort, '__call__'): + res.sort(sort) + else: + res.sort() + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return self.strpath == str(other) + +class FNMatcher: + def __init__(self, pattern): + self.pattern = pattern + def __call__(self, path): + """return true if the basename/fullname matches the glob-'pattern'. + + * matches everything + ? matches any single character + [seq] matches any character in seq + [!seq] matches any char not in seq + + if the pattern contains a path-separator then the full path + is used for pattern matching and a '*' is prepended to the + pattern. + + if the pattern doesn't contain a path-separator the pattern + is only matched against the basename. + """ + pattern = self.pattern + if pattern.find(path.sep) == -1: + name = path.basename + else: + name = str(path) # path.strpath # XXX svn? + pattern = '*' + path.sep + pattern + from fnmatch import fnmatch + return fnmatch(name, pattern) + Added: pypy/branch/py11/py/impl/path/gateway/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/gateway/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/py11/py/impl/path/gateway/channeltest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/gateway/channeltest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,65 @@ +import threading + + +class PathServer: + + def __init__(self, channel): + self.channel = channel + self.C2P = {} + self.next_id = 0 + threading.Thread(target=self.serve).start() + + def p2c(self, path): + id = self.next_id + self.next_id += 1 + self.C2P[id] = path + return id + + def command_LIST(self, id, *args): + path = self.C2P[id] + answer = [(self.p2c(p), p.basename) for p in path.listdir(*args)] + self.channel.send(answer) + + def command_DEL(self, id): + del self.C2P[id] + + def command_GET(self, id, spec): + path = self.C2P[id] + self.channel.send(path._getbyspec(spec)) + + def command_READ(self, id): + path = self.C2P[id] + self.channel.send(path.read()) + + def command_JOIN(self, id, resultid, *args): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.join(*args) + + def command_DIRPATH(self, id, resultid): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.dirpath() + + def serve(self): + try: + while 1: + msg = self.channel.receive() + meth = getattr(self, 'command_' + msg[0]) + meth(*msg[1:]) + except EOFError: + pass + +if __name__ == '__main__': + import py + gw = execnet.PopenGateway() + channel = gw._channelfactory.new() + srv = PathServer(channel) + c = gw.remote_exec(""" + import remotepath + p = remotepath.RemotePath(channel.receive(), channel.receive()) + channel.send(len(p.listdir())) + """) + c.send(channel) + c.send(srv.p2c(py.path.local('/tmp'))) + print(c.receive()) Added: pypy/branch/py11/py/impl/path/gateway/channeltest2.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/gateway/channeltest2.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,21 @@ +import py +from remotepath import RemotePath + + +SRC = open('channeltest.py', 'r').read() + +SRC += ''' +import py +srv = PathServer(channel.receive()) +channel.send(srv.p2c(py.path.local("/tmp"))) +''' + + +#gw = execnet.SshGateway('codespeak.net') +gw = execnet.PopenGateway() +gw.remote_init_threads(5) +c = gw.remote_exec(SRC, stdout=py.std.sys.stdout, stderr=py.std.sys.stderr) +subchannel = gw._channelfactory.new() +c.send(subchannel) + +p = RemotePath(subchannel, c.receive()) Added: pypy/branch/py11/py/impl/path/gateway/remotepath.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/gateway/remotepath.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,47 @@ +import py, itertools +from py.impl.path import common + +COUNTER = itertools.count() + +class RemotePath(common.PathBase): + sep = '/' + + def __init__(self, channel, id, basename=None): + self._channel = channel + self._id = id + self._basename = basename + self._specs = {} + + def __del__(self): + self._channel.send(('DEL', self._id)) + + def __repr__(self): + return 'RemotePath(%s)' % self.basename + + def listdir(self, *args): + self._channel.send(('LIST', self._id) + args) + return [RemotePath(self._channel, id, basename) + for (id, basename) in self._channel.receive()] + + def dirpath(self): + id = ~COUNTER.next() + self._channel.send(('DIRPATH', self._id, id)) + return RemotePath(self._channel, id) + + def join(self, *args): + id = ~COUNTER.next() + self._channel.send(('JOIN', self._id, id) + args) + return RemotePath(self._channel, id) + + def _getbyspec(self, spec): + parts = spec.split(',') + ask = [x for x in parts if x not in self._specs] + if ask: + self._channel.send(('GET', self._id, ",".join(ask))) + for part, value in zip(ask, self._channel.receive()): + self._specs[part] = value + return [self._specs[x] for x in parts] + + def read(self): + self._channel.send(('READ', self._id)) + return self._channel.receive() Added: pypy/branch/py11/py/impl/path/local.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/local.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,799 @@ +""" +local path implementation. +""" +import sys, os, stat, re, atexit +import py +from py.impl.path import common + +iswin32 = sys.platform == "win32" + +class Stat(object): + def __getattr__(self, name): + return getattr(self._osstatresult, "st_" + name) + + def __init__(self, path, osstatresult): + self.path = path + self._osstatresult = osstatresult + + def owner(self): + if iswin32: + raise NotImplementedError("XXX win32") + import pwd + entry = py.error.checked_call(pwd.getpwuid, self.uid) + return entry[0] + owner = property(owner, None, None, "owner of path") + + def group(self): + """ return group name of file. """ + if iswin32: + raise NotImplementedError("XXX win32") + import grp + entry = py.error.checked_call(grp.getgrgid, self.gid) + return entry[0] + group = property(group) + +class PosixPath(common.PathBase): + def chown(self, user, group, rec=0): + """ change ownership to the given user and group. + user and group may be specified by a number or + by a name. if rec is True change ownership + recursively. + """ + uid = getuserid(user) + gid = getgroupid(group) + if rec: + for x in self.visit(rec=lambda x: x.check(link=0)): + if x.check(link=0): + py.error.checked_call(os.chown, str(x), uid, gid) + py.error.checked_call(os.chown, str(self), uid, gid) + + def readlink(self): + """ return value of a symbolic link. """ + return py.error.checked_call(os.readlink, self.strpath) + + def mklinkto(self, oldname): + """ posix style hard link to another name. """ + py.error.checked_call(os.link, str(oldname), str(self)) + + def mksymlinkto(self, value, absolute=1): + """ create a symbolic link with the given value (pointing to another name). """ + if absolute: + py.error.checked_call(os.symlink, str(value), self.strpath) + else: + base = self.common(value) + # with posix local paths '/' is always a common base + relsource = self.__class__(value).relto(base) + reldest = self.relto(base) + n = reldest.count(self.sep) + target = self.sep.join(('..', )*n + (relsource, )) + py.error.checked_call(os.symlink, target, self.strpath) + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return py.error.checked_call(os.path.samefile, str(self), str(other)) + +def getuserid(user): + import pwd + if not isinstance(user, int): + user = pwd.getpwnam(user)[2] + return user + +def getgroupid(group): + import grp + if not isinstance(group, int): + group = grp.getgrnam(group)[2] + return group + +FSBase = not iswin32 and PosixPath or common.PathBase + +class LocalPath(FSBase): + """ object oriented interface to os.path and other local filesystem + related information. + """ + sep = os.sep + class Checkers(common.Checkers): + def _stat(self): + try: + return self._statcache + except AttributeError: + try: + self._statcache = self.path.stat() + except py.error.ELOOP: + self._statcache = self.path.lstat() + return self._statcache + + def dir(self): + return stat.S_ISDIR(self._stat().mode) + + def file(self): + return stat.S_ISREG(self._stat().mode) + + def exists(self): + return self._stat() + + def link(self): + st = self.path.lstat() + return stat.S_ISLNK(st.mode) + + def __new__(cls, path=None): + """ Initialize and return a local Path instance. + + Path can be relative to the current directory. + If it is None then the current working directory is taken. + Note that Path instances always carry an absolute path. + Note also that passing in a local path object will simply return + the exact same path object. Use new() to get a new copy. + """ + if isinstance(path, common.PathBase): + if path.__class__ == cls: + return path + path = path.strpath + # initialize the path + self = object.__new__(cls) + if not path: + self.strpath = os.getcwd() + elif isinstance(path, py.builtin._basestring): + self.strpath = os.path.abspath(os.path.normpath(str(path))) + else: + raise ValueError("can only pass None, Path instances " + "or non-empty strings to LocalPath") + assert isinstance(self.strpath, str) + return self + + def __hash__(self): + return hash(self.strpath) + + def __eq__(self, other): + s1 = str(self) + s2 = str(other) + if iswin32: + s1 = s1.lower() + s2 = s2.lower() + return s1 == s2 + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return str(self) < str(other) + + def remove(self, rec=1): + """ remove a file or directory (or a directory tree if rec=1). """ + if self.check(dir=1, link=0): + if rec: + # force remove of readonly files on windows + if iswin32: + self.chmod(448, rec=1) # octcal 0700 + py.error.checked_call(py.std.shutil.rmtree, self.strpath) + else: + py.error.checked_call(os.rmdir, self.strpath) + else: + if iswin32: + self.chmod(448) # octcal 0700 + py.error.checked_call(os.remove, self.strpath) + + def computehash(self, hashtype="md5", chunksize=524288): + """ return hexdigest of hashvalue for this file. """ + try: + try: + import hashlib as mod + except ImportError: + if hashtype == "sha1": + hashtype = "sha" + mod = __import__(hashtype) + hash = getattr(mod, hashtype)() + except (AttributeError, ImportError): + raise ValueError("Don't know how to compute %r hash" %(hashtype,)) + f = self.open('rb') + try: + while 1: + buf = f.read(chunksize) + if not buf: + return hash.hexdigest() + hash.update(buf) + finally: + f.close() + + def new(self, **kw): + """ create a modified version of this path. + the following keyword arguments modify various path parts: + + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + drive, dirname, basename, purebasename,ext = self._getbyspec( + "drive,dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + try: + ext = kw['ext'] + except KeyError: + pass + else: + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('drive', drive) + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + obj.strpath = os.path.normpath( + "%(drive)s%(dirname)s%(sep)s%(basename)s" % kw) + return obj + + def _getbyspec(self, spec): + """ return a sequence of specified path parts. 'spec' is + a comma separated string containing path part names. + according to the following convention: + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + + args = filter(None, spec.split(',') ) + append = res.append + for name in args: + if name == 'drive': + append(parts[0]) + elif name == 'dirname': + append(self.sep.join(['']+parts[1:-1])) + else: + basename = parts[-1] + if name == 'basename': + append(basename) + else: + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + append(purebasename) + elif name == 'ext': + append(ext) + else: + raise ValueError("invalid part specification %r" % name) + return res + + def join(self, *args, **kwargs): + """ return a new path by appending all 'args' as path + components. if abs=1 is used restart from root if any + of the args is an absolute path. + """ + if not args: + return self + strpath = self.strpath + sep = self.sep + strargs = [str(x) for x in args] + if kwargs.get('abs', 0): + for i in range(len(strargs)-1, -1, -1): + if os.path.isabs(strargs[i]): + strpath = strargs[i] + strargs = strargs[i+1:] + break + for arg in strargs: + arg = arg.strip(sep) + if iswin32: + # allow unix style paths even on windows. + arg = arg.strip('/') + arg = arg.replace('/', sep) + if arg: + if not strpath.endswith(sep): + strpath += sep + strpath += arg + obj = self.new() + obj.strpath = os.path.normpath(strpath) + return obj + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return py.error.checked_call(open, self.strpath, mode) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + res = [] + for name in py.error.checked_call(os.listdir, self.strpath): + childurl = self.join(name) + if fil is None or fil(childurl): + res.append(childurl) + self._sortlist(res, sort) + return res + + def size(self): + """ return size of the underlying file object """ + return self.stat().size + + def mtime(self): + """ return last modification time of the path. """ + return self.stat().mtime + + def copy(self, target, archive=False): + """ copy path to target.""" + assert not archive, "XXX archive-mode not supported" + if self.check(file=1): + if target.check(dir=1): + target = target.join(self.basename) + assert self!=target + copychunked(self, target) + else: + def rec(p): + return p.check(link=0) + for x in self.visit(rec=rec): + relpath = x.relto(self) + newx = target.join(relpath) + newx.dirpath().ensure(dir=1) + if x.check(link=1): + newx.mksymlinkto(x.readlink()) + elif x.check(file=1): + copychunked(x, newx) + elif x.check(dir=1): + newx.ensure(dir=1) + + def rename(self, target): + """ rename this path to target. """ + return py.error.checked_call(os.rename, str(self), str(target)) + + def dump(self, obj, bin=1): + """ pickle object into path location""" + f = self.open('wb') + try: + py.error.checked_call(py.std.pickle.dump, obj, f, bin) + finally: + f.close() + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + p = self.join(*args) + py.error.checked_call(os.mkdir, str(p)) + return p + + def write(self, data, mode='w'): + """ write data into path. """ + if 'b' in mode: + if not py.builtin._isbytes(data): + raise ValueError("can only process bytes") + else: + if not py.builtin._istext(data): + if not py.builtin._isbytes(data): + data = str(data) + else: + data = py.builtin._totext(data, sys.getdefaultencoding()) + f = self.open(mode) + try: + f.write(data) + finally: + f.close() + + def _ensuredirs(self): + parent = self.dirpath() + if parent == self: + return self + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + try: + self.mkdir() + except py.error.EEXIST: + # race condition: file/dir created by another thread/process. + # complain if it is not a dir + if self.check(dir=0): + raise + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if kwargs.get('dir', 0): + return p._ensuredirs() + else: + p.dirpath()._ensuredirs() + if not p.check(file=1): + p.open('w').close() + return p + + def stat(self): + """ Return an os.stat() tuple. """ + return Stat(self, py.error.checked_call(os.stat, self.strpath)) + + def lstat(self): + """ Return an os.lstat() tuple. """ + return Stat(self, py.error.checked_call(os.lstat, self.strpath)) + + def setmtime(self, mtime=None): + """ set modification time for the given path. if 'mtime' is None + (the default) then the file's mtime is set to current time. + + Note that the resolution for 'mtime' is platform dependent. + """ + if mtime is None: + return py.error.checked_call(os.utime, self.strpath, mtime) + try: + return py.error.checked_call(os.utime, self.strpath, (-1, mtime)) + except py.error.EINVAL: + return py.error.checked_call(os.utime, self.strpath, (self.atime(), mtime)) + + def chdir(self): + """ change directory to self and return old current directory """ + old = self.__class__() + py.error.checked_call(os.chdir, self.strpath) + return old + + def realpath(self): + """ return a new path which contains no symbolic links.""" + return self.__class__(os.path.realpath(self.strpath)) + + def atime(self): + """ return last access time of the path. """ + return self.stat().atime + + def __repr__(self): + return 'local(%r)' % self.strpath + + def __str__(self): + """ return string representation of the Path. """ + return self.strpath + + def pypkgpath(self, pkgname=None): + """ return the path's package path by looking for the given + pkgname. If pkgname is None then look for the last + directory upwards which still contains an __init__.py. + Return None if a pkgpath can not be determined. + """ + pkgpath = None + for parent in self.parts(reverse=True): + if pkgname is None: + if parent.check(file=1): + continue + if parent.join('__init__.py').check(): + pkgpath = parent + continue + return pkgpath + else: + if parent.basename == pkgname: + return parent + return pkgpath + + def _prependsyspath(self, path): + s = str(path) + if s != sys.path[0]: + #print "prepending to sys.path", s + sys.path.insert(0, s) + + def chmod(self, mode, rec=0): + """ change permissions to the given mode. If mode is an + integer it directly encodes the os-specific modes. + if rec is True perform recursively. + """ + if not isinstance(mode, int): + raise TypeError("mode %r must be an integer" % (mode,)) + if rec: + for x in self.visit(rec=rec): + py.error.checked_call(os.chmod, str(x), mode) + py.error.checked_call(os.chmod, str(self), mode) + + def pyimport(self, modname=None, ensuresyspath=True): + """ return path as an imported python module. + if modname is None, look for the containing package + and construct an according module name. + The module will be put/looked up in sys.modules. + """ + if not self.check(): + raise py.error.ENOENT(self) + #print "trying to import", self + pkgpath = None + if modname is None: + pkgpath = self.pypkgpath() + if pkgpath is not None: + if ensuresyspath: + self._prependsyspath(pkgpath.dirpath()) + pkg = __import__(pkgpath.basename, None, None, []) + names = self.new(ext='').relto(pkgpath.dirpath()) + names = names.split(self.sep) + modname = ".".join(names) + else: + # no package scope, still make it possible + if ensuresyspath: + self._prependsyspath(self.dirpath()) + modname = self.purebasename + mod = __import__(modname, None, None, ['__doc__']) + modfile = mod.__file__ + if modfile[-4:] in ('.pyc', '.pyo'): + modfile = modfile[:-1] + elif modfile.endswith('$py.class'): + modfile = modfile[:-9] + '.py' + if not self.samefile(modfile): + raise EnvironmentError("mismatch:\n" + "imported module %r\n" + "does not stem from %r\n" + "maybe __init__.py files are missing?" % (mod, str(self))) + return mod + else: + try: + return sys.modules[modname] + except KeyError: + # we have a custom modname, do a pseudo-import + mod = py.std.types.ModuleType(modname) + mod.__file__ = str(self) + sys.modules[modname] = mod + try: + py.builtin.execfile(str(self), mod.__dict__) + except: + del sys.modules[modname] + raise + return mod + + def sysexec(self, *argv, **popen_opts): + """ return stdout text from executing a system child process, + where the 'self' path points to executable. + The process is directly invoked and not through a system shell. + """ + from subprocess import Popen, PIPE + argv = map(str, argv) + popen_opts['stdout'] = popen_opts['stderr'] = PIPE + proc = Popen([str(self)] + list(argv), **popen_opts) + stdout, stderr = proc.communicate() + ret = proc.wait() + if py.builtin._isbytes(stdout): + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + if ret != 0: + if py.builtin._isbytes(stderr): + stderr = py.builtin._totext(stderr, sys.getdefaultencoding()) + raise py.process.cmdexec.Error(ret, ret, str(self), + stdout, stderr,) + return stdout + + def sysfind(cls, name, checker=None): + """ return a path object found by looking at the systems + underlying PATH specification. If the checker is not None + it will be invoked to filter matching paths. If a binary + cannot be found, None is returned + Note: This is probably not working on plain win32 systems + but may work on cygwin. + """ + if os.path.isabs(name): + p = py.path.local(name) + if p.check(file=1): + return p + else: + if iswin32: + paths = py.std.os.environ['Path'].split(';') + if '' not in paths and '.' not in paths: + paths.append('.') + try: + systemroot = os.environ['SYSTEMROOT'] + except KeyError: + pass + else: + paths = [re.sub('%SystemRoot%', systemroot, path) + for path in paths] + tryadd = '', '.exe', '.com', '.bat' # XXX add more? + else: + paths = py.std.os.environ['PATH'].split(':') + tryadd = ('',) + + for x in paths: + for addext in tryadd: + p = py.path.local(x).join(name, abs=True) + addext + try: + if p.check(file=1): + if checker: + if not checker(p): + continue + return p + except py.error.EACCES: + pass + return None + sysfind = classmethod(sysfind) + + def _gethomedir(cls): + try: + x = os.environ['HOME'] + except KeyError: + x = os.environ['HOMEPATH'] + return cls(x) + _gethomedir = classmethod(_gethomedir) + + #""" + #special class constructors for local filesystem paths + #""" + def get_temproot(cls): + """ return the system's temporary directory + (where tempfiles are usually created in) + """ + return py.path.local(py.std.tempfile.gettempdir()) + get_temproot = classmethod(get_temproot) + + def mkdtemp(cls): + """ return a Path object pointing to a fresh new temporary directory + (which we created ourself). + """ + import tempfile + tries = 10 + for i in range(tries): + dname = tempfile.mktemp() + dpath = cls(tempfile.mktemp()) + try: + dpath.mkdir() + except (py.error.EEXIST, py.error.EPERM, py.error.EACCES): + continue + return dpath + raise py.error.ENOENT(dpath, "could not create tempdir, %d tries" % tries) + mkdtemp = classmethod(mkdtemp) + + def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800): # two days + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = cls.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError): # AttributeError on win32 + pass + + return udir + make_numbered_dir = classmethod(make_numbered_dir) + +def copychunked(src, dest): + chunksize = 524288 # half a meg of bytes + fsrc = src.open('rb') + try: + fdest = dest.open('wb') + try: + while 1: + buf = fsrc.read(chunksize) + if not buf: + break + fdest.write(buf) + finally: + fdest.close() + finally: + fsrc.close() + +def autopath(globs=None): + """ (deprecated) return the (local) path of the "current" file pointed to by globals or - if it is none - alternatively the callers frame globals. + + the path will always point to a .py file or to None. + the path will have the following payload: + pkgdir is the last parent directory path containing __init__.py + """ + py.log._apiwarn("1.1", "py.magic.autopath deprecated, " + "use py.path.local(__file__) and maybe pypkgpath/pyimport().") + if globs is None: + globs = sys._getframe(1).f_globals + try: + __file__ = globs['__file__'] + except KeyError: + if not sys.argv[0]: + raise ValueError("cannot compute autopath in interactive mode") + __file__ = os.path.abspath(sys.argv[0]) + + ret = py.path.local(__file__) + if ret.ext in ('.pyc', '.pyo'): + ret = ret.new(ext='.py') + current = pkgdir = ret.dirpath() + while 1: + if current.join('__init__.py').check(): + pkgdir = current + current = current.dirpath() + if pkgdir != current: + continue + elif str(current) not in sys.path: + sys.path.insert(0, str(current)) + break + ret.pkgdir = pkgdir + return ret + Added: pypy/branch/py11/py/impl/path/local.py.orig ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/local.py.orig Tue Nov 10 19:17:34 2009 @@ -0,0 +1,802 @@ +""" +local path implementation. +""" +import sys, os, stat, re, atexit +import py +from py.impl.path import common + +iswin32 = sys.platform == "win32" + +class Stat(object): + def __getattr__(self, name): + return getattr(self._osstatresult, "st_" + name) + + def __init__(self, path, osstatresult): + self.path = path + self._osstatresult = osstatresult + + def owner(self): + if iswin32: + raise NotImplementedError("XXX win32") + import pwd + entry = py.error.checked_call(pwd.getpwuid, self.uid) + return entry[0] + owner = property(owner, None, None, "owner of path") + + def group(self): + """ return group name of file. """ + if iswin32: + raise NotImplementedError("XXX win32") + import grp + entry = py.error.checked_call(grp.getgrgid, self.gid) + return entry[0] + group = property(group) + +class PosixPath(common.PathBase): + def chown(self, user, group, rec=0): + """ change ownership to the given user and group. + user and group may be specified by a number or + by a name. if rec is True change ownership + recursively. + """ + uid = getuserid(user) + gid = getgroupid(group) + if rec: + for x in self.visit(rec=lambda x: x.check(link=0)): + if x.check(link=0): + py.error.checked_call(os.chown, str(x), uid, gid) + py.error.checked_call(os.chown, str(self), uid, gid) + + def readlink(self): + """ return value of a symbolic link. """ + return py.error.checked_call(os.readlink, self.strpath) + + def mklinkto(self, oldname): + """ posix style hard link to another name. """ + py.error.checked_call(os.link, str(oldname), str(self)) + + def mksymlinkto(self, value, absolute=1): + """ create a symbolic link with the given value (pointing to another name). """ + if absolute: + py.error.checked_call(os.symlink, str(value), self.strpath) + else: + base = self.common(value) + # with posix local paths '/' is always a common base + relsource = self.__class__(value).relto(base) + reldest = self.relto(base) + n = reldest.count(self.sep) + target = self.sep.join(('..', )*n + (relsource, )) + py.error.checked_call(os.symlink, target, self.strpath) + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return py.error.checked_call(os.path.samefile, str(self), str(other)) + +def getuserid(user): + import pwd + if not isinstance(user, int): + user = pwd.getpwnam(user)[2] + return user + +def getgroupid(group): + import grp + if not isinstance(group, int): + group = grp.getgrnam(group)[2] + return group + +FSBase = not iswin32 and PosixPath or common.PathBase + +class LocalPath(FSBase): + """ object oriented interface to os.path and other local filesystem + related information. + """ + sep = os.sep + class Checkers(common.Checkers): + def _stat(self): + try: + return self._statcache + except AttributeError: + try: + self._statcache = self.path.stat() + except py.error.ELOOP: + self._statcache = self.path.lstat() + return self._statcache + + def dir(self): + return stat.S_ISDIR(self._stat().mode) + + def file(self): + return stat.S_ISREG(self._stat().mode) + + def exists(self): + return self._stat() + + def link(self): + st = self.path.lstat() + return stat.S_ISLNK(st.mode) + + def __new__(cls, path=None): + """ Initialize and return a local Path instance. + + Path can be relative to the current directory. + If it is None then the current working directory is taken. + Note that Path instances always carry an absolute path. + Note also that passing in a local path object will simply return + the exact same path object. Use new() to get a new copy. + """ + if isinstance(path, common.PathBase): + if path.__class__ == cls: + return path + path = path.strpath + # initialize the path + self = object.__new__(cls) + if not path: + self.strpath = os.getcwd() + elif isinstance(path, py.builtin._basestring): + self.strpath = os.path.abspath(os.path.normpath(str(path))) + else: + raise ValueError("can only pass None, Path instances " + "or non-empty strings to LocalPath") + assert isinstance(self.strpath, str) + return self + + def __hash__(self): + return hash(self.strpath) + + def __eq__(self, other): + s1 = str(self) + s2 = str(other) + if iswin32: + s1 = s1.lower() + s2 = s2.lower() + return s1 == s2 + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return str(self) < str(other) + + def remove(self, rec=1): + """ remove a file or directory (or a directory tree if rec=1). """ + if self.check(dir=1, link=0): + if rec: + # force remove of readonly files on windows + if iswin32: + self.chmod(448, rec=1) # octcal 0700 + py.error.checked_call(py.std.shutil.rmtree, self.strpath) + else: + py.error.checked_call(os.rmdir, self.strpath) + else: + if iswin32: + self.chmod(448) # octcal 0700 + py.error.checked_call(os.remove, self.strpath) + + def computehash(self, hashtype="md5", chunksize=524288): + """ return hexdigest of hashvalue for this file. """ + try: + try: + import hashlib as mod + except ImportError: + if hashtype == "sha1": + hashtype = "sha" + mod = __import__(hashtype) + hash = getattr(mod, hashtype)() + except (AttributeError, ImportError): + raise ValueError("Don't know how to compute %r hash" %(hashtype,)) + f = self.open('rb') + try: + while 1: + buf = f.read(chunksize) + if not buf: + return hash.hexdigest() + hash.update(buf) + finally: + f.close() + + def new(self, **kw): + """ create a modified version of this path. + the following keyword arguments modify various path parts: + + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + drive, dirname, basename, purebasename,ext = self._getbyspec( + "drive,dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + try: + ext = kw['ext'] + except KeyError: + pass + else: + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('drive', drive) + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + obj.strpath = os.path.normpath( + "%(drive)s%(dirname)s%(sep)s%(basename)s" % kw) + return obj + + def _getbyspec(self, spec): + """ return a sequence of specified path parts. 'spec' is + a comma separated string containing path part names. + according to the following convention: + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + + args = filter(None, spec.split(',') ) + append = res.append + for name in args: + if name == 'drive': + append(parts[0]) + elif name == 'dirname': + append(self.sep.join(['']+parts[1:-1])) + else: + basename = parts[-1] + if name == 'basename': + append(basename) + else: + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + append(purebasename) + elif name == 'ext': + append(ext) + else: + raise ValueError("invalid part specification %r" % name) + return res + + def join(self, *args, **kwargs): + """ return a new path by appending all 'args' as path + components. if abs=1 is used restart from root if any + of the args is an absolute path. + """ + if not args: + return self + strpath = self.strpath + sep = self.sep + strargs = [str(x) for x in args] + if kwargs.get('abs', 0): + for i in range(len(strargs)-1, -1, -1): + if os.path.isabs(strargs[i]): + strpath = strargs[i] + strargs = strargs[i+1:] + break + for arg in strargs: + arg = arg.strip(sep) + if iswin32: + # allow unix style paths even on windows. + arg = arg.strip('/') + arg = arg.replace('/', sep) + if arg: + if not strpath.endswith(sep): + strpath += sep + strpath += arg + obj = self.new() + obj.strpath = os.path.normpath(strpath) + return obj + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return py.error.checked_call(open, self.strpath, mode) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + res = [] + for name in py.error.checked_call(os.listdir, self.strpath): + childurl = self.join(name) + if fil is None or fil(childurl): + res.append(childurl) + self._sortlist(res, sort) + return res + + def size(self): + """ return size of the underlying file object """ + return self.stat().size + + def mtime(self): + """ return last modification time of the path. """ + return self.stat().mtime + + def copy(self, target, archive=False): + """ copy path to target.""" + assert not archive, "XXX archive-mode not supported" + if self.check(file=1): + if target.check(dir=1): + target = target.join(self.basename) + assert self!=target + copychunked(self, target) + else: + def rec(p): + return p.check(link=0) + for x in self.visit(rec=rec): + relpath = x.relto(self) + newx = target.join(relpath) + newx.dirpath().ensure(dir=1) + if x.check(link=1): + newx.mksymlinkto(x.readlink()) + elif x.check(file=1): + copychunked(x, newx) + elif x.check(dir=1): + newx.ensure(dir=1) + + def rename(self, target): + """ rename this path to target. """ + return py.error.checked_call(os.rename, str(self), str(target)) + + def dump(self, obj, bin=1): + """ pickle object into path location""" + f = self.open('wb') + try: + py.error.checked_call(py.std.pickle.dump, obj, f, bin) + finally: + f.close() + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + p = self.join(*args) + py.error.checked_call(os.mkdir, str(p)) + return p + + def write(self, data, mode='w'): + """ write data into path. """ + if 'b' in mode: + if not py.builtin._isbytes(data): + raise ValueError("can only process bytes") + else: + if not py.builtin._istext(data): + if not py.builtin._isbytes(data): + data = str(data) + else: + try: + data = py.builtin._totext(data, sys.getdefaultencoding()) + except UnicodeDecodeError: + pass + f = self.open(mode) + try: + f.write(data) + finally: + f.close() + + def _ensuredirs(self): + parent = self.dirpath() + if parent == self: + return self + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + try: + self.mkdir() + except py.error.EEXIST: + # race condition: file/dir created by another thread/process. + # complain if it is not a dir + if self.check(dir=0): + raise + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if kwargs.get('dir', 0): + return p._ensuredirs() + else: + p.dirpath()._ensuredirs() + if not p.check(file=1): + p.open('w').close() + return p + + def stat(self): + """ Return an os.stat() tuple. """ + return Stat(self, py.error.checked_call(os.stat, self.strpath)) + + def lstat(self): + """ Return an os.lstat() tuple. """ + return Stat(self, py.error.checked_call(os.lstat, self.strpath)) + + def setmtime(self, mtime=None): + """ set modification time for the given path. if 'mtime' is None + (the default) then the file's mtime is set to current time. + + Note that the resolution for 'mtime' is platform dependent. + """ + if mtime is None: + return py.error.checked_call(os.utime, self.strpath, mtime) + try: + return py.error.checked_call(os.utime, self.strpath, (-1, mtime)) + except py.error.EINVAL: + return py.error.checked_call(os.utime, self.strpath, (self.atime(), mtime)) + + def chdir(self): + """ change directory to self and return old current directory """ + old = self.__class__() + py.error.checked_call(os.chdir, self.strpath) + return old + + def realpath(self): + """ return a new path which contains no symbolic links.""" + return self.__class__(os.path.realpath(self.strpath)) + + def atime(self): + """ return last access time of the path. """ + return self.stat().atime + + def __repr__(self): + return 'local(%r)' % self.strpath + + def __str__(self): + """ return string representation of the Path. """ + return self.strpath + + def pypkgpath(self, pkgname=None): + """ return the path's package path by looking for the given + pkgname. If pkgname is None then look for the last + directory upwards which still contains an __init__.py. + Return None if a pkgpath can not be determined. + """ + pkgpath = None + for parent in self.parts(reverse=True): + if pkgname is None: + if parent.check(file=1): + continue + if parent.join('__init__.py').check(): + pkgpath = parent + continue + return pkgpath + else: + if parent.basename == pkgname: + return parent + return pkgpath + + def _prependsyspath(self, path): + s = str(path) + if s != sys.path[0]: + #print "prepending to sys.path", s + sys.path.insert(0, s) + + def chmod(self, mode, rec=0): + """ change permissions to the given mode. If mode is an + integer it directly encodes the os-specific modes. + if rec is True perform recursively. + """ + if not isinstance(mode, int): + raise TypeError("mode %r must be an integer" % (mode,)) + if rec: + for x in self.visit(rec=rec): + py.error.checked_call(os.chmod, str(x), mode) + py.error.checked_call(os.chmod, str(self), mode) + + def pyimport(self, modname=None, ensuresyspath=True): + """ return path as an imported python module. + if modname is None, look for the containing package + and construct an according module name. + The module will be put/looked up in sys.modules. + """ + if not self.check(): + raise py.error.ENOENT(self) + #print "trying to import", self + pkgpath = None + if modname is None: + pkgpath = self.pypkgpath() + if pkgpath is not None: + if ensuresyspath: + self._prependsyspath(pkgpath.dirpath()) + pkg = __import__(pkgpath.basename, None, None, []) + names = self.new(ext='').relto(pkgpath.dirpath()) + names = names.split(self.sep) + modname = ".".join(names) + else: + # no package scope, still make it possible + if ensuresyspath: + self._prependsyspath(self.dirpath()) + modname = self.purebasename + mod = __import__(modname, None, None, ['__doc__']) + modfile = mod.__file__ + if modfile[-4:] in ('.pyc', '.pyo'): + modfile = modfile[:-1] + elif modfile.endswith('$py.class'): + modfile = modfile[:-9] + '.py' + if not self.samefile(modfile): + raise EnvironmentError("mismatch:\n" + "imported module %r\n" + "does not stem from %r\n" + "maybe __init__.py files are missing?" % (mod, str(self))) + return mod + else: + try: + return sys.modules[modname] + except KeyError: + # we have a custom modname, do a pseudo-import + mod = py.std.types.ModuleType(modname) + mod.__file__ = str(self) + sys.modules[modname] = mod + try: + py.builtin.execfile(str(self), mod.__dict__) + except: + del sys.modules[modname] + raise + return mod + + def sysexec(self, *argv, **popen_opts): + """ return stdout text from executing a system child process, + where the 'self' path points to executable. + The process is directly invoked and not through a system shell. + """ + from subprocess import Popen, PIPE + argv = map(str, argv) + popen_opts['stdout'] = popen_opts['stderr'] = PIPE + proc = Popen([str(self)] + list(argv), **popen_opts) + stdout, stderr = proc.communicate() + ret = proc.wait() + if py.builtin._isbytes(stdout): + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + if ret != 0: + if py.builtin._isbytes(stderr): + stderr = py.builtin._totext(stderr, sys.getdefaultencoding()) + raise py.process.cmdexec.Error(ret, ret, str(self), + stdout, stderr,) + return stdout + + def sysfind(cls, name, checker=None): + """ return a path object found by looking at the systems + underlying PATH specification. If the checker is not None + it will be invoked to filter matching paths. If a binary + cannot be found, None is returned + Note: This is probably not working on plain win32 systems + but may work on cygwin. + """ + if os.path.isabs(name): + p = py.path.local(name) + if p.check(file=1): + return p + else: + if iswin32: + paths = py.std.os.environ['Path'].split(';') + if '' not in paths and '.' not in paths: + paths.append('.') + try: + systemroot = os.environ['SYSTEMROOT'] + except KeyError: + pass + else: + paths = [re.sub('%SystemRoot%', systemroot, path) + for path in paths] + tryadd = '', '.exe', '.com', '.bat' # XXX add more? + else: + paths = py.std.os.environ['PATH'].split(':') + tryadd = ('',) + + for x in paths: + for addext in tryadd: + p = py.path.local(x).join(name, abs=True) + addext + try: + if p.check(file=1): + if checker: + if not checker(p): + continue + return p + except py.error.EACCES: + pass + return None + sysfind = classmethod(sysfind) + + def _gethomedir(cls): + try: + x = os.environ['HOME'] + except KeyError: + x = os.environ['HOMEPATH'] + return cls(x) + _gethomedir = classmethod(_gethomedir) + + #""" + #special class constructors for local filesystem paths + #""" + def get_temproot(cls): + """ return the system's temporary directory + (where tempfiles are usually created in) + """ + return py.path.local(py.std.tempfile.gettempdir()) + get_temproot = classmethod(get_temproot) + + def mkdtemp(cls): + """ return a Path object pointing to a fresh new temporary directory + (which we created ourself). + """ + import tempfile + tries = 10 + for i in range(tries): + dname = tempfile.mktemp() + dpath = cls(tempfile.mktemp()) + try: + dpath.mkdir() + except (py.error.EEXIST, py.error.EPERM, py.error.EACCES): + continue + return dpath + raise py.error.ENOENT(dpath, "could not create tempdir, %d tries" % tries) + mkdtemp = classmethod(mkdtemp) + + def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800): # two days + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = cls.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError): # AttributeError on win32 + pass + + return udir + make_numbered_dir = classmethod(make_numbered_dir) + +def copychunked(src, dest): + chunksize = 524288 # half a meg of bytes + fsrc = src.open('rb') + try: + fdest = dest.open('wb') + try: + while 1: + buf = fsrc.read(chunksize) + if not buf: + break + fdest.write(buf) + finally: + fdest.close() + finally: + fsrc.close() + +def autopath(globs=None): + """ (deprecated) return the (local) path of the "current" file pointed to by globals or - if it is none - alternatively the callers frame globals. + + the path will always point to a .py file or to None. + the path will have the following payload: + pkgdir is the last parent directory path containing __init__.py + """ + py.log._apiwarn("1.1", "py.magic.autopath deprecated, " + "use py.path.local(__file__) and maybe pypkgpath/pyimport().") + if globs is None: + globs = sys._getframe(1).f_globals + try: + __file__ = globs['__file__'] + except KeyError: + if not sys.argv[0]: + raise ValueError("cannot compute autopath in interactive mode") + __file__ = os.path.abspath(sys.argv[0]) + + ret = py.path.local(__file__) + if ret.ext in ('.pyc', '.pyo'): + ret = ret.new(ext='.py') + current = pkgdir = ret.dirpath() + while 1: + if current.join('__init__.py').check(): + pkgdir = current + current = current.dirpath() + if pkgdir != current: + continue + elif str(current) not in sys.path: + sys.path.insert(0, str(current)) + break + ret.pkgdir = pkgdir + return ret + Added: pypy/branch/py11/py/impl/path/svnurl.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/svnurl.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,365 @@ +""" +module defining a subversion path object based on the external +command 'svn'. This modules aims to work with svn 1.3 and higher +but might also interact well with earlier versions. +""" + +import os, sys, time, re +import py +from py import path, process +from py.impl.path import common +from py.impl.path import svnwc as svncommon +from py.impl.path.cacheutil import BuildcostAccessCache, AgingCache + +DEBUG=False + +class SvnCommandPath(svncommon.SvnPathBase): + """ path implementation that offers access to (possibly remote) subversion + repositories. """ + + _lsrevcache = BuildcostAccessCache(maxentries=128) + _lsnorevcache = AgingCache(maxentries=1000, maxseconds=60.0) + + def __new__(cls, path, rev=None, auth=None): + self = object.__new__(cls) + if isinstance(path, cls): + rev = path.rev + auth = path.auth + path = path.strpath + svncommon.checkbadchars(path) + path = path.rstrip('/') + self.strpath = path + self.rev = rev + self.auth = auth + return self + + def __repr__(self): + if self.rev == -1: + return 'svnurl(%r)' % self.strpath + else: + return 'svnurl(%r, %r)' % (self.strpath, self.rev) + + def _svnwithrev(self, cmd, *args): + """ execute an svn command, append our own url and revision """ + if self.rev is None: + return self._svnwrite(cmd, *args) + else: + args = ['-r', self.rev] + list(args) + return self._svnwrite(cmd, *args) + + def _svnwrite(self, cmd, *args): + """ execute an svn command, append our own url """ + l = ['svn %s' % cmd] + args = ['"%s"' % self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._encodedurl()) + # fixing the locale because we can't otherwise parse + string = " ".join(l) + if DEBUG: + print("execing %s" % string) + out = self._svncmdexecauth(string) + return out + + def _svncmdexecauth(self, cmd): + """ execute an svn command 'as is' """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._cmdexec(cmd) + + def _cmdexec(self, cmd): + try: + out = process.cmdexec(cmd) + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if (e.err.find('File Exists') != -1 or + e.err.find('File already exists') != -1): + raise py.error.EEXIST(self) + raise + return out + + def _svnpopenauth(self, cmd): + """ execute an svn command, return a pipe for reading stdin """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._popen(cmd) + + def _popen(self, cmd): + return os.popen(cmd) + + def _encodedurl(self): + return self._escape(self.strpath) + + def _norev_delentry(self, path): + auth = self.auth and self.auth.makecmdoptions() or None + self._lsnorevcache.delentry((str(path), auth)) + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + if mode not in ("r", "rU",): + raise ValueError("mode %r not supported" % (mode,)) + assert self.check(file=1) # svn cat returns an empty file otherwise + if self.rev is None: + return self._svnpopenauth('svn cat "%s"' % ( + self._escape(self.strpath), )) + else: + return self._svnpopenauth('svn cat -r %s "%s"' % ( + self.rev, self._escape(self.strpath))) + + def dirpath(self, *args, **kwargs): + """ return the directory path of the current path joined + with any given path arguments. + """ + l = self.strpath.split(self.sep) + if len(l) < 4: + raise py.error.EINVAL(self, "base is not valid") + elif len(l) == 4: + return self.join(*args, **kwargs) + else: + return self.new(basename='').join(*args, **kwargs) + + # modifying methods (cache must be invalidated) + def mkdir(self, *args, **kwargs): + """ create & return the directory joined with args. + pass a 'msg' keyword argument to set the commit message. + """ + commit_msg = kwargs.get('msg', "mkdir by py lib invocation") + createpath = self.join(*args) + createpath._svnwrite('mkdir', '-m', commit_msg) + self._norev_delentry(createpath.dirpath()) + return createpath + + def copy(self, target, msg='copied by py lib invocation'): + """ copy path to target with checkin message msg.""" + if getattr(target, 'rev', None) is not None: + raise py.error.EINVAL(target, "revisions are immutable") + self._svncmdexecauth('svn copy -m "%s" "%s" "%s"' %(msg, + self._escape(self), self._escape(target))) + self._norev_delentry(target.dirpath()) + + def rename(self, target, msg="renamed by py lib invocation"): + """ rename this path to target with checkin message msg. """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn move -m "%s" --force "%s" "%s"' %( + msg, self._escape(self), self._escape(target))) + self._norev_delentry(self.dirpath()) + self._norev_delentry(self) + + def remove(self, rec=1, msg='removed by py lib invocation'): + """ remove a file or directory (or a directory tree if rec=1) with +checkin message msg.""" + if self.rev is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn rm -m "%s" "%s"' %(msg, self._escape(self))) + self._norev_delentry(self.dirpath()) + + def export(self, topath): + """ export to a local path + + topath should not exist prior to calling this, returns a + py.path.local instance + """ + topath = py.path.local(topath) + args = ['"%s"' % (self._escape(self),), + '"%s"' % (self._escape(topath),)] + if self.rev is not None: + args = ['-r', str(self.rev)] + args + self._svncmdexecauth('svn export %s' % (' '.join(args),)) + return topath + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). If you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + target = self.join(*args) + dir = kwargs.get('dir', 0) + for x in target.parts(reverse=True): + if x.check(): + break + else: + raise py.error.ENOENT(target, "has not any valid base!") + if x == target: + if not x.check(dir=dir): + raise dir and py.error.ENOTDIR(x) or py.error.EISDIR(x) + return x + tocreate = target.relto(x) + basename = tocreate.split(self.sep, 1)[0] + tempdir = py.path.local.mkdtemp() + try: + tempdir.ensure(tocreate, dir=dir) + cmd = 'svn import -m "%s" "%s" "%s"' % ( + "ensure %s" % self._escape(tocreate), + self._escape(tempdir.join(basename)), + x.join(basename)._encodedurl()) + self._svncmdexecauth(cmd) + self._norev_delentry(x) + finally: + tempdir.remove() + return target + + # end of modifying methods + def _propget(self, name): + res = self._svnwithrev('propget', name) + return res[:-1] # strip trailing newline + + def _proplist(self): + res = self._svnwithrev('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return svncommon.PropListDict(self, lines) + + def _listdir_nameinfo(self): + """ return sequence of name-info directory entries of self """ + def builder(): + try: + res = self._svnwithrev('ls', '-v') + except process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('non-existent in that revision') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('File not found') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('not part of a repository')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('Unable to open')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.lower().find('method not allowed')!=-1: + raise py.error.EACCES(self, e.err) + raise py.error.Error(e.err) + lines = res.split('\n') + nameinfo_seq = [] + for lsline in lines: + if lsline: + info = InfoSvnCommand(lsline) + if info._name != '.': # svn 1.5 produces '.' dirs, + nameinfo_seq.append((info._name, info)) + nameinfo_seq.sort() + return nameinfo_seq + auth = self.auth and self.auth.makecmdoptions() or None + if self.rev is not None: + return self._lsrevcache.getorbuild((self.strpath, self.rev, auth), + builder) + else: + return self._lsnorevcache.getorbuild((self.strpath, auth), + builder) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + nameinfo_seq = self._listdir_nameinfo() + if len(nameinfo_seq) == 1: + name, info = nameinfo_seq[0] + if name == self.basename and info.kind == 'file': + #if not self.check(dir=1): + raise py.error.ENOTDIR(self) + paths = [self.join(name) for (name, info) in nameinfo_seq] + if fil: + paths = [x for x in paths if fil(x)] + self._sortlist(paths, sort) + return paths + + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() #make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + xmlpipe = self._svnpopenauth('svn log --xml %s %s "%s"' % + (rev_opt, verbose_opt, self.strpath)) + from xml.dom import minidom + tree = minidom.parse(xmlpipe) + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(svncommon.LogEntry(logentry)) + return result + +#01234567890123456789012345678901234567890123467 +# 2256 hpk 165 Nov 24 17:55 __init__.py +# XXX spotted by Guido, SVN 1.3.0 has different aligning, breaks the code!!! +# 1312 johnny 1627 May 05 14:32 test_decorators.py +# +class InfoSvnCommand: + # the '0?' part in the middle is an indication of whether the resource is + # locked, see 'svn help ls' + lspattern = re.compile( + r'^ *(?P\d+) +(?P.+?) +(0? *(?P\d+))? ' + '*(?P\w+ +\d{2} +[\d:]+) +(?P.*)$') + def __init__(self, line): + # this is a typical line from 'svn ls http://...' + #_ 1127 jum 0 Jul 13 15:28 branch/ + match = self.lspattern.match(line) + data = match.groupdict() + self._name = data['file'] + if self._name[-1] == '/': + self._name = self._name[:-1] + self.kind = 'dir' + else: + self.kind = 'file' + #self.has_props = l.pop(0) == 'P' + self.created_rev = int(data['rev']) + self.last_author = data['author'] + self.size = data['size'] and int(data['size']) or 0 + self.mtime = parse_time_with_missing_year(data['date']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + +#____________________________________________________ +# +# helper functions +#____________________________________________________ +def parse_time_with_missing_year(timestr): + """ analyze the time part from a single line of "svn ls -v" + the svn output doesn't show the year makes the 'timestr' + ambigous. + """ + import calendar + t_now = time.gmtime() + + tparts = timestr.split() + month = time.strptime(tparts.pop(0), '%b')[1] + day = time.strptime(tparts.pop(0), '%d')[2] + last = tparts.pop(0) # year or hour:minute + try: + year = time.strptime(last, '%Y')[0] + hour = minute = 0 + except ValueError: + hour, minute = time.strptime(last, '%H:%M')[3:5] + year = t_now[0] + + t_result = (year, month, day, hour, minute, 0,0,0,0) + if t_result > t_now: + year -= 1 + t_result = (year, month, day, hour, minute, 0,0,0,0) + return calendar.timegm(t_result) + +class PathEntry: + def __init__(self, ppart): + self.strpath = ppart.firstChild.nodeValue.encode('UTF-8') + self.action = ppart.getAttribute('action').encode('UTF-8') + if self.action == 'A': + self.copyfrom_path = ppart.getAttribute('copyfrom-path').encode('UTF-8') + if self.copyfrom_path: + self.copyfrom_rev = int(ppart.getAttribute('copyfrom-rev')) + Added: pypy/branch/py11/py/impl/path/svnwc.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/path/svnwc.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,1236 @@ +""" +svn-Command based Implementation of a Subversion WorkingCopy Path. + + SvnWCCommandPath is the main class. + +""" + +import os, sys, time, re, calendar +import py +import subprocess +from py.impl.path import common + +#----------------------------------------------------------- +# Caching latest repository revision and repo-paths +# (getting them is slow with the current implementations) +# +# XXX make mt-safe +#----------------------------------------------------------- + +class cache: + proplist = {} + info = {} + entries = {} + prop = {} + +class RepoEntry: + def __init__(self, url, rev, timestamp): + self.url = url + self.rev = rev + self.timestamp = timestamp + + def __str__(self): + return "repo: %s;%s %s" %(self.url, self.rev, self.timestamp) + +class RepoCache: + """ The Repocache manages discovered repository paths + and their revisions. If inside a timeout the cache + will even return the revision of the root. + """ + timeout = 20 # seconds after which we forget that we know the last revision + + def __init__(self): + self.repos = [] + + def clear(self): + self.repos = [] + + def put(self, url, rev, timestamp=None): + if rev is None: + return + if timestamp is None: + timestamp = time.time() + + for entry in self.repos: + if url == entry.url: + entry.timestamp = timestamp + entry.rev = rev + #print "set repo", entry + break + else: + entry = RepoEntry(url, rev, timestamp) + self.repos.append(entry) + #print "appended repo", entry + + def get(self, url): + now = time.time() + for entry in self.repos: + if url.startswith(entry.url): + if now < entry.timestamp + self.timeout: + #print "returning immediate Etrny", entry + return entry.url, entry.rev + return entry.url, -1 + return url, -1 + +repositories = RepoCache() + + +# svn support code + +ALLOWED_CHARS = "_ -/\\=$.~+" #add characters as necessary when tested +if sys.platform == "win32": + ALLOWED_CHARS += ":" +ALLOWED_CHARS_HOST = ALLOWED_CHARS + '@:' + +def _getsvnversion(ver=[]): + try: + return ver[0] + except IndexError: + v = py.process.cmdexec("svn -q --version") + v.strip() + v = '.'.join(v.split('.')[:2]) + ver.append(v) + return v + +def _escape_helper(text): + text = str(text) + if py.std.sys.platform != 'win32': + text = str(text).replace('$', '\\$') + return text + +def _check_for_bad_chars(text, allowed_chars=ALLOWED_CHARS): + for c in str(text): + if c.isalnum(): + continue + if c in allowed_chars: + continue + return True + return False + +def checkbadchars(url): + # (hpk) not quite sure about the exact purpose, guido w.? + proto, uri = url.split("://", 1) + if proto != "file": + host, uripath = uri.split('/', 1) + # only check for bad chars in the non-protocol parts + if (_check_for_bad_chars(host, ALLOWED_CHARS_HOST) \ + or _check_for_bad_chars(uripath, ALLOWED_CHARS)): + raise ValueError("bad char in %r" % (url, )) + + +#_______________________________________________________________ + +class SvnPathBase(common.PathBase): + """ Base implementation for SvnPath implementations. """ + sep = '/' + + def _geturl(self): + return self.strpath + url = property(_geturl, None, None, "url of this svn-path.") + + def __str__(self): + """ return a string representation (including rev-number) """ + return self.strpath + + def __hash__(self): + return hash(self.strpath) + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + obj.rev = kw.get('rev', self.rev) + obj.auth = kw.get('auth', self.auth) + dirname, basename, purebasename, ext = self._getbyspec( + "dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + ext = kw.setdefault('ext', ext) + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + if kw['basename']: + obj.strpath = "%(dirname)s%(sep)s%(basename)s" % kw + else: + obj.strpath = "%(dirname)s" % kw + return obj + + def _getbyspec(self, spec): + """ get specified parts of the path. 'arg' is a string + with comma separated path parts. The parts are returned + in exactly the order of the specification. + + you may specify the following parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + for name in spec.split(','): + name = name.strip() + if name == 'dirname': + res.append(self.sep.join(parts[:-1])) + elif name == 'basename': + res.append(parts[-1]) + else: + basename = parts[-1] + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + res.append(purebasename) + elif name == 'ext': + res.append(ext) + else: + raise NameError("Don't know part %r" % name) + return res + + def __eq__(self, other): + """ return true if path and rev attributes each match """ + return (str(self) == str(other) and + (self.rev == other.rev or self.rev == other.rev)) + + def __ne__(self, other): + return not self == other + + def join(self, *args): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + + args = tuple([arg.strip(self.sep) for arg in args]) + parts = (self.strpath, ) + args + newpath = self.__class__(self.sep.join(parts), self.rev, self.auth) + return newpath + + def propget(self, name): + """ return the content of the given property. """ + value = self._propget(name) + return value + + def proplist(self): + """ list all property names. """ + content = self._proplist() + return content + + def info(self): + """ return an Info structure with svn-provided information. """ + parent = self.dirpath() + nameinfo_seq = parent._listdir_nameinfo() + bn = self.basename + for name, info in nameinfo_seq: + if name == bn: + return info + raise py.error.ENOENT(self) + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + # shared help methods + + def _escape(self, cmd): + return _escape_helper(cmd) + + + #def _childmaxrev(self): + # """ return maximum revision number of childs (or self.rev if no childs) """ + # rev = self.rev + # for name, info in self._listdir_nameinfo(): + # rev = max(rev, info.created_rev) + # return rev + + #def _getlatestrevision(self): + # """ return latest repo-revision for this path. """ + # url = self.strpath + # path = self.__class__(url, None) + # + # # we need a long walk to find the root-repo and revision + # while 1: + # try: + # rev = max(rev, path._childmaxrev()) + # previous = path + # path = path.dirpath() + # except (IOError, process.cmdexec.Error): + # break + # if rev is None: + # raise IOError, "could not determine newest repo revision for %s" % self + # return rev + + class Checkers(common.Checkers): + def dir(self): + try: + return self.path.info().kind == 'dir' + except py.error.Error: + return self._listdirworks() + + def _listdirworks(self): + try: + self.path.listdir() + except py.error.ENOENT: + return False + else: + return True + + def file(self): + try: + return self.path.info().kind == 'file' + except py.error.ENOENT: + return False + + def exists(self): + try: + return self.path.info() + except py.error.ENOENT: + return self._listdirworks() + +def parse_apr_time(timestr): + i = timestr.rfind('.') + if i == -1: + raise ValueError("could not parse %s" % timestr) + timestr = timestr[:i] + parsedtime = time.strptime(timestr, "%Y-%m-%dT%H:%M:%S") + return time.mktime(parsedtime) + +class PropListDict(dict): + """ a Dictionary which fetches values (InfoSvnCommand instances) lazily""" + def __init__(self, path, keynames): + dict.__init__(self, [(x, None) for x in keynames]) + self.path = path + + def __getitem__(self, key): + value = dict.__getitem__(self, key) + if value is None: + value = self.path.propget(key) + dict.__setitem__(self, key, value) + return value + +def fixlocale(): + if sys.platform != 'win32': + return 'LC_ALL=C ' + return '' + +# some nasty chunk of code to solve path and url conversion and quoting issues +ILLEGAL_CHARS = '* | \ / : < > ? \t \n \x0b \x0c \r'.split(' ') +if os.sep in ILLEGAL_CHARS: + ILLEGAL_CHARS.remove(os.sep) +ISWINDOWS = sys.platform == 'win32' +_reg_allow_disk = re.compile(r'^([a-z]\:\\)?[^:]+$', re.I) +def _check_path(path): + illegal = ILLEGAL_CHARS[:] + sp = path.strpath + if ISWINDOWS: + illegal.remove(':') + if not _reg_allow_disk.match(sp): + raise ValueError('path may not contain a colon (:)') + for char in sp: + if char not in string.printable or char in illegal: + raise ValueError('illegal character %r in path' % (char,)) + +def path_to_fspath(path, addat=True): + _check_path(path) + sp = path.strpath + if addat and path.rev != -1: + sp = '%s@%s' % (sp, path.rev) + elif addat: + sp = '%s at HEAD' % (sp,) + return sp + +def url_from_path(path): + fspath = path_to_fspath(path, False) + quote = py.std.urllib.quote + if ISWINDOWS: + match = _reg_allow_disk.match(fspath) + fspath = fspath.replace('\\', '/') + if match.group(1): + fspath = '/%s%s' % (match.group(1).replace('\\', '/'), + quote(fspath[len(match.group(1)):])) + else: + fspath = quote(fspath) + else: + fspath = quote(fspath) + if path.rev != -1: + fspath = '%s@%s' % (fspath, path.rev) + else: + fspath = '%s at HEAD' % (fspath,) + return 'file://%s' % (fspath,) + +class SvnAuth(object): + """ container for auth information for Subversion """ + def __init__(self, username, password, cache_auth=True, interactive=True): + self.username = username + self.password = password + self.cache_auth = cache_auth + self.interactive = interactive + + def makecmdoptions(self): + uname = self.username.replace('"', '\\"') + passwd = self.password.replace('"', '\\"') + ret = [] + if uname: + ret.append('--username="%s"' % (uname,)) + if passwd: + ret.append('--password="%s"' % (passwd,)) + if not self.cache_auth: + ret.append('--no-auth-cache') + if not self.interactive: + ret.append('--non-interactive') + return ' '.join(ret) + + def __str__(self): + return "" %(self.username,) + +rex_blame = re.compile(r'\s*(\d+)\s*(\S+) (.*)') + +class SvnWCCommandPath(common.PathBase): + """ path implementation offering access/modification to svn working copies. + It has methods similar to the functions in os.path and similar to the + commands of the svn client. + """ + sep = os.sep + + def __new__(cls, wcpath=None, auth=None): + self = object.__new__(cls) + if isinstance(wcpath, cls): + if wcpath.__class__ == cls: + return wcpath + wcpath = wcpath.localpath + if _check_for_bad_chars(str(wcpath), + ALLOWED_CHARS): + raise ValueError("bad char in wcpath %s" % (wcpath, )) + self.localpath = py.path.local(wcpath) + self.auth = auth + return self + + strpath = property(lambda x: str(x.localpath), None, None, "string path") + + def __eq__(self, other): + return self.localpath == getattr(other, 'localpath', None) + + def _geturl(self): + if getattr(self, '_url', None) is None: + info = self.info() + self._url = info.url #SvnPath(info.url, info.rev) + assert isinstance(self._url, py.builtin._basestring) + return self._url + + url = property(_geturl, None, None, "url of this WC item") + + def _escape(self, cmd): + return _escape_helper(cmd) + + def dump(self, obj): + """ pickle object into path location""" + return self.localpath.dump(obj) + + def svnurl(self): + """ return current SvnPath for this WC-item. """ + info = self.info() + return py.path.svnurl(info.url) + + def __repr__(self): + return "svnwc(%r)" % (self.strpath) # , self._url) + + def __str__(self): + return str(self.localpath) + + def _makeauthoptions(self): + if self.auth is None: + return '' + return self.auth.makecmdoptions() + + def _authsvn(self, cmd, args=None): + args = args and list(args) or [] + args.append(self._makeauthoptions()) + return self._svn(cmd, *args) + + def _svn(self, cmd, *args): + l = ['svn %s' % cmd] + args = [self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._escape(self.strpath)) + # try fixing the locale because we can't otherwise parse + string = fixlocale() + " ".join(l) + try: + try: + key = 'LC_MESSAGES' + hold = os.environ.get(key) + os.environ[key] = 'C' + out = py.process.cmdexec(string) + finally: + if hold: + os.environ[key] = hold + else: + del os.environ[key] + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + strerr = e.err.lower() + if strerr.find('file not found') != -1: + raise py.error.ENOENT(self) + if (strerr.find('file exists') != -1 or + strerr.find('file already exists') != -1 or + strerr.find("can't create directory") != -1): + raise py.error.EEXIST(self) + raise + return out + + def switch(self, url): + """ switch to given URL. """ + self._authsvn('switch', [url]) + + def checkout(self, url=None, rev=None): + """ checkout from url to local wcpath. """ + args = [] + if url is None: + url = self.url + if rev is None or rev == -1: + if (py.std.sys.platform != 'win32' and + _getsvnversion() == '1.3'): + url += "@HEAD" + else: + if _getsvnversion() == '1.3': + url += "@%d" % rev + else: + args.append('-r' + str(rev)) + args.append(url) + self._authsvn('co', args) + + def update(self, rev='HEAD'): + """ update working copy item to given revision. (None -> HEAD). """ + self._authsvn('up', ['-r', rev, "--non-interactive"],) + + def write(self, content, mode='w'): + """ write content into local filesystem wc. """ + self.localpath.write(content, mode) + + def dirpath(self, *args): + """ return the directory Path of the current Path. """ + return self.__class__(self.localpath.dirpath(*args), auth=self.auth) + + def _ensuredirs(self): + parent = self.dirpath() + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + self.mkdir() + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'directory=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if p.check(): + if p.check(versioned=False): + p.add() + return p + if kwargs.get('dir', 0): + return p._ensuredirs() + parent = p.dirpath() + parent._ensuredirs() + p.write("") + p.add() + return p + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + if args: + return self.join(*args).mkdir() + else: + self._svn('mkdir') + return self + + def add(self): + """ add ourself to svn """ + self._svn('add') + + def remove(self, rec=1, force=1): + """ remove a file or a directory tree. 'rec'ursive is + ignored and considered always true (because of + underlying svn semantics. + """ + assert rec, "svn cannot remove non-recursively" + if not self.check(versioned=True): + # not added to svn (anymore?), just remove + py.path.local(self).remove() + return + flags = [] + if force: + flags.append('--force') + self._svn('remove', *flags) + + def copy(self, target): + """ copy path to target.""" + py.process.cmdexec("svn copy %s %s" %(str(self), str(target))) + + def rename(self, target): + """ rename this path to target. """ + py.process.cmdexec("svn move --force %s %s" %(str(self), str(target))) + + def lock(self): + """ set a lock (exclusive) on the resource """ + out = self._authsvn('lock').strip() + if not out: + # warning or error, raise exception + raise Exception(out[4:]) + + def unlock(self): + """ unset a previously set lock """ + out = self._authsvn('unlock').strip() + if out.startswith('svn:'): + # warning or error, raise exception + raise Exception(out[4:]) + + def cleanup(self): + """ remove any locks from the resource """ + # XXX should be fixed properly!!! + try: + self.unlock() + except: + pass + + def status(self, updates=0, rec=0, externals=0): + """ return (collective) Status object for this file. """ + # http://svnbook.red-bean.com/book.html#svn-ch-3-sect-4.3.1 + # 2201 2192 jum test + # XXX + if externals: + raise ValueError("XXX cannot perform status() " + "on external items yet") + else: + #1.2 supports: externals = '--ignore-externals' + externals = '' + if rec: + rec= '' + else: + rec = '--non-recursive' + + # XXX does not work on all subversion versions + #if not externals: + # externals = '--ignore-externals' + + if updates: + updates = '-u' + else: + updates = '' + + try: + cmd = 'status -v --xml --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + except py.process.cmdexec.Error: + cmd = 'status -v --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + rootstatus = WCStatus(self).fromstring(out, self) + else: + rootstatus = XMLWCStatus(self).fromstring(out, self) + return rootstatus + + def diff(self, rev=None): + """ return a diff of the current path against revision rev (defaulting + to the last one). + """ + args = [] + if rev is not None: + args.append("-r %d" % rev) + out = self._authsvn('diff', args) + return out + + def blame(self): + """ return a list of tuples of three elements: + (revision, commiter, line) + """ + out = self._svn('blame') + result = [] + blamelines = out.splitlines() + reallines = py.path.svnurl(self.url).readlines() + for i, (blameline, line) in enumerate( + zip(blamelines, reallines)): + m = rex_blame.match(blameline) + if not m: + raise ValueError("output line %r of svn blame does not match " + "expected format" % (line, )) + rev, name, _ = m.groups() + result.append((int(rev), name, line)) + return result + + _rex_commit = re.compile(r'.*Committed revision (\d+)\.$', re.DOTALL) + def commit(self, msg='', rec=1): + """ commit with support for non-recursive commits """ + # XXX i guess escaping should be done better here?!? + cmd = 'commit -m "%s" --force-log' % (msg.replace('"', '\\"'),) + if not rec: + cmd += ' -N' + out = self._authsvn(cmd) + try: + del cache.info[self] + except KeyError: + pass + if out: + m = self._rex_commit.match(out) + return int(m.group(1)) + + def propset(self, name, value, *args): + """ set property name to value on this path. """ + d = py.path.local.mkdtemp() + try: + p = d.join('value') + p.write(value) + self._svn('propset', name, '--file', str(p), *args) + finally: + d.remove() + + def propget(self, name): + """ get property name on this path. """ + res = self._svn('propget', name) + return res[:-1] # strip trailing newline + + def propdel(self, name): + """ delete property name on this path. """ + res = self._svn('propdel', name) + return res[:-1] # strip trailing newline + + def proplist(self, rec=0): + """ return a mapping of property names to property values. +If rec is True, then return a dictionary mapping sub-paths to such mappings. +""" + if rec: + res = self._svn('proplist -R') + return make_recursive_propdict(self, res) + else: + res = self._svn('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return PropListDict(self, lines) + + def revert(self, rec=0): + """ revert the local changes of this path. if rec is True, do so +recursively. """ + if rec: + result = self._svn('revert -R') + else: + result = self._svn('revert') + return result + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + if kw: + localpath = self.localpath.new(**kw) + else: + localpath = self.localpath + return self.__class__(localpath, auth=self.auth) + + def join(self, *args, **kwargs): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + localpath = self.localpath.join(*args, **kwargs) + return self.__class__(localpath, auth=self.auth) + + def info(self, usecache=1): + """ return an Info structure with svn-provided information. """ + info = usecache and cache.info.get(self) + if not info: + try: + output = self._svn('info') + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('Path is not a working copy directory') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find("is not under version control") != -1: + raise py.error.ENOENT(self, e.err) + raise + # XXX SVN 1.3 has output on stderr instead of stdout (while it does + # return 0!), so a bit nasty, but we assume no output is output + # to stderr... + if (output.strip() == '' or + output.lower().find('not a versioned resource') != -1): + raise py.error.ENOENT(self, output) + info = InfoSvnWCCommand(output) + + # Can't reliably compare on Windows without access to win32api + if py.std.sys.platform != 'win32': + if info.path != self.localpath: + raise py.error.ENOENT(self, "not a versioned resource:" + + " %s != %s" % (info.path, self.localpath)) + cache.info[self] = info + self.rev = info.rev + return info + + def listdir(self, fil=None, sort=None): + """ return a sequence of Paths. + + listdir will return either a tuple or a list of paths + depending on implementation choices. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + # XXX unify argument naming with LocalPath.listdir + def notsvn(path): + return path.basename != '.svn' + + paths = [self.__class__(p, auth=self.auth) + for p in self.localpath.listdir() + if notsvn(p) and (not fil or fil(p))] + self._sortlist(paths, sort) + return paths + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return open(self.strpath, mode) + + def _getbyspec(self, spec): + return self.localpath._getbyspec(spec) + + class Checkers(py.path.local.Checkers): + def __init__(self, path): + self.svnwcpath = path + self.path = path.localpath + def versioned(self): + try: + s = self.svnwcpath.info() + except (py.error.ENOENT, py.error.EEXIST): + return False + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('is not a working copy')!=-1: + return False + if e.err.lower().find('not a versioned resource') != -1: + return False + raise + else: + return True + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() # make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + locale_env = fixlocale() + # some blather on stderr + auth_opt = self._makeauthoptions() + #stdin, stdout, stderr = os.popen3(locale_env + + # 'svn log --xml %s %s %s "%s"' % ( + # rev_opt, verbose_opt, auth_opt, + # self.strpath)) + cmd = locale_env + 'svn log --xml %s %s %s "%s"' % ( + rev_opt, verbose_opt, auth_opt, self.strpath) + + popen = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + stdout, stderr = popen.communicate() + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + minidom,ExpatError = importxml() + try: + tree = minidom.parseString(stdout) + except ExpatError: + raise ValueError('no such revision') + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(LogEntry(logentry)) + return result + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + def __hash__(self): + return hash((self.strpath, self.__class__, self.auth)) + + +class WCStatus: + attrnames = ('modified','added', 'conflict', 'unchanged', 'external', + 'deleted', 'prop_modified', 'unknown', 'update_available', + 'incomplete', 'kindmismatch', 'ignored', 'locked', 'replaced' + ) + + def __init__(self, wcpath, rev=None, modrev=None, author=None): + self.wcpath = wcpath + self.rev = rev + self.modrev = modrev + self.author = author + + for name in self.attrnames: + setattr(self, name, []) + + def allpath(self, sort=True, **kw): + d = {} + for name in self.attrnames: + if name not in kw or kw[name]: + for path in getattr(self, name): + d[path] = 1 + l = d.keys() + if sort: + l.sort() + return l + + # XXX a bit scary to assume there's always 2 spaces between username and + # path, however with win32 allowing spaces in user names there doesn't + # seem to be a more solid approach :( + _rex_status = re.compile(r'\s+(\d+|-)\s+(\S+)\s+(.+?)\s{2,}(.*)') + + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ return a new WCStatus object from data 's' + """ + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + for line in data.split('\n'): + if not line.strip(): + continue + #print "processing %r" % line + flags, rest = line[:8], line[8:] + # first column + c0,c1,c2,c3,c4,c5,x6,c7 = flags + #if '*' in line: + # print "flags", repr(flags), "rest", repr(rest) + + if c0 in '?XI': + fn = line.split(None, 1)[1] + if c0 == '?': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.unknown.append(wcpath) + elif c0 == 'X': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(fn, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + elif c0 == 'I': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.ignored.append(wcpath) + + continue + + #elif c0 in '~!' or c4 == 'S': + # raise NotImplementedError("received flag %r" % c0) + + m = WCStatus._rex_status.match(rest) + if not m: + if c7 == '*': + fn = rest.strip() + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.update_available.append(wcpath) + continue + if line.lower().find('against revision:')!=-1: + update_rev = int(rest.split(':')[1].strip()) + continue + if line.lower().find('status on external') > -1: + # XXX not sure what to do here... perhaps we want to + # store some state instead of just continuing, as right + # now it makes the top-level external get added twice + # (once as external, once as 'normal' unchanged item) + # because of the way SVN presents external items + continue + # keep trying + raise ValueError("could not parse line %r" % line) + else: + rev, modrev, author, fn = m.groups() + wcpath = rootwcpath.join(fn, abs=1) + #assert wcpath.check() + if c0 == 'M': + assert wcpath.check(file=1), "didn't expect a directory with changed content here" + rootstatus.modified.append(wcpath) + elif c0 == 'A' or c3 == '+' : + rootstatus.added.append(wcpath) + elif c0 == 'D': + rootstatus.deleted.append(wcpath) + elif c0 == 'C': + rootstatus.conflict.append(wcpath) + elif c0 == '~': + rootstatus.kindmismatch.append(wcpath) + elif c0 == '!': + rootstatus.incomplete.append(wcpath) + elif c0 == 'R': + rootstatus.replaced.append(wcpath) + elif not c0.strip(): + rootstatus.unchanged.append(wcpath) + else: + raise NotImplementedError("received flag %r" % c0) + + if c1 == 'M': + rootstatus.prop_modified.append(wcpath) + # XXX do we cover all client versions here? + if c2 == 'L' or c5 == 'K': + rootstatus.locked.append(wcpath) + if c7 == '*': + rootstatus.update_available.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + if update_rev: + rootstatus.update_rev = update_rev + continue + return rootstatus + fromstring = staticmethod(fromstring) + +class XMLWCStatus(WCStatus): + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ parse 'data' (XML string as outputted by svn st) into a status obj + """ + # XXX for externals, the path is shown twice: once + # with external information, and once with full info as if + # the item was a normal non-external... the current way of + # dealing with this issue is by ignoring it - this does make + # externals appear as external items as well as 'normal', + # unchanged ones in the status object so this is far from ideal + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + minidom, ExpatError = importxml() + try: + doc = minidom.parseString(data) + except ExpatError: + e = sys.exc_info()[1] + raise ValueError(str(e)) + urevels = doc.getElementsByTagName('against') + if urevels: + rootstatus.update_rev = urevels[-1].getAttribute('revision') + for entryel in doc.getElementsByTagName('entry'): + path = entryel.getAttribute('path') + statusel = entryel.getElementsByTagName('wc-status')[0] + itemstatus = statusel.getAttribute('item') + + if itemstatus == 'unversioned': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.unknown.append(wcpath) + continue + elif itemstatus == 'external': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(path, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + continue + elif itemstatus == 'ignored': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.ignored.append(wcpath) + continue + elif itemstatus == 'incomplete': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.incomplete.append(wcpath) + continue + + rev = statusel.getAttribute('revision') + if itemstatus == 'added' or itemstatus == 'none': + rev = '0' + modrev = '?' + author = '?' + date = '' + else: + #print entryel.toxml() + commitel = entryel.getElementsByTagName('commit')[0] + if commitel: + modrev = commitel.getAttribute('revision') + author = '' + author_els = commitel.getElementsByTagName('author') + if author_els: + for c in author_els[0].childNodes: + author += c.nodeValue + date = '' + for c in commitel.getElementsByTagName('date')[0]\ + .childNodes: + date += c.nodeValue + + wcpath = rootwcpath.join(path, abs=1) + + assert itemstatus != 'modified' or wcpath.check(file=1), ( + 'did\'t expect a directory with changed content here') + + itemattrname = { + 'normal': 'unchanged', + 'unversioned': 'unknown', + 'conflicted': 'conflict', + 'none': 'added', + }.get(itemstatus, itemstatus) + + attr = getattr(rootstatus, itemattrname) + attr.append(wcpath) + + propsstatus = statusel.getAttribute('props') + if propsstatus not in ('none', 'normal'): + rootstatus.prop_modified.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + rootstatus.date = date + + # handle repos-status element (remote info) + rstatusels = entryel.getElementsByTagName('repos-status') + if rstatusels: + rstatusel = rstatusels[0] + ritemstatus = rstatusel.getAttribute('item') + if ritemstatus in ('added', 'modified'): + rootstatus.update_available.append(wcpath) + + lockels = entryel.getElementsByTagName('lock') + if len(lockels): + rootstatus.locked.append(wcpath) + + return rootstatus + fromstring = staticmethod(fromstring) + +class InfoSvnWCCommand: + def __init__(self, output): + # Path: test + # URL: http://codespeak.net/svn/std.path/trunk/dist/std.path/test + # Repository UUID: fd0d7bf2-dfb6-0310-8d31-b7ecfe96aada + # Revision: 2151 + # Node Kind: directory + # Schedule: normal + # Last Changed Author: hpk + # Last Changed Rev: 2100 + # Last Changed Date: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + # Properties Last Updated: 2003-11-03 14:47:48 +0100 (Mon, 03 Nov 2003) + + d = {} + for line in output.split('\n'): + if not line.strip(): + continue + key, value = line.split(':', 1) + key = key.lower().replace(' ', '') + value = value.strip() + d[key] = value + try: + self.url = d['url'] + except KeyError: + raise ValueError("Not a versioned resource") + #raise ValueError, "Not a versioned resource %r" % path + self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind'] + self.rev = int(d['revision']) + self.path = py.path.local(d['path']) + self.size = self.path.size() + if 'lastchangedrev' in d: + self.created_rev = int(d['lastchangedrev']) + if 'lastchangedauthor' in d: + self.last_author = d['lastchangedauthor'] + if 'lastchangeddate' in d: + self.mtime = parse_wcinfotime(d['lastchangeddate']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + +def parse_wcinfotime(timestr): + """ Returns seconds since epoch, UTC. """ + # example: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + m = re.match(r'(\d+-\d+-\d+ \d+:\d+:\d+) ([+-]\d+) .*', timestr) + if not m: + raise ValueError("timestring %r does not match" % timestr) + timestr, timezone = m.groups() + # do not handle timezone specially, return value should be UTC + parsedtime = time.strptime(timestr, "%Y-%m-%d %H:%M:%S") + return calendar.timegm(parsedtime) + +def make_recursive_propdict(wcroot, + output, + rex = re.compile("Properties on '(.*)':")): + """ Return a dictionary of path->PropListDict mappings. """ + lines = [x for x in output.split('\n') if x] + pdict = {} + while lines: + line = lines.pop(0) + m = rex.match(line) + if not m: + raise ValueError("could not parse propget-line: %r" % line) + path = m.groups()[0] + wcpath = wcroot.join(path, abs=1) + propnames = [] + while lines and lines[0].startswith(' '): + propname = lines.pop(0).strip() + propnames.append(propname) + assert propnames, "must have found properties!" + pdict[wcpath] = PropListDict(wcpath, propnames) + return pdict + + +def importxml(cache=[]): + if cache: + return cache + from xml.dom import minidom + from xml.parsers.expat import ExpatError + cache.extend([minidom, ExpatError]) + return cache + +class LogEntry: + def __init__(self, logentry): + self.rev = int(logentry.getAttribute('revision')) + for lpart in filter(None, logentry.childNodes): + if lpart.nodeType == lpart.ELEMENT_NODE: + if lpart.nodeName == 'author': + self.author = lpart.firstChild.nodeValue + elif lpart.nodeName == 'msg': + if lpart.firstChild: + self.msg = lpart.firstChild.nodeValue + else: + self.msg = '' + elif lpart.nodeName == 'date': + #2003-07-29T20:05:11.598637Z + timestr = lpart.firstChild.nodeValue + self.date = parse_apr_time(timestr) + elif lpart.nodeName == 'paths': + self.strpaths = [] + for ppart in filter(None, lpart.childNodes): + if ppart.nodeType == ppart.ELEMENT_NODE: + self.strpaths.append(PathEntry(ppart)) + def __repr__(self): + return '' % ( + self.rev, self.author, self.date) + + Added: pypy/branch/py11/py/impl/process/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/process/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +""" high-level sub-process handling """ Added: pypy/branch/py11/py/impl/process/cmdexec.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/process/cmdexec.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,44 @@ +""" + +""" + +import os, sys +import subprocess +import py +from subprocess import Popen, PIPE + +def cmdexec(cmd): + """ return output of executing 'cmd' in a separate process. + + raise cmdexec.ExecutionFailed exeception if the command failed. + the exception will provide an 'err' attribute containing + the error-output from the command. + """ + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + out = py.builtin._totext(out, sys.getdefaultencoding()) + err = py.builtin._totext(err, sys.getdefaultencoding()) + status = process.poll() + if status: + raise ExecutionFailed(status, status, cmd, out, err) + return out + +class ExecutionFailed(py.error.Error): + def __init__(self, status, systemstatus, cmd, out, err): + Exception.__init__(self) + self.status = status + self.systemstatus = systemstatus + self.cmd = cmd + self.err = err + self.out = out + + def __str__(self): + return "ExecutionFailed: %d %s\n%s" %(self.status, self.cmd, self.err) + +# export the exception under the name 'py.process.cmdexec.Error' +cmdexec.Error = ExecutionFailed +try: + ExecutionFailed.__module__ = 'py.process.cmdexec' + ExecutionFailed.__name__ = 'Error' +except (AttributeError, TypeError): + pass Added: pypy/branch/py11/py/impl/process/forkedfunc.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/process/forkedfunc.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,108 @@ + +""" + ForkedFunc provides a way to run a function in a forked process + and get at its return value, stdout and stderr output as well + as signals and exitstatusus. + + XXX see if tempdir handling is sane +""" + +import py +import os +import sys +import marshal + +class ForkedFunc(object): + EXITSTATUS_EXCEPTION = 3 + def __init__(self, fun, args=None, kwargs=None, nice_level=0): + if args is None: + args = [] + if kwargs is None: + kwargs = {} + self.fun = fun + self.args = args + self.kwargs = kwargs + self.tempdir = tempdir = py.path.local.mkdtemp() + self.RETVAL = tempdir.ensure('retval') + self.STDOUT = tempdir.ensure('stdout') + self.STDERR = tempdir.ensure('stderr') + + pid = os.fork() + if pid: # in parent process + self.pid = pid + else: # in child process + self._child(nice_level) + + def _child(self, nice_level): + # right now we need to call a function, but first we need to + # map all IO that might happen + # make sure sys.stdout points to file descriptor one + sys.stdout = stdout = self.STDOUT.open('w') + sys.stdout.flush() + fdstdout = stdout.fileno() + if fdstdout != 1: + os.dup2(fdstdout, 1) + sys.stderr = stderr = self.STDERR.open('w') + fdstderr = stderr.fileno() + if fdstderr != 2: + os.dup2(fdstderr, 2) + retvalf = self.RETVAL.open("wb") + EXITSTATUS = 0 + try: + if nice_level: + os.nice(nice_level) + try: + retval = self.fun(*self.args, **self.kwargs) + retvalf.write(marshal.dumps(retval)) + except: + excinfo = py.code.ExceptionInfo() + stderr.write(excinfo.exconly()) + EXITSTATUS = self.EXITSTATUS_EXCEPTION + finally: + stdout.close() + stderr.close() + retvalf.close() + os.close(1) + os.close(2) + os._exit(EXITSTATUS) + + def waitfinish(self, waiter=os.waitpid): + pid, systemstatus = waiter(self.pid, 0) + if systemstatus: + if os.WIFSIGNALED(systemstatus): + exitstatus = os.WTERMSIG(systemstatus) + 128 + else: + exitstatus = os.WEXITSTATUS(systemstatus) + #raise ExecutionFailed(status, systemstatus, cmd, + # ''.join(out), ''.join(err)) + else: + exitstatus = 0 + signal = systemstatus & 0x7f + if not exitstatus and not signal: + retval = self.RETVAL.open('rb') + try: + retval_data = retval.read() + finally: + retval.close() + retval = marshal.loads(retval_data) + else: + retval = None + stdout = self.STDOUT.read() + stderr = self.STDERR.read() + self._removetemp() + return Result(exitstatus, signal, retval, stdout, stderr) + + def _removetemp(self): + if self.tempdir.check(): + self.tempdir.remove() + + def __del__(self): + self._removetemp() + +class Result(object): + def __init__(self, exitstatus, signal, retval, stdout, stderr): + self.exitstatus = exitstatus + self.signal = signal + self.retval = retval + self.out = stdout + self.err = stderr Added: pypy/branch/py11/py/impl/process/killproc.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/process/killproc.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,23 @@ +import py +import os, sys + +if sys.platform == "win32": + try: + import ctypes + except ImportError: + def dokill(pid): + py.process.cmdexec("taskkill /F /PID %d" %(pid,)) + else: + def dokill(pid): + PROCESS_TERMINATE = 1 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_TERMINATE, False, pid) + ctypes.windll.kernel32.TerminateProcess(handle, -1) + ctypes.windll.kernel32.CloseHandle(handle) +else: + def dokill(pid): + os.kill(pid, 15) + +def kill(pid): + """ kill process by id. """ + dokill(pid) Added: pypy/branch/py11/py/impl/std.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/std.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,18 @@ +import sys + +class Std(object): + """ makes top-level python modules available as an attribute, + importing them on first access. + """ + + def __init__(self): + self.__dict__ = sys.modules + + def __getattr__(self, name): + try: + m = __import__(name) + except ImportError: + raise AttributeError("py.std: could not import %s" % name) + return m + +std = Std() Added: pypy/branch/py11/py/impl/test/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +""" versatile unit-testing tool + libraries """ Added: pypy/branch/py11/py/impl/test/cmdline.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/cmdline.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,23 @@ +import py +import sys + +# +# main entry point +# + +def main(args=None): + if args is None: + args = sys.argv[1:] + config = py.test.config + try: + config.parse(args) + config.pluginmanager.do_configure(config) + session = config.initsession() + exitstatus = session.main() + config.pluginmanager.do_unconfigure(config) + raise SystemExit(exitstatus) + except config.Error: + e = sys.exc_info()[1] + sys.stderr.write("ERROR: %s\n" %(e.args[0],)) + raise SystemExit(3) + Added: pypy/branch/py11/py/impl/test/collect.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/collect.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,466 @@ +""" +base test collection objects. +Collectors and test Items form a tree +that is usually built iteratively. +""" +import py + +def configproperty(name): + def fget(self): + #print "retrieving %r property from %s" %(name, self.fspath) + return self.config.getvalue(name, self.fspath) + return property(fget) + +class Node(object): + """ base class for Nodes in the collection tree. + Collector nodes have children and + Item nodes are terminal. + + All nodes of the collection tree carry a _config + attribute for these reasons: + - to access custom Collection Nodes from a project + (defined in conftest's) + - to pickle themselves relatively to the "topdir" + - configuration/options for setup/teardown + stdout/stderr capturing and execution of test items + """ + def __init__(self, name, parent=None): + self.name = name + self.parent = parent + self.config = getattr(parent, 'config', None) + self.fspath = getattr(parent, 'fspath', None) + + def _checkcollectable(self): + if not hasattr(self, 'fspath'): + self.parent._memocollect() # to reraise exception + + # + # note to myself: Pickling is uh. + # + def __getstate__(self): + return (self.name, self.parent) + def __setstate__(self, nameparent): + name, parent = nameparent + try: + colitems = parent._memocollect() + except KeyboardInterrupt: + raise + except Exception: + # seems our parent can't collect us + # so let's be somewhat operable + # _checkcollectable() is to tell outsiders about the fact + self.name = name + self.parent = parent + self.config = parent.config + #self._obj = "could not unpickle" + else: + for colitem in colitems: + if colitem.name == name: + # we are a copy that will not be returned + # by our parent + self.__dict__ = colitem.__dict__ + break + + def __repr__(self): + if getattr(self.config.option, 'debug', False): + return "<%s %r %0x>" %(self.__class__.__name__, + getattr(self, 'name', None), id(self)) + else: + return "<%s %r>" %(self.__class__.__name__, + getattr(self, 'name', None)) + + # methods for ordering nodes + + def __eq__(self, other): + if not isinstance(other, Node): + return False + return self.name == other.name and self.parent == other.parent + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.name, self.parent)) + + def setup(self): + pass + + def teardown(self): + pass + + def _memoizedcall(self, attrname, function): + exattrname = "_ex_" + attrname + failure = getattr(self, exattrname, None) + if failure is not None: + py.builtin._reraise(failure[0], failure[1], failure[2]) + if hasattr(self, attrname): + return getattr(self, attrname) + try: + res = function() + except (KeyboardInterrupt, SystemExit): + raise + except: + failure = py.std.sys.exc_info() + setattr(self, exattrname, failure) + raise + setattr(self, attrname, res) + return res + + def listchain(self, rootfirst=False): + """ return list of all parent collectors up to self, + starting form root of collection tree. """ + l = [self] + while 1: + x = l[-1] + if x.parent is not None: + l.append(x.parent) + else: + if not rootfirst: + l.reverse() + return l + + def listnames(self): + return [x.name for x in self.listchain()] + + def getparent(self, cls): + current = self + while current and not isinstance(current, cls): + current = current.parent + return current + + def _getitembynames(self, namelist): + cur = self + for name in namelist: + if name: + next = cur.collect_by_name(name) + if next is None: + existingnames = [x.name for x in self._memocollect()] + msg = ("Collector %r does not have name %r " + "existing names are: %s" % + (cur, name, existingnames)) + raise AssertionError(msg) + cur = next + return cur + + + def _getfsnode(self, path): + # this method is usually called from + # config.getfsnode() which returns a colitem + # from filename arguments + # + # pytest's collector tree does not neccessarily + # follow the filesystem and we thus need to do + # some special matching code here because + # _getitembynames() works by colitem names, not + # basenames. + if path == self.fspath: + return self + basenames = path.relto(self.fspath).split(path.sep) + cur = self + while basenames: + basename = basenames.pop(0) + assert basename + fspath = cur.fspath.join(basename) + colitems = cur._memocollect() + l = [] + for colitem in colitems: + if colitem.fspath == fspath or colitem.name == basename: + l.append(colitem) + if not l: + raise self.config.Error("can't collect: %s" %(fspath,)) + if basenames: + if len(l) > 1: + msg = ("Collector %r has more than one %r colitem " + "existing colitems are: %s" % + (cur, fspath, colitems)) + raise self.config.Error("xxx-too many test types for: %s" % (fspath, )) + cur = l[0] + else: + if len(l) > 1: + cur = l + else: + cur = l[0] + break + return cur + + def readkeywords(self): + return dict([(x, True) for x in self._keywords()]) + + def _keywords(self): + return [self.name] + + def _skipbykeyword(self, keywordexpr): + """ return True if they given keyword expression means to + skip this collector/item. + """ + if not keywordexpr: + return + chain = self.listchain() + for key in filter(None, keywordexpr.split()): + eor = key[:1] == '-' + if eor: + key = key[1:] + if not (eor ^ self._matchonekeyword(key, chain)): + return True + + def _matchonekeyword(self, key, chain): + elems = key.split(".") + # XXX O(n^2), anyone cares? + chain = [item.readkeywords() for item in chain if item._keywords()] + for start, _ in enumerate(chain): + if start + len(elems) > len(chain): + return False + for num, elem in enumerate(elems): + for keyword in chain[num + start]: + ok = False + if elem in keyword: + ok = True + break + if not ok: + break + if num == len(elems) - 1 and ok: + return True + return False + + def _prunetraceback(self, traceback): + return traceback + + def _totrail(self): + """ provide a trail relative to the topdir, + which can be used to reconstruct the + collector (possibly on a different host + starting from a different topdir). + """ + chain = self.listchain() + topdir = self.config.topdir + relpath = chain[0].fspath.relto(topdir) + if not relpath: + if chain[0].fspath == topdir: + relpath = "." + else: + raise ValueError("%r not relative to topdir %s" + %(chain[0].fspath, topdir)) + return relpath, tuple([x.name for x in chain[1:]]) + + def _fromtrail(trail, config): + relpath, names = trail + fspath = config.topdir.join(relpath) + col = config.getfsnode(fspath) + return col._getitembynames(names) + _fromtrail = staticmethod(_fromtrail) + + def _repr_failure_py(self, excinfo, outerr=None): + assert outerr is None, "XXX deprecated" + excinfo.traceback = self._prunetraceback(excinfo.traceback) + # XXX temporary hack: getrepr() should not take a 'style' argument + # at all; it should record all data in all cases, and the style + # should be parametrized in toterminal(). + if self.config.option.tbstyle == "short": + style = "short" + else: + style = "long" + return excinfo.getrepr(funcargs=True, + showlocals=self.config.option.showlocals, + style=style) + + repr_failure = _repr_failure_py + shortfailurerepr = "F" + +class Collector(Node): + """ + Collector instances create children through collect() + and thus iteratively build a tree. attributes:: + + parent: attribute pointing to the parent collector + (or None if this is the root collector) + name: basename of this collector object + """ + Directory = configproperty('Directory') + Module = configproperty('Module') + + def collect(self): + """ returns a list of children (items and collectors) + for this collection node. + """ + raise NotImplementedError("abstract") + + def collect_by_name(self, name): + """ return a child matching the given name, else None. """ + for colitem in self._memocollect(): + if colitem.name == name: + return colitem + + def repr_failure(self, excinfo, outerr=None): + """ represent a failure. """ + assert outerr is None, "XXX deprecated" + return self._repr_failure_py(excinfo) + + def _memocollect(self): + """ internal helper method to cache results of calling collect(). """ + return self._memoizedcall('_collected', self.collect) + + # ********************************************************************** + # DEPRECATED METHODS + # ********************************************************************** + + def _deprecated_collect(self): + # avoid recursion: + # collect -> _deprecated_collect -> custom run() -> + # super().run() -> collect + attrname = '_depcollectentered' + if hasattr(self, attrname): + return + setattr(self, attrname, True) + method = getattr(self.__class__, 'run', None) + if method is not None and method != Collector.run: + warnoldcollect(function=method) + names = self.run() + return [x for x in [self.join(name) for name in names] if x] + + def run(self): + """ DEPRECATED: returns a list of names available from this collector. + You can return an empty list. Callers of this method + must take care to catch exceptions properly. + """ + return [colitem.name for colitem in self._memocollect()] + + def join(self, name): + """ DEPRECATED: return a child collector or item for the given name. + If the return value is None there is no such child. + """ + return self.collect_by_name(name) + + def _prunetraceback(self, traceback): + if hasattr(self, 'fspath'): + path = self.fspath + ntraceback = traceback.cut(path=self.fspath) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._dir) + traceback = ntraceback.filter() + return traceback + +class FSCollector(Collector): + def __init__(self, fspath, parent=None): + fspath = py.path.local(fspath) + super(FSCollector, self).__init__(fspath.basename, parent) + self.fspath = fspath + + def __getstate__(self): + if self.parent is None: + # the root node needs to pickle more context info + topdir = self.config.topdir + relpath = self.fspath.relto(topdir) + if not relpath: + if self.fspath == topdir: + relpath = "." + else: + raise ValueError("%r not relative to topdir %s" + %(self.fspath, topdir)) + return (self.name, self.config, relpath) + else: + return (self.name, self.parent) + + def __setstate__(self, picklestate): + if len(picklestate) == 3: + # root node + name, config, relpath = picklestate + fspath = config.topdir.join(relpath) + fsnode = config.getfsnode(fspath) + self.__dict__.update(fsnode.__dict__) + else: + name, parent = picklestate + self.__init__(parent.fspath.join(name), parent=parent) + +class File(FSCollector): + """ base class for collecting tests from a file. """ + +class Directory(FSCollector): + def recfilter(self, path): + if path.check(dir=1, dotfile=0): + return path.basename not in ('CVS', '_darcs', '{arch}') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + l = [] + for path in self.fspath.listdir(sort=True): + res = self.consider(path) + if res is not None: + if isinstance(res, (list, tuple)): + l.extend(res) + else: + l.append(res) + return l + + def _ignore(self, path): + ignore_paths = self.config.getconftest_pathlist("collect_ignore", path=path) + return ignore_paths and path in ignore_paths + # XXX more refined would be: + if ignore_paths: + for p in ignore_paths: + if path == p or path.relto(p): + return True + + def consider(self, path): + if self._ignore(path): + return + if path.check(file=1): + res = self.consider_file(path) + elif path.check(dir=1): + res = self.consider_dir(path) + else: + res = None + if isinstance(res, list): + # throw out identical results + l = [] + for x in res: + if x not in l: + assert x.parent == self, "wrong collection tree construction" + l.append(x) + res = l + return res + + def consider_file(self, path): + return self.config.hook.pytest_collect_file(path=path, parent=self) + + def consider_dir(self, path, usefilters=None): + if usefilters is not None: + py.log._apiwarn("0.99", "usefilters argument not needed") + return self.config.hook.pytest_collect_directory( + path=path, parent=self) + +class Item(Node): + """ a basic test item. """ + def _deprecated_testexecution(self): + if self.__class__.run != Item.run: + warnoldtestrun(function=self.run) + elif self.__class__.execute != Item.execute: + warnoldtestrun(function=self.execute) + else: + return False + self.run() + return True + + def run(self): + """ deprecated, here because subclasses might call it. """ + return self.execute(self.obj) + + def execute(self, obj): + """ deprecated, here because subclasses might call it. """ + return obj() + + def reportinfo(self): + return self.fspath, None, "" + +def warnoldcollect(function=None): + py.log._apiwarn("1.0", + "implement collector.collect() instead of " + "collector.run() and collector.join()", + stacklevel=2, function=function) + +def warnoldtestrun(function=None): + py.log._apiwarn("1.0", + "implement item.runtest() instead of " + "item.run() and item.execute()", + stacklevel=2, function=function) Added: pypy/branch/py11/py/impl/test/compat.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/compat.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,58 @@ +import py + +from py.test.collect import Function + +class TestCaseUnit(Function): + """ compatibility Unit executor for TestCase methods + honouring setUp and tearDown semantics. + """ + def runtest(self, _deprecated=None): + boundmethod = self.obj + instance = py.builtin._getimself(boundmethod) + instance.setUp() + try: + boundmethod() + finally: + instance.tearDown() + +class TestCase(object): + """compatibility class of unittest's TestCase. """ + Function = TestCaseUnit + + def setUp(self): + pass + + def tearDown(self): + pass + + def fail(self, msg=None): + """ fail immediate with given message. """ + py.test.fail(msg) + + def assertRaises(self, excclass, func, *args, **kwargs): + py.test.raises(excclass, func, *args, **kwargs) + failUnlessRaises = assertRaises + + # dynamically construct (redundant) methods + aliasmap = [ + ('x', 'not x', 'assert_, failUnless'), + ('x', 'x', 'failIf'), + ('x,y', 'x!=y', 'failUnlessEqual,assertEqual, assertEquals'), + ('x,y', 'x==y', 'failIfEqual,assertNotEqual, assertNotEquals'), + ] + items = [] + for sig, expr, names in aliasmap: + names = map(str.strip, names.split(',')) + sigsubst = expr.replace('y', '%s').replace('x', '%s') + for name in names: + items.append(""" + def %(name)s(self, %(sig)s, msg=""): + __tracebackhide__ = True + if %(expr)s: + py.test.fail(msg=msg + (%(sigsubst)r %% (%(sig)s))) + """ % locals() ) + + source = "".join(items) + exec(py.code.Source(source).compile()) + +__all__ = ['TestCase'] Added: pypy/branch/py11/py/impl/test/config.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/config.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,307 @@ +import py, os +from py.impl.test.conftesthandle import Conftest + +from py.impl.test import parseopt + +def ensuretemp(string, dir=1): + """ return temporary directory path with + the given string as the trailing part. + """ + return py.test.config.ensuretemp(string, dir=dir) + +class CmdOptions(object): + """ pure container instance for holding cmdline options + as attributes. + """ + def __repr__(self): + return "" %(self.__dict__,) + +class Error(Exception): + """ Test Configuration Error. """ + +class Config(object): + """ test configuration object, provides access to config valueso, + the pluginmanager and plugin api. + """ + Option = py.std.optparse.Option + Error = Error + basetemp = None + _sessionclass = None + + def __init__(self, pluginmanager=None, topdir=None): + self.option = CmdOptions() + self.topdir = topdir + self._parser = parseopt.Parser( + usage="usage: %prog [options] [file_or_dir] [file_or_dir] [...]", + processopt=self._processopt, + ) + if pluginmanager is None: + pluginmanager = py.test._PluginManager() + assert isinstance(pluginmanager, py.test._PluginManager) + self.pluginmanager = pluginmanager + self._conftest = Conftest(onimport=self._onimportconftest) + self.hook = pluginmanager.hook + + def _onimportconftest(self, conftestmodule): + self.trace("loaded conftestmodule %r" %(conftestmodule,)) + self.pluginmanager.consider_conftest(conftestmodule) + + def trace(self, msg): + if getattr(self.option, 'traceconfig', None): + self.hook.pytest_trace(category="config", msg=msg) + + def _processopt(self, opt): + if hasattr(opt, 'default') and opt.dest: + val = os.environ.get("PYTEST_OPTION_" + opt.dest.upper(), None) + if val is not None: + if opt.type == "int": + val = int(val) + elif opt.type == "long": + val = long(val) + elif opt.type == "float": + val = float(val) + elif not opt.type and opt.action in ("store_true", "store_false"): + val = eval(val) + opt.default = val + else: + name = "option_" + opt.dest + try: + opt.default = self._conftest.rget(name) + except (ValueError, KeyError): + pass + if not hasattr(self.option, opt.dest): + setattr(self.option, opt.dest, opt.default) + + def _preparse(self, args): + self._conftest.setinitial(args) + self.pluginmanager.consider_preparse(args) + self.pluginmanager.consider_env() + self.pluginmanager.do_addoption(self._parser) + + def parse(self, args): + """ parse cmdline arguments into this config object. + Note that this can only be called once per testing process. + """ + assert not hasattr(self, 'args'), ( + "can only parse cmdline args at most once per Config object") + self._preparse(args) + args = self._parser.parse_setoption(args, self.option) + if not args: + args.append(py.std.os.getcwd()) + self.topdir = gettopdir(args) + self.args = [py.path.local(x) for x in args] + + # config objects are usually pickled across system + # barriers but they contain filesystem paths. + # upon getstate/setstate we take care to do everything + # relative to "topdir". + def __getstate__(self): + l = [] + for path in self.args: + path = py.path.local(path) + l.append(path.relto(self.topdir)) + return l, self.option + + def __setstate__(self, repr): + # warning global side effects: + # * registering to py lib plugins + # * setting py.test.config + self.__init__( + pluginmanager=py.test._PluginManager(py._com.comregistry), + topdir=py.path.local(), + ) + # we have to set py.test.config because preparse() + # might load conftest files which have + # py.test.config.addoptions() lines in them + py.test.config = self + args, cmdlineopts = repr + args = [self.topdir.join(x) for x in args] + self.option = cmdlineopts + self._preparse(args) + self.args = args + + def ensuretemp(self, string, dir=True): + return self.getbasetemp().ensure(string, dir=dir) + + def getbasetemp(self): + if self.basetemp is None: + basetemp = self.option.basetemp + if basetemp: + basetemp = py.path.local(basetemp) + if not basetemp.check(dir=1): + basetemp.mkdir() + else: + basetemp = py.path.local.make_numbered_dir(prefix='pytest-') + self.basetemp = basetemp + return self.basetemp + + def mktemp(self, basename, numbered=False): + basetemp = self.getbasetemp() + if not numbered: + return basetemp.mkdir(basename) + else: + return py.path.local.make_numbered_dir(prefix=basename + "-", + keep=0, rootdir=basetemp, lock_timeout=None) + + def getcolitems(self): + return [self.getfsnode(arg) for arg in self.args] + + def getfsnode(self, path): + path = py.path.local(path) + if not path.check(): + raise self.Error("file not found: %s" %(path,)) + # we want our possibly custom collection tree to start at pkgroot + pkgpath = path.pypkgpath() + if pkgpath is None: + pkgpath = path.check(file=1) and path.dirpath() or path + Dir = self._conftest.rget("Directory", pkgpath) + col = Dir(pkgpath) + col.config = self + return col._getfsnode(path) + + def getconftest_pathlist(self, name, path=None): + """ return a matching value, which needs to be sequence + of filenames that will be returned as a list of Path + objects (they can be relative to the location + where they were found). + """ + try: + mod, relroots = self._conftest.rget_with_confmod(name, path) + except KeyError: + return None + modpath = py.path.local(mod.__file__).dirpath() + l = [] + for relroot in relroots: + relroot = relroot.replace("/", py.path.local.sep) + l.append(modpath.join(relroot, abs=True)) + return l + + def addoptions(self, groupname, *specs): + """ add a named group of options to the current testing session. + This function gets invoked during testing session initialization. + """ + py.log._apiwarn("1.0", "define plugins to add options", stacklevel=2) + group = self._parser.addgroup(groupname) + for opt in specs: + group._addoption_instance(opt) + return self.option + + def addoption(self, *optnames, **attrs): + return self._parser.addoption(*optnames, **attrs) + + def getvalueorskip(self, name, path=None): + """ return getvalue() or call py.test.skip if no value exists. """ + try: + val = self.getvalue(name, path) + if val is None: + raise KeyError(name) + return val + except KeyError: + py.test.skip("no %r value found" %(name,)) + + def getvalue(self, name, path=None): + """ return 'name' value looked up from the 'options' + and then from the first conftest file found up + the path (including the path itself). + if path is None, lookup the value in the initial + conftest modules found during command line parsing. + """ + try: + return getattr(self.option, name) + except AttributeError: + return self._conftest.rget(name, path) + + def setsessionclass(self, cls): + if self._sessionclass is not None: + raise ValueError("sessionclass already set to: %r" %( + self._sessionclass)) + self._sessionclass = cls + + def initsession(self): + """ return an initialized session object. """ + cls = self._sessionclass + if cls is None: + from py.impl.test.session import Session + cls = Session + session = cls(self) + self.trace("instantiated session %r" % session) + return session + + def _reparse(self, args): + """ this is used from tests that want to re-invoke parse(). """ + #assert args # XXX should not be empty + global config_per_process + oldconfig = py.test.config + try: + config_per_process = py.test.config = Config() + config_per_process.basetemp = self.mktemp("reparse", numbered=True) + config_per_process.parse(args) + return config_per_process + finally: + config_per_process = py.test.config = oldconfig + + def getxspecs(self): + xspeclist = [] + for xspec in self.getvalue("tx"): + i = xspec.find("*") + try: + num = int(xspec[:i]) + except ValueError: + xspeclist.append(xspec) + else: + xspeclist.extend([xspec[i+1:]] * num) + if not xspeclist: + raise self.Error("MISSING test execution (tx) nodes: please specify --tx") + import execnet + return [execnet.XSpec(x) for x in xspeclist] + + def getrsyncdirs(self): + config = self + roots = config.option.rsyncdir + conftestroots = config.getconftest_pathlist("rsyncdirs") + if conftestroots: + roots.extend(conftestroots) + pydirs = [x.realpath() for x in py._pydirs] + roots = [py.path.local(root) for root in roots] + for root in roots: + if not root.check(): + raise config.Error("rsyncdir doesn't exist: %r" %(root,)) + if pydirs is not None and root.basename in ("py", "_py"): + pydirs.remove(root) # otherwise it's a conflict + roots.extend(pydirs) + return roots + +# +# helpers +# + +def checkmarshal(name, value): + try: + py.std.marshal.dumps(value) + except ValueError: + raise ValueError("%s=%r is not marshallable" %(name, value)) + +def gettopdir(args): + """ return the top directory for the given paths. + if the common base dir resides in a python package + parent directory of the root package is returned. + """ + args = [py.path.local(arg) for arg in args] + p = args and args[0] or None + for x in args[1:]: + p = p.common(x) + assert p, "cannot determine common basedir of %s" %(args,) + pkgdir = p.pypkgpath() + if pkgdir is None: + if p.check(file=1): + p = p.dirpath() + return p + else: + return pkgdir.dirpath() + + +# this is the one per-process instance of py.test configuration +config_per_process = Config( + pluginmanager=py.test._PluginManager(py._com.comregistry) +) + Added: pypy/branch/py11/py/impl/test/conftesthandle.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/conftesthandle.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,82 @@ +import py +defaultconftestpath = py.path.local(__file__).dirpath("defaultconftest.py") + +class Conftest(object): + """ the single place for accessing values and interacting + towards conftest modules from py.test objects. + + Note that triggering Conftest instances to import + conftest.py files may result in added cmdline options. + XXX + """ + def __init__(self, path=None, onimport=None): + self._path2confmods = {} + self._onimport = onimport + if path is not None: + self.setinitial([path]) + + def setinitial(self, args): + """ try to find a first anchor path for looking up global values + from conftests. This function is usually called _before_ + argument parsing. conftest files may add command line options + and we thus have no completely safe way of determining + which parts of the arguments are actually related to options + and which are file system paths. We just try here to get + bootstrapped ... + """ + current = py.path.local() + for arg in args + [current]: + anchor = current.join(arg, abs=1) + if anchor.check(): # we found some file object + self._path2confmods[None] = self.getconftestmodules(anchor) + break + else: + assert 0, "no root of filesystem?" + + def getconftestmodules(self, path): + """ return a list of imported conftest modules for the given path. """ + try: + clist = self._path2confmods[path] + except KeyError: + if path is None: + raise ValueError("missing default conftest.") + dp = path.dirpath() + if dp == path: + return [self.importconftest(defaultconftestpath)] + clist = self.getconftestmodules(dp) + conftestpath = path.join("conftest.py") + if conftestpath.check(file=1): + clist.append(self.importconftest(conftestpath)) + self._path2confmods[path] = clist + # be defensive: avoid changes from caller side to + # affect us by always returning a copy of the actual list + return clist[:] + + def rget(self, name, path=None): + mod, value = self.rget_with_confmod(name, path) + return value + + def rget_with_confmod(self, name, path=None): + modules = self.getconftestmodules(path) + modules.reverse() + for mod in modules: + try: + return mod, getattr(mod, name) + except AttributeError: + continue + raise KeyError(name) + + def importconftest(self, conftestpath): + # Using caching here looks redundant since ultimately + # sys.modules caches already + assert conftestpath.check(), conftestpath + if not conftestpath.dirpath('__init__.py').check(file=1): + # HACK: we don't want any "globally" imported conftest.py, + # prone to conflicts and subtle problems + modname = str(conftestpath).replace('.', conftestpath.sep) + mod = conftestpath.pyimport(modname=modname) + else: + mod = conftestpath.pyimport() + if self._onimport: + self._onimport(mod) + return mod Added: pypy/branch/py11/py/impl/test/defaultconftest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/defaultconftest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,14 @@ +import py + +Module = py.test.collect.Module +Directory = py.test.collect.Directory +File = py.test.collect.File + +# python collectors +Class = py.test.collect.Class +Generator = py.test.collect.Generator +Function = py.test.collect.Function +Instance = py.test.collect.Instance + +pytest_plugins = "default runner capture terminal mark skipping tmpdir monkeypatch recwarn pdb pastebin unittest helpconfig nose assertion".split() + Added: pypy/branch/py11/py/impl/test/dist/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/py11/py/impl/test/dist/dsession.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/dsession.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,280 @@ +""" + + EXPERIMENTAL dsession session (for dist/non-dist unification) + +""" + +import py +from py.impl.test.session import Session +from py.impl.test import outcome +from py.impl.test.dist.nodemanage import NodeManager +queue = py.builtin._tryimport('queue', 'Queue') + +debug_file = None # open('/tmp/loop.log', 'w') +def debug(*args): + if debug_file is not None: + s = " ".join(map(str, args)) + debug_file.write(s+"\n") + debug_file.flush() + +class LoopState(object): + def __init__(self, dsession, colitems): + self.dsession = dsession + self.colitems = colitems + self.exitstatus = None + # loopstate.dowork is False after reschedule events + # because otherwise we might very busily loop + # waiting for a host to become ready. + self.dowork = True + self.shuttingdown = False + self.testsfailed = False + + def __repr__(self): + return "" % ( + self.exitstatus, self.shuttingdown, len(self.colitems)) + + def pytest_runtest_logreport(self, report): + if report.item in self.dsession.item2nodes: + if report.when != "teardown": # otherwise we already managed it + self.dsession.removeitem(report.item, report.node) + if report.failed: + self.testsfailed = True + + def pytest_collectreport(self, report): + if report.passed: + self.colitems.extend(report.result) + + def pytest_testnodeready(self, node): + self.dsession.addnode(node) + + def pytest_testnodedown(self, node, error=None): + pending = self.dsession.removenode(node) + if pending: + if error: + crashitem = pending[0] + debug("determined crashitem", crashitem) + self.dsession.handle_crashitem(crashitem, node) + # XXX recovery handling for "each"? + # currently pending items are not retried + if self.dsession.config.option.dist == "load": + self.colitems.extend(pending[1:]) + + def pytest_rescheduleitems(self, items): + self.colitems.extend(items) + self.dowork = False # avoid busywait + +class DSession(Session): + """ + Session drives the collection and running of tests + and generates test events for reporters. + """ + MAXITEMSPERHOST = 15 + + def __init__(self, config): + self.queue = queue.Queue() + self.node2pending = {} + self.item2nodes = {} + super(DSession, self).__init__(config=config) + + #def pytest_configure(self, __multicall__, config): + # __multicall__.execute() + # try: + # config.getxspecs() + # except config.Error: + # print + # raise config.Error("dist mode %r needs test execution environments, " + # "none found." %(config.option.dist)) + + def main(self, colitems=None): + colitems = self.getinitialitems(colitems) + self.sessionstarts() + self.setup() + exitstatus = self.loop(colitems) + self.teardown() + self.sessionfinishes(exitstatus=exitstatus) + return exitstatus + + def loop_once(self, loopstate): + if loopstate.shuttingdown: + return self.loop_once_shutdown(loopstate) + colitems = loopstate.colitems + if loopstate.dowork and colitems: + self.triggertesting(loopstate.colitems) + colitems[:] = [] + # we use a timeout here so that control-C gets through + while 1: + try: + eventcall = self.queue.get(timeout=2.0) + break + except queue.Empty: + continue + loopstate.dowork = True + + callname, args, kwargs = eventcall + if callname is not None: + call = getattr(self.config.hook, callname) + assert not args + call(**kwargs) + + # termination conditions + if ((loopstate.testsfailed and self.config.option.exitfirst) or + (not self.item2nodes and not colitems and not self.queue.qsize())): + self.triggershutdown() + loopstate.shuttingdown = True + elif not self.node2pending: + loopstate.exitstatus = outcome.EXIT_NOHOSTS + + def loop_once_shutdown(self, loopstate): + # once we are in shutdown mode we dont send + # events other than HostDown upstream + eventname, args, kwargs = self.queue.get() + if eventname == "pytest_testnodedown": + self.config.hook.pytest_testnodedown(**kwargs) + self.removenode(kwargs['node']) + elif eventname == "pytest_runtest_logreport": + # might be some teardown report + self.config.hook.pytest_runtest_logreport(**kwargs) + if not self.node2pending: + # finished + if loopstate.testsfailed: + loopstate.exitstatus = outcome.EXIT_TESTSFAILED + else: + loopstate.exitstatus = outcome.EXIT_OK + #self.config.pluginmanager.unregister(loopstate) + + def _initloopstate(self, colitems): + loopstate = LoopState(self, colitems) + self.config.pluginmanager.register(loopstate) + return loopstate + + def loop(self, colitems): + try: + loopstate = self._initloopstate(colitems) + loopstate.dowork = False # first receive at least one HostUp events + while 1: + self.loop_once(loopstate) + if loopstate.exitstatus is not None: + exitstatus = loopstate.exitstatus + break + except KeyboardInterrupt: + excinfo = py.code.ExceptionInfo() + self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + exitstatus = outcome.EXIT_INTERRUPTED + except: + self.config.pluginmanager.notify_exception() + exitstatus = outcome.EXIT_INTERNALERROR + self.config.pluginmanager.unregister(loopstate) + if exitstatus == 0 and self._testsfailed: + exitstatus = outcome.EXIT_TESTSFAILED + return exitstatus + + def triggershutdown(self): + for node in self.node2pending: + node.shutdown() + + def addnode(self, node): + assert node not in self.node2pending + self.node2pending[node] = [] + + def removenode(self, node): + try: + pending = self.node2pending.pop(node) + except KeyError: + # this happens if we didn't receive a testnodeready event yet + return [] + for item in pending: + l = self.item2nodes[item] + l.remove(node) + if not l: + del self.item2nodes[item] + return pending + + def triggertesting(self, colitems): + colitems = self.filteritems(colitems) + senditems = [] + for next in colitems: + if isinstance(next, py.test.collect.Item): + senditems.append(next) + else: + self.config.hook.pytest_collectstart(collector=next) + colrep = self.config.hook.pytest_make_collect_report(collector=next) + self.queueevent("pytest_collectreport", report=colrep) + if self.config.option.dist == "each": + self.senditems_each(senditems) + else: + # XXX assert self.config.option.dist == "load" + self.senditems_load(senditems) + + def queueevent(self, eventname, **kwargs): + self.queue.put((eventname, (), kwargs)) + + def senditems_each(self, tosend): + if not tosend: + return + room = self.MAXITEMSPERHOST + for node, pending in self.node2pending.items(): + room = min(self.MAXITEMSPERHOST - len(pending), room) + sending = tosend[:room] + for node, pending in self.node2pending.items(): + node.sendlist(sending) + pending.extend(sending) + for item in sending: + nodes = self.item2nodes.setdefault(item, []) + assert node not in nodes + nodes.append(node) + self.config.hook.pytest_itemstart(item=item, node=node) + tosend[:] = tosend[room:] # update inplace + if tosend: + # we have some left, give it to the main loop + self.queueevent("pytest_rescheduleitems", items=tosend) + + def senditems_load(self, tosend): + if not tosend: + return + for node, pending in self.node2pending.items(): + room = self.MAXITEMSPERHOST - len(pending) + if room > 0: + sending = tosend[:room] + node.sendlist(sending) + for item in sending: + #assert item not in self.item2node, ( + # "sending same item %r to multiple " + # "not implemented" %(item,)) + self.item2nodes.setdefault(item, []).append(node) + self.config.hook.pytest_itemstart(item=item, node=node) + pending.extend(sending) + tosend[:] = tosend[room:] # update inplace + if not tosend: + break + if tosend: + # we have some left, give it to the main loop + self.queueevent("pytest_rescheduleitems", items=tosend) + + def removeitem(self, item, node): + if item not in self.item2nodes: + raise AssertionError(item, self.item2nodes) + nodes = self.item2nodes[item] + if node in nodes: # the node might have gone down already + nodes.remove(node) + if not nodes: + del self.item2nodes[item] + pending = self.node2pending[node] + pending.remove(item) + + def handle_crashitem(self, item, node): + runner = item.config.pluginmanager.getplugin("runner") + info = "!!! Node %r crashed during running of test %r" %(node, item) + rep = runner.ItemTestReport(item=item, excinfo=info, when="???") + rep.node = node + self.config.hook.pytest_runtest_logreport(report=rep) + + def setup(self): + """ setup any neccessary resources ahead of the test run. """ + self.nodemanager = NodeManager(self.config) + self.nodemanager.setup_nodes(putevent=self.queue.put) + if self.config.option.dist == "each": + self.nodemanager.wait_nodesready(5.0) + + def teardown(self): + """ teardown any resources after a test run. """ + self.nodemanager.teardown_nodes() Added: pypy/branch/py11/py/impl/test/dist/gwmanage.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/gwmanage.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,124 @@ +""" + instantiating, managing and rsyncing to test hosts +""" + +import py +import sys, os +import execnet +from execnet.gateway_base import RemoteError + +class GatewayManager: + RemoteError = RemoteError + def __init__(self, specs, hook, defaultchdir="pyexecnetcache"): + self.gateways = [] + self.specs = [] + self.hook = hook + for spec in specs: + if not isinstance(spec, execnet.XSpec): + spec = execnet.XSpec(spec) + if not spec.chdir and not spec.popen: + spec.chdir = defaultchdir + self.specs.append(spec) + + def makegateways(self): + assert not self.gateways + for spec in self.specs: + gw = execnet.makegateway(spec) + self.gateways.append(gw) + gw.id = "[%s]" % len(self.gateways) + self.hook.pytest_gwmanage_newgateway( + gateway=gw, platinfo=gw._rinfo()) + + def getgateways(self, remote=True, inplacelocal=True): + if not self.gateways and self.specs: + self.makegateways() + l = [] + for gw in self.gateways: + if gw.spec._samefilesystem(): + if inplacelocal: + l.append(gw) + else: + if remote: + l.append(gw) + return execnet.MultiGateway(gateways=l) + + def multi_exec(self, source, inplacelocal=True): + """ remote execute code on all gateways. + @param inplacelocal=False: don't send code to inplacelocal hosts. + """ + multigw = self.getgateways(inplacelocal=inplacelocal) + return multigw.remote_exec(source) + + def multi_chdir(self, basename, inplacelocal=True): + """ perform a remote chdir to the given path, may be relative. + @param inplacelocal=False: don't send code to inplacelocal hosts. + """ + self.multi_exec("import os ; os.chdir(%r)" % basename, + inplacelocal=inplacelocal).waitclose() + + def rsync(self, source, notify=None, verbose=False, ignores=None): + """ perform rsync to all remote hosts. + """ + rsync = HostRSync(source, verbose=verbose, ignores=ignores) + seen = py.builtin.set() + gateways = [] + for gateway in self.gateways: + spec = gateway.spec + if not spec._samefilesystem(): + if spec not in seen: + def finished(): + if notify: + notify("rsyncrootready", spec, source) + rsync.add_target_host(gateway, finished=finished) + seen.add(spec) + gateways.append(gateway) + if seen: + self.hook.pytest_gwmanage_rsyncstart( + source=source, + gateways=gateways, + ) + rsync.send() + self.hook.pytest_gwmanage_rsyncfinish( + source=source, + gateways=gateways, + ) + + def exit(self): + while self.gateways: + gw = self.gateways.pop() + gw.exit() + +class HostRSync(execnet.RSync): + """ RSyncer that filters out common files + """ + def __init__(self, sourcedir, *args, **kwargs): + self._synced = {} + ignores= None + if 'ignores' in kwargs: + ignores = kwargs.pop('ignores') + self._ignores = ignores or [] + super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs) + + def filter(self, path): + path = py.path.local(path) + if not path.ext in ('.pyc', '.pyo'): + if not path.basename.endswith('~'): + if path.check(dotfile=0): + for x in self._ignores: + if path == x: + break + else: + return True + + def add_target_host(self, gateway, finished=None): + remotepath = os.path.basename(self._sourcedir) + super(HostRSync, self).add_target(gateway, remotepath, + finishedcallback=finished, + delete=True,) + + def _report_send_file(self, gateway, modified_rel_path): + if self._verbose: + path = os.path.basename(self._sourcedir) + "/" + modified_rel_path + remotepath = gateway.spec.chdir + py.builtin.print_('%s:%s <= %s' % + (gateway.spec, remotepath, path)) Added: pypy/branch/py11/py/impl/test/dist/mypickle.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/mypickle.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,191 @@ +""" + + Pickling support for two processes that want to exchange + *immutable* object instances. Immutable in the sense + that the receiving side of an object can modify its + copy but when it sends it back the original sending + side will continue to see its unmodified version + (and no actual state will go over the wire). + + This module also implements an experimental + execnet pickling channel using this idea. + +""" + +import py +import sys, os, struct +#debug = open("log-mypickle-%d" % os.getpid(), 'w') + +if sys.version_info >= (3,0): + makekey = lambda x: x + fromkey = lambda x: x + from pickle import _Pickler as Pickler + from pickle import _Unpickler as Unpickler +else: + makekey = str + fromkey = int + from pickle import Pickler, Unpickler + + +class MyPickler(Pickler): + """ Pickler with a custom memoize() + to take care of unique ID creation. + See the usage in ImmutablePickler + XXX we could probably extend Pickler + and Unpickler classes to directly + update the other'S memos. + """ + def __init__(self, file, protocol, uneven): + Pickler.__init__(self, file, protocol) + self.uneven = uneven + + def memoize(self, obj): + if self.fast: + return + assert id(obj) not in self.memo + memo_len = len(self.memo) + key = memo_len * 2 + self.uneven + self.write(self.put(key)) + self.memo[id(obj)] = key, obj + + #if sys.version_info < (3,0): + # def save_string(self, obj, pack=struct.pack): + # obj = unicode(obj) + # self.save_unicode(obj, pack=pack) + # Pickler.dispatch[str] = save_string + +class ImmutablePickler: + def __init__(self, uneven, protocol=0): + """ ImmutablePicklers are instantiated in Pairs. + The two sides need to create unique IDs + while pickling their objects. This is + done by using either even or uneven + numbers, depending on the instantiation + parameter. + """ + self._picklememo = {} + self._unpicklememo = {} + self._protocol = protocol + self.uneven = uneven and 1 or 0 + + def selfmemoize(self, obj): + # this is for feeding objects to ourselfes + # which be the case e.g. if you want to pickle + # from a forked process back to the original + f = py.io.BytesIO() + pickler = MyPickler(f, self._protocol, uneven=self.uneven) + pickler.memo = self._picklememo + pickler.memoize(obj) + self._updateunpicklememo() + + def dumps(self, obj): + f = py.io.BytesIO() + pickler = MyPickler(f, self._protocol, uneven=self.uneven) + pickler.memo = self._picklememo + pickler.dump(obj) + if obj is not None: + self._updateunpicklememo() + #print >>debug, "dumped", obj + #print >>debug, "picklememo", self._picklememo + return f.getvalue() + + def loads(self, string): + f = py.io.BytesIO(string) + unpickler = Unpickler(f) + unpickler.memo = self._unpicklememo + res = unpickler.load() + self._updatepicklememo() + #print >>debug, "loaded", res + #print >>debug, "unpicklememo", self._unpicklememo + return res + + def _updatepicklememo(self): + for x, obj in self._unpicklememo.items(): + self._picklememo[id(obj)] = (fromkey(x), obj) + + def _updateunpicklememo(self): + for key,obj in self._picklememo.values(): + key = makekey(key) + if key in self._unpicklememo: + assert self._unpicklememo[key] is obj + self._unpicklememo[key] = obj + +NO_ENDMARKER_WANTED = object() + +class UnpickleError(Exception): + """ Problems while unpickling. """ + def __init__(self, formatted): + self.formatted = formatted + Exception.__init__(self, formatted) + def __str__(self): + return self.formatted + +class PickleChannel(object): + """ PickleChannels wrap execnet channels + and allow to send/receive by using + "immutable pickling". + """ + _unpicklingerror = None + def __init__(self, channel): + self._channel = channel + # we use the fact that each side of a + # gateway connection counts with uneven + # or even numbers depending on which + # side it is (for the purpose of creating + # unique ids - which is what we need it here for) + uneven = channel.gateway._channelfactory.count % 2 + self._ipickle = ImmutablePickler(uneven=uneven) + self.RemoteError = channel.RemoteError + + def send(self, obj): + from execnet.gateway_base import Channel + if not isinstance(obj, Channel): + pickled_obj = self._ipickle.dumps(obj) + self._channel.send(pickled_obj) + else: + self._channel.send(obj) + + def receive(self): + pickled_obj = self._channel.receive() + return self._unpickle(pickled_obj) + + def _unpickle(self, pickled_obj): + if isinstance(pickled_obj, self._channel.__class__): + return pickled_obj + return self._ipickle.loads(pickled_obj) + + def _getremoteerror(self): + return self._unpicklingerror or self._channel._getremoteerror() + + def close(self): + return self._channel.close() + + def isclosed(self): + return self._channel.isclosed() + + def waitclose(self, timeout=None): + return self._channel.waitclose(timeout=timeout) + + def setcallback(self, callback, endmarker=NO_ENDMARKER_WANTED): + if endmarker is NO_ENDMARKER_WANTED: + def unpickle_callback(pickled_obj): + obj = self._unpickle(pickled_obj) + callback(obj) + self._channel.setcallback(unpickle_callback) + return + uniqueendmarker = object() + def unpickle_callback(pickled_obj): + if pickled_obj is uniqueendmarker: + return callback(endmarker) + try: + obj = self._unpickle(pickled_obj) + except KeyboardInterrupt: + raise + except: + excinfo = py.code.ExceptionInfo() + formatted = str(excinfo.getrepr(showlocals=True,funcargs=True)) + self._unpicklingerror = UnpickleError(formatted) + callback(endmarker) + else: + callback(obj) + self._channel.setcallback(unpickle_callback, uniqueendmarker) Added: pypy/branch/py11/py/impl/test/dist/nodemanage.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/nodemanage.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,81 @@ +import py +import sys, os +from py.impl.test.dist.txnode import TXNode +from py.impl.test.dist.gwmanage import GatewayManager + + +class NodeManager(object): + def __init__(self, config, specs=None): + self.config = config + if specs is None: + specs = self.config.getxspecs() + self.roots = self.config.getrsyncdirs() + self.gwmanager = GatewayManager(specs, config.hook) + self.nodes = [] + self._nodesready = py.std.threading.Event() + + def trace(self, msg): + self.config.hook.pytest_trace(category="nodemanage", msg=msg) + + def config_getignores(self): + return self.config.getconftest_pathlist("rsyncignore") + + def rsync_roots(self): + """ make sure that all remote gateways + have the same set of roots in their + current directory. + """ + self.makegateways() + options = { + 'ignores': self.config_getignores(), + 'verbose': self.config.option.verbose, + } + if self.roots: + # send each rsync root + for root in self.roots: + self.gwmanager.rsync(root, **options) + else: + XXX # do we want to care for situations without explicit rsyncdirs? + # we transfer our topdir as the root + self.gwmanager.rsync(self.config.topdir, **options) + # and cd into it + self.gwmanager.multi_chdir(self.config.topdir.basename, inplacelocal=False) + + def makegateways(self): + # we change to the topdir sot that + # PopenGateways will have their cwd + # such that unpickling configs will + # pick it up as the right topdir + # (for other gateways this chdir is irrelevant) + self.trace("making gateways") + old = self.config.topdir.chdir() + try: + self.gwmanager.makegateways() + finally: + old.chdir() + + def setup_nodes(self, putevent): + self.rsync_roots() + self.trace("setting up nodes") + for gateway in self.gwmanager.gateways: + node = TXNode(gateway, self.config, putevent, slaveready=self._slaveready) + gateway.node = node # to keep node alive + self.trace("started node %r" % node) + + def _slaveready(self, node): + #assert node.gateway == node.gateway + #assert node.gateway.node == node + self.nodes.append(node) + self.trace("%s slave node ready %r" % (node.gateway.id, node)) + if len(self.nodes) == len(self.gwmanager.gateways): + self._nodesready.set() + + def wait_nodesready(self, timeout=None): + self._nodesready.wait(timeout) + if not self._nodesready.isSet(): + raise IOError("nodes did not get ready for %r secs" % timeout) + + def teardown_nodes(self): + # XXX do teardown nodes? + self.gwmanager.exit() + Added: pypy/branch/py11/py/impl/test/dist/txnode.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/dist/txnode.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,153 @@ +""" + Manage setup, running and local representation of remote nodes/processes. +""" +import py +from py.impl.test.dist.mypickle import PickleChannel + +class TXNode(object): + """ Represents a Test Execution environment in the controlling process. + - sets up a slave node through an execnet gateway + - manages sending of test-items and receival of results and events + - creates events when the remote side crashes + """ + ENDMARK = -1 + + def __init__(self, gateway, config, putevent, slaveready=None): + self.config = config + self.putevent = putevent + self.gateway = gateway + self.channel = install_slave(gateway, config) + self._sendslaveready = slaveready + self.channel.setcallback(self.callback, endmarker=self.ENDMARK) + self._down = False + + def __repr__(self): + id = self.gateway.id + status = self._down and 'true' or 'false' + return "" %(id, status) + + def notify(self, eventname, *args, **kwargs): + assert not args + self.putevent((eventname, args, kwargs)) + + def callback(self, eventcall): + """ this gets called for each object we receive from + the other side and if the channel closes. + + Note that channel callbacks run in the receiver + thread of execnet gateways - we need to + avoid raising exceptions or doing heavy work. + """ + try: + if eventcall == self.ENDMARK: + err = self.channel._getremoteerror() + if not self._down: + if not err: + err = "Not properly terminated" + self.notify("pytest_testnodedown", node=self, error=err) + self._down = True + return + eventname, args, kwargs = eventcall + if eventname == "slaveready": + if self._sendslaveready: + self._sendslaveready(self) + self.notify("pytest_testnodeready", node=self) + elif eventname == "slavefinished": + self._down = True + self.notify("pytest_testnodedown", error=None, node=self) + elif eventname == "pytest_runtest_logreport": + rep = kwargs['report'] + rep.node = self + self.notify("pytest_runtest_logreport", report=rep) + else: + self.notify(eventname, *args, **kwargs) + except KeyboardInterrupt: + # should not land in receiver-thread + raise + except: + excinfo = py.code.ExceptionInfo() + py.builtin.print_("!" * 20, excinfo) + self.config.pluginmanager.notify_exception(excinfo) + + def send(self, item): + assert item is not None + self.channel.send(item) + + def sendlist(self, itemlist): + self.channel.send(itemlist) + + def shutdown(self): + self.channel.send(None) + +# setting up slave code +def install_slave(gateway, config): + channel = gateway.remote_exec(source=""" + import os, sys + sys.path.insert(0, os.getcwd()) + from py.impl.test.dist.mypickle import PickleChannel + from py.impl.test.dist.txnode import SlaveNode + channel = PickleChannel(channel) + slavenode = SlaveNode(channel) + slavenode.run() + """) + channel = PickleChannel(channel) + basetemp = None + if gateway.spec.popen: + popenbase = config.ensuretemp("popen") + basetemp = py.path.local.make_numbered_dir(prefix="slave-", + keep=0, rootdir=popenbase) + basetemp = str(basetemp) + channel.send((config, basetemp)) + return channel + +class SlaveNode(object): + def __init__(self, channel): + self.channel = channel + + def __repr__(self): + return "<%s channel=%s>" %(self.__class__.__name__, self.channel) + + def sendevent(self, eventname, *args, **kwargs): + self.channel.send((eventname, args, kwargs)) + + def pytest_runtest_logreport(self, report): + self.sendevent("pytest_runtest_logreport", report=report) + + def run(self): + channel = self.channel + self.config, basetemp = channel.receive() + if basetemp: + self.config.basetemp = py.path.local(basetemp) + self.config.pluginmanager.do_configure(self.config) + self.config.pluginmanager.register(self) + self.runner = self.config.pluginmanager.getplugin("pytest_runner") + self.sendevent("slaveready") + try: + while 1: + task = channel.receive() + if task is None: + self.sendevent("slavefinished") + break + if isinstance(task, list): + for item in task: + self.run_single(item=item) + else: + self.run_single(item=task) + except KeyboardInterrupt: + raise + except: + er = py.code.ExceptionInfo().getrepr(funcargs=True, showlocals=True) + self.sendevent("pytest_internalerror", excrepr=er) + raise + + def run_single(self, item): + call = self.runner.CallInfo(item._checkcollectable, when='setup') + if call.excinfo: + # likely it is not collectable here because of + # platform/import-dependency induced skips + # we fake a setup-error report with the obtained exception + # and do not care about capturing or non-runner hooks + rep = self.runner.pytest_runtest_makereport(item=item, call=call) + self.pytest_runtest_logreport(rep) + return + item.config.hook.pytest_runtest_protocol(item=item) Added: pypy/branch/py11/py/impl/test/funcargs.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/funcargs.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,194 @@ +import py + +def getfuncargnames(function): + argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0] + startindex = py.std.inspect.ismethod(function) and 1 or 0 + defaults = getattr(function, 'func_defaults', + getattr(function, '__defaults__', None)) or () + numdefaults = len(defaults) + if numdefaults: + return argnames[startindex:-numdefaults] + return argnames[startindex:] + +def fillfuncargs(function): + """ fill missing funcargs. """ + request = FuncargRequest(pyfuncitem=function) + request._fillfuncargs() + + +_notexists = object() +class CallSpec: + def __init__(self, funcargs, id, param): + self.funcargs = funcargs + self.id = id + if param is not _notexists: + self.param = param + def __repr__(self): + return "" %( + self.id, getattr(self, 'param', '?'), self.funcargs) + +class Metafunc: + def __init__(self, function, config=None, cls=None, module=None): + self.config = config + self.module = module + self.function = function + self.funcargnames = getfuncargnames(function) + self.cls = cls + self.module = module + self._calls = [] + self._ids = py.builtin.set() + + def addcall(self, funcargs=None, id=_notexists, param=_notexists): + assert funcargs is None or isinstance(funcargs, dict) + if id is None: + raise ValueError("id=None not allowed") + if id is _notexists: + id = len(self._calls) + id = str(id) + if id in self._ids: + raise ValueError("duplicate id %r" % id) + self._ids.add(id) + self._calls.append(CallSpec(funcargs, id, param)) + +class FunctionCollector(py.test.collect.Collector): + def __init__(self, name, parent, calls): + super(FunctionCollector, self).__init__(name, parent) + self.calls = calls + self.obj = getattr(self.parent.obj, name) + + def collect(self): + l = [] + for callspec in self.calls: + name = "%s[%s]" %(self.name, callspec.id) + function = self.parent.Function(name=name, parent=self, + callspec=callspec, callobj=self.obj) + l.append(function) + return l + + def reportinfo(self): + try: + return self._fslineno, self.name + except AttributeError: + pass + fspath, lineno = py.code.getfslineno(self.obj) + self._fslineno = fspath, lineno + return fspath, lineno, self.name + + +class FuncargRequest: + _argprefix = "pytest_funcarg__" + _argname = None + + class Error(LookupError): + """ error on performing funcarg request. """ + + def __init__(self, pyfuncitem): + self._pyfuncitem = pyfuncitem + self.function = pyfuncitem.obj + self.module = pyfuncitem.getparent(py.test.collect.Module).obj + clscol = pyfuncitem.getparent(py.test.collect.Class) + self.cls = clscol and clscol.obj or None + self.instance = py.builtin._getimself(self.function) + self.config = pyfuncitem.config + self.fspath = pyfuncitem.fspath + if hasattr(pyfuncitem, '_requestparam'): + self.param = pyfuncitem._requestparam + self._plugins = self.config.pluginmanager.getplugins() + self._plugins.append(self.module) + if self.instance is not None: + self._plugins.append(self.instance) + self._funcargs = self._pyfuncitem.funcargs.copy() + self._name2factory = {} + self._currentarg = None + + def _fillfuncargs(self): + argnames = getfuncargnames(self.function) + if argnames: + assert not getattr(self._pyfuncitem, '_args', None), ( + "yielded functions cannot have funcargs") + for argname in argnames: + if argname not in self._pyfuncitem.funcargs: + self._pyfuncitem.funcargs[argname] = self.getfuncargvalue(argname) + + def cached_setup(self, setup, teardown=None, scope="module", extrakey=None): + """ cache and return result of calling setup(). + + The requested argument name, the scope and the ``extrakey`` + determine the cache key. The scope also determines when + teardown(result) will be called. valid scopes are: + scope == 'function': when the single test function run finishes. + scope == 'module': when tests in a different module are run + scope == 'session': when tests of the session have run. + """ + if not hasattr(self.config, '_setupcache'): + self.config._setupcache = {} # XXX weakref? + cachekey = (self._currentarg, self._getscopeitem(scope), extrakey) + cache = self.config._setupcache + try: + val = cache[cachekey] + except KeyError: + val = setup() + cache[cachekey] = val + if teardown is not None: + def finalizer(): + del cache[cachekey] + teardown(val) + self._addfinalizer(finalizer, scope=scope) + return val + + def getfuncargvalue(self, argname): + try: + return self._funcargs[argname] + except KeyError: + pass + if argname not in self._name2factory: + self._name2factory[argname] = self.config.pluginmanager.listattr( + plugins=self._plugins, + attrname=self._argprefix + str(argname) + ) + #else: we are called recursively + if not self._name2factory[argname]: + self._raiselookupfailed(argname) + funcargfactory = self._name2factory[argname].pop() + oldarg = self._currentarg + self._currentarg = argname + try: + self._funcargs[argname] = res = funcargfactory(request=self) + finally: + self._currentarg = oldarg + return res + + def _getscopeitem(self, scope): + if scope == "function": + return self._pyfuncitem + elif scope == "module": + return self._pyfuncitem.getparent(py.test.collect.Module) + elif scope == "session": + return None + raise ValueError("unknown finalization scope %r" %(scope,)) + + def _addfinalizer(self, finalizer, scope): + colitem = self._getscopeitem(scope) + self.config._setupstate.addfinalizer( + finalizer=finalizer, colitem=colitem) + + def addfinalizer(self, finalizer): + """ call the given finalizer after test function finished execution. """ + self._addfinalizer(finalizer, scope="function") + + def __repr__(self): + return "" %(self._pyfuncitem) + + def _raiselookupfailed(self, argname): + available = [] + for plugin in self._plugins: + for name in vars(plugin): + if name.startswith(self._argprefix): + name = name[len(self._argprefix):] + if name not in available: + available.append(name) + fspath, lineno, msg = self._pyfuncitem.reportinfo() + line = "%s:%s" %(fspath, lineno) + msg = "funcargument %r not found for: %s" %(argname, line) + msg += "\n available funcargs: %s" %(", ".join(available),) + raise self.Error(msg) Added: pypy/branch/py11/py/impl/test/looponfail/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/looponfail/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/py11/py/impl/test/looponfail/remote.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/looponfail/remote.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,162 @@ +""" + LooponfailingSession and Helpers. + + NOTE that one really has to avoid loading and depending on + application modules within the controlling process + (the one that starts repeatedly test processes) + otherwise changes to source code can crash + the controlling process which should never happen. +""" +import py +import sys +import execnet +from py.impl.test.session import Session +from py.impl.test.dist.mypickle import PickleChannel +from py.impl.test.looponfail import util + +class LooponfailingSession(Session): + def __init__(self, config): + super(LooponfailingSession, self).__init__(config=config) + self.rootdirs = [self.config.topdir] # xxx dist_rsync_roots? + self.statrecorder = util.StatRecorder(self.rootdirs) + self.remotecontrol = RemoteControl(self.config) + self.out = py.io.TerminalWriter() + + def main(self, initialitems=None): + try: + self.loopstate = loopstate = LoopState(initialitems) + self.remotecontrol.setup() + while 1: + self.loop_once(loopstate) + if not loopstate.colitems and loopstate.wasfailing: + continue # the last failures passed, let's rerun all + self.statrecorder.waitonchange(checkinterval=2.0) + except KeyboardInterrupt: + print + + def loop_once(self, loopstate): + colitems = loopstate.colitems + loopstate.wasfailing = colitems and len(colitems) + loopstate.colitems = self.remotecontrol.runsession(colitems or ()) + self.remotecontrol.setup() + +class LoopState: + def __init__(self, colitems=None): + self.colitems = colitems + +class RemoteControl(object): + def __init__(self, config): + self.config = config + + def trace(self, *args): + if self.config.option.debug: + msg = " ".join([str(x) for x in args]) + py.builtin.print_("RemoteControl:", msg) + + def initgateway(self): + return execnet.PopenGateway() + + def setup(self, out=None): + if out is None: + out = py.io.TerminalWriter() + if hasattr(self, 'gateway'): + raise ValueError("already have gateway %r" % self.gateway) + self.trace("setting up slave session") + old = self.config.topdir.chdir() + try: + self.gateway = self.initgateway() + finally: + old.chdir() + channel = self.gateway.remote_exec(source=""" + from py.impl.test.dist.mypickle import PickleChannel + from py.impl.test.looponfail.remote import slave_runsession + outchannel = channel.gateway.newchannel() + channel.send(outchannel) + channel = PickleChannel(channel) + config, fullwidth, hasmarkup = channel.receive() + import sys + sys.stdout = sys.stderr = outchannel.makefile('w') + slave_runsession(channel, config, fullwidth, hasmarkup) + """) + remote_outchannel = channel.receive() + def write(s): + out._file.write(s) + out._file.flush() + remote_outchannel.setcallback(write) + channel = self.channel = PickleChannel(channel) + channel.send((self.config, out.fullwidth, out.hasmarkup)) + self.trace("set up of slave session complete") + + def ensure_teardown(self): + if hasattr(self, 'channel'): + if not self.channel.isclosed(): + self.trace("closing", self.channel) + self.channel.close() + del self.channel + if hasattr(self, 'gateway'): + self.trace("exiting", self.gateway) + self.gateway.exit() + del self.gateway + + def runsession(self, colitems=()): + try: + self.trace("sending", colitems) + trails = colitems + self.channel.send(trails) + try: + return self.channel.receive() + except self.channel.RemoteError: + e = sys.exc_info()[1] + self.trace("ERROR", e) + raise + finally: + self.ensure_teardown() + +def slave_runsession(channel, config, fullwidth, hasmarkup): + """ we run this on the other side. """ + if config.option.debug: + def DEBUG(*args): + print(" ".join(map(str, args))) + else: + def DEBUG(*args): pass + + DEBUG("SLAVE: received configuration, using topdir:", config.topdir) + #config.option.session = None + config.option.looponfail = False + config.option.usepdb = False + trails = channel.receive() + config.pluginmanager.do_configure(config) + DEBUG("SLAVE: initsession()") + session = config.initsession() + # XXX configure the reporter object's terminal writer more directly + # XXX and write a test for this remote-terminal setting logic + config.pytest_terminal_hasmarkup = hasmarkup + config.pytest_terminal_fullwidth = fullwidth + if trails: + colitems = [] + for trail in trails: + try: + colitem = py.test.collect.Collector._fromtrail(trail, config) + except AssertionError: + #XXX send info for "test disappeared" or so + continue + colitems.append(colitem) + else: + colitems = None + session.shouldclose = channel.isclosed + + class Failures(list): + def pytest_runtest_logreport(self, report): + if report.failed: + self.append(report) + pytest_collectreport = pytest_runtest_logreport + + failreports = Failures() + session.pluginmanager.register(failreports) + + DEBUG("SLAVE: starting session.main()") + session.main(colitems) + session.config.hook.pytest_looponfailinfo( + failreports=list(failreports), + rootdirs=[config.topdir]) + channel.send([rep.getnode()._totrail() for rep in failreports]) Added: pypy/branch/py11/py/impl/test/looponfail/util.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/looponfail/util.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,53 @@ +import py + +class StatRecorder: + def __init__(self, rootdirlist): + self.rootdirlist = rootdirlist + self.statcache = {} + self.check() # snapshot state + + def fil(self, p): + return p.ext in ('.py', '.txt', '.c', '.h') + def rec(self, p): + return p.check(dotfile=0) + + def waitonchange(self, checkinterval=1.0): + while 1: + changed = self.check() + if changed: + return + py.std.time.sleep(checkinterval) + + def check(self, removepycfiles=True): + changed = False + statcache = self.statcache + newstat = {} + for rootdir in self.rootdirlist: + for path in rootdir.visit(self.fil, self.rec): + oldstat = statcache.get(path, None) + if oldstat is not None: + del statcache[path] + try: + newstat[path] = curstat = path.stat() + except py.error.ENOENT: + if oldstat: + del statcache[path] + changed = True + else: + if oldstat: + if oldstat.mtime != curstat.mtime or \ + oldstat.size != curstat.size: + changed = True + py.builtin.print_("# MODIFIED", path) + if removepycfiles and path.ext == ".py": + pycfile = path + "c" + if pycfile.check(): + pycfile.remove() + + else: + changed = True + if statcache: + changed = True + self.statcache = newstat + return changed + Added: pypy/branch/py11/py/impl/test/outcome.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/outcome.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,122 @@ +""" + Test OutcomeExceptions and helpers for creating them. + py.test.skip|fail|raises helper implementations + +""" + +import py +import sys + +class OutcomeException(Exception): + """ OutcomeException and its subclass instances indicate and + contain info about test and collection outcomes. + """ + def __init__(self, msg=None, excinfo=None): + self.msg = msg + self.excinfo = excinfo + + def __repr__(self): + if self.msg: + return repr(self.msg) + return "<%s instance>" %(self.__class__.__name__,) + __str__ = __repr__ + +class Passed(OutcomeException): + pass + +class Skipped(OutcomeException): + # XXX slighly hackish: on 3k we fake to live in the builtins + # in order to have Skipped exception printing shorter/nicer + __module__ = 'builtins' + +class Failed(OutcomeException): + pass + +class ExceptionFailure(Failed): + def __init__(self, expr, expected, msg=None, excinfo=None): + Failed.__init__(self, msg=msg, excinfo=excinfo) + self.expr = expr + self.expected = expected + +class Exit(KeyboardInterrupt): + """ for immediate program exits without tracebacks and reporter/summary. """ + def __init__(self, msg="unknown reason"): + self.msg = msg + KeyboardInterrupt.__init__(self, msg) + +# exposed helper methods + +def exit(msg): + """ exit testing process immediately. """ + __tracebackhide__ = True + raise Exit(msg) + +def skip(msg=""): + """ skip with the given message. """ + __tracebackhide__ = True + raise Skipped(msg=msg) + +def fail(msg="unknown failure"): + """ fail with the given Message. """ + __tracebackhide__ = True + raise Failed(msg=msg) + +def raises(ExpectedException, *args, **kwargs): + """ raise AssertionError, if target code does not raise the expected + exception. + """ + __tracebackhide__ = True + assert args + if isinstance(args[0], str): + code, = args + assert isinstance(code, str) + frame = sys._getframe(1) + loc = frame.f_locals.copy() + loc.update(kwargs) + #print "raises frame scope: %r" % frame.f_locals + try: + code = py.code.Source(code).compile() + py.builtin.exec_(code, frame.f_globals, loc) + # XXX didn'T mean f_globals == f_locals something special? + # this is destroyed here ... + except ExpectedException: + return py.code.ExceptionInfo() + else: + func = args[0] + try: + func(*args[1:], **kwargs) + except ExpectedException: + return py.code.ExceptionInfo() + k = ", ".join(["%s=%r" % x for x in kwargs.items()]) + if k: + k = ', ' + k + expr = '%s(%r%s)' %(func.__name__, args, k) + raise ExceptionFailure(msg="DID NOT RAISE", + expr=args, expected=ExpectedException) + +def importorskip(modname, minversion=None): + """ return imported module or perform a dynamic skip() """ + compile(modname, '', 'eval') # to catch syntaxerrors + try: + mod = __import__(modname, None, None, ['__doc__']) + except ImportError: + py.test.skip("could not import %r" %(modname,)) + if minversion is None: + return mod + verattr = getattr(mod, '__version__', None) + if isinstance(minversion, str): + minver = minversion.split(".") + else: + minver = list(minversion) + if verattr is None or verattr.split(".") < minver: + py.test.skip("module %r has __version__ %r, required is: %r" %( + modname, verattr, minversion)) + return mod + + +# exitcodes for the command line +EXIT_OK = 0 +EXIT_TESTSFAILED = 1 +EXIT_INTERRUPTED = 2 +EXIT_INTERNALERROR = 3 +EXIT_NOHOSTS = 4 Added: pypy/branch/py11/py/impl/test/parseopt.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/parseopt.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,103 @@ +""" +thin wrapper around Python's optparse.py +adding some extra checks and ways to systematically +have Environment variables provide default values +for options. basic usage: + + >>> parser = Parser() + >>> parser.addoption("--hello", action="store_true", dest="hello") + >>> option, args = parser.parse(['--hello']) + >>> option.hello + True + >>> args + [] + +""" +import py +import optparse + +class Parser: + """ Parser for command line arguments. """ + + def __init__(self, usage=None, processopt=None): + self._anonymous = OptionGroup("custom options", parser=self) + self._groups = [] + self._processopt = processopt + self._usage = usage + self.epilog = "" + + def processoption(self, option): + if self._processopt: + if option.dest: + self._processopt(option) + + def addnote(self, note): + self._notes.append(note) + + def getgroup(self, name, description="", after=None): + for group in self._groups: + if group.name == name: + return group + group = OptionGroup(name, description, parser=self) + i = 0 + for i, grp in enumerate(self._groups): + if grp.name == after: + break + self._groups.insert(i+1, group) + return group + + addgroup = getgroup + def addgroup(self, name, description=""): + py.log._apiwarn("1.1", "use getgroup() which gets-or-creates") + return self.getgroup(name, description) + + def addoption(self, *opts, **attrs): + """ add an optparse-style option. """ + self._anonymous.addoption(*opts, **attrs) + + def parse(self, args): + optparser = optparse.OptionParser(usage=self._usage) + # make sure anaonymous group is at the end + optparser.epilog = self.epilog + groups = self._groups + [self._anonymous] + for group in groups: + if group.options: + desc = group.description or group.name + optgroup = optparse.OptionGroup(optparser, desc) + optgroup.add_options(group.options) + optparser.add_option_group(optgroup) + return optparser.parse_args([str(x) for x in args]) + + def parse_setoption(self, args, option): + parsedoption, args = self.parse(args) + for name, value in parsedoption.__dict__.items(): + setattr(option, name, value) + return args + + +class OptionGroup: + def __init__(self, name, description="", parser=None): + self.name = name + self.description = description + self.options = [] + self.parser = parser + + def addoption(self, *optnames, **attrs): + """ add an option to this group. """ + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=False) + + def _addoption(self, *optnames, **attrs): + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=True) + + def _addoption_instance(self, option, shortupper=False): + if not shortupper: + for opt in option._short_opts: + if opt[0] == '-' and opt[1].islower(): + raise ValueError("lowercase shortoptions reserved") + if self.parser: + self.parser.processoption(option) + self.options.append(option) + + Added: pypy/branch/py11/py/impl/test/pluginmanager.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/pluginmanager.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,300 @@ +""" +managing loading and interacting with pytest plugins. +""" +import py +from py.plugin import hookspec +from py.impl.test.outcome import Skipped + +def check_old_use(mod, modname): + clsname = modname[len('pytest_'):].capitalize() + "Plugin" + assert not hasattr(mod, clsname), (mod, clsname) + +class PluginManager(object): + class Error(Exception): + """signals a plugin specific error.""" + def __init__(self, comregistry=None): + if comregistry is None: + comregistry = py._com.Registry() + self.comregistry = comregistry + self._name2plugin = {} + + self.hook = py._com.HookRelay( + hookspecs=hookspec, + registry=self.comregistry) + + def _getpluginname(self, plugin, name): + if name is None: + if hasattr(plugin, '__name__'): + name = plugin.__name__.split(".")[-1] + else: + name = id(plugin) + return name + + def register(self, plugin, name=None): + assert not self.isregistered(plugin) + name = self._getpluginname(plugin, name) + if name in self._name2plugin: + return False + self._name2plugin[name] = plugin + self.hook.pytest_plugin_registered(plugin=plugin) + self._checkplugin(plugin) + self.comregistry.register(plugin) + return True + + def unregister(self, plugin): + self.hook.pytest_plugin_unregistered(plugin=plugin) + self.comregistry.unregister(plugin) + for name, value in list(self._name2plugin.items()): + if value == plugin: + del self._name2plugin[name] + + def isregistered(self, plugin, name=None): + if self._getpluginname(plugin, name) in self._name2plugin: + return True + for val in self._name2plugin.values(): + if plugin == val: + return True + + def getplugins(self): + return list(self.comregistry) + + def getplugin(self, name): + try: + return self._name2plugin[name] + except KeyError: + impname = canonical_importname(name) + return self._name2plugin[impname] + + # API for bootstrapping + # + def _envlist(self, varname): + val = py.std.os.environ.get(varname, None) + if val is not None: + return val.split(',') + return () + + def consider_env(self): + for spec in self._envlist("PYTEST_PLUGINS"): + self.import_plugin(spec) + + def consider_preparse(self, args): + for opt1,opt2 in zip(args, args[1:]): + if opt1 == "-p": + self.import_plugin(opt2) + + def consider_conftest(self, conftestmodule): + cls = getattr(conftestmodule, 'ConftestPlugin', None) + if cls is not None: + raise ValueError("%r: 'ConftestPlugins' only existed till 1.0.0b1, " + "were removed in 1.0.0b2" % (cls,)) + if self.register(conftestmodule, name=conftestmodule.__file__): + self.consider_module(conftestmodule) + + def consider_module(self, mod): + attr = getattr(mod, "pytest_plugins", ()) + if attr: + if not isinstance(attr, (list, tuple)): + attr = (attr,) + for spec in attr: + self.import_plugin(spec) + + def import_plugin(self, spec): + assert isinstance(spec, str) + modname = canonical_importname(spec) + if modname in self._name2plugin: + return + try: + mod = importplugin(modname) + except KeyboardInterrupt: + raise + except Skipped: + e = py.std.sys.exc_info()[1] + self._warn("could not import plugin %r, reason: %r" %( + (modname, e.msg))) + else: + check_old_use(mod, modname) + self.register(mod) + self.consider_module(mod) + + def _warn(self, msg): + print ("===WARNING=== %s" % (msg,)) + + def _checkplugin(self, plugin): + # ===================================================== + # check plugin hooks + # ===================================================== + methods = collectattr(plugin) + hooks = collectattr(hookspec) + stringio = py.io.TextIO() + def Print(*args): + if args: + stringio.write(" ".join(map(str, args))) + stringio.write("\n") + + fail = False + while methods: + name, method = methods.popitem() + #print "checking", name + if isgenerichook(name): + continue + if name not in hooks: + Print("found unknown hook:", name) + fail = True + else: + method_args = getargs(method) + if '__multicall__' in method_args: + method_args.remove('__multicall__') + hook = hooks[name] + hookargs = getargs(hook) + for arg in method_args: + if arg not in hookargs: + Print("argument %r not available" %(arg, )) + Print("actual definition: %s" %(formatdef(method))) + Print("available hook arguments: %s" % + ", ".join(hookargs)) + fail = True + break + #if not fail: + # print "matching hook:", formatdef(method) + if fail: + name = getattr(plugin, '__name__', plugin) + raise self.Error("%s:\n%s" %(name, stringio.getvalue())) + # + # + # API for interacting with registered and instantiated plugin objects + # + # + def listattr(self, attrname, plugins=None, extra=()): + return self.comregistry.listattr(attrname, plugins=plugins, extra=extra) + + def notify_exception(self, excinfo=None): + if excinfo is None: + excinfo = py.code.ExceptionInfo() + excrepr = excinfo.getrepr(funcargs=True, showlocals=True) + return self.hook.pytest_internalerror(excrepr=excrepr) + + def do_addoption(self, parser): + mname = "pytest_addoption" + methods = self.comregistry.listattr(mname, reverse=True) + mc = py._com.MultiCall(methods, {'parser': parser}) + mc.execute() + + def pytest_plugin_registered(self, plugin): + if hasattr(self, '_config'): + self.call_plugin(plugin, "pytest_addoption", + {'parser': self._config._parser}) + self.call_plugin(plugin, "pytest_configure", + {'config': self._config}) + #dic = self.call_plugin(plugin, "pytest_namespace") + #self._updateext(dic) + + def call_plugin(self, plugin, methname, kwargs): + return py._com.MultiCall( + methods=self.listattr(methname, plugins=[plugin]), + kwargs=kwargs, firstresult=True).execute() + + def _updateext(self, dic): + if dic: + for name, value in dic.items(): + setattr(py.test, name, value) + + def do_configure(self, config): + assert not hasattr(self, '_config') + config.pluginmanager.register(self) + self._config = config + config.hook.pytest_configure(config=self._config) + for dic in config.hook.pytest_namespace() or []: + self._updateext(dic) + + def do_unconfigure(self, config): + config = self._config + del self._config + config.hook.pytest_unconfigure(config=config) + config.pluginmanager.unregister(self) + +# +# XXX old code to automatically load classes +# +def canonical_importname(name): + name = name.lower() + modprefix = "pytest_" + if not name.startswith(modprefix): + name = modprefix + name + return name + +def importplugin(importspec): + try: + return __import__(importspec) + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + try: + return __import__("py.plugin.%s" %(importspec), + None, None, '__doc__') + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + #print "syspath:", py.std.sys.path + #print "curdir:", py.std.os.getcwd() + return __import__(importspec) # show the original exception + + + +def isgenerichook(name): + return name == "pytest_plugins" or \ + name.startswith("pytest_funcarg__") + +def getargs(func): + args = py.std.inspect.getargs(py.code.getrawcode(func))[0] + startindex = py.std.inspect.ismethod(func) and 1 or 0 + return args[startindex:] + +def collectattr(obj, prefixes=("pytest_",)): + methods = {} + for apiname in dir(obj): + for prefix in prefixes: + if apiname.startswith(prefix): + methods[apiname] = getattr(obj, apiname) + return methods + +def formatdef(func): + return "%s%s" %( + func.__name__, + py.std.inspect.formatargspec(*py.std.inspect.getargspec(func)) + ) + +if __name__ == "__main__": + import py.plugin + basedir = py._dir.join('_plugin') + name2text = {} + for p in basedir.listdir("pytest_*"): + if p.ext == ".py" or ( + p.check(dir=1) and p.join("__init__.py").check()): + impname = p.purebasename + if impname.find("__") != -1: + continue + try: + plugin = importplugin(impname) + except (ImportError, py.impl.test.outcome.Skipped): + name2text[impname] = "IMPORT ERROR" + else: + doc = plugin.__doc__ or "" + doc = doc.strip() + name2text[impname] = doc + + for name in sorted(name2text.keys()): + text = name2text[name] + if name[0] == "_": + continue + print ("%-20s %s" % (name, text.split("\n")[0])) + + #text = py.std.textwrap.wrap(name2text[name], + # width = 80, + # initial_indent="%s: " % name, + # replace_whitespace = False) + #for line in text: + # print line + + Added: pypy/branch/py11/py/impl/test/pycollect.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/pycollect.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,371 @@ +""" +Python related collection nodes. Here is an example of +a tree of collectors and test items that this modules provides:: + + Module # File + Class + Instance + Function + Generator + ... + Function + Generator + Function + + DoctestFile # File + DoctestFileContent # acts as Item + +""" +import py +import inspect +from py.impl.test.collect import configproperty, warnoldcollect +from py.impl.test import funcargs + +class PyobjMixin(object): + def obj(): + def fget(self): + try: + return self._obj + except AttributeError: + self._obj = obj = self._getobj() + return obj + def fset(self, value): + self._obj = value + return property(fget, fset, None, "underlying python object") + obj = obj() + + def _getobj(self): + return getattr(self.parent.obj, self.name) + + def getmodpath(self, stopatmodule=True, includemodule=False): + """ return python path relative to the containing module. """ + chain = self.listchain() + chain.reverse() + parts = [] + for node in chain: + if isinstance(node, Instance): + continue + name = node.name + if isinstance(node, Module): + assert name.endswith(".py") + name = name[:-3] + if stopatmodule: + if includemodule: + parts.append(name) + break + parts.append(name) + parts.reverse() + s = ".".join(parts) + return s.replace(".[", "[") + + def _getfslineno(self): + try: + return self._fslineno + except AttributeError: + pass + obj = self.obj + # xxx let decorators etc specify a sane ordering + if hasattr(obj, 'place_as'): + obj = obj.place_as + + self._fslineno = py.code.getfslineno(obj) + return self._fslineno + + def reportinfo(self): + fspath, lineno = self._getfslineno() + modpath = self.getmodpath() + return fspath, lineno, modpath + +class PyCollectorMixin(PyobjMixin, py.test.collect.Collector): + Class = configproperty('Class') + Instance = configproperty('Instance') + Function = configproperty('Function') + Generator = configproperty('Generator') + + def funcnamefilter(self, name): + return name.startswith('test') + def classnamefilter(self, name): + return name.startswith('Test') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + name2items = self._buildname2items() + colitems = list(name2items.values()) + colitems.sort(key=lambda item: item.reportinfo()[:2]) + return colitems + + def _buildname2items(self): + # NB. we avoid random getattrs and peek in the __dict__ instead + d = {} + dicts = [getattr(self.obj, '__dict__', {})] + for basecls in inspect.getmro(self.obj.__class__): + dicts.append(basecls.__dict__) + seen = {} + for dic in dicts: + for name, obj in dic.items(): + if name in seen: + continue + seen[name] = True + if name[0] != "_": + res = self.makeitem(name, obj) + if res is not None: + d[name] = res + return d + + def _deprecated_join(self, name): + if self.__class__.join != py.test.collect.Collector.join: + warnoldcollect() + return self.join(name) + + def makeitem(self, name, obj): + return self.config.hook.pytest_pycollect_makeitem( + collector=self, name=name, obj=obj) + + def _istestclasscandidate(self, name, obj): + if self.classnamefilter(name) and \ + inspect.isclass(obj): + if hasinit(obj): + # XXX WARN + return False + return True + + def _genfunctions(self, name, funcobj): + module = self.getparent(Module).obj + clscol = self.getparent(Class) + cls = clscol and clscol.obj or None + metafunc = funcargs.Metafunc(funcobj, config=self.config, + cls=cls, module=module) + gentesthook = self.config.hook._makecall( + "pytest_generate_tests", extralookup=module) + gentesthook(metafunc=metafunc) + if not metafunc._calls: + return self.Function(name, parent=self) + return funcargs.FunctionCollector(name=name, + parent=self, calls=metafunc._calls) + + +class Module(py.test.collect.File, PyCollectorMixin): + def _getobj(self): + return self._memoizedcall('_obj', self._importtestmodule) + + def _importtestmodule(self): + # we assume we are only called once per module + mod = self.fspath.pyimport() + #print "imported test module", mod + self.config.pluginmanager.consider_module(mod) + return mod + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.test.skip("%r is disabled" %(self.obj,)) + if hasattr(self.obj, 'setup_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.setup_module)[0]: + self.obj.setup_module(self.obj) + else: + self.obj.setup_module() + + def teardown(self): + if hasattr(self.obj, 'teardown_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.teardown_module)[0]: + self.obj.teardown_module(self.obj) + else: + self.obj.teardown_module() + +class Class(PyCollectorMixin, py.test.collect.Collector): + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + return [self.Instance(name="()", parent=self)] + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.test.skip("%r is disabled" %(self.obj,)) + setup_class = getattr(self.obj, 'setup_class', None) + if setup_class is not None: + setup_class = getattr(setup_class, 'im_func', setup_class) + setup_class(self.obj) + + def teardown(self): + teardown_class = getattr(self.obj, 'teardown_class', None) + if teardown_class is not None: + teardown_class = getattr(teardown_class, 'im_func', teardown_class) + teardown_class(self.obj) + +class Instance(PyCollectorMixin, py.test.collect.Collector): + def _getobj(self): + return self.parent.obj() + def Function(self): + return getattr(self.obj, 'Function', + PyCollectorMixin.Function.__get__(self)) # XXX for python 2.2 + def _keywords(self): + return [] + Function = property(Function) + + #def __repr__(self): + # return "<%s of '%s'>" %(self.__class__.__name__, + # self.parent.obj.__name__) + + def newinstance(self): + self.obj = self._getobj() + return self.obj + +class FunctionMixin(PyobjMixin): + """ mixin for the code common to Function and Generator. + """ + + def setup(self): + """ perform setup for this test function. """ + if inspect.ismethod(self.obj): + name = 'setup_method' + else: + name = 'setup_function' + if isinstance(self.parent, Instance): + obj = self.parent.newinstance() + self.obj = self._getobj() + else: + obj = self.parent.obj + setup_func_or_method = getattr(obj, name, None) + if setup_func_or_method is not None: + setup_func_or_method(self.obj) + + def teardown(self): + """ perform teardown for this test function. """ + if inspect.ismethod(self.obj): + name = 'teardown_method' + else: + name = 'teardown_function' + obj = self.parent.obj + teardown_func_or_meth = getattr(obj, name, None) + if teardown_func_or_meth is not None: + teardown_func_or_meth(self.obj) + + def _prunetraceback(self, traceback): + if hasattr(self, '_obj') and not self.config.option.fulltrace: + code = py.code.Code(self.obj) + path, firstlineno = code.path, code.firstlineno + ntraceback = traceback.cut(path=path, firstlineno=firstlineno) + if ntraceback == traceback: + ntraceback = ntraceback.cut(path=path) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._dir) + traceback = ntraceback.filter() + return traceback + + def repr_failure(self, excinfo, outerr=None): + assert outerr is None, "XXX outerr usage is deprecated" + return self._repr_failure_py(excinfo) + + shortfailurerepr = "F" + +class Generator(FunctionMixin, PyCollectorMixin, py.test.collect.Collector): + def collect(self): + # test generators are seen as collectors but they also + # invoke setup/teardown on popular request + # (induced by the common "test_*" naming shared with normal tests) + self.config._setupstate.prepare(self) + l = [] + seen = {} + for i, x in enumerate(self.obj()): + name, call, args = self.getcallargs(x) + if not py.builtin.callable(call): + raise TypeError("%r yielded non callable test %r" %(self.obj, call,)) + if name is None: + name = "[%d]" % i + else: + name = "['%s']" % name + if name in seen: + raise ValueError("%r generated tests with non-unique name %r" %(self, name)) + seen[name] = True + l.append(self.Function(name, self, args=args, callobj=call)) + return l + + def getcallargs(self, obj): + if not isinstance(obj, (tuple, list)): + obj = (obj,) + # explict naming + if isinstance(obj[0], py.builtin._basestring): + name = obj[0] + obj = obj[1:] + else: + name = None + call, args = obj[0], obj[1:] + return name, call, args + + +# +# Test Items +# +_dummy = object() +class Function(FunctionMixin, py.test.collect.Item): + """ a Function Item is responsible for setting up + and executing a Python callable test object. + """ + _genid = None + def __init__(self, name, parent=None, args=None, + callspec=None, callobj=_dummy): + super(Function, self).__init__(name, parent) + self._args = args + if self._isyieldedfunction(): + assert not callspec, "yielded functions (deprecated) cannot have funcargs" + else: + if callspec is not None: + self.funcargs = callspec.funcargs or {} + self._genid = callspec.id + if hasattr(callspec, "param"): + self._requestparam = callspec.param + else: + self.funcargs = {} + if callobj is not _dummy: + self._obj = callobj + + def _isyieldedfunction(self): + return self._args is not None + + def readkeywords(self): + d = super(Function, self).readkeywords() + d.update(py.builtin._getfuncdict(self.obj)) + return d + + def runtest(self): + """ execute the underlying test function. """ + self.config.hook.pytest_pyfunc_call(pyfuncitem=self) + + def setup(self): + super(Function, self).setup() + if hasattr(self, 'funcargs'): + funcargs.fillfuncargs(self) + + def __eq__(self, other): + try: + return (self.name == other.name and + self._args == other._args and + self.parent == other.parent and + self.obj == other.obj and + getattr(self, '_genid', None) == + getattr(other, '_genid', None) + ) + except AttributeError: + pass + return False + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.parent, self.name)) + +def hasinit(obj): + init = getattr(obj, '__init__', None) + if init: + if not isinstance(init, type(object.__init__)): + return True Added: pypy/branch/py11/py/impl/test/session.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/test/session.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,126 @@ +""" basic test session implementation. + +* drives collection of tests +* triggers executions of tests +* produces events used by reporting +""" + +import py +from py.impl.test import outcome + +# imports used for genitems() +Item = py.test.collect.Item +Collector = py.test.collect.Collector + +class Session(object): + """ + Session drives the collection and running of tests + and generates test events for reporters. + """ + def __init__(self, config): + self.config = config + self.pluginmanager = config.pluginmanager # shortcut + self.pluginmanager.register(self) + self._testsfailed = False + self._nomatch = False + self.shouldstop = False + + def genitems(self, colitems, keywordexpr=None): + """ yield Items from iterating over the given colitems. """ + while colitems: + next = colitems.pop(0) + if isinstance(next, (tuple, list)): + colitems[:] = list(next) + colitems + continue + assert self.pluginmanager is next.config.pluginmanager + if isinstance(next, Item): + remaining = self.filteritems([next]) + if remaining: + self.config.hook.pytest_itemstart(item=next) + yield next + else: + assert isinstance(next, Collector) + self.config.hook.pytest_collectstart(collector=next) + rep = self.config.hook.pytest_make_collect_report(collector=next) + if rep.passed: + for x in self.genitems(rep.result, keywordexpr): + yield x + self.config.hook.pytest_collectreport(report=rep) + if self.shouldstop: + break + + def filteritems(self, colitems): + """ return items to process (some may be deselected)""" + keywordexpr = self.config.option.keyword + if not keywordexpr or self._nomatch: + return colitems + if keywordexpr[-1] == ":": + keywordexpr = keywordexpr[:-1] + remaining = [] + deselected = [] + for colitem in colitems: + if isinstance(colitem, Item): + if colitem._skipbykeyword(keywordexpr): + deselected.append(colitem) + continue + remaining.append(colitem) + if deselected: + self.config.hook.pytest_deselected(items=deselected) + if self.config.option.keyword.endswith(":"): + self._nomatch = True + return remaining + + def collect(self, colitems): + keyword = self.config.option.keyword + for x in self.genitems(colitems, keyword): + yield x + + def sessionstarts(self): + """ setup any neccessary resources ahead of the test run. """ + self.config.hook.pytest_sessionstart(session=self) + + def pytest_runtest_logreport(self, report): + if report.failed: + self._testsfailed = True + if self.config.option.exitfirst: + self.shouldstop = True + pytest_collectreport = pytest_runtest_logreport + + def sessionfinishes(self, exitstatus): + """ teardown any resources after a test run. """ + self.config.hook.pytest_sessionfinish( + session=self, + exitstatus=exitstatus, + ) + + def getinitialitems(self, colitems): + if colitems is None: + colitems = [self.config.getfsnode(arg) + for arg in self.config.args] + return colitems + + def main(self, colitems=None): + """ main loop for running tests. """ + colitems = self.getinitialitems(colitems) + self.shouldstop = False + self.sessionstarts() + exitstatus = outcome.EXIT_OK + captured_excinfo = None + try: + for item in self.collect(colitems): + if self.shouldstop: + break + if not self.config.option.collectonly: + item.config.hook.pytest_runtest_protocol(item=item) + except KeyboardInterrupt: + excinfo = py.code.ExceptionInfo() + self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + exitstatus = outcome.EXIT_INTERRUPTED + except: + excinfo = py.code.ExceptionInfo() + self.config.pluginmanager.notify_exception(captured_excinfo) + exitstatus = outcome.EXIT_INTERNALERROR + if exitstatus == 0 and self._testsfailed: + exitstatus = outcome.EXIT_TESTSFAILED + self.sessionfinishes(exitstatus=exitstatus) + return exitstatus Added: pypy/branch/py11/py/impl/xmlgen.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/impl/xmlgen.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,243 @@ +""" +module for generating and serializing xml and html structures +by using simple python objects. + +(c) holger krekel, holger at merlinux eu. 2009 +""" +import py +import sys, re + +if sys.version_info >= (3,0): + def u(s): + return s + def unicode(x): + if hasattr(x, '__unicode__'): + return x.__unicode__() + return str(x) +else: + def u(s): + return unicode(s) + unicode = unicode + + +class NamespaceMetaclass(type): + def __getattr__(self, name): + if name[:1] == '_': + raise AttributeError(name) + if self == Namespace: + raise ValueError("Namespace class is abstract") + tagspec = self.__tagspec__ + if tagspec is not None and name not in tagspec: + raise AttributeError(name) + classattr = {} + if self.__stickyname__: + classattr['xmlname'] = name + cls = type(name, (self.__tagclass__,), classattr) + setattr(self, name, cls) + return cls + +class Tag(list): + class Attr(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __init__(self, *args, **kwargs): + super(Tag, self).__init__(args) + self.attr = self.Attr(**kwargs) + + def __unicode__(self): + return self.unicode(indent=0) + __str__ = __unicode__ + + def unicode(self, indent=2): + l = [] + SimpleUnicodeVisitor(l.append, indent).visit(self) + return "".join(l) + + def __repr__(self): + name = self.__class__.__name__ + return "<%r tag object %d>" % (name, id(self)) + +Namespace = NamespaceMetaclass('Namespace', (object, ), { + '__tagspec__': None, + '__tagclass__': Tag, + '__stickyname__': False, +}) + +class HtmlTag(Tag): + def unicode(self, indent=2): + l = [] + HtmlVisitor(l.append, indent, shortempty=False).visit(self) + return u("").join(l) + +# exported plain html namespace +class html(Namespace): + __tagclass__ = HtmlTag + __stickyname__ = True + __tagspec__ = dict([(x,1) for x in ( + 'a,abbr,acronym,address,applet,area,b,bdo,big,blink,' + 'blockquote,body,br,button,caption,center,cite,code,col,' + 'colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed,' + 'fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html,' + 'i,iframe,img,input,ins,kbd,label,legend,li,link,listing,' + 'map,marquee,menu,meta,multicol,nobr,noembed,noframes,' + 'noscript,object,ol,optgroup,option,p,pre,q,s,script,' + 'select,small,span,strike,strong,style,sub,sup,table,' + 'tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp,' + 'base,basefont,frame,hr,isindex,param,samp,var' + ).split(',') if x]) + + class Style(object): + def __init__(self, **kw): + for x, y in kw.items(): + x = x.replace('_', '-') + setattr(self, x, y) + + +class raw(object): + """just a box that can contain a unicode string that will be + included directly in the output""" + def __init__(self, uniobj): + self.uniobj = uniobj + +class SimpleUnicodeVisitor(object): + """ recursive visitor to write unicode. """ + def __init__(self, write, indent=0, curindent=0, shortempty=True): + self.write = write + self.cache = {} + self.visited = {} # for detection of recursion + self.indent = indent + self.curindent = curindent + self.parents = [] + self.shortempty = shortempty # short empty tags or not + + def visit(self, node): + """ dispatcher on node's class/bases name. """ + cls = node.__class__ + try: + visitmethod = self.cache[cls] + except KeyError: + for subclass in cls.__mro__: + visitmethod = getattr(self, subclass.__name__, None) + if visitmethod is not None: + break + else: + visitmethod = self.object + self.cache[cls] = visitmethod + visitmethod(node) + + def object(self, obj): + #self.write(obj) + self.write(escape(unicode(obj))) + + def raw(self, obj): + self.write(obj.uniobj) + + def list(self, obj): + assert id(obj) not in self.visited + self.visited[id(obj)] = 1 + map(self.visit, obj) + + def Tag(self, tag): + assert id(tag) not in self.visited + try: + tag.parent = self.parents[-1] + except IndexError: + tag.parent = None + self.visited[id(tag)] = 1 + tagname = getattr(tag, 'xmlname', tag.__class__.__name__) + if self.curindent and not self._isinline(tagname): + self.write("\n" + u(' ') * self.curindent) + if tag: + self.curindent += self.indent + self.write(u('<%s%s>') % (tagname, self.attributes(tag))) + self.parents.append(tag) + for x in tag: + self.visit(x) + self.parents.pop() + self.write(u('') % tagname) + self.curindent -= self.indent + else: + nameattr = tagname+self.attributes(tag) + if self._issingleton(tagname): + self.write(u('<%s/>') % (nameattr,)) + else: + self.write(u('<%s>') % (nameattr, tagname)) + + def attributes(self, tag): + # serialize attributes + attrlist = dir(tag.attr) + attrlist.sort() + l = [] + for name in attrlist: + res = self.repr_attribute(tag.attr, name) + if res is not None: + l.append(res) + l.extend(self.getstyle(tag)) + return u("").join(l) + + def repr_attribute(self, attrs, name): + if name[:2] != '__': + value = getattr(attrs, name) + if name.endswith('_'): + name = name[:-1] + return ' %s="%s"' % (name, escape(unicode(value))) + + def getstyle(self, tag): + """ return attribute list suitable for styling. """ + try: + styledict = tag.style.__dict__ + except AttributeError: + return [] + else: + stylelist = [x+': ' + y for x,y in styledict.items()] + return [u(' style="%s"') % u('; ').join(stylelist)] + + def _issingleton(self, tagname): + """can (and will) be overridden in subclasses""" + return self.shortempty + + def _isinline(self, tagname): + """can (and will) be overridden in subclasses""" + return False + +class HtmlVisitor(SimpleUnicodeVisitor): + + single = dict([(x, 1) for x in + ('br,img,area,param,col,hr,meta,link,base,' + 'input,frame').split(',')]) + inline = dict([(x, 1) for x in + ('a abbr acronym b basefont bdo big br cite code dfn em font ' + 'i img input kbd label q s samp select small span strike ' + 'strong sub sup textarea tt u var'.split(' '))]) + + def repr_attribute(self, attrs, name): + if name == 'class_': + value = getattr(attrs, name) + if value is None: + return + return super(HtmlVisitor, self).repr_attribute(attrs, name) + + def _issingleton(self, tagname): + return tagname in self.single + + def _isinline(self, tagname): + return tagname in self.inline + + +class _escape: + def __init__(self): + self.escape = { + u('"') : u('"'), u('<') : u('<'), u('>') : u('>'), + u('&') : u('&'), u("'") : u('''), + } + self.charef_rex = re.compile(u("|").join(self.escape.keys())) + + def _replacer(self, match): + return self.escape[match.group(0)] + + def __call__(self, ustring): + """ xml-escape the given unicode string. """ + return self.charef_rex.sub(self._replacer, ustring) + +escape = _escape() Added: pypy/branch/py11/py/plugin/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/__init__.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1 @@ +# Added: pypy/branch/py11/py/plugin/hookspec.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/hookspec.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,175 @@ +""" +hook specifications for py.test plugins +""" + +# ------------------------------------------------------------------------- +# Command line and configuration +# ------------------------------------------------------------------------- + +def pytest_addoption(parser): + """ called before commandline parsing. """ + +def pytest_namespace(): + """ return dict of name->object which will get stored at py.test. namespace""" + +def pytest_configure(config): + """ called after command line options have been parsed. + and all plugins and initial conftest files been loaded. + """ + +def pytest_unconfigure(config): + """ called before test process is exited. """ + +# ------------------------------------------------------------------------- +# collection hooks +# ------------------------------------------------------------------------- + +def pytest_collect_directory(path, parent): + """ return Collection node or None for the given path. """ + +def pytest_collect_file(path, parent): + """ return Collection node or None for the given path. """ + +def pytest_collectstart(collector): + """ collector starts collecting. """ + +def pytest_collectreport(report): + """ collector finished collecting. """ + +def pytest_deselected(items): + """ called for test items deselected by keyword. """ + +def pytest_make_collect_report(collector): + """ perform a collection and return a collection. """ +pytest_make_collect_report.firstresult = True + +# XXX rename to item_collected()? meaning in distribution context? +def pytest_itemstart(item, node=None): + """ test item gets collected. """ + +# ------------------------------------------------------------------------- +# Python test function related hooks +# ------------------------------------------------------------------------- + +def pytest_pycollect_makeitem(collector, name, obj): + """ return custom item/collector for a python object in a module, or None. """ +pytest_pycollect_makeitem.firstresult = True + +def pytest_pyfunc_call(pyfuncitem): + """ perform function call to the with the given function arguments. """ +pytest_pyfunc_call.firstresult = True + +def pytest_generate_tests(metafunc): + """ generate (multiple) parametrized calls to a test function.""" + +# ------------------------------------------------------------------------- +# generic runtest related hooks +# ------------------------------------------------------------------------- + +def pytest_runtest_protocol(item): + """ implement fixture, run and report protocol. """ +pytest_runtest_protocol.firstresult = True + +def pytest_runtest_setup(item): + """ called before pytest_runtest_call(). """ + +def pytest_runtest_call(item): + """ execute test item. """ + +def pytest_runtest_teardown(item): + """ called after pytest_runtest_call(). """ + +def pytest_runtest_makereport(item, call): + """ make ItemTestReport for the given item and call outcome. """ +pytest_runtest_makereport.firstresult = True + +def pytest_runtest_logreport(report): + """ process item test report. """ + +# special handling for final teardown - somewhat internal for now +def pytest__teardown_final(session): + """ called before test session finishes. """ +pytest__teardown_final.firstresult = True + +def pytest__teardown_final_logerror(report): + """ called if runtest_teardown_final failed. """ + +# ------------------------------------------------------------------------- +# test session related hooks +# ------------------------------------------------------------------------- + +def pytest_sessionstart(session): + """ before session.main() is called. """ + +def pytest_sessionfinish(session, exitstatus): + """ whole test run finishes. """ + +# ------------------------------------------------------------------------- +# hooks for influencing reporting (invoked from pytest_terminal) +# ------------------------------------------------------------------------- + +def pytest_report_teststatus(report): + """ return result-category, shortletter and verbose word for reporting.""" +pytest_report_teststatus.firstresult = True + +def pytest_terminal_summary(terminalreporter): + """ add additional section in terminal summary reporting. """ + +def pytest_report_iteminfo(item): + """ return (fspath, lineno, name) for the item. + the information is used for result display and to sort tests + """ +pytest_report_iteminfo.firstresult = True + +# ------------------------------------------------------------------------- +# doctest hooks +# ------------------------------------------------------------------------- + +def pytest_doctest_prepare_content(content): + """ return processed content for a given doctest""" +pytest_doctest_prepare_content.firstresult = True + +# ------------------------------------------------------------------------- +# distributed testing +# ------------------------------------------------------------------------- + +def pytest_gwmanage_newgateway(gateway, platinfo): + """ called on new raw gateway creation. """ + +def pytest_gwmanage_rsyncstart(source, gateways): + """ called before rsyncing a directory to remote gateways takes place. """ + +def pytest_gwmanage_rsyncfinish(source, gateways): + """ called after rsyncing a directory to remote gateways takes place. """ + +def pytest_testnodeready(node): + """ Test Node is ready to operate. """ + +def pytest_testnodedown(node, error): + """ Test Node is down. """ + +def pytest_rescheduleitems(items): + """ reschedule Items from a node that went down. """ + +def pytest_looponfailinfo(failreports, rootdirs): + """ info for repeating failing tests. """ + + +# ------------------------------------------------------------------------- +# error handling and internal debugging hooks +# ------------------------------------------------------------------------- + +def pytest_plugin_registered(plugin): + """ a new py lib plugin got registered. """ + +def pytest_plugin_unregistered(plugin): + """ a py lib plugin got unregistered. """ + +def pytest_internalerror(excrepr): + """ called for internal errors. """ + +def pytest_keyboard_interrupt(excinfo): + """ called for keyboard interrupt. """ + +def pytest_trace(category, msg): + """ called for debug info. """ Added: pypy/branch/py11/py/plugin/pytest__pytest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest__pytest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,101 @@ +import py + +def pytest_funcarg___pytest(request): + return PytestArg(request) + +class PytestArg: + def __init__(self, request): + self.request = request + self.monkeypatch = self.request.getfuncargvalue("monkeypatch") + self.comregistry = py._com.Registry() + self.monkeypatch.setattr(py._com, 'comregistry', self.comregistry) + + def gethookrecorder(self, hookspecs, registry=None): + if registry is not None: + self.monkeypatch.setattr(py._com, 'comregistry', registry) + self.comregistry = registry + hookrecorder = HookRecorder(self.comregistry) + hookrecorder.start_recording(hookspecs) + self.request.addfinalizer(hookrecorder.finish_recording) + return hookrecorder + +class ParsedCall: + def __init__(self, name, locals): + assert '_name' not in locals + self.__dict__.update(locals) + self.__dict__.pop('self') + self._name = name + + def __repr__(self): + d = self.__dict__.copy() + del d['_name'] + return "" %(self._name, d) + +class HookRecorder: + def __init__(self, comregistry): + self._comregistry = comregistry + self.calls = [] + self._recorders = {} + + def start_recording(self, hookspecs): + assert hookspecs not in self._recorders + class RecordCalls: + _recorder = self + for name, method in vars(hookspecs).items(): + if name[0] != "_": + setattr(RecordCalls, name, self._makecallparser(method)) + recorder = RecordCalls() + self._recorders[hookspecs] = recorder + self._comregistry.register(recorder) + self.hook = py._com.HookRelay(hookspecs, registry=self._comregistry) + + def finish_recording(self): + for recorder in self._recorders.values(): + self._comregistry.unregister(recorder) + self._recorders.clear() + + def _makecallparser(self, method): + name = method.__name__ + args, varargs, varkw, default = py.std.inspect.getargspec(method) + if not args or args[0] != "self": + args.insert(0, 'self') + fspec = py.std.inspect.formatargspec(args, varargs, varkw, default) + # we use exec because we want to have early type + # errors on wrong input arguments, using + # *args/**kwargs delays this and gives errors + # elsewhere + exec (py.code.compile(""" + def %(name)s%(fspec)s: + self._recorder.calls.append( + ParsedCall(%(name)r, locals())) + """ % locals())) + return locals()[name] + + def getcalls(self, names): + if isinstance(names, str): + names = names.split() + for name in names: + for cls in self._recorders: + if name in vars(cls): + break + else: + raise ValueError("callname %r not found in %r" %( + name, self._recorders.keys())) + l = [] + for call in self.calls: + if call._name in names: + l.append(call) + return l + + def popcall(self, name): + for i, call in enumerate(self.calls): + if call._name == name: + del self.calls[i] + return call + raise ValueError("could not find call %r" %(name, )) + + def getcall(self, name): + l = self.getcalls(name) + assert len(l) == 1, (name, l) + return l[0] + Added: pypy/branch/py11/py/plugin/pytest_assertion.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_assertion.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,31 @@ +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group._addoption('--no-assert', action="store_true", default=False, + dest="noassert", + help="disable python assert expression reinterpretation."), + +def pytest_configure(config): + #if sys.platform.startswith("java"): + # return # XXX assertions don't work yet with jython 2.5.1 + + if not config.getvalue("noassert") and not config.getvalue("nomagic"): + warn_about_missing_assertion() + config._oldassertion = py.builtin.builtins.AssertionError + py.builtin.builtins.AssertionError = py.code._AssertionError + +def pytest_unconfigure(config): + if hasattr(config, '_oldassertion'): + py.builtin.builtins.AssertionError = config._oldassertion + del config._oldassertion + +def warn_about_missing_assertion(): + try: + assert False + except AssertionError: + pass + else: + py.std.warnings.warn("Assertions are turned off!" + " (are you using python -O?)") Added: pypy/branch/py11/py/plugin/pytest_capture.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_capture.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,277 @@ +""" +configurable per-test stdout/stderr capturing mechanisms. + +This plugin captures stdout/stderr output for each test separately. +In case of test failures this captured output is shown grouped +togtther with the test. + +The plugin also provides test function arguments that help to +assert stdout/stderr output from within your tests, see the +`funcarg example`_. + + +Capturing of input/output streams during tests +--------------------------------------------------- + +By default ``sys.stdout`` and ``sys.stderr`` are substituted with +temporary streams during the execution of tests and setup/teardown code. +During the whole testing process it will re-use the same temporary +streams allowing to play well with the logging module which easily +takes ownership on these streams. + +Also, 'sys.stdin' is substituted with a file-like "null" object that +does not return any values. This is to immediately error out +on tests that wait on reading something from stdin. + +You can influence output capturing mechanisms from the command line:: + + py.test -s # disable all capturing + py.test --capture=sys # replace sys.stdout/stderr with in-mem files + py.test --capture=fd # point filedescriptors 1 and 2 to temp file + +If you set capturing values in a conftest file like this:: + + # conftest.py + option_capture = 'fd' + +then all tests in that directory will execute with "fd" style capturing. + +sys-level capturing +------------------------------------------ + +Capturing on 'sys' level means that ``sys.stdout`` and ``sys.stderr`` +will be replaced with in-memory files (``py.io.TextIO`` to be precise) +that capture writes and decode non-unicode strings to a unicode object +(using a default, usually, UTF-8, encoding). + +FD-level capturing and subprocesses +------------------------------------------ + +The ``fd`` based method means that writes going to system level files +based on the standard file descriptors will be captured, for example +writes such as ``os.write(1, 'hello')`` will be captured properly. +Capturing on fd-level will include output generated from +any subprocesses created during a test. + +.. _`funcarg example`: + +Example Usage of the capturing Function arguments +--------------------------------------------------- + +You can use the `capsys funcarg`_ and `capfd funcarg`_ to +capture writes to stdout and stderr streams. Using the +funcargs frees your test from having to care about setting/resetting +the old streams and also interacts well with py.test's own +per-test capturing. Here is an example test function: + +.. sourcecode:: python + + def test_myoutput(capsys): + print ("hello") + sys.stderr.write("world\\n") + out, err = capsys.readouterr() + assert out == "hello\\n" + assert err == "world\\n" + print "next" + out, err = capsys.readouterr() + assert out == "next\\n" + +The ``readouterr()`` call snapshots the output so far - +and capturing will be continued. After the test +function finishes the original streams will +be restored. If you want to capture on +the filedescriptor level you can use the ``capfd`` function +argument which offers the same interface. +""" + +import py +import os + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--capture', action="store", default=None, + metavar="method", type="choice", choices=['fd', 'sys', 'no'], + help="set capturing method during tests: fd (default)|sys|no.") + group._addoption('-s', action="store_const", const="no", dest="capture", + help="shortcut for --capture=no.") + +def addouterr(rep, outerr): + repr = getattr(rep, 'longrepr', None) + if not hasattr(repr, 'addsection'): + return + for secname, content in zip(["out", "err"], outerr): + if content: + repr.addsection("Captured std%s" % secname, content.rstrip()) + +def pytest_configure(config): + config.pluginmanager.register(CaptureManager(), 'capturemanager') + +class CaptureManager: + def __init__(self): + self._method2capture = {} + + def _maketempfile(self): + f = py.std.tempfile.TemporaryFile() + newf = py.io.dupfile(f, encoding="UTF-8") + return newf + + def _makestringio(self): + return py.io.TextIO() + + def _startcapture(self, method): + if method == "fd": + return py.io.StdCaptureFD( + out=self._maketempfile(), err=self._maketempfile() + ) + elif method == "sys": + return py.io.StdCapture( + out=self._makestringio(), err=self._makestringio() + ) + else: + raise ValueError("unknown capturing method: %r" % method) + + def _getmethod(self, config, fspath): + if config.option.capture: + method = config.option.capture + else: + try: + method = config._conftest.rget("option_capture", path=fspath) + except KeyError: + method = "fd" + if method == "fd" and not hasattr(os, 'dup'): # e.g. jython + method = "sys" + return method + + def resumecapture_item(self, item): + method = self._getmethod(item.config, item.fspath) + if not hasattr(item, 'outerr'): + item.outerr = ('', '') # we accumulate outerr on the item + return self.resumecapture(method) + + def resumecapture(self, method): + if hasattr(self, '_capturing'): + raise ValueError("cannot resume, already capturing with %r" % + (self._capturing,)) + if method != "no": + cap = self._method2capture.get(method) + if cap is None: + cap = self._startcapture(method) + self._method2capture[method] = cap + else: + cap.resume() + self._capturing = method + + def suspendcapture(self): + self.deactivate_funcargs() + method = self._capturing + if method != "no": + cap = self._method2capture[method] + outerr = cap.suspend() + else: + outerr = "", "" + del self._capturing + return outerr + + def activate_funcargs(self, pyfuncitem): + if not hasattr(pyfuncitem, 'funcargs'): + return + assert not hasattr(self, '_capturing_funcargs') + l = [] + for name, obj in pyfuncitem.funcargs.items(): + if name in ('capsys', 'capfd'): + obj._start() + l.append(obj) + if l: + self._capturing_funcargs = l + + def deactivate_funcargs(self): + if hasattr(self, '_capturing_funcargs'): + for capfuncarg in self._capturing_funcargs: + capfuncarg._finalize() + del self._capturing_funcargs + + def pytest_make_collect_report(self, __multicall__, collector): + method = self._getmethod(collector.config, collector.fspath) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + addouterr(rep, outerr) + return rep + + def pytest_runtest_setup(self, item): + self.resumecapture_item(item) + + def pytest_runtest_call(self, item): + self.resumecapture_item(item) + self.activate_funcargs(item) + + def pytest_runtest_teardown(self, item): + self.resumecapture_item(item) + + def pytest_runtest_teardown(self, item): + self.resumecapture_item(item) + + def pytest__teardown_final(self, __multicall__, session): + method = self._getmethod(session.config, None) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + if rep: + addouterr(rep, outerr) + return rep + + def pytest_keyboard_interrupt(self, excinfo): + if hasattr(self, '_capturing'): + self.suspendcapture() + + def pytest_runtest_makereport(self, __multicall__, item, call): + self.deactivate_funcargs() + rep = __multicall__.execute() + outerr = self.suspendcapture() + outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + if not rep.passed: + addouterr(rep, outerr) + if not rep.passed or rep.when == "teardown": + outerr = ('', '') + item.outerr = outerr + return rep + +def pytest_funcarg__capsys(request): + """captures writes to sys.stdout/sys.stderr and makes + them available successively via a ``capsys.readouterr()`` method + which returns a ``(out, err)`` tuple of captured snapshot strings. + """ + return CaptureFuncarg(request, py.io.StdCapture) + +def pytest_funcarg__capfd(request): + """captures writes to file descriptors 1 and 2 and makes + snapshotted ``(out, err)`` string tuples available + via the ``capsys.readouterr()`` method. + """ + return CaptureFuncarg(request, py.io.StdCaptureFD) + + +class CaptureFuncarg: + def __init__(self, request, captureclass): + self._cclass = captureclass + #request.addfinalizer(self._finalize) + + def _start(self): + self.capture = self._cclass() + + def _finalize(self): + if hasattr(self, 'capture'): + self.capture.reset() + del self.capture + + def readouterr(self): + return self.capture.readouterr() + + def close(self): + self.capture.reset() + del self.capture + Added: pypy/branch/py11/py/plugin/pytest_default.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_default.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,149 @@ +""" default hooks and general py.test options. """ + +import sys +import py + +try: + import execnet +except ImportError: + execnet = None + +def pytest_pyfunc_call(__multicall__, pyfuncitem): + if not __multicall__.execute(): + testfunction = pyfuncitem.obj + if pyfuncitem._isyieldedfunction(): + testfunction(*pyfuncitem._args) + else: + funcargs = pyfuncitem.funcargs + testfunction(**funcargs) + +def pytest_collect_file(path, parent): + ext = path.ext + pb = path.purebasename + if pb.startswith("test_") or pb.endswith("_test") or \ + path in parent.config.args: + if ext == ".py": + return parent.Module(path, parent=parent) + +def pytest_collect_directory(path, parent): + # XXX reconsider the following comment + # not use parent.Directory here as we generally + # want dir/conftest.py to be able to + # define Directory(dir) already + if not parent.recfilter(path): # by default special ".cvs", ... + # check if cmdline specified this dir or a subdir directly + for arg in parent.config.args: + if path == arg or arg.relto(path): + break + else: + return + Directory = parent.config.getvalue('Directory', path) + return Directory(path, parent=parent) + +def pytest_report_iteminfo(item): + return item.reportinfo() + +def pytest_addoption(parser): + group = parser.getgroup("general", "running and selection options") + group._addoption('-x', '--exitfirst', + action="store_true", dest="exitfirst", default=False, + help="exit instantly on first error or failed test."), + group._addoption('-k', + action="store", dest="keyword", default='', + help="only run test items matching the given " + "space separated keywords. precede a keyword with '-' to negate. " + "Terminate the expression with ':' to treat a match as a signal " + "to run all subsequent tests. ") + group._addoption('-p', action="append", dest="plugins", default = [], + help=("load the specified plugin after command line parsing. ")) + if execnet: + group._addoption('-f', '--looponfail', + action="store_true", dest="looponfail", default=False, + help="run tests, re-run failing test set until all pass.") + + group = parser.getgroup("debugconfig", + "test process debugging and configuration") + group.addoption('--basetemp', dest="basetemp", default=None, metavar="dir", + help="base temporary directory for this test run.") + + if execnet: + add_dist_options(parser) + else: + parser.epilog = ( + "'execnet' package required for --looponfailing / distributed testing.") + +def add_dist_options(parser): + # see http://pytest.org/help/dist") + group = parser.getgroup("dist", "distributed testing") + group._addoption('--dist', metavar="distmode", + action="store", choices=['load', 'each', 'no'], + type="choice", dest="dist", default="no", + help=("set mode for distributing tests to exec environments.\n\n" + "each: send each test to each available environment.\n\n" + "load: send each test to available environment.\n\n" + "(default) no: run tests inprocess, don't distribute.")) + group._addoption('--tx', dest="tx", action="append", default=[], metavar="xspec", + help=("add a test execution environment. some examples: " + "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 " + "--tx ssh=user at codespeak.net//chdir=testcache")) + group._addoption('-d', + action="store_true", dest="distload", default=False, + help="load-balance tests. shortcut for '--dist=load'") + group._addoption('-n', dest="numprocesses", metavar="numprocesses", + action="store", type="int", + help="shortcut for '--dist=load --tx=NUM*popen'") + group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", + help="add directory for rsyncing to remote tx nodes.") + +def pytest_configure(config): + fixoptions(config) + setsession(config) + +def fixoptions(config): + if execnet: + if config.option.numprocesses: + config.option.dist = "load" + config.option.tx = ['popen'] * int(config.option.numprocesses) + if config.option.distload: + config.option.dist = "load" + +def setsession(config): + val = config.getvalue + if val("collectonly"): + from py.impl.test.session import Session + config.setsessionclass(Session) + elif execnet: + if val("looponfail"): + from py.impl.test.looponfail.remote import LooponfailingSession + config.setsessionclass(LooponfailingSession) + elif val("dist") != "no": + from py.impl.test.dist.dsession import DSession + config.setsessionclass(DSession) + +# pycollect related hooks and code, should move to pytest_pycollect.py + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + res = __multicall__.execute() + if res is not None: + return res + if collector._istestclasscandidate(name, obj): + res = collector._deprecated_join(name) + if res is not None: + return res + return collector.Class(name, parent=collector) + elif collector.funcnamefilter(name) and hasattr(obj, '__call__'): + res = collector._deprecated_join(name) + if res is not None: + return res + if is_generator(obj): + # XXX deprecation warning + return collector.Generator(name, parent=collector) + else: + return collector._genfunctions(name, obj) + +def is_generator(func): + try: + return py.code.getrawcode(func).co_flags & 32 # generator function + except AttributeError: # builtin functions have no bytecode + # assume them to not be generators + return False Added: pypy/branch/py11/py/plugin/pytest_doctest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_doctest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,86 @@ +""" +collect and execute doctests from modules and test files. + +Usage +------------- + +By default all files matching the ``test_*.txt`` pattern will +be run with the ``doctest`` module. If you issue:: + + py.test --doctest-modules + +all python files in your projects will be doctest-run +as well. +""" + +import py +from py.impl.code.code import TerminalRepr, ReprFileLocation +import doctest + +def pytest_addoption(parser): + group = parser.getgroup("doctest options") + group.addoption("--doctest-modules", + action="store_true", default=False, + help="search all python files for doctests", + dest="doctestmodules") + +def pytest_collect_file(path, parent): + if path.ext == ".py": + if parent.config.getvalue("doctestmodules"): + return DoctestModule(path, parent) + if path.check(fnmatch="test_*.txt"): + return DoctestTextfile(path, parent) + +class ReprFailDoctest(TerminalRepr): + def __init__(self, reprlocation, lines): + self.reprlocation = reprlocation + self.lines = lines + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + self.reprlocation.toterminal(tw) + +class DoctestItem(py.test.collect.Item): + def __init__(self, path, parent): + name = self.__class__.__name__ + ":" + path.basename + super(DoctestItem, self).__init__(name=name, parent=parent) + self.fspath = path + + def repr_failure(self, excinfo): + if excinfo.errisinstance(doctest.DocTestFailure): + doctestfailure = excinfo.value + example = doctestfailure.example + test = doctestfailure.test + filename = test.filename + lineno = test.lineno + example.lineno + 1 + message = excinfo.type.__name__ + reprlocation = ReprFileLocation(filename, lineno, message) + checker = doctest.OutputChecker() + REPORT_UDIFF = doctest.REPORT_UDIFF + filelines = py.path.local(filename).readlines(cr=0) + i = max(test.lineno, max(0, lineno - 10)) # XXX? + lines = [] + for line in filelines[i:lineno]: + lines.append("%03d %s" % (i+1, line)) + i += 1 + lines += checker.output_difference(example, + doctestfailure.got, REPORT_UDIFF).split("\n") + return ReprFailDoctest(reprlocation, lines) + elif excinfo.errisinstance(doctest.UnexpectedException): + excinfo = py.code.ExceptionInfo(excinfo.value.exc_info) + return super(DoctestItem, self).repr_failure(excinfo) + else: + return super(DoctestItem, self).repr_failure(excinfo) + +class DoctestTextfile(DoctestItem): + def runtest(self): + if not self._deprecated_testexecution(): + failed, tot = doctest.testfile( + str(self.fspath), module_relative=False, + raise_on_error=True, verbose=0) + +class DoctestModule(DoctestItem): + def runtest(self): + module = self.fspath.pyimport() + failed, tot = doctest.testmod( + module, raise_on_error=True, verbose=0) Added: pypy/branch/py11/py/plugin/pytest_figleaf.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_figleaf.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,51 @@ +""" +write and report coverage data with 'figleaf'. + +""" +import py + +py.test.importorskip("figleaf.annotate_html") +import figleaf + +def pytest_addoption(parser): + group = parser.getgroup('figleaf options') + group.addoption('-F', action='store_true', default=False, + dest = 'figleaf', + help=('trace python coverage with figleaf and write HTML ' + 'for files below the current working dir')) + group.addoption('--figleaf-data', action='store', default='.figleaf', + dest='figleafdata', + help='path to coverage tracing file.') + group.addoption('--figleaf-html', action='store', default='html', + dest='figleafhtml', + help='path to the coverage html dir.') + +def pytest_configure(config): + figleaf.start() + +def pytest_terminal_summary(terminalreporter): + config = terminalreporter.config + datafile = py.path.local(config.getvalue('figleafdata')) + tw = terminalreporter._tw + tw.sep('-', 'figleaf') + tw.line('Writing figleaf data to %s' % (datafile)) + figleaf.stop() + figleaf.write_coverage(str(datafile)) + coverage = get_coverage(datafile, config) + reportdir = py.path.local(config.getvalue('figleafhtml')) + tw.line('Writing figleaf html to file://%s' % (reportdir)) + figleaf.annotate_html.prepare_reportdir(str(reportdir)) + exclude = [] + figleaf.annotate_html.report_as_html(coverage, + str(reportdir), exclude, {}) + +def get_coverage(datafile, config): + # basepath = config.topdir + basepath = py.path.local() + data = figleaf.read_coverage(str(datafile)) + d = {} + coverage = figleaf.combine_coverage(d, data) + for path in coverage.keys(): + if not py.path.local(path).relto(basepath): + del coverage[path] + return coverage Added: pypy/branch/py11/py/plugin/pytest_helpconfig.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_helpconfig.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,63 @@ +""" provide version info, conftest/environment config names. +""" +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup('debugconfig') + group.addoption("--help-config", action="store_true", dest="helpconfig", + help="show available conftest.py and ENV-variable names.") + group.addoption('--version', action="store_true", + help="display py lib version and import information.") + +def pytest_configure(__multicall__, config): + if config.option.version: + p = py.path.local(py.__file__).dirpath() + sys.stderr.write("This is py.test version %s, imported from %s\n" % + (py.__version__, p)) + sys.exit(0) + if not config.option.helpconfig: + return + __multicall__.execute() + options = [] + for group in config._parser._groups: + options.extend(group.options) + widths = [0] * 10 + tw = py.io.TerminalWriter() + tw.sep("-") + tw.line("%-13s | %-18s | %-25s | %s" %( + "cmdline name", "conftest.py name", "ENV-variable name", "help")) + tw.sep("-") + + options = [opt for opt in options if opt._long_opts] + options.sort(key=lambda x: x._long_opts) + for opt in options: + if not opt._long_opts: + continue + optstrings = list(opt._long_opts) # + list(opt._short_opts) + optstrings = filter(None, optstrings) + optstring = "|".join(optstrings) + line = "%-13s | %-18s | %-25s | %s" %( + optstring, + "option_%s" % opt.dest, + "PYTEST_OPTION_%s" % opt.dest.upper(), + opt.help and opt.help or "", + ) + tw.line(line[:tw.fullwidth]) + for name, help in conftest_options: + line = "%-13s | %-18s | %-25s | %s" %( + "", + name, + "", + help, + ) + tw.line(line[:tw.fullwidth]) + + tw.sep("-") + sys.exit(0) + +conftest_options = ( + ('pytest_plugins', 'list of plugin names to load'), + ('collect_ignore', '(relative) paths ignored during collection'), + ('rsyncdirs', 'to-be-rsynced directories for dist-testing'), +) Added: pypy/branch/py11/py/plugin/pytest_hooklog.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_hooklog.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,33 @@ +""" log invocations of extension hooks to a file. """ +import py + +def pytest_addoption(parser): + parser.addoption("--hooklog", dest="hooklog", default=None, + help="write hook calls to the given file.") + +def pytest_configure(config): + hooklog = config.getvalue("hooklog") + if hooklog: + config._hooklogfile = open(hooklog, 'w') + config._hooklog_oldperformcall = config.hook._performcall + config.hook._performcall = (lambda name, multicall: + logged_call(name=name, multicall=multicall, config=config)) + +def logged_call(name, multicall, config): + f = config._hooklogfile + f.write("%s(**%s)\n" % (name, multicall.kwargs)) + try: + res = config._hooklog_oldperformcall(name=name, multicall=multicall) + except: + f.write("-> exception") + raise + f.write("-> %r" % (res,)) + return res + +def pytest_unconfigure(config): + try: + del config.hook.__dict__['_performcall'] + except KeyError: + pass + else: + config._hooklogfile.close() Added: pypy/branch/py11/py/plugin/pytest_mark.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_mark.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,148 @@ +""" +generic mechanism for marking python functions. + +By using the ``py.test.mark`` helper you can instantiate +decorators that will set named meta data on test functions. + +Marking a single function +---------------------------------------------------- + +You can "mark" a test function with meta data like this:: + + @py.test.mark.webtest + def test_send_http(): + ... + +This will set a "Marker" instance as a function attribute named "webtest". +You can also specify parametrized meta data like this:: + + @py.test.mark.webtest(firefox=30) + def test_receive(): + ... + +The named marker can be accessed like this later:: + + test_receive.webtest.kwargs['firefox'] == 30 + +In addition to set key-value pairs you can also use positional arguments:: + + @py.test.mark.webtest("triangular") + def test_receive(): + ... + +and later access it with ``test_receive.webtest.args[0] == 'triangular``. + +.. _`scoped-marking`: + +Marking classes or modules +---------------------------------------------------- + +To mark all methods of a class set a ``pytestmark`` attribute like this:: + + import py + + class TestClass: + pytestmark = py.test.mark.webtest + +You can re-use the same markers that you would use for decorating +a function - in fact this marker decorator will be applied +to all test methods of the class. + +You can also set a module level marker:: + + import py + pytestmark = py.test.mark.webtest + +in which case then the marker decorator will be applied to all functions and +methods defined in the module. + +The order in which marker functions are called is this:: + + per-function (upon import of module already) + per-class + per-module + +Later called markers may overwrite previous key-value settings. +Positional arguments are all appended to the same 'args' list +of the Marker object. + +Using "-k MARKNAME" to select tests +---------------------------------------------------- + +You can use the ``-k`` command line option to select +tests:: + + py.test -k webtest # will only run tests marked as webtest + +""" +import py + +def pytest_namespace(): + return {'mark': Mark()} + + +class Mark(object): + def __getattr__(self, name): + if name[0] == "_": + raise AttributeError(name) + return MarkerDecorator(name) + +class MarkerDecorator: + """ decorator for setting function attributes. """ + def __init__(self, name): + self.markname = name + self.kwargs = {} + self.args = [] + + def __repr__(self): + d = self.__dict__.copy() + name = d.pop('markname') + return "" %(name, d) + + def __call__(self, *args, **kwargs): + if args: + if len(args) == 1 and hasattr(args[0], '__call__'): + func = args[0] + holder = getattr(func, self.markname, None) + if holder is None: + holder = Marker(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) + return func + else: + self.args.extend(args) + self.kwargs.update(kwargs) + return self + +class Marker: + def __init__(self, name, args, kwargs): + self._name = name + self.args = args + self.kwargs = kwargs + + def __getattr__(self, name): + if name[0] != '_' and name in self.kwargs: + py.log._apiwarn("1.1", "use .kwargs attribute to access key-values") + return self.kwargs[name] + raise AttributeError(name) + + def __repr__(self): + return "" % ( + self._name, self.args, self.kwargs) + + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + item = __multicall__.execute() + if isinstance(item, py.test.collect.Function): + cls = collector.getparent(py.test.collect.Class) + mod = collector.getparent(py.test.collect.Module) + func = item.obj + func = getattr(func, '__func__', func) # py3 + func = getattr(func, 'im_func', func) # py2 + for parent in [x for x in (mod, cls) if x]: + marker = getattr(parent.obj, 'pytestmark', None) + if isinstance(marker, MarkerDecorator): + marker(func) + return item Added: pypy/branch/py11/py/plugin/pytest_monkeypatch.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_monkeypatch.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,142 @@ +""" +safely patch object attributes, dicts and environment variables. + +Usage +---------------- + +Use the `monkeypatch funcarg`_ to tweak your global test environment +for running a particular test. You can safely set/del an attribute, +dictionary item or environment variable by respective methods +on the monkeypatch funcarg. If you want e.g. to set an ENV1 variable +and have os.path.expanduser return a particular directory, you can +write it down like this: + +.. sourcecode:: python + + def test_mytest(monkeypatch): + monkeypatch.setenv('ENV1', 'myval') + monkeypatch.setattr(os.path, 'expanduser', lambda x: '/tmp/xyz') + ... # your test code that uses those patched values implicitely + +After the test function finished all modifications will be undone, +because the ``monkeypatch.undo()`` method is registered as a finalizer. + +``monkeypatch.setattr/delattr/delitem/delenv()`` all +by default raise an Exception if the target does not exist. +Pass ``raising=False`` if you want to skip this check. + +prepending to PATH or other environment variables +--------------------------------------------------------- + +To prepend a value to an already existing environment parameter: + +.. sourcecode:: python + + def test_mypath_finding(monkeypatch): + monkeypatch.setenv('PATH', 'x/y', prepend=":") + # in bash language: export PATH=x/y:$PATH + +calling "undo" finalization explicitely +----------------------------------------- + +At the end of function execution py.test invokes +a teardown hook which undoes all monkeypatch changes. +If you do not want to wait that long you can call +finalization explicitely:: + + monkeypatch.undo() + +This will undo previous changes. This call consumes the +undo stack. Calling it a second time has no effect unless +you start monkeypatching after the undo call. + +.. _`monkeypatch blog post`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ +""" + +import py, os, sys + +def pytest_funcarg__monkeypatch(request): + """The returned ``monkeypatch`` funcarg provides these + helper methods to modify objects, dictionaries or os.environ:: + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=False) + monkeypatch.delenv(name, value, raising=True) + monkeypatch.syspath_prepend(path) + + All modifications will be undone when the requesting + test function finished its execution. For the ``del`` + methods the ``raising`` parameter determines if a + KeyError or AttributeError will be raised if the + deletion has no target. + """ + monkeypatch = MonkeyPatch() + request.addfinalizer(monkeypatch.undo) + return monkeypatch + +notset = object() + +class MonkeyPatch: + def __init__(self): + self._setattr = [] + self._setitem = [] + + def setattr(self, obj, name, value, raising=True): + oldval = getattr(obj, name, notset) + if raising and oldval is notset: + raise AttributeError("%r has no attribute %r" %(obj, name)) + self._setattr.insert(0, (obj, name, oldval)) + setattr(obj, name, value) + + def delattr(self, obj, name, raising=True): + if not hasattr(obj, name): + if raising: + raise AttributeError(name) + else: + self._setattr.insert(0, (obj, name, getattr(obj, name, notset))) + delattr(obj, name) + + def setitem(self, dic, name, value): + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + dic[name] = value + + def delitem(self, dic, name, raising=True): + if name not in dic: + if raising: + raise KeyError(name) + else: + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + del dic[name] + + def setenv(self, name, value, prepend=None): + value = str(value) + if prepend and name in os.environ: + value = value + prepend + os.environ[name] + self.setitem(os.environ, name, value) + + def delenv(self, name, raising=True): + self.delitem(os.environ, name, raising=raising) + + def syspath_prepend(self, path): + if not hasattr(self, '_savesyspath'): + self._savesyspath = sys.path[:] + sys.path.insert(0, str(path)) + + def undo(self): + for obj, name, value in self._setattr: + if value is not notset: + setattr(obj, name, value) + else: + delattr(obj, name) + self._setattr[:] = [] + for dictionary, name, value in self._setitem: + if value is notset: + del dictionary[name] + else: + dictionary[name] = value + self._setitem[:] = [] + if hasattr(self, '_savesyspath'): + sys.path[:] = self._savesyspath Added: pypy/branch/py11/py/plugin/pytest_nose.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_nose.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,98 @@ +"""nose-compatibility plugin: allow to run nose test suites natively. + +This is an experimental plugin for allowing to run tests written +in 'nosetests style with py.test. + +Usage +------------- + +type:: + + py.test # instead of 'nosetests' + +and you should be able to run nose style tests and at the same +time can make full use of py.test's capabilities. + +Supported nose Idioms +---------------------- + +* setup and teardown at module/class/method level +* SkipTest exceptions and markers +* setup/teardown decorators +* yield-based tests and their setup +* general usage of nose utilities + +Unsupported idioms / issues +---------------------------------- + +- nose-style doctests are not collected and executed correctly, + also fixtures don't work. + +- no nose-configuration is recognized + +If you find other issues or have suggestions please run:: + + py.test --pastebin=all + +and send the resulting URL to a py.test contact channel, +at best to the mailing list. +""" +import py +import inspect +import sys + +def pytest_runtest_makereport(__multicall__, item, call): + SkipTest = getattr(sys.modules.get('nose', None), 'SkipTest', None) + if SkipTest: + if call.excinfo and call.excinfo.errisinstance(SkipTest): + # let's substitute the excinfo with a py.test.skip one + call2 = call.__class__(lambda: py.test.skip(str(call.excinfo.value)), call.when) + call.excinfo = call2.excinfo + +def pytest_report_iteminfo(item): + # nose 0.11.1 uses decorators for "raises" and other helpers. + # for reporting progress by filename we fish for the filename + if isinstance(item, py.test.collect.Function): + obj = item.obj + if hasattr(obj, 'compat_co_firstlineno'): + fn = sys.modules[obj.__module__].__file__ + if fn.endswith(".pyc"): + fn = fn[:-1] + #assert 0 + #fn = inspect.getsourcefile(obj) or inspect.getfile(obj) + lineno = obj.compat_co_firstlineno + return py.path.local(fn), lineno, obj.__module__ + +def pytest_runtest_setup(item): + if isinstance(item, (py.test.collect.Function)): + if isinstance(item.parent, py.test.collect.Generator): + gen = item.parent + if not hasattr(gen, '_nosegensetup'): + call_optional(gen.obj, 'setup') + if isinstance(gen.parent, py.test.collect.Instance): + call_optional(gen.parent.obj, 'setup') + gen._nosegensetup = True + if not call_optional(item.obj, 'setup'): + # call module level setup if there is no object level one + call_optional(item.parent.obj, 'setup') + +def pytest_runtest_teardown(item): + if isinstance(item, py.test.collect.Function): + if not call_optional(item.obj, 'teardown'): + call_optional(item.parent.obj, 'teardown') + #if hasattr(item.parent, '_nosegensetup'): + # #call_optional(item._nosegensetup, 'teardown') + # del item.parent._nosegensetup + +def pytest_make_collect_report(collector): + if isinstance(collector, py.test.collect.Generator): + call_optional(collector.obj, 'setup') + +def call_optional(obj, name): + method = getattr(obj, name, None) + if method: + ismethod = inspect.ismethod(method) + rawcode = py.code.getrawcode(method) + if not rawcode.co_varnames[ismethod:]: + method() + return True Added: pypy/branch/py11/py/plugin/pytest_pastebin.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_pastebin.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,83 @@ +""" +submit failure or test session information to a pastebin service. + +Usage +---------- + +**Creating a URL for each test failure**:: + + py.test --pastebin=failed + +This will submit test run information to a remote Paste service and +provide a URL for each failure. You may select tests as usual or add +for example ``-x`` if you only want to send one particular failure. + +**Creating a URL for a whole test session log**:: + + py.test --pastebin=all + +Currently only pasting to the http://paste.pocoo.org service is implemented. + +""" +import py, sys + +class url: + base = "http://paste.pocoo.org" + xmlrpc = base + "/xmlrpc/" + show = base + "/show/" + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--pastebin', metavar="mode", + action='store', dest="pastebin", default=None, + type="choice", choices=['failed', 'all'], + help="send failed|all info to Pocoo pastebin service.") + +def pytest_configure(__multicall__, config): + import tempfile + __multicall__.execute() + if config.option.pastebin == "all": + config._pastebinfile = tempfile.TemporaryFile('w+') + tr = config.pluginmanager.getplugin('terminalreporter') + oldwrite = tr._tw.write + def tee_write(s, **kwargs): + oldwrite(s, **kwargs) + config._pastebinfile.write(str(s)) + tr._tw.write = tee_write + +def pytest_unconfigure(config): + if hasattr(config, '_pastebinfile'): + config._pastebinfile.seek(0) + sessionlog = config._pastebinfile.read() + config._pastebinfile.close() + del config._pastebinfile + proxyid = getproxy().newPaste("python", sessionlog) + pastebinurl = "%s%s" % (url.show, proxyid) + sys.stderr.write("pastebin session-log: %s\n" % pastebinurl) + tr = config.pluginmanager.getplugin('terminalreporter') + del tr._tw.__dict__['write'] + +def getproxy(): + return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes + +def pytest_terminal_summary(terminalreporter): + if terminalreporter.config.option.pastebin != "failed": + return + tr = terminalreporter + if 'failed' in tr.stats: + terminalreporter.write_sep("=", "Sending information to Paste Service") + if tr.config.option.debug: + terminalreporter.write_line("xmlrpcurl: %s" %(url.xmlrpc,)) + serverproxy = getproxy() + for rep in terminalreporter.stats.get('failed'): + try: + msg = rep.longrepr.reprtraceback.reprentries[-1].reprfileloc + except AttributeError: + msg = tr._getfailureheadline(rep) + tw = py.io.TerminalWriter(stringio=True) + rep.toterminal(tw) + s = tw.stringio.getvalue() + assert len(s) + proxyid = serverproxy.newPaste("python", s) + pastebinurl = "%s%s" % (url.show, proxyid) + tr.write_line("%s --> %s" %(msg, pastebinurl)) Added: pypy/branch/py11/py/plugin/pytest_pdb.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_pdb.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,114 @@ +""" +interactive debugging with the Python Debugger. +""" +import py +import pdb, sys, linecache +from py.impl.test.outcome import Skipped +try: + import execnet +except ImportError: + execnet = None + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--pdb', + action="store_true", dest="usepdb", default=False, + help="start pdb (the Python debugger) on errors.") + + +def pytest_configure(config): + if config.option.usepdb: + if execnet: + if config.getvalue("looponfail"): + raise config.Error("--pdb incompatible with --looponfail.") + if config.option.dist != "no": + raise config.Error("--pdb incompatible with distributing tests.") + config.pluginmanager.register(PdbInvoke()) + +class PdbInvoke: + def pytest_runtest_makereport(self, item, call): + if call.excinfo and not call.excinfo.errisinstance(Skipped): + # play well with capturing, slightly hackish + capman = item.config.pluginmanager.getplugin('capturemanager') + capman.suspendcapture() + + tw = py.io.TerminalWriter() + repr = call.excinfo.getrepr() + repr.toterminal(tw) + post_mortem(call.excinfo._excinfo[2]) + + capman.resumecapture_item(item) + +class Pdb(py.std.pdb.Pdb): + def do_list(self, arg): + self.lastcmd = 'list' + last = None + if arg: + try: + x = eval(arg, {}, {}) + if type(x) == type(()): + first, last = x + first = int(first) + last = int(last) + if last < first: + # Assume it's a count + last = first + last + else: + first = max(1, int(x) - 5) + except: + print ('*** Error in argument: %s' % repr(arg)) + return + elif self.lineno is None: + first = max(1, self.curframe.f_lineno - 5) + else: + first = self.lineno + 1 + if last is None: + last = first + 10 + filename = self.curframe.f_code.co_filename + breaklist = self.get_file_breaks(filename) + try: + for lineno in range(first, last+1): + # start difference from normal do_line + line = self._getline(filename, lineno) + # end difference from normal do_line + if not line: + print ('[EOF]') + break + else: + s = repr(lineno).rjust(3) + if len(s) < 4: s = s + ' ' + if lineno in breaklist: s = s + 'B' + else: s = s + ' ' + if lineno == self.curframe.f_lineno: + s = s + '->' + sys.stdout.write(s + '\t' + line) + self.lineno = lineno + except KeyboardInterrupt: + pass + do_l = do_list + + def _getline(self, filename, lineno): + if hasattr(filename, "__source__"): + try: + return filename.__source__.lines[lineno - 1] + "\n" + except IndexError: + return None + return linecache.getline(filename, lineno) + + def get_stack(self, f, t): + # Modified from bdb.py to be able to walk the stack beyond generators, + # which does not work in the normal pdb :-( + stack, i = pdb.Pdb.get_stack(self, f, t) + if f is None: + i = max(0, len(stack) - 1) + return stack, i + +def post_mortem(t): + # modified from pdb.py for the new get_stack() implementation + p = Pdb() + p.reset() + p.interaction(None, t) + +def set_trace(): + # again, a copy of the version in pdb.py + Pdb().set_trace(sys._getframe().f_back) Added: pypy/branch/py11/py/plugin/pytest_pylint.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_pylint.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,36 @@ +"""pylint plugin + +XXX: Currently in progress, NOT IN WORKING STATE. +""" +import py + +pylint = py.test.importorskip("pylint.lint") + +def pytest_addoption(parser): + group = parser.getgroup('pylint options') + group.addoption('--pylint', action='store_true', + default=False, dest='pylint', + help='run pylint on python files.') + +def pytest_collect_file(path, parent): + if path.ext == ".py": + if parent.config.getvalue('pylint'): + return PylintItem(path, parent) + +#def pytest_terminal_summary(terminalreporter): +# print 'placeholder for pylint output' + +class PylintItem(py.test.collect.Item): + def runtest(self): + capture = py.io.StdCaptureFD() + try: + linter = pylint.lint.PyLinter() + linter.check(str(self.fspath)) + finally: + out, err = capture.reset() + rating = out.strip().split('\n')[-1] + sys.stdout.write(">>>") + print(rating) + assert 0 + + Added: pypy/branch/py11/py/plugin/pytest_pytester.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_pytester.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,459 @@ +""" +funcargs and support code for testing py.test's own functionality. +""" + +import py +import sys, os +import inspect +from py.impl.test.config import Config as pytestConfig +from py.plugin import hookspec +from py.builtin import print_ + +pytest_plugins = '_pytest' + +def pytest_funcarg__linecomp(request): + return LineComp() + +def pytest_funcarg__LineMatcher(request): + return LineMatcher + +def pytest_funcarg__testdir(request): + tmptestdir = TmpTestdir(request) + return tmptestdir + +def pytest_funcarg__reportrecorder(request): + reprec = ReportRecorder(py._com.comregistry) + request.addfinalizer(lambda: reprec.comregistry.unregister(reprec)) + return reprec + +class RunResult: + def __init__(self, ret, outlines, errlines): + self.ret = ret + self.outlines = outlines + self.errlines = errlines + self.stdout = LineMatcher(outlines) + self.stderr = LineMatcher(errlines) + +class TmpTestdir: + def __init__(self, request): + self.request = request + self._pytest = request.getfuncargvalue("_pytest") + # XXX remove duplication with tmpdir plugin + basetmp = request.config.ensuretemp("testdir") + name = request.function.__name__ + for i in range(100): + try: + tmpdir = basetmp.mkdir(name + str(i)) + except py.error.EEXIST: + continue + break + # we need to create another subdir + # because Directory.collect() currently loads + # conftest.py from sibling directories + self.tmpdir = tmpdir.mkdir(name) + self.plugins = [] + self._syspathremove = [] + self.chdir() # always chdir + assert hasattr(self, '_olddir') + self.request.addfinalizer(self.finalize) + + def __repr__(self): + return "" % (self.tmpdir,) + + def Config(self, comregistry=None, topdir=None): + if topdir is None: + topdir = self.tmpdir.dirpath() + return pytestConfig(comregistry, topdir=topdir) + + def finalize(self): + for p in self._syspathremove: + py.std.sys.path.remove(p) + if hasattr(self, '_olddir'): + self._olddir.chdir() + # delete modules that have been loaded from tmpdir + for name, mod in list(sys.modules.items()): + if mod: + fn = getattr(mod, '__file__', None) + if fn and fn.startswith(str(self.tmpdir)): + del sys.modules[name] + + def getreportrecorder(self, obj): + if isinstance(obj, py._com.Registry): + registry = obj + elif hasattr(obj, 'comregistry'): + registry = obj.comregistry + elif hasattr(obj, 'pluginmanager'): + registry = obj.pluginmanager.comregistry + elif hasattr(obj, 'config'): + registry = obj.config.pluginmanager.comregistry + else: + raise ValueError("obj %r provides no comregistry" %(obj,)) + assert isinstance(registry, py._com.Registry) + reprec = ReportRecorder(registry) + reprec.hookrecorder = self._pytest.gethookrecorder(hookspec, registry) + reprec.hook = reprec.hookrecorder.hook + return reprec + + def chdir(self): + old = self.tmpdir.chdir() + if not hasattr(self, '_olddir'): + self._olddir = old + + def _makefile(self, ext, args, kwargs): + items = list(kwargs.items()) + if args: + source = "\n".join(map(str, args)) + basename = self.request.function.__name__ + items.insert(0, (basename, source)) + ret = None + for name, value in items: + p = self.tmpdir.join(name).new(ext=ext) + source = py.code.Source(value) + p.write(str(py.code.Source(value)).lstrip()) + if ret is None: + ret = p + return ret + + + def makefile(self, ext, *args, **kwargs): + return self._makefile(ext, args, kwargs) + + def makeconftest(self, source): + return self.makepyfile(conftest=source) + + def makepyfile(self, *args, **kwargs): + return self._makefile('.py', args, kwargs) + + def maketxtfile(self, *args, **kwargs): + return self._makefile('.txt', args, kwargs) + + def syspathinsert(self, path=None): + if path is None: + path = self.tmpdir + py.std.sys.path.insert(0, str(path)) + self._syspathremove.append(str(path)) + + def mkdir(self, name): + return self.tmpdir.mkdir(name) + + def mkpydir(self, name): + p = self.mkdir(name) + p.ensure("__init__.py") + return p + + def genitems(self, colitems): + return list(self.session.genitems(colitems)) + + def inline_genitems(self, *args): + #config = self.parseconfig(*args) + config = self.parseconfig(*args) + session = config.initsession() + rec = self.getreportrecorder(config) + colitems = [config.getfsnode(arg) for arg in config.args] + items = list(session.genitems(colitems)) + return items, rec + + def runitem(self, source): + # used from runner functional tests + item = self.getitem(source) + # the test class where we are called from wants to provide the runner + testclassinstance = py.builtin._getimself(self.request.function) + runner = testclassinstance.getrunner() + return runner(item) + + def inline_runsource(self, source, *cmdlineargs): + p = self.makepyfile(source) + l = list(cmdlineargs) + [p] + return self.inline_run(*l) + + def inline_runsource1(self, *args): + args = list(args) + source = args.pop() + p = self.makepyfile(source) + l = list(args) + [p] + reprec = self.inline_run(*l) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 1, reports + return reports[0] + + def inline_run(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + session = config.initsession() + reprec = self.getreportrecorder(config) + session.main() + config.pluginmanager.do_unconfigure(config) + return reprec + + def config_preparse(self): + config = self.Config() + for plugin in self.plugins: + if isinstance(plugin, str): + config.pluginmanager.import_plugin(plugin) + else: + if isinstance(plugin, dict): + plugin = PseudoPlugin(plugin) + if not config.pluginmanager.isregistered(plugin): + config.pluginmanager.register(plugin) + return config + + def parseconfig(self, *args): + if not args: + args = (self.tmpdir,) + config = self.config_preparse() + args = list(args) + ["--basetemp=%s" % self.tmpdir.dirpath('basetemp')] + config.parse(args) + return config + + def parseconfigure(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + return config + + def getitem(self, source, funcname="test_func"): + modcol = self.getmodulecol(source) + moditems = modcol.collect() + for item in modcol.collect(): + if item.name == funcname: + return item + else: + assert 0, "%r item not found in module:\n%s" %(funcname, source) + + def getitems(self, source): + modcol = self.getmodulecol(source) + return list(modcol.config.initsession().genitems([modcol])) + #assert item is not None, "%r item not found in module:\n%s" %(funcname, source) + #return item + + def getfscol(self, path, configargs=()): + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + return self.config.getfsnode(path) + + def getmodulecol(self, source, configargs=(), withinit=False): + kw = {self.request.function.__name__: py.code.Source(source).strip()} + path = self.makepyfile(**kw) + if withinit: + self.makepyfile(__init__ = "#") + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + #self.config.pluginmanager.do_configure(config=self.config) + # XXX + self.config.pluginmanager.import_plugin("runner") + plugin = self.config.pluginmanager.getplugin("runner") + plugin.pytest_configure(config=self.config) + + return self.config.getfsnode(path) + + def prepare(self): + p = self.tmpdir.join("conftest.py") + if not p.check(): + plugins = [x for x in self.plugins if isinstance(x, str)] + if not plugins: + return + p.write("import py ; pytest_plugins = %r" % plugins) + else: + if self.plugins: + print ("warning, ignoring reusing existing %s" % p) + + def popen(self, cmdargs, stdout, stderr, **kw): + if not hasattr(py.std, 'subprocess'): + py.test.skip("no subprocess module") + env = os.environ.copy() + env['PYTHONPATH'] = ":".join(filter(None, [ + str(os.getcwd()), env.get('PYTHONPATH', '')])) + kw['env'] = env + #print "env", env + return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw) + + def run(self, *cmdargs): + self.prepare() + old = self.tmpdir.chdir() + #print "chdir", self.tmpdir + try: + return self._run(*cmdargs) + finally: + old.chdir() + + def _run(self, *cmdargs): + cmdargs = [str(x) for x in cmdargs] + p1 = py.path.local("stdout") + p2 = py.path.local("stderr") + print_("running", cmdargs, "curdir=", py.path.local()) + f1 = p1.open("w") + f2 = p2.open("w") + popen = self.popen(cmdargs, stdout=f1, stderr=f2, + close_fds=(sys.platform != "win32")) + ret = popen.wait() + f1.close() + f2.close() + out, err = p1.readlines(cr=0), p2.readlines(cr=0) + if err: + for line in err: + py.builtin.print_(line, file=sys.stderr) + if out: + for line in out: + py.builtin.print_(line, file=sys.stdout) + return RunResult(ret, out, err) + + def runpybin(self, scriptname, *args): + fullargs = self._getpybinargs(scriptname) + args + return self.run(*fullargs) + + def _getpybinargs(self, scriptname): + bindir = py._dir.dirpath('bin') + if not bindir.check(): + script = py.path.local.sysfind(scriptname) + else: + script = bindir.join(scriptname) + assert script.check() + return py.std.sys.executable, script + + def runpython(self, script): + return self.run(py.std.sys.executable, script) + + def runpytest(self, *args): + p = py.path.local.make_numbered_dir(prefix="runpytest-", + keep=None, rootdir=self.tmpdir) + args = ('--basetemp=%s' % p, ) + args + return self.runpybin("py.test", *args) + + def spawn_pytest(self, string, expect_timeout=10.0): + pexpect = py.test.importorskip("pexpect", "2.3") + basetemp = self.tmpdir.mkdir("pexpect") + invoke = "%s %s" % self._getpybinargs("py.test") + cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string) + child = pexpect.spawn(cmd, logfile=basetemp.join("spawn.out").open("w")) + child.timeout = expect_timeout + return child + +class PseudoPlugin: + def __init__(self, vars): + self.__dict__.update(vars) + +class ReportRecorder(object): + def __init__(self, comregistry): + self.comregistry = comregistry + comregistry.register(self) + + def getcall(self, name): + return self.hookrecorder.getcall(name) + + def popcall(self, name): + return self.hookrecorder.popcall(name) + + def getcalls(self, names): + """ return list of ParsedCall instances matching the given eventname. """ + return self.hookrecorder.getcalls(names) + + # functionality for test reports + + def getreports(self, names="pytest_runtest_logreport pytest_collectreport"): + return [x.report for x in self.getcalls(names)] + + def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport"): + """ return a testreport whose dotted import path matches """ + l = [] + for rep in self.getreports(names=names): + colitem = rep.getnode() + if not inamepart or inamepart in colitem.listnames(): + l.append(rep) + if not l: + raise ValueError("could not find test report matching %r: no test reports at all!" % + (inamepart,)) + if len(l) > 1: + raise ValueError("found more than one testreport matching %r: %s" %( + inamepart, l)) + return l[0] + + def getfailures(self, names='pytest_runtest_logreport pytest_collectreport'): + return [rep for rep in self.getreports(names) if rep.failed] + + def getfailedcollections(self): + return self.getfailures('pytest_collectreport') + + def listoutcomes(self): + passed = [] + skipped = [] + failed = [] + for rep in self.getreports("pytest_runtest_logreport"): + if rep.passed: + if rep.when == "call": + passed.append(rep) + elif rep.skipped: + skipped.append(rep) + elif rep.failed: + failed.append(rep) + return passed, skipped, failed + + def countoutcomes(self): + return [len(x) for x in self.listoutcomes()] + + def assertoutcome(self, passed=0, skipped=0, failed=0): + realpassed, realskipped, realfailed = self.listoutcomes() + assert passed == len(realpassed) + assert skipped == len(realskipped) + assert failed == len(realfailed) + + def clear(self): + self.hookrecorder.calls[:] = [] + + def unregister(self): + self.comregistry.unregister(self) + self.hookrecorder.finish_recording() + +class LineComp: + def __init__(self): + self.stringio = py.io.TextIO() + + def assert_contains_lines(self, lines2): + """ assert that lines2 are contained (linearly) in lines1. + return a list of extralines found. + """ + __tracebackhide__ = True + val = self.stringio.getvalue() + self.stringio.truncate(0) # remove what we got + lines1 = val.split("\n") + return LineMatcher(lines1).fnmatch_lines(lines2) + +class LineMatcher: + def __init__(self, lines): + self.lines = lines + + def str(self): + return "\n".join(self.lines) + + def fnmatch_lines(self, lines2): + if isinstance(lines2, str): + lines2 = py.code.Source(lines2) + if isinstance(lines2, py.code.Source): + lines2 = lines2.strip().lines + + from fnmatch import fnmatch + __tracebackhide__ = True + lines1 = self.lines[:] + nextline = None + extralines = [] + for line in lines2: + nomatchprinted = False + while lines1: + nextline = lines1.pop(0) + if line == nextline: + print_("exact match:", repr(line)) + break + elif fnmatch(nextline, line): + print_("fnmatch:", repr(line)) + print_(" with:", repr(nextline)) + break + else: + if not nomatchprinted: + print_("nomatch:", repr(line)) + nomatchprinted = True + print_(" and:", repr(nextline)) + extralines.append(nextline) + else: + if line != nextline: + #__tracebackhide__ = True + raise AssertionError("expected line not found: %r" % line) + extralines.extend(lines1) + return extralines Added: pypy/branch/py11/py/plugin/pytest_recwarn.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_recwarn.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,120 @@ +""" +helpers for asserting deprecation and other warnings. + +Example usage +--------------------- + +You can use the ``recwarn`` funcarg to track +warnings within a test function: + +.. sourcecode:: python + + def test_hello(recwarn): + from warnings import warn + warn("hello", DeprecationWarning) + w = recwarn.pop(DeprecationWarning) + assert issubclass(w.category, DeprecationWarning) + assert 'hello' in str(w.message) + assert w.filename + assert w.lineno + +You can also call a global helper for checking +taht a certain function call yields a Deprecation +warning: + +.. sourcecode:: python + + import py + + def test_global(): + py.test.deprecated_call(myfunction, 17) + + +""" + +import py +import os + +def pytest_funcarg__recwarn(request): + """Return a WarningsRecorder instance that provides these methods: + + * ``pop(category=None)``: return last warning matching the category. + * ``clear()``: clear list of warnings + """ + warnings = WarningsRecorder() + request.addfinalizer(warnings.finalize) + return warnings + +def pytest_namespace(): + return {'deprecated_call': deprecated_call} + +def deprecated_call(func, *args, **kwargs): + """ assert that calling func(*args, **kwargs) + triggers a DeprecationWarning. + """ + warningmodule = py.std.warnings + l = [] + oldwarn_explicit = getattr(warningmodule, 'warn_explicit') + def warn_explicit(*args, **kwargs): + l.append(args) + oldwarn_explicit(*args, **kwargs) + oldwarn = getattr(warningmodule, 'warn') + def warn(*args, **kwargs): + l.append(args) + oldwarn(*args, **kwargs) + + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + try: + ret = func(*args, **kwargs) + finally: + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + if not l: + #print warningmodule + raise AssertionError("%r did not produce DeprecationWarning" %(func,)) + return ret + + +class RecordedWarning: + def __init__(self, message, category, filename, lineno, line): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.line = line + +class WarningsRecorder: + def __init__(self): + warningmodule = py.std.warnings + self.list = [] + def showwarning(message, category, filename, lineno, line=0): + self.list.append(RecordedWarning( + message, category, filename, lineno, line)) + try: + self.old_showwarning(message, category, + filename, lineno, line=line) + except TypeError: + # < python2.6 + self.old_showwarning(message, category, filename, lineno) + self.old_showwarning = warningmodule.showwarning + warningmodule.showwarning = showwarning + + def pop(self, cls=Warning): + """ pop the first recorded warning, raise exception if not exists.""" + for i, w in enumerate(self.list): + if issubclass(w.category, cls): + return self.list.pop(i) + __tracebackhide__ = True + assert 0, "%r not found in %r" %(cls, self.list) + + #def resetregistry(self): + # import warnings + # warnings.onceregistry.clear() + # warnings.__warningregistry__.clear() + + def clear(self): + self.list[:] = [] + + def finalize(self): + py.std.warnings.showwarning = self.old_showwarning Added: pypy/branch/py11/py/plugin/pytest_restdoc.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_restdoc.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,432 @@ +""" +perform ReST syntax, local and remote reference tests on .rst/.txt files. +""" +import py +import sys, os, re + +def pytest_addoption(parser): + group = parser.getgroup("ReST", "ReST documentation check options") + group.addoption('-R', '--urlcheck', + action="store_true", dest="urlcheck", default=False, + help="urlopen() remote links found in ReST text files.") + group.addoption('--urltimeout', action="store", metavar="secs", + type="int", dest="urlcheck_timeout", default=5, + help="timeout in seconds for remote urlchecks") + group.addoption('--forcegen', + action="store_true", dest="forcegen", default=False, + help="force generation of html files.") + +def pytest_collect_file(path, parent): + if path.ext in (".txt", ".rst"): + project = getproject(path) + if project is not None: + return ReSTFile(path, parent=parent, project=project) + +def getproject(path): + for parent in path.parts(reverse=True): + confrest = parent.join("confrest.py") + if confrest.check(): + Project = confrest.pyimport().Project + return Project(parent) + +class ReSTFile(py.test.collect.File): + def __init__(self, fspath, parent, project=None): + super(ReSTFile, self).__init__(fspath=fspath, parent=parent) + if project is None: + project = getproject(fspath) + assert project is not None + self.project = project + + def collect(self): + return [ + ReSTSyntaxTest(self.project, "ReSTSyntax", parent=self), + LinkCheckerMaker("checklinks", parent=self), + DoctestText("doctest", parent=self), + ] + +def deindent(s, sep='\n'): + leastspaces = -1 + lines = s.split(sep) + for line in lines: + if not line.strip(): + continue + spaces = len(line) - len(line.lstrip()) + if leastspaces == -1 or spaces < leastspaces: + leastspaces = spaces + if leastspaces == -1: + return s + for i, line in enumerate(lines): + if not line.strip(): + lines[i] = '' + else: + lines[i] = line[leastspaces:] + return sep.join(lines) + +class ReSTSyntaxTest(py.test.collect.Item): + def __init__(self, project, *args, **kwargs): + super(ReSTSyntaxTest, self).__init__(*args, **kwargs) + self.project = project + + def reportinfo(self): + return self.fspath, None, "syntax check" + + def runtest(self): + self.restcheck(py.path.svnwc(self.fspath)) + + def restcheck(self, path): + py.test.importorskip("docutils") + self.register_linkrole() + from docutils.utils import SystemMessage + try: + self._checkskip(path, self.project.get_htmloutputpath(path)) + self.project.process(path) + except KeyboardInterrupt: + raise + except SystemMessage: + # we assume docutils printed info on stdout + py.test.fail("docutils processing failed, see captured stderr") + + def register_linkrole(self): + #directive.register_linkrole('api', self.resolve_linkrole) + #directive.register_linkrole('source', self.resolve_linkrole) +# +# # XXX fake sphinx' "toctree" and refs +# directive.register_linkrole('ref', self.resolve_linkrole) + + from docutils.parsers.rst import directives + def toctree_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + toctree_directive.content = 1 + toctree_directive.options = {'maxdepth': int, 'glob': directives.flag, + 'hidden': directives.flag} + directives.register_directive('toctree', toctree_directive) + self.register_pygments() + + def register_pygments(self): + # taken from pygments-main/external/rst-directive.py + from docutils.parsers.rst import directives + try: + from pygments.formatters import HtmlFormatter + except ImportError: + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + pygments_directive.options = {} + else: + # The default formatter + DEFAULT = HtmlFormatter(noclasses=True) + # Add name -> formatter pairs for every variant you want to use + VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), + } + + from docutils import nodes + + from pygments import highlight + from pygments.lexers import get_lexer_by_name, TextLexer + + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + try: + lexer = get_lexer_by_name(arguments[0]) + except ValueError: + # no lexer found - use the text one instead of an exception + lexer = TextLexer() + # take an arbitrary option if more than one is given + formatter = options and VARIANTS[options.keys()[0]] or DEFAULT + parsed = highlight('\n'.join(content), lexer, formatter) + return [nodes.raw('', parsed, format='html')] + + pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS]) + + pygments_directive.arguments = (1, 0, 1) + pygments_directive.content = 1 + directives.register_directive('sourcecode', pygments_directive) + + def resolve_linkrole(self, name, text, check=True): + apigen_relpath = self.project.apigen_relpath + + if name == 'api': + if text == 'py': + return ('py', apigen_relpath + 'api/index.html') + else: + assert text.startswith('py.'), ( + 'api link "%s" does not point to the py package') % (text,) + dotted_name = text + if dotted_name.find('(') > -1: + dotted_name = dotted_name[:text.find('(')] + # remove pkg root + path = dotted_name.split('.')[1:] + dotted_name = '.'.join(path) + obj = py + if check: + for chunk in path: + try: + obj = getattr(obj, chunk) + except AttributeError: + raise AssertionError( + 'problem with linkrole :api:`%s`: can not resolve ' + 'dotted name %s' % (text, dotted_name,)) + return (text, apigen_relpath + 'api/%s.html' % (dotted_name,)) + elif name == 'source': + assert text.startswith('py/'), ('source link "%s" does not point ' + 'to the py package') % (text,) + relpath = '/'.join(text.split('/')[1:]) + if check: + pkgroot = py._impldir + abspath = pkgroot.join(relpath) + assert pkgroot.join(relpath).check(), ( + 'problem with linkrole :source:`%s`: ' + 'path %s does not exist' % (text, relpath)) + if relpath.endswith('/') or not relpath: + relpath += 'index.html' + else: + relpath += '.html' + return (text, apigen_relpath + 'source/%s' % (relpath,)) + elif name == 'ref': + return ("", "") + + def _checkskip(self, lpath, htmlpath=None): + if not self.config.getvalue("forcegen"): + lpath = py.path.local(lpath) + if htmlpath is not None: + htmlpath = py.path.local(htmlpath) + if lpath.ext == '.txt': + htmlpath = htmlpath or lpath.new(ext='.html') + if htmlpath.check(file=1) and htmlpath.mtime() >= lpath.mtime(): + py.test.skip("html file is up to date, use --forcegen to regenerate") + #return [] # no need to rebuild + +class DoctestText(py.test.collect.Item): + def reportinfo(self): + return self.fspath, None, "doctest" + + def runtest(self): + content = self._normalize_linesep() + newcontent = self.config.hook.pytest_doctest_prepare_content(content=content) + if newcontent is not None: + content = newcontent + s = content + l = [] + prefix = '.. >>> ' + mod = py.std.types.ModuleType(self.fspath.purebasename) + skipchunk = False + for line in deindent(s).split('\n'): + stripped = line.strip() + if skipchunk and line.startswith(skipchunk): + py.builtin.print_("skipping", line) + continue + skipchunk = False + if stripped.startswith(prefix): + try: + py.builtin.exec_(py.code.Source( + stripped[len(prefix):]).compile(), mod.__dict__) + except ValueError: + e = sys.exc_info()[1] + if e.args and e.args[0] == "skipchunk": + skipchunk = " " * (len(line) - len(line.lstrip())) + else: + raise + else: + l.append(line) + docstring = "\n".join(l) + mod.__doc__ = docstring + failed, tot = py.std.doctest.testmod(mod, verbose=1) + if failed: + py.test.fail("doctest %s: %s failed out of %s" %( + self.fspath, failed, tot)) + + def _normalize_linesep(self): + # XXX quite nasty... but it works (fixes win32 issues) + s = self.fspath.read() + linesep = '\n' + if '\r' in s: + if '\n' not in s: + linesep = '\r' + else: + linesep = '\r\n' + s = s.replace(linesep, '\n') + return s + +class LinkCheckerMaker(py.test.collect.Collector): + def collect(self): + return list(self.genlinkchecks()) + + def genlinkchecks(self): + path = self.fspath + # generating functions + args as single tests + timeout = self.config.getvalue("urlcheck_timeout") + for lineno, line in enumerate(path.readlines()): + line = line.strip() + if line.startswith('.. _'): + if line.startswith('.. _`'): + delim = '`:' + else: + delim = ':' + l = line.split(delim, 1) + if len(l) != 2: + continue + tryfn = l[1].strip() + name = "%s:%d" %(tryfn, lineno) + if tryfn.startswith('http:') or tryfn.startswith('https'): + if self.config.getvalue("urlcheck"): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno, timeout), checkfunc=urlcheck) + elif tryfn.startswith('webcal:'): + continue + else: + i = tryfn.find('#') + if i != -1: + checkfn = tryfn[:i] + else: + checkfn = tryfn + if checkfn.strip() and (1 or checkfn.endswith('.html')): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno), checkfunc=localrefcheck) + +class CheckLink(py.test.collect.Item): + def __init__(self, name, parent, args, checkfunc): + super(CheckLink, self).__init__(name, parent) + self.args = args + self.checkfunc = checkfunc + + def runtest(self): + return self.checkfunc(*self.args) + + def reportinfo(self, basedir=None): + return (self.fspath, self.args[2], "checklink: %s" % self.args[0]) + +def urlcheck(tryfn, path, lineno, TIMEOUT_URLOPEN): + old = py.std.socket.getdefaulttimeout() + py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN) + try: + try: + py.builtin.print_("trying remote", tryfn) + py.std.urllib2.urlopen(tryfn) + finally: + py.std.socket.setdefaulttimeout(old) + except (py.std.urllib2.URLError, py.std.urllib2.HTTPError): + e = sys.exc_info()[1] + if getattr(e, 'code', None) in (401, 403): # authorization required, forbidden + py.test.skip("%s: %s" %(tryfn, str(e))) + else: + py.test.fail("remote reference error %r in %s:%d\n%s" %( + tryfn, path.basename, lineno+1, e)) + +def localrefcheck(tryfn, path, lineno): + # assume it should be a file + i = tryfn.find('#') + if tryfn.startswith('javascript:'): + return # don't check JS refs + if i != -1: + anchor = tryfn[i+1:] + tryfn = tryfn[:i] + else: + anchor = '' + fn = path.dirpath(tryfn) + ishtml = fn.ext == '.html' + fn = ishtml and fn.new(ext='.txt') or fn + py.builtin.print_("filename is", fn) + if not fn.check(): # not ishtml or not fn.check(): + if not py.path.local(tryfn).check(): # the html could be there + py.test.fail("reference error %r in %s:%d" %( + tryfn, path.basename, lineno+1)) + if anchor: + source = unicode(fn.read(), 'latin1') + source = source.lower().replace('-', ' ') # aehem + + anchor = anchor.replace('-', ' ') + match2 = ".. _`%s`:" % anchor + match3 = ".. _%s:" % anchor + candidates = (anchor, match2, match3) + py.builtin.print_("candidates", repr(candidates)) + for line in source.split('\n'): + line = line.strip() + if line in candidates: + break + else: + py.test.fail("anchor reference error %s#%s in %s:%d" %( + tryfn, anchor, path.basename, lineno+1)) + +if hasattr(sys.stdout, 'fileno') and os.isatty(sys.stdout.fileno()): + def log(msg): + print(msg) +else: + def log(msg): + pass + +def convert_rest_html(source, source_path, stylesheet=None, encoding='latin1'): + """ return html latin1-encoded document for the given input. + source a ReST-string + sourcepath where to look for includes (basically) + stylesheet path (to be used if any) + """ + from docutils.core import publish_string + kwargs = { + 'stylesheet' : stylesheet, + 'stylesheet_path': None, + 'traceback' : 1, + 'embed_stylesheet': 0, + 'output_encoding' : encoding, + #'halt' : 0, # 'info', + 'halt_level' : 2, + } + # docutils uses os.getcwd() :-( + source_path = os.path.abspath(str(source_path)) + prevdir = os.getcwd() + try: + #os.chdir(os.path.dirname(source_path)) + return publish_string(source, source_path, writer_name='html', + settings_overrides=kwargs) + finally: + os.chdir(prevdir) + +def process(txtpath, encoding='latin1'): + """ process a textfile """ + log("processing %s" % txtpath) + assert txtpath.check(ext='.txt') + if isinstance(txtpath, py.path.svnwc): + txtpath = txtpath.localpath + htmlpath = txtpath.new(ext='.html') + #svninfopath = txtpath.localpath.new(ext='.svninfo') + + style = txtpath.dirpath('style.css') + if style.check(): + stylesheet = style.basename + else: + stylesheet = None + content = unicode(txtpath.read(), encoding) + doc = convert_rest_html(content, txtpath, stylesheet=stylesheet, encoding=encoding) + htmlpath.open('wb').write(doc) + #log("wrote %r" % htmlpath) + #if txtpath.check(svnwc=1, versioned=1): + # info = txtpath.info() + # svninfopath.dump(info) + +if sys.version_info > (3, 0): + def _uni(s): return s +else: + def _uni(s): + return unicode(s) + +rex1 = re.compile(r'.*(.*).*', re.MULTILINE | re.DOTALL) +rex2 = re.compile(r'.*
(.*)
.*', re.MULTILINE | re.DOTALL) + +def strip_html_header(string, encoding='utf8'): + """ return the content of the body-tag """ + uni = unicode(string, encoding) + for rex in rex1,rex2: + match = rex.search(uni) + if not match: + break + uni = match.group(1) + return uni + +class Project: # used for confrest.py files + def __init__(self, sourcepath): + self.sourcepath = sourcepath + def process(self, path): + return process(path) + def get_htmloutputpath(self, path): + return path.new(ext='html') Added: pypy/branch/py11/py/plugin/pytest_resultlog.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_resultlog.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,94 @@ +"""resultlog plugin for machine-readable logging of test results. + Useful for buildbot integration code. +""" + +import py +from py.builtin import print_ + +def pytest_addoption(parser): + group = parser.getgroup("resultlog", "resultlog plugin options") + group.addoption('--resultlog', action="store", dest="resultlog", metavar="path", default=None, + help="path for machine-readable result log.") + +def pytest_configure(config): + resultlog = config.option.resultlog + if resultlog: + logfile = open(resultlog, 'w', 1) # line buffered + config._resultlog = ResultLog(config, logfile) + config.pluginmanager.register(config._resultlog) + +def pytest_unconfigure(config): + resultlog = getattr(config, '_resultlog', None) + if resultlog: + resultlog.logfile.close() + del config._resultlog + config.pluginmanager.unregister(resultlog) + +def generic_path(item): + chain = item.listchain() + gpath = [chain[0].name] + fspath = chain[0].fspath + fspart = False + for node in chain[1:]: + newfspath = node.fspath + if newfspath == fspath: + if fspart: + gpath.append(':') + fspart = False + else: + gpath.append('.') + else: + gpath.append('/') + fspart = True + name = node.name + if name[0] in '([': + gpath.pop() + gpath.append(name) + fspath = newfspath + return ''.join(gpath) + +class ResultLog(object): + def __init__(self, config, logfile): + self.config = config + self.logfile = logfile # preferably line buffered + + def write_log_entry(self, testpath, shortrepr, longrepr): + print_("%s %s" % (shortrepr, testpath), file=self.logfile) + for line in longrepr.splitlines(): + print_(" %s" % line, file=self.logfile) + + def log_outcome(self, node, shortrepr, longrepr): + testpath = generic_path(node) + self.write_log_entry(testpath, shortrepr, longrepr) + + def pytest_runtest_logreport(self, report): + res = self.config.hook.pytest_report_teststatus(report=report) + if res is not None: + code = res[1] + else: + code = report.shortrepr + if code == 'x': + longrepr = str(report.longrepr) + elif code == 'P': + longrepr = '' + elif report.passed: + longrepr = "" + elif report.failed: + longrepr = str(report.longrepr) + elif report.skipped: + longrepr = str(report.longrepr.reprcrash.message) + self.log_outcome(report.item, code, longrepr) + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + code = "F" + else: + assert report.skipped + code = "S" + longrepr = str(report.longrepr.reprcrash) + self.log_outcome(report.collector, code, longrepr) + + def pytest_internalerror(self, excrepr): + path = excrepr.reprcrash.path + self.write_log_entry(path, '!', str(excrepr)) Added: pypy/branch/py11/py/plugin/pytest_runner.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_runner.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,294 @@ +""" +collect and run test items and create reports. +""" + +import py +from py.impl.test.outcome import Skipped + +# +# pytest plugin hooks + +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--boxed', + action="store_true", dest="boxed", default=False, + help="box each test run in a separate process") + +# XXX move to pytest_sessionstart and fix py.test owns tests +def pytest_configure(config): + config._setupstate = SetupState() + +def pytest_sessionfinish(session, exitstatus): + if hasattr(session.config, '_setupstate'): + hook = session.config.hook + rep = hook.pytest__teardown_final(session=session) + if rep: + hook.pytest__teardown_final_logerror(report=rep) + +def pytest_make_collect_report(collector): + result = excinfo = None + try: + result = collector._memocollect() + except KeyboardInterrupt: + raise + except: + excinfo = py.code.ExceptionInfo() + return CollectReport(collector, result, excinfo) + +def pytest_runtest_protocol(item): + if item.config.getvalue("boxed"): + reports = forked_run_report(item) + for rep in reports: + item.config.hook.pytest_runtest_logreport(report=rep) + else: + runtestprotocol(item) + return True + +def runtestprotocol(item, log=True): + rep = call_and_report(item, "setup", log) + reports = [rep] + if rep.passed: + reports.append(call_and_report(item, "call", log)) + reports.append(call_and_report(item, "teardown", log)) + return reports + +def pytest_runtest_setup(item): + item.config._setupstate.prepare(item) + +def pytest_runtest_call(item): + if not item._deprecated_testexecution(): + item.runtest() + +def pytest_runtest_makereport(item, call): + return ItemTestReport(item, call.excinfo, call.when) + +def pytest_runtest_teardown(item): + item.config._setupstate.teardown_exact(item) + +def pytest__teardown_final(session): + call = CallInfo(session.config._setupstate.teardown_all, when="teardown") + if call.excinfo: + rep = TeardownErrorReport(call.excinfo) + return rep + +def pytest_report_teststatus(report): + if report.when in ("setup", "teardown"): + if report.failed: + # category, shortletter, verbose-word + return "error", "E", "ERROR" + elif report.skipped: + return "skipped", "s", "SKIPPED" + else: + return "", "", "" +# +# Implementation + +def call_and_report(item, when, log=True): + call = call_runtest_hook(item, when) + hook = item.config.hook + report = hook.pytest_runtest_makereport(item=item, call=call) + if log and (when == "call" or not report.passed): + hook.pytest_runtest_logreport(report=report) + return report + +def call_runtest_hook(item, when): + hookname = "pytest_runtest_" + when + hook = getattr(item.config.hook, hookname) + return CallInfo(lambda: hook(item=item), when=when) + +class CallInfo: + excinfo = None + def __init__(self, func, when): + self.when = when + try: + self.result = func() + except KeyboardInterrupt: + raise + except: + self.excinfo = py.code.ExceptionInfo() + + def __repr__(self): + if self.excinfo: + status = "exception: %s" % str(self.excinfo.value) + else: + status = "result: %r" % (self.result,) + return "" % (self.when, status) + +def forked_run_report(item): + # for now, we run setup/teardown in the subprocess + # XXX optionally allow sharing of setup/teardown + EXITSTATUS_TESTEXIT = 4 + from py.impl.test.dist.mypickle import ImmutablePickler + ipickle = ImmutablePickler(uneven=0) + ipickle.selfmemoize(item.config) + # XXX workaround the issue that 2.6 cannot pickle + # instances of classes defined in global conftest.py files + ipickle.selfmemoize(item) + def runforked(): + try: + reports = runtestprotocol(item, log=False) + except KeyboardInterrupt: + py.std.os._exit(EXITSTATUS_TESTEXIT) + return ipickle.dumps(reports) + + ff = py.process.ForkedFunc(runforked) + result = ff.waitfinish() + if result.retval is not None: + return ipickle.loads(result.retval) + else: + if result.exitstatus == EXITSTATUS_TESTEXIT: + py.test.exit("forked test item %s raised Exit" %(item,)) + return [report_process_crash(item, result)] + +def report_process_crash(item, result): + path, lineno = item._getfslineno() + info = "%s:%s: running the test CRASHED with signal %d" %( + path, lineno, result.signal) + return ItemTestReport(item, excinfo=info, when="???") + +class BaseReport(object): + def __repr__(self): + l = ["%s=%s" %(key, value) + for key, value in self.__dict__.items()] + return "<%s %s>" %(self.__class__.__name__, " ".join(l),) + + def toterminal(self, out): + longrepr = self.longrepr + if hasattr(longrepr, 'toterminal'): + longrepr.toterminal(out) + else: + out.line(str(longrepr)) + +class ItemTestReport(BaseReport): + failed = passed = skipped = False + + def __init__(self, item, excinfo=None, when=None): + self.item = item + self.when = when + if item and when != "setup": + self.keywords = item.readkeywords() + else: + # if we fail during setup it might mean + # we are not able to access the underlying object + # this might e.g. happen if we are unpickled + # and our parent collector did not collect us + # (because it e.g. skipped for platform reasons) + self.keywords = {} + if not excinfo: + self.passed = True + self.shortrepr = "." + else: + if not isinstance(excinfo, py.code.ExceptionInfo): + self.failed = True + shortrepr = "?" + longrepr = excinfo + elif excinfo.errisinstance(Skipped): + self.skipped = True + shortrepr = "s" + longrepr = self.item._repr_failure_py(excinfo) + else: + self.failed = True + shortrepr = self.item.shortfailurerepr + if self.when == "call": + longrepr = self.item.repr_failure(excinfo) + else: # exception in setup or teardown + longrepr = self.item._repr_failure_py(excinfo) + shortrepr = shortrepr.lower() + self.shortrepr = shortrepr + self.longrepr = longrepr + + def __repr__(self): + status = (self.passed and "passed" or + self.skipped and "skipped" or + self.failed and "failed" or + "CORRUPT") + l = [repr(self.item.name), "when=%r" % self.when, "outcome %r" % status,] + if hasattr(self, 'node'): + l.append("txnode=%s" % self.node.gateway.id) + info = " " .join(map(str, l)) + return "" % info + + def getnode(self): + return self.item + +class CollectReport(BaseReport): + skipped = failed = passed = False + + def __init__(self, collector, result, excinfo=None): + self.collector = collector + if not excinfo: + self.passed = True + self.result = result + else: + self.longrepr = self.collector._repr_failure_py(excinfo) + if excinfo.errisinstance(Skipped): + self.skipped = True + self.reason = str(excinfo.value) + else: + self.failed = True + + def getnode(self): + return self.collector + +class TeardownErrorReport(BaseReport): + skipped = passed = False + failed = True + when = "teardown" + def __init__(self, excinfo): + self.longrepr = excinfo.getrepr(funcargs=True) + +class SetupState(object): + """ shared state for setting up/tearing down test items or collectors. """ + def __init__(self): + self.stack = [] + self._finalizers = {} + + def addfinalizer(self, finalizer, colitem): + """ attach a finalizer to the given colitem. + if colitem is None, this will add a finalizer that + is called at the end of teardown_all(). + """ + assert hasattr(finalizer, '__call__') + #assert colitem in self.stack + self._finalizers.setdefault(colitem, []).append(finalizer) + + def _pop_and_teardown(self): + colitem = self.stack.pop() + self._teardown_with_finalization(colitem) + + def _callfinalizers(self, colitem): + finalizers = self._finalizers.pop(colitem, None) + while finalizers: + fin = finalizers.pop() + fin() + + def _teardown_with_finalization(self, colitem): + self._callfinalizers(colitem) + if colitem: + colitem.teardown() + for colitem in self._finalizers: + assert colitem is None or colitem in self.stack + + def teardown_all(self): + while self.stack: + self._pop_and_teardown() + self._teardown_with_finalization(None) + assert not self._finalizers + + def teardown_exact(self, item): + if self.stack and item == self.stack[-1]: + self._pop_and_teardown() + else: + self._callfinalizers(item) + + def prepare(self, colitem): + """ setup objects along the collector chain to the test-method + and teardown previously setup objects.""" + needed_collectors = colitem.listchain() + while self.stack: + if self.stack == needed_collectors[:len(self.stack)]: + break + self._pop_and_teardown() + for col in needed_collectors[len(self.stack):]: + col.setup() + self.stack.append(col) Added: pypy/branch/py11/py/plugin/pytest_skipping.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_skipping.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,232 @@ +""" +advanced skipping for python test functions, classes or modules. + +With this plugin you can mark test functions for conditional skipping +or as "xfail", expected-to-fail. Skipping a test will avoid running it +at all while xfail-marked tests will run and result in an inverted outcome: +a pass becomes a failure and a fail becomes a semi-passing one. + +The need for skipping a test is usually connected to a condition. +If a test fails under all conditions then it's probably better +to mark your test as 'xfail'. + +By passing ``--report=xfailed,skipped`` to the terminal reporter +you will see summary information on skips and xfail-run tests +at the end of a test run. + +.. _skipif: + +Skipping a single function +------------------------------------------- + +Here is an example for marking a test function to be skipped +when run on a Python3 interpreter:: + + @py.test.mark.skipif("sys.version_info >= (3,0)") + def test_function(): + ... + +During test function setup the skipif condition is +evaluated by calling ``eval(expr, namespace)``. The namespace +contains the ``sys`` and ``os`` modules and the test +``config`` object. The latter allows you to skip based +on a test configuration value e.g. like this:: + + @py.test.mark.skipif("not config.getvalue('db')") + def test_function(...): + ... + +Create a shortcut for your conditional skip decorator +at module level like this:: + + win32only = py.test.mark.skipif("sys.platform != 'win32'") + + @win32only + def test_function(): + ... + + +skip groups of test functions +-------------------------------------- + +As with all metadata function marking you can do it at +`whole class- or module level`_. Here is an example +for skipping all methods of a test class based on platform:: + + class TestPosixCalls: + pytestmark = py.test.mark.skipif("sys.platform == 'win32'") + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +The ``pytestmark`` decorator will be applied to each test function. + +.. _`whole class- or module level`: mark.html#scoped-marking + + +mark a test function as **expected to fail** +------------------------------------------------------- + +You can use the ``xfail`` marker to indicate that you +expect the test to fail:: + + @py.test.mark.xfail + def test_function(): + ... + +This test will be run but no traceback will be reported +when it fails. Instead terminal reporting will list it in the +"expected to fail" or "unexpectedly passing" sections. + +Same as with skipif_ you can also selectively expect a failure +depending on platform:: + + @py.test.mark.xfail("sys.version_info >= (3,0)") + + def test_function(): + ... + + +skipping on a missing import dependency +-------------------------------------------------- + +You can use the following import helper at module level +or within a test or test setup function:: + + docutils = py.test.importorskip("docutils") + +If ``docutils`` cannot be imported here, this will lead to a +skip outcome of the test. You can also skip dependeing if +if a library does not come with a high enough version:: + + docutils = py.test.importorskip("docutils", minversion="0.3") + +The version will be read from the specified module's ``__version__`` attribute. + +imperative skip from within a test or setup function +------------------------------------------------------ + +If for some reason you cannot declare skip-conditions +you can also imperatively produce a Skip-outcome from +within test or setup code. Example:: + + def test_function(): + if not valid_config(): + py.test.skip("unsuppored configuration") + +""" +# XXX py.test.skip, .importorskip and the Skipped class +# should also be defined in this plugin, requires thought/changes + +import py + +def pytest_runtest_setup(item): + expr, result = evalexpression(item, 'skipif') + if result: + py.test.skip(expr) + +def pytest_runtest_makereport(__multicall__, item, call): + if call.when != "call": + return + expr, result = evalexpression(item, 'xfail') + rep = __multicall__.execute() + if result: + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = expr + else: + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep + +# called by terminalreporter progress reporting +def pytest_report_teststatus(report): + if 'xfail' in report.keywords: + if report.skipped: + return "xfailed", "x", "xfail" + elif report.failed: + return "xpassed", "P", "xpass" + +# called by the terminalreporter instance/plugin +def pytest_terminal_summary(terminalreporter): + show_xfailed(terminalreporter) + show_skipped(terminalreporter) + +def show_xfailed(terminalreporter): + tr = terminalreporter + xfailed = tr.stats.get("xfailed") + if xfailed: + if not tr.hasopt('xfailed'): + if tr.config.getvalue("verbose"): + tr.write_line( + "%d expected failures, use --report=xfailed for more info" % + len(xfailed)) + return + tr.write_sep("_", "expected failures") + for rep in xfailed: + entry = rep.longrepr.reprcrash + modpath = rep.item.getmodpath(includemodule=True) + pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) + reason = rep.longrepr.reprcrash.message + i = reason.find("\n") + if i != -1: + reason = reason[:i] + tr._tw.line("%s %s" %(pos, reason)) + + xpassed = terminalreporter.stats.get("xpassed") + if xpassed: + tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") + for rep in xpassed: + fspath, lineno, modpath = rep.item.reportinfo() + pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) + tr._tw.line(pos) + + +def evalexpression(item, keyword): + if isinstance(item, py.test.collect.Function): + markholder = getattr(item.obj, keyword, None) + result = False + if markholder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} + expr, result = None, True + for expr in markholder.args: + if isinstance(expr, str): + result = eval(expr, d) + else: + result = expr + if not result: + break + return expr, result + return None, False + +def folded_skips(skipped): + d = {} + for event in skipped: + entry = event.longrepr.reprcrash + key = entry.path, entry.lineno, entry.message + d.setdefault(key, []).append(event) + l = [] + for key, events in d.items(): + l.append((len(events),) + key) + return l + +def show_skipped(terminalreporter): + tr = terminalreporter + skipped = tr.stats.get('skipped', []) + if skipped: + if not tr.hasopt('skipped'): + if tr.config.getvalue("verbose"): + tr.write_line( + "%d skipped tests, use --report=skipped for more info" % + len(skipped)) + return + fskips = folded_skips(skipped) + if fskips: + tr.write_sep("_", "skipped test summary") + for num, fspath, lineno, reason in fskips: + tr._tw.line("%s:%d: [%d] %s" %(fspath, lineno, num, reason)) Added: pypy/branch/py11/py/plugin/pytest_terminal.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_terminal.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,468 @@ +""" +Implements terminal reporting of the full testing process. + +This is a good source for looking at the various reporting hooks. +""" +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", after="general") + group._addoption('-v', '--verbose', action="count", + dest="verbose", default=0, help="increase verbosity."), + group._addoption('-l', '--showlocals', + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group.addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="comma separated reporting options") + group._addoption('--tb', metavar="style", + action="store", dest="tbstyle", default='long', + type="choice", choices=['long', 'short', 'no'], + help="traceback verboseness (long/short/no).") + group._addoption('--fulltrace', + action="store_true", dest="fulltrace", default=False, + help="don't cut any tracebacks (default is to cut).") + + group = parser.getgroup("debugconfig") + group.addoption('--collectonly', + action="store_true", dest="collectonly", + help="only collect tests, don't execute them."), + group.addoption('--traceconfig', + action="store_true", dest="traceconfig", default=False, + help="trace considerations of conftest.py files."), + group._addoption('--nomagic', + action="store_true", dest="nomagic", default=False, + help="don't reinterpret asserts, no traceback cutting. ") + group.addoption('--debug', + action="store_true", dest="debug", default=False, + help="generate and show internal debugging information.") + + +def pytest_configure(config): + if config.option.collectonly: + reporter = CollectonlyReporter(config) + else: + reporter = TerminalReporter(config) + # XXX see remote.py's XXX + for attr in 'pytest_terminal_hasmarkup', 'pytest_terminal_fullwidth': + if hasattr(config, attr): + #print "SETTING TERMINAL OPTIONS", attr, getattr(config, attr) + name = attr.split("_")[-1] + assert hasattr(self.reporter._tw, name), name + setattr(reporter._tw, name, getattr(config, attr)) + config.pluginmanager.register(reporter, 'terminalreporter') + +def getreportopt(optvalue): + d = {} + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + val = True + if setting.startswith("no"): + val = False + setting = setting[2:] + d[setting] = val + return d + +class TerminalReporter: + def __init__(self, config, file=None): + self.config = config + self.stats = {} + self.curdir = py.path.local() + if file is None: + file = py.std.sys.stdout + self._tw = py.io.TerminalWriter(file) + self.currentfspath = None + self.gateway2info = {} + self._reportopt = getreportopt(config.getvalue('report')) + + def hasopt(self, name): + return self._reportopt.get(name, False) + + def write_fspath_result(self, fspath, res): + fspath = self.curdir.bestrelpath(fspath) + if fspath != self.currentfspath: + self._tw.line() + relpath = self.curdir.bestrelpath(fspath) + self._tw.write(relpath + " ") + self.currentfspath = fspath + self._tw.write(res) + + def write_ensure_prefix(self, prefix, extra="", **kwargs): + if self.currentfspath != prefix: + self._tw.line() + self.currentfspath = prefix + self._tw.write(prefix) + if extra: + self._tw.write(extra, **kwargs) + self.currentfspath = -2 + + def ensure_newline(self): + if self.currentfspath: + self._tw.line() + self.currentfspath = None + + def write_line(self, line, **markup): + line = str(line) + self.ensure_newline() + self._tw.line(line, **markup) + + def write_sep(self, sep, title=None, **markup): + self.ensure_newline() + self._tw.sep(sep, title, **markup) + + def getcategoryletterword(self, rep): + res = self.config.hook.pytest_report_teststatus(report=rep) + if res: + return res + for cat in 'skipped failed passed ???'.split(): + if getattr(rep, cat, None): + break + return cat, self.getoutcomeletter(rep), self.getoutcomeword(rep) + + def getoutcomeletter(self, rep): + return rep.shortrepr + + def getoutcomeword(self, rep): + if rep.passed: + return "PASS", dict(green=True) + elif rep.failed: + return "FAIL", dict(red=True) + elif rep.skipped: + return "SKIP" + else: + return "???", dict(red=True) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.write_line("INTERNALERROR> " + line) + + def pytest_gwmanage_newgateway(self, gateway, platinfo): + #self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec)) + d = {} + d['version'] = repr_pythonversion(platinfo.version_info) + d['id'] = gateway.id + d['spec'] = gateway.spec._spec + d['platform'] = platinfo.platform + if self.config.option.verbose: + d['extra'] = "- " + platinfo.executable + else: + d['extra'] = "" + d['cwd'] = platinfo.cwd + infoline = ("%(id)s %(spec)s -- platform %(platform)s, " + "Python %(version)s " + "cwd: %(cwd)s" + "%(extra)s" % d) + self.write_line(infoline) + self.gateway2info[gateway] = infoline + + def pytest_gwmanage_rsyncstart(self, source, gateways): + targets = ", ".join([gw.id for gw in gateways]) + msg = "rsyncstart: %s -> %s" %(source, targets) + if not self.config.option.verbose: + msg += " # use --verbose to see rsync progress" + self.write_line(msg) + + def pytest_gwmanage_rsyncfinish(self, source, gateways): + targets = ", ".join([gw.id for gw in gateways]) + self.write_line("rsyncfinish: %s -> %s" %(source, targets)) + + def pytest_plugin_registered(self, plugin): + if self.config.option.traceconfig: + msg = "PLUGIN registered: %s" %(plugin,) + # XXX this event may happen during setup/teardown time + # which unfortunately captures our output here + # which garbles our output if we use self.write_line + self.write_line(msg) + + def pytest_testnodeready(self, node): + self.write_line("%s txnode ready to receive tests" %(node.gateway.id,)) + + def pytest_testnodedown(self, node, error): + if error: + self.write_line("%s node down, error: %s" %(node.gateway.id, error)) + + def pytest_trace(self, category, msg): + if self.config.option.debug or \ + self.config.option.traceconfig and category.find("config") != -1: + self.write_line("[%s] %s" %(category, msg)) + + def pytest_rescheduleitems(self, items): + if self.config.option.debug: + self.write_sep("!", "RESCHEDULING %s " %(items,)) + + def pytest_deselected(self, items): + self.stats.setdefault('deselected', []).append(items) + + def pytest_itemstart(self, item, node=None): + if getattr(self.config.option, 'dist', 'no') != "no": + # for dist-testing situations itemstart means we + # queued the item for sending, not interesting (unless debugging) + if self.config.option.debug: + line = self._reportinfoline(item) + extra = "" + if node: + extra = "-> " + str(node.gateway.id) + self.write_ensure_prefix(line, extra) + else: + if self.config.option.verbose: + line = self._reportinfoline(item) + self.write_ensure_prefix(line, "") + else: + # ensure that the path is printed before the + # 1st test of a module starts running + + self.write_fspath_result(self._getfspath(item), "") + + def pytest__teardown_final_logerror(self, report): + self.stats.setdefault("error", []).append(report) + + def pytest_runtest_logreport(self, report): + rep = report + cat, letter, word = self.getcategoryletterword(rep) + if not letter and not word: + # probably passed setup/teardown + return + if isinstance(word, tuple): + word, markup = word + else: + markup = {} + self.stats.setdefault(cat, []).append(rep) + if not self.config.option.verbose: + self.write_fspath_result(self._getfspath(rep.item), letter) + else: + line = self._reportinfoline(rep.item) + if not hasattr(rep, 'node'): + self.write_ensure_prefix(line, word, **markup) + else: + self.ensure_newline() + if hasattr(rep, 'node'): + self._tw.write("%s " % rep.node.gateway.id) + self._tw.write(word, **markup) + self._tw.write(" " + line) + self.currentfspath = -2 + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.stats.setdefault("error", []).append(report) + msg = report.longrepr.reprcrash.message + self.write_fspath_result(report.collector.fspath, "E") + elif report.skipped: + self.stats.setdefault("skipped", []).append(report) + self.write_fspath_result(report.collector.fspath, "S") + + def pytest_sessionstart(self, session): + self.write_sep("=", "test session starts", bold=True) + self._sessionstarttime = py.std.time.time() + + verinfo = ".".join(map(str, sys.version_info[:3])) + msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg += " -- pytest-%s" % (py.__version__) + if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): + msg += " -- " + str(sys.executable) + self.write_line(msg) + + if self.config.option.debug or self.config.option.traceconfig: + self.write_line("using py lib: %s" % (py.path.local(py.__file__).dirpath())) + if self.config.option.traceconfig: + self.write_line("active plugins:") + plugins = [] + items = self.config.pluginmanager._name2plugin.items() + for name, plugin in items: + repr_plugin = repr(plugin) + fullwidth = getattr(self._tw, 'fullwidth', 65000) + if len(repr_plugin)+26 > fullwidth: + repr_plugin = repr_plugin[:(fullwidth-30)] + '...' + self.write_line(" %-20s: %s" %(name, repr_plugin)) + for i, testarg in enumerate(self.config.args): + self.write_line("test object %d: %s" %(i+1, testarg)) + + def pytest_sessionfinish(self, exitstatus, __multicall__): + __multicall__.execute() + self._tw.line("") + if exitstatus in (0, 1, 2): + self.summary_errors() + self.summary_failures() + self.config.hook.pytest_terminal_summary(terminalreporter=self) + if exitstatus == 2: + self._report_keyboardinterrupt() + self.summary_deselected() + self.summary_stats() + + def pytest_keyboard_interrupt(self, excinfo): + self._keyboardinterrupt_memo = excinfo.getrepr() + + def _report_keyboardinterrupt(self): + self.write_sep("!", "KEYBOARD INTERRUPT") + excrepr = self._keyboardinterrupt_memo + if self.config.option.verbose: + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) + + def pytest_looponfailinfo(self, failreports, rootdirs): + if failreports: + self.write_sep("#", "LOOPONFAILING", red=True) + for report in failreports: + try: + loc = report.longrepr.reprcrash + except AttributeError: + loc = str(report.longrepr)[:50] + self.write_line(loc, red=True) + self.write_sep("#", "waiting for changes") + for rootdir in rootdirs: + self.write_line("### Watching: %s" %(rootdir,), bold=True) + + def _reportinfoline(self, item): + collect_fspath = self._getfspath(item) + fspath, lineno, msg = self._getreportinfo(item) + if fspath and fspath != collect_fspath: + fspath = "%s <- %s" % ( + self.curdir.bestrelpath(collect_fspath), + self.curdir.bestrelpath(fspath)) + elif fspath: + fspath = self.curdir.bestrelpath(fspath) + if lineno is not None: + lineno += 1 + if fspath and lineno and msg: + line = "%(fspath)s:%(lineno)s: %(msg)s" + elif fspath and msg: + line = "%(fspath)s: %(msg)s" + elif fspath and lineno: + line = "%(fspath)s:%(lineno)s %(extrapath)s" + else: + line = "[noreportinfo]" + return line % locals() + " " + + def _getfailureheadline(self, rep): + if hasattr(rep, "collector"): + return str(rep.collector.fspath) + elif hasattr(rep, 'item'): + fspath, lineno, msg = self._getreportinfo(rep.item) + return msg + else: + return "test session" + + def _getreportinfo(self, item): + try: + return item.__reportinfo + except AttributeError: + pass + reportinfo = item.config.hook.pytest_report_iteminfo(item=item) + # cache on item + item.__reportinfo = reportinfo + return reportinfo + + def _getfspath(self, item): + try: + return item.fspath + except AttributeError: + fspath, lineno, msg = self._getreportinfo(item) + return fspath + + # + # summaries for sessionfinish + # + + def summary_failures(self): + if 'failed' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "FAILURES") + for rep in self.stats['failed']: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def summary_errors(self): + if 'error' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "ERRORS") + for rep in self.stats['error']: + msg = self._getfailureheadline(rep) + if not hasattr(rep, 'when'): + # collect + msg = "ERROR during collection " + msg + elif rep.when == "setup": + msg = "ERROR at setup of " + msg + elif rep.when == "teardown": + msg = "ERROR at teardown of " + msg + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def write_platinfo(self, rep): + if hasattr(rep, 'node'): + self.write_line(self.gateway2info.get( + rep.node.gateway, + "node %r (platinfo not found? strange)") + [:self._tw.fullwidth-1]) + + def summary_stats(self): + session_duration = py.std.time.time() - self._sessionstarttime + + keys = "failed passed skipped deselected".split() + for key in self.stats.keys(): + if key not in keys: + keys.append(key) + parts = [] + for key in keys: + val = self.stats.get(key, None) + if val: + parts.append("%d %s" %(len(val), key)) + line = ", ".join(parts) + # XXX coloring + self.write_sep("=", "%s in %.2f seconds" %(line, session_duration)) + + def summary_deselected(self): + if 'deselected' in self.stats: + self.write_sep("=", "%d tests deselected by %r" %( + len(self.stats['deselected']), self.config.option.keyword), bold=True) + + +class CollectonlyReporter: + INDENT = " " + + def __init__(self, config, out=None): + self.config = config + if out is None: + out = py.std.sys.stdout + self.out = py.io.TerminalWriter(out) + self.indent = "" + self._failed = [] + + def outindent(self, line): + self.out.line(self.indent + str(line)) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.out.line("INTERNALERROR> " + line) + + def pytest_collectstart(self, collector): + self.outindent(collector) + self.indent += self.INDENT + + def pytest_itemstart(self, item, node=None): + self.outindent(item) + + def pytest_collectreport(self, report): + if not report.passed: + self.outindent("!!! %s !!!" % report.longrepr.reprcrash.message) + self._failed.append(report) + self.indent = self.indent[:-len(self.INDENT)] + + def pytest_sessionfinish(self, session, exitstatus): + if self._failed: + self.out.sep("!", "collection failures") + for rep in self._failed: + rep.toterminal(self.out) + + +def repr_pythonversion(v=None): + if v is None: + v = sys.version_info + try: + return "%s.%s.%s-%s-%s" % v + except (TypeError, ValueError): + return str(v) + Added: pypy/branch/py11/py/plugin/pytest_tmpdir.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_tmpdir.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,21 @@ +"""provide temporary directories to test functions. + +usage example:: + + def test_plugin(tmpdir): + tmpdir.join("hello").write("hello") + +.. _`py.path.local`: ../../path.html + +""" +import py + +def pytest_funcarg__tmpdir(request): + """return a temporary directory path object + unique to each test function invocation, + created as a sub directory of the base temporary + directory. The returned object is a `py.path.local`_ + path object. + """ + name = request.function.__name__ + return request.config.mktemp(name, numbered=True) Added: pypy/branch/py11/py/plugin/pytest_unittest.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/plugin/pytest_unittest.py Tue Nov 10 19:17:34 2009 @@ -0,0 +1,79 @@ +""" +automatically discover and run traditional "unittest.py" style tests. + +Usage +---------------- + +This plugin collects and runs Python `unittest.py style`_ tests. +It will automatically collect ``unittest.TestCase`` subclasses +and their ``test`` methods from the test modules of a project +(usually following the ``test_*.py`` pattern). + +This plugin is enabled by default. + +.. _`unittest.py style`: http://docs.python.org/library/unittest.html +""" +import py +import sys + +def pytest_pycollect_makeitem(collector, name, obj): + if 'unittest' not in sys.modules: + return # nobody derived unittest.TestCase + try: + isunit = issubclass(obj, py.std.unittest.TestCase) + except TypeError: + pass + else: + if isunit: + return UnitTestCase(name, parent=collector) + +class UnitTestCase(py.test.collect.Class): + def collect(self): + return [UnitTestCaseInstance("()", self)] + + def setup(self): + pass + + def teardown(self): + pass + +_dummy = object() +class UnitTestCaseInstance(py.test.collect.Instance): + def collect(self): + loader = py.std.unittest.TestLoader() + names = loader.getTestCaseNames(self.obj.__class__) + l = [] + for name in names: + callobj = getattr(self.obj, name) + if py.builtin.callable(callobj): + l.append(UnitTestFunction(name, parent=self)) + return l + + def _getobj(self): + x = self.parent.obj + return self.parent.obj(methodName='run') + +class UnitTestFunction(py.test.collect.Function): + def __init__(self, name, parent, args=(), obj=_dummy, sort_value=None): + super(UnitTestFunction, self).__init__(name, parent) + self._args = args + if obj is not _dummy: + self._obj = obj + self._sort_value = sort_value + if hasattr(self.parent, 'newinstance'): + self.parent.newinstance() + self.obj = self._getobj() + + def runtest(self): + target = self.obj + args = self._args + target(*args) + + def setup(self): + instance = py.builtin._getimself(self.obj) + instance.setUp() + + def teardown(self): + instance = py.builtin._getimself(self.obj) + instance.tearDown() + From magcius at codespeak.net Wed Nov 11 00:55:37 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 11 Nov 2009 00:55:37 +0100 (CET) Subject: [pypy-svn] r69137 - pypy/branch/avm/pypy/translator/avm2/test Message-ID: <20091110235537.B1205168105@codespeak.net> Author: magcius Date: Wed Nov 11 00:55:36 2009 New Revision: 69137 Modified: pypy/branch/avm/pypy/translator/avm2/test/harness.py pypy/branch/avm/pypy/translator/avm2/test/runtest.py pypy/branch/avm/pypy/translator/avm2/test/test.abc pypy/branch/avm/pypy/translator/avm2/test/test.swf pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py Log: Some tests run, for a very limited definition of 'some'. Modified: pypy/branch/avm/pypy/translator/avm2/test/harness.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/harness.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/harness.py Wed Nov 11 00:55:36 2009 @@ -2,7 +2,7 @@ import py from pypy.translator.avm1 import swf as s, tags as t, records as r from pypy.translator.avm2.test import browsertest as b -from pypy.translator.avm2 import avm2gen as g, constants as c, abc_ as a +from pypy.translator.avm2 import avm2gen as g, constants as c, abc_ as a, traits fl_dis_ns = c.Namespace("flash.display", c.TYPE_NAMESPACE_PackageNamespace) @@ -10,21 +10,14 @@ def __init__(self, name, gen): self.testname = name self.swf = s.SwfData() + self.swf.add_tag(t.FileAttributes()) self.swf.add_tag(t.SetBackgroundColor(0x333333)) - self.swf.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "", + self.swf.add_tag(t.DefineEditText(r.Rect(0, 600, 0, 400), "tt", "Testing %s." % (name,), color=r.RGBA(0xFFFFFF))) self.swf.add_tag(t.PlaceObject2(1, 2, name="edittext")) self.abc = t.DoABC() self.actions = g.Avm2ilasm(gen.db, self.abc) - self.maincls = self.actions.begin_class(c.QName("PyPyTest_EntryPoint"), c.packagedQName("flash.display", "Sprite"), [ - c.packagedQName("flash.display", "Sprite"), - c.packagedQName("flash.display", "DisplayObjectContainer"), - c.packagedQName("flash.display", "InteractiveObject"), - c.packagedQName("flash.display", "DisplayObject"), - c.packagedQName("flash.events", "EventDispatcher"), - c.QName("Object"), - ]) self.swf.add_tag(self.abc) self.swf.add_tag(t.SymbolClass({0:"PyPyTest_EntryPoint"})) self.swf.add_tag(t.ShowFrame()) @@ -35,8 +28,7 @@ return self.actions.push_var('edittext') self.actions.push_this() - self.actions.push_const('edittext') - self.actions.emit('callproperty', c.Multiname("getChildByName", c.PROP_NAMESPACE_SET), 1) + self.actions.emit('getlex', c.QName("edittext")) self.actions.store_var('edittext') def update_text(self): @@ -70,8 +62,17 @@ self.actions.store_var('text') def start_test(self): + self.maincls = self.actions.begin_class(c.QName("PyPyTest_EntryPoint"), c.packagedQName("flash.display", "Sprite"), [ + c.packagedQName("flash.display", "Sprite"), + c.packagedQName("flash.display", "DisplayObjectContainer"), + c.packagedQName("flash.display", "InteractiveObject"), + c.packagedQName("flash.display", "DisplayObject"), + c.packagedQName("flash.events", "EventDispatcher"), + c.QName("Object"), + ]) self.maincls.make_iinit() self.get_edittext() + self.maincls.add_instance_trait(traits.AbcSlotTrait(c.QName('edittext'), c.packagedQName("flash.text", "TextField"))) self.actions.push_var('edittext') self.actions.emit('getproperty', c.QName('text')) self.actions.store_var('text') @@ -90,7 +91,7 @@ self.actions.emit('setproperty', c.QName("method")) self.actions.push_var('request') self.actions.emit('findpropstrict', c.packagedQName("flash.net", "URLVariables")) - self.actions.emit('constructprop', c.packagedQName("flash.net", "URLVariables"), 1) + self.actions.emit('constructprop', c.packagedQName("flash.net", "URLVariables"), 0) self.actions.emit('setproperty', c.QName("data")) self.actions.push_var('request') self.actions.emit('getproperty', c.QName("data")) @@ -110,5 +111,5 @@ f = open("test.abc", "w") f.write(a.AbcFile.serialize(self.abc)) f.close() - py.test.fail("debug") - #return b.browsertest(self.testname, self.swf) + #py.test.fail("debug") + return b.browsertest(self.testname, self.swf) Modified: pypy/branch/avm/pypy/translator/avm2/test/runtest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/runtest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/runtest.py Wed Nov 11 00:55:36 2009 @@ -20,19 +20,18 @@ # gen.call_method("fromCharCode") def compile_function(func, name, annotation=[], graph=None, backendopt=True, - auto_raise_exc=False, exctrans=False, - annotatorpolicy=None, nowrap=False): + exctrans=False, annotatorpolicy=None): olddefs = patch_os() - gen = _build_gen(func, annotation, name, graph, backendopt, - exctrans, annotatorpolicy, nowrap) + gen = _build_gen(func, annotation,graph, backendopt, + exctrans, annotatorpolicy) harness = TestHarness(name, gen) gen.ilasm = harness.actions gen.generate_source() unpatch_os(olddefs) # restore original values return gen, harness -def _build_gen(func, annotation, name, graph=None, backendopt=True, exctrans=False, - annotatorpolicy=None, nowrap=False): +def _build_gen(func, annotation, graph=None, backendopt=True, exctrans=False, + annotatorpolicy=None): try: func = func.im_func except AttributeError: @@ -69,14 +68,13 @@ self._genoo = None self._harness = None - def _compile(self, fn, args, ann=None, backendopt=True, auto_raise_exc=False, exctrans=False): + def _compile(self, fn, args, ann=None, backendopt=True, exctrans=False): if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] self._genoo, self._harness = compile_function(fn, "%s.%s" % (self.__class__.__name__, fn.func_name), ann, backendopt=backendopt, - auto_raise_exc=auto_raise_exc, exctrans=exctrans) self._func = fn self._ann = ann @@ -141,3 +139,14 @@ def read_attr(self, obj, name): py.test.skip('read_attr not supported on gencli tests') + +class InstanceWrapper: + def __init__(self, class_name): + self.class_name = class_name + +class ExceptionWrapper: + def __init__(self, class_name): + self.class_name = class_name + + def __repr__(self): + return 'ExceptionWrapper(%s)' % repr(self.class_name) Modified: pypy/branch/avm/pypy/translator/avm2/test/test.abc ============================================================================== Binary files. No diff available. Modified: pypy/branch/avm/pypy/translator/avm2/test/test.swf ============================================================================== Binary files. No diff available. Modified: pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/test_runtest.py Wed Nov 11 00:55:36 2009 @@ -4,13 +4,7 @@ from pypy.translator.avm2.test.runtest import AVM2Test class TestRunTest(BaseTestRunTest, AVM2Test): - - def test_auto_raise_exc(self): - def fn(): - raise ValueError - f = self._compile(fn, [], auto_raise_exc=True) - py.test.raises(ValueError, f) - + def test_big_arglist(self): def fn(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9): return a0 From magcius at codespeak.net Wed Nov 11 00:58:17 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 11 Nov 2009 00:58:17 +0100 (CET) Subject: [pypy-svn] r69138 - in pypy/branch/avm/pypy/translator: avm1 avm1/.ropeproject avm1/test avm2 avm2/.ropeproject Message-ID: <20091110235817.86316168105@codespeak.net> Author: magcius Date: Wed Nov 11 00:58:15 2009 New Revision: 69138 Added: pypy/branch/avm/pypy/translator/avm1/ pypy/branch/avm/pypy/translator/avm1/.ropeproject/ pypy/branch/avm/pypy/translator/avm1/.ropeproject/config.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/.ropeproject/globalnames (contents, props changed) pypy/branch/avm/pypy/translator/avm1/.ropeproject/history (contents, props changed) pypy/branch/avm/pypy/translator/avm1/.ropeproject/objectdb (contents, props changed) pypy/branch/avm/pypy/translator/avm1/__init__.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/authoring.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/avm1.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/avm1gen.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/constant.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/database.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/function.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/genavm.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/metavm.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/opcodes.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/records.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/records_flymake.py pypy/branch/avm/pypy/translator/avm1/swf.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/tags.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/ pypy/branch/avm/pypy/translator/avm1/test/__init__.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/autopath.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/bootstrap.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/browsertest.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/harness.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/harness_flymake.py pypy/branch/avm/pypy/translator/avm1/test/mylib.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/runtest.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/runtest2.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/simpletest.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_backendopt.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_bool.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_builtin.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_carbonpython.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_cast.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_class.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_constant.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_cts.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_dict.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_dotnet.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_exception.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_float.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_harness.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_int.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_list.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_objectmodel.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_oo.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_op.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_overflow.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_pbc.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_primitive.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_query.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_range.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_runtest.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_snippet.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_streamio.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_string.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_tuple.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_unicode.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/test/test_weakref.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/types_.py (contents, props changed) pypy/branch/avm/pypy/translator/avm1/util.py (contents, props changed) Removed: pypy/branch/avm/pypy/translator/avm2/.ropeproject/ Modified: pypy/branch/avm/pypy/translator/avm2/abc_.py pypy/branch/avm/pypy/translator/avm2/assembler.py pypy/branch/avm/pypy/translator/avm2/avm2gen.py pypy/branch/avm/pypy/translator/avm2/class_.py pypy/branch/avm/pypy/translator/avm2/constant.py pypy/branch/avm/pypy/translator/avm2/constants.py pypy/branch/avm/pypy/translator/avm2/database.py pypy/branch/avm/pypy/translator/avm2/function.py pypy/branch/avm/pypy/translator/avm2/genavm.py pypy/branch/avm/pypy/translator/avm2/instructions.py pypy/branch/avm/pypy/translator/avm2/metavm.py pypy/branch/avm/pypy/translator/avm2/opcodes.py pypy/branch/avm/pypy/translator/avm2/traits.py pypy/branch/avm/pypy/translator/avm2/types_.py pypy/branch/avm/pypy/translator/avm2/util.py Log: I forgot the avm1 module. Added: pypy/branch/avm/pypy/translator/avm1/.ropeproject/config.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/.ropeproject/config.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,85 @@ +# The default ``config.py`` + + +def set_prefs(prefs): + """This function is called before opening the project""" + + # Specify which files and folders to ignore in the project. + # Changes to ignored resources are not added to the history and + # VCSs. Also they are not returned in `Project.get_files()`. + # Note that ``?`` and ``*`` match all characters but slashes. + # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc' + # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc' + # '.svn': matches 'pkg/.svn' and all of its children + # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o' + # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o' + prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject', + '.hg', '.svn', '_svn', '.git'] + + # Specifies which files should be considered python files. It is + # useful when you have scripts inside your project. Only files + # ending with ``.py`` are considered to be python files by + # default. + #prefs['python_files'] = ['*.py'] + + # Custom source folders: By default rope searches the project + # for finding source folders (folders that should be searched + # for finding modules). You can add paths to that list. Note + # that rope guesses project source folders correctly most of the + # time; use this if you have any problems. + # The folders should be relative to project root and use '/' for + # separating folders regardless of the platform rope is running on. + # 'src/my_source_folder' for instance. + #prefs.add('source_folders', 'src') + + # You can extend python path for looking up modules + #prefs.add('python_path', '~/python/') + + # Should rope save object information or not. + prefs['save_objectdb'] = True + prefs['compress_objectdb'] = False + + # If `True`, rope analyzes each module when it is being saved. + prefs['automatic_soa'] = True + # The depth of calls to follow in static object analysis + prefs['soa_followed_calls'] = 0 + + # If `False` when running modules or unit tests "dynamic object + # analysis" is turned off. This makes them much faster. + prefs['perform_doa'] = True + + # Rope can check the validity of its object DB when running. + prefs['validate_objectdb'] = True + + # How many undos to hold? + prefs['max_history_items'] = 32 + + # Shows whether to save history across sessions. + prefs['save_history'] = True + prefs['compress_history'] = False + + # Set the number spaces used for indenting. According to + # :PEP:`8`, it is best to use 4 spaces. Since most of rope's + # unit-tests use 4 spaces it is more reliable, too. + prefs['indent_size'] = 4 + + # Builtin and c-extension modules that are allowed to be imported + # and inspected by rope. + prefs['extension_modules'] = [] + + # Add all standard c-extensions to extension_modules list. + prefs['import_dynload_stdmods'] = True + + # If `True` modules with syntax errors are considered to be empty. + # The default value is `False`; When `False` syntax errors raise + # `rope.base.exceptions.ModuleSyntaxError` exception. + prefs['ignore_syntax_errors'] = False + + # If `True`, rope ignores unresolvable imports. Otherwise, they + # appear in the importing namespace. + prefs['ignore_bad_imports'] = False + + +def project_opened(project): + """This function is called after opening the project""" + # Do whatever you like here! Added: pypy/branch/avm/pypy/translator/avm1/.ropeproject/globalnames ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/.ropeproject/globalnames Wed Nov 11 00:58:15 2009 @@ -0,0 +1,10 @@ +?}qUavm1q]q(UActionStringEqualsqUActionDefineLocalqUActionMultiplyqUActionCallFunctionqUActionShiftLeftqUActionSetVariableq UActionAsciiToCharq +UActionGetPropertyq UActionGetTargetPathq UActionInitObjectq UActionCallNewMethodqU ActionGetURL2qU RegisterErrorqU ActionGreaterqURegisterqUValueqURegisterByIndexqU ActionTypeofqUActionIfqU +ActionStopqU ActionAndqUBranchingActionBaseqU ActionTraceqU ActionExtendsqU +ActionLessqUConstantIndexDescriptorqUActionStringAddqUIndexqU ActionNotq U ActionEndDragq!U ShortActionq"UActionRandomNumberq#U +ActionJumpq$UActionNewObjectq%U Undefinedq&UActionCallMethodq'UActionMBCharToAsciiq(U ActionBitOrq)UActionGotoFrameq*U ActionReturnq+U ActionDivideq,UActionSetPropertyq-UActionTypedAddq.UActionPreviousFrameq/UActionOrq0UActionConvertToNumberq1UActionDelThreadVarsq2U ActionEqualsq3U ActionDelVarq4UActionDecrementq5UActionEnumerateq6U ActionGetTimeq7UActionWaitForFrame2q8UBlockq9UActionInitArrayq:U ActionGetURLq;UActionGetVariableqUActionWaitForFrameq?U ActionPopq at UActionConvertToStringqAUActionSetTargetqBUActionMBStringExtractqCUActionConstantPoolqDURegisterByValueqEUActionMBAsciiToCharqFUActionStringExtractqGUActionNextFrameqHU ActionThrowqIUActionStopSoundsqJUActionStartDragqKUActionImplementsOpqLUActionShiftRightqMUglobal_registersqNUActionGotoFrame2qOU +ActionWithqPU +ActionSwapqQU ActionBitXorqRUActionqSUConstantqTUActionCharToAsciiqUUActionGotoLabelqVUActionCloneSpriteqWUActionDuplicateqXUActionShiftUnsignedqYUActionMBStringLengthqZUActionStoreRegisterq[U ActionBitAndq\UActionDefineLocalValq]UActionGetMemberq^USealedBlockErrorq_UActionIncrementq`U ActionAddqaU ActionTryqbU +ActionPushqcUActionToIntegerqdUActionTypedEqualsqeU +ActionCallqfUActionStringLessqgUActionSetMemberqhU +ActionPlayqiU DataTypesqjUActionStringLengthqkU ActionCastOpqlUActionStrictEqualsqmUNullqnUActionSubtractqoUActionSetTarget2qpUActionDefineFunctionqqU ActionModuloqrUActionStringGreaterqsUActionToggleQualityqtUActionDefineFunction2ques. \ No newline at end of file Added: pypy/branch/avm/pypy/translator/avm1/.ropeproject/history ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/.ropeproject/history Wed Nov 11 00:58:15 2009 @@ -0,0 +1 @@ +?]q(]q]qe. \ No newline at end of file Added: pypy/branch/avm/pypy/translator/avm1/.ropeproject/objectdb ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/.ropeproject/objectdb Wed Nov 11 00:58:15 2009 @@ -0,0 +1,4 @@ +?}qUavm1.pyq}qUBlock.__init__qcrope.base.oi.memorydb +ScopeInfo +q)?q}qUinstanceqUdefinedq hUActionDefineFunction2q +??Uunknown?q ?q Uunknown?q s}q?bss. \ No newline at end of file Added: pypy/branch/avm/pypy/translator/avm1/__init__.py ============================================================================== Added: pypy/branch/avm/pypy/translator/avm1/authoring.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/authoring.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,46 @@ + +from pypy.translator.avm.tags import ShapeWithStyle +from pypy.translator.avm.records import StyleChangeRecord, CurvedEdgeRecord, StraightEdgeRecord + +class Circle(object): + + def __init__(self, x, y, radius, linestyle, fillstyle0, fillstyle1): + self.x = x + self.y = y + self.radius = radius + self.linestyle = linestyle + self.fillstyle0 = fillstyle0 + self.fillstyle1 = fillstyle1 + + def to_shape(self): + shape = ShapeWithStyle() + + if self.linestyle: shape.add_line_style(self.linestyle) + if self.fillstyle0: shape.add_fill_style(self.fillstyle0) + if self.fillstyle1: shape.add_fill_style(self.fillstyle1) + + # Precalculated: + # math.tan(math.radians(22.5)) = 0.41421356237309503 + # math.sin(math.radians(45)) = 0.70710678118654746 + + c = self.radius * 0.41421356237309503 + a = self.radius * 0.70710678118654746 - c + + shape.add_shape_record(StyleChangeRecord(self.x + self.radius, self.y, self.linestyle, self.fillstyle0, self.fillstyle1)) + # 0 to PI/2 + shape.add_shape_record(CurvedEdgeRecord(0, -c, -a, -a)) + shape.add_shape_record(CurvedEdgeRecord(-a, -a, -c, 0)) + + # PI/2 to PI + shape.add_shape_record(CurvedEdgeRecord(-c, 0, -a, a)) + shape.add_shape_record(CurvedEdgeRecord(-a, a, 0, c)) + + # PI to 3PI/2 + shape.add_shape_record(CurvedEdgeRecord(0, c, a, a)) + shape.add_shape_record(CurvedEdgeRecord(a, a, c, 0)) + + # 3PI/2 to 2PI + shape.add_shape_record(CurvedEdgeRecord(c, 0, a, -a)) + shape.add_shape_record(CurvedEdgeRecord(a, -a, 0, -c)) + + return shape Added: pypy/branch/avm/pypy/translator/avm1/avm1.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/avm1.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,637 @@ + +# AVM1 = ActionScript Virtual Machine 1 +# Used for ActionScript 1 and 2 + +from pypy.translator.avm1.util import BitStream +from collections import namedtuple +import struct + +DataType = namedtuple("DataType", "id name size") + +STRING = DataType(0, "string", "Z") +FLOAT = DataType(1, "float", "f") +NULL = DataType(2, "null", "!") +UNDEFINED = DataType(3, "undefined", "!") +REGISTER = DataType(4, "register", "B") +BOOLEAN = DataType(5, "boolean", "B") +DOUBLE = DataType(6, "double", "d") +INTEGER = DataType(7, "integer", "l") +CONSTANT8 = DataType(8, "constant 8", "B") +CONSTANT16 = DataType(9, "constant 16", "H") + +preload = dict(this="preload_this", + arguments="preload_args", + super="preload_super", + _root="preload_root", + _parent="preload_parent", + _global="preload_global") + +class Action(object): + + ACTION_NAME = "NotImplemented" + ACTION_ID = 0x00 + + offset = 0 + label_name = "" + + def serialize(self): + inner_data = self.gen_data() + outer_data = self.gen_outer_data() + header = struct.pack(" 0 and action.ACTION_NAME == "ActionPush" and self.actions[-1].ACTION_NAME == "ActionPush": + old_action = self.actions[-1] + old_len = len(old_action) + self.actions[-1].values.extend(action.values) + self.current_offset += len(old_action) - old_len + return old_action + + # Two nots negate. Take them out. + if len(self.actions) > 0 and action.ACTION_NAME == "ActionNot" and self.actions[-1].ACTION_NAME == "ActionNot": + self.actions.pop() + self.current_offset -= 1 # len(ShortAction) is 1 + return None + + if not isinstance(action, Block): # Don't add block length until we've finalized. + self.current_offset += len(action) + + self.actions.append(action) + return action + + def serialize(self): + if not self._sealed: + raise SealedBlockError("Block must be sealed before it can be serialized") + if len(self.code) > 0: + return self.code + bytes = [] + block_offset = 0 + for action in self.actions: + if isinstance(action, Block): + block_offset += len(action) + action.offset += block_offset + action.get_block_props_late(self) + bytes += action.serialize() + if self.insert_end: + bytes += "\0" + self.code = "".join(bytes) + return self.code + + def new_label(self): + self.label_count += 1 + name = Block.AUTO_LABEL_TEMPLATE % self.label_count + self.labels[name] = -1 + return name + + def set_label_here(self, name): + self.labels[name] = self.current_offset + + def new_label_here(self): + name = self.new_label() + self.labels[name] = self.current_offset + return name + +class ActionCall(Action): + ACTION_NAME = "ActionCall" + ACTION_ID = 0x9e + +class ActionDefineFunction(Action, Block): + ACTION_NAME = "ActionDefineFunction" + ACTION_ID = 0x9b + FUNCTION_TYPE = 1 + + def __init__(self, toplevel, name, parameters): + Block.__init__(self, toplevel, False) + self.function_name = name + self.params = parameters + + def gen_data(self): + self.block_data = Block.serialize(self) + bytes = [self.function_name, "\0", struct.pack("H", len(self.params))] + bytes += [p + "\0" for p in self.params] + bytes += struct.pack("H", len(self.block_data)) + return "".join(bytes) + + def gen_outer_data(self): + return self.block_data + +class ActionDefineFunction2(Action, Block): + ACTION_NAME = "ActionDefineFunction2" + ACTION_ID = 0x8e + MAX_REGISTERS = 256 + FUNCTION_TYPE = 2 + + def __init__(self, toplevel, name, parameters): + Block.__init__(self, toplevel, False) + self.function_name = name + self.params = parameters + self.preload_register_count = 1 # Start at 1. + + # Flags + self.registers = [None] + self.preload_parent = False + self.preload_root = False + self.suppress_super = True + self.preload_super = False + self.suppress_args = True + self.preload_args = False + self.suppress_this = True + self.preload_this = False + self.preload_global = False + self.eval_flags() + + for name in parameters: + self.registers.append(name) + + def eval_flags(self): + + # According to the docs, this is the order of register allocation. + if self.preload_this and "this" not in self.registers: + self.suppress_this = False + self.registers.insert(1, "this") + + if self.preload_args and "arguments" not in self.registers: + self.suppress_args = False + self.registers.insert(2, "arguments") + + if self.preload_super and "super" not in self.registers: + self.suppress_super = False + self.registers.insert(3, "super") + + if self.preload_root and "_root" not in self.registers: + self.registers.insert(4, "_root") + + if self.preload_parent and "_parent" not in self.registers: + self.registers.insert(5, "_parent") + + if self.preload_global and "_global" not in self.registers: + self.registers.insert(6, "_global") + + def gen_data(self): + + bits = BitStream() + bits.write_bit(self.preload_parent) + bits.write_bit(self.preload_root) + bits.write_bit(self.suppress_super) + bits.write_bit(self.preload_super) + bits.write_bit(self.suppress_args) + bits.write_bit(self.preload_args) + bits.write_bit(self.suppress_this) + bits.write_bit(self.preload_this) + bits.zero_fill(7) # skip over 7 Reserved bits + bits.write_bit(self.preload_global) + + self.block_data = Block.serialize(self) + bytes = [self.function_name, "\0", + struct.pack("HB", len(self.params), len(self.registers)), + bits.serialize()] + + for name in self.params: + bytes += [chr(self.registers.index(name)), name, "\0"] + + bytes += [struct.pack("H", len(self.block_data))] + return "".join(bytes) + + def gen_outer_data(self): + return self.block_data + +class ActionGetURL(Action): + ACTION_NAME = "ActionGetURL" + ACTION_ID = 0x83 + + def __init__(self, url, target=""): + self.url = url + self.target = target + + def gen_data(self): + return "%s\0%s\0" % (self.url, self.target) + +class ActionGetURL2(Action): + ACTION_NAME = "ActionGetURL2" + ACTION_ID = 0x9a + + METHODS = {"": 0, "GET": 1, "POST": 2} + + def __init__(self, method, load_target=False, load_variables=False): + self.method = method + self.load_target = load_target + self.load_variables = load_variables + + def gen_data(self): + # The SWF 10 spec document is incorrect. + # method goes at the low end + # and the flags at the high end + bits = BitStream() + bits.write_bit(self.load_variables) + bits.write_bit(self.load_target) + bits.zero_fill(4) + bits.write_int_value(self.METHODS[self.method.upper()], 2) + return bits.serialize() + +class ActionGotoFrame(Action): + ACTION_NAME = "ActionGotoFrame" + ACTION_ID = 0x81 + + def __init__(self, index): + self.index = index + + def gen_data(self): + return struct.pack("H", self.index) + +class ActionGotoFrame2(Action): + ACTION_NAME = "ActionGotoFrame2" + ACTION_ID = 0x9f + + def __init__(self, play=False, scene_bias=0): + self.play = play + self.scene_bias = scene_bias + + def gen_data(self): + bits = BitStream() + bits.zero_fill(6) + bits.write_bit(self.scene_bias > 0) + bits.write_bit(self.play) + + if self.scene_bias > 0: + return bits.serialize() + struct.pack(" 0: + print "BRANCH:", self.branch_label, block.labels[self.branch_label], self.offset + self.branch_offset = block.labels[self.branch_label] - self.offset - len(self) + + def gen_data(self): + return struct.pack("h", self.branch_offset) + +class ActionJump(BranchingActionBase): + ACTION_NAME = "ActionJump" + ACTION_ID = 0x99 + +class ActionIf(BranchingActionBase): + ACTION_NAME = "ActionIf" + ACTION_ID = 0x9d + +class ActionPush(Action): + ACTION_NAME = "ActionPush" + ACTION_ID = 0x96 + + USE_CONSTANTS = False + + def __init__(self, *args): + self.values = [] + self.add_element(*args) + + def add_element(self, element): + if hasattr(element, "__iter__") and not isinstance(element, (basestring, tuple)): + for t in element: + self.add_element(t) + else: + if element in (NULL, UNDEFINED): + element = (None, element) + assert isinstance(element, tuple) + self.values.append(element) + + def get_block_props_early(self, block): + if not ActionPush.USE_CONSTANTS: return + for index, (value, type) in enumerate(self.values): + if type == STRING: + constant_index = block.constants.add_constant(value) + self.values[index] = (constant_index, CONSTANT8 if constant_index < 256 else CONSTANT16) + + def gen_data(self): + bytes = [] + for value, type in self.values: + bytes += chr(type.id) + if type.size == "Z": + bytes += [value, "\0"] + elif type.size != "!": + bytes += struct.pack("<"+type.size, value) + return "".join(bytes) + +class ActionSetTarget(Action): + ACTION_NAME = "ActionSetTarget" + ACTION_ID = 0x8b + + def __init__(self, target): + self.target = target + + def gen_data(self): + return self.target + "\0" + +class ActionStoreRegister(Action): + ACTION_NAME = "ActionStoreRegister" + ACTION_ID = 0x87 + + def __init__(self, index): + self.index = index + + def gen_data(self): + return chr(self.index) + +class ActionTry(Action): + ACTION_NAME = "ActionTry" + ACTION_ID = 0x8f + + def __init__(self, catch_object, try_block=None, catch_block=None, finally_block=None): + + self.catch_object = catch_object + + self.try_block = try_block or Block() + self.catch_block = catch_block or Block() + self.finally_block = finally_block or Block() + + def gen_data(self): + has_catch_block = len(self.catch_block.actions) > 0 + bits = BitStream() + bits.zero_fill(5) + bits.write_bit(isinstance(self.catch_object, int)) + bits.write_bit(len(self.finally_block.actions) > 0) + bits.write_bit(has_catch_block) + bytes = [bits.serialize()] + bytes += [struct.pack("3H", + len(self.try_block) + 5 if has_catch_block else 0, + len(self.catch_block), + len(self.finally_block))] + bytes += [self.catch_object, "" if isinstance(self.catch_object, int) else "\0"] + return bytes + + def gen_outer_data(self): + bytes = [self.try_block.serialize()] + if len(self.catch_block.actions) > 0: + bytes += ActionJump(len(self.catch_block)).serialize() + bytes += self.catch_block.serialize() + bytes += self.finally_block.serialize() + +class ActionWaitForFrame(Action): + ACTION_NAME = "ActionWaitForFrame" + ACTION_ID = 0x8a + + def __init__(self, index, skip_count=0): + self.index = index + self.skip_count = skip_count + + def gen_data(self): + return struct.pack("HB", self.index, self.skip_count) + +class ActionWaitForFrame2(Action): + ACTION_NAME = "ActionWaitForFrame2" + ACTION_ID = 0x8d + + def __init__(self, skip_count=0): + self.skip_count = skip_count + + def gen_data(self): + return chr(self.skip_count) + +class ActionWith(Action): + ACTION_NAME = "ActionWith" + ACTION_ID = 0x94 + + def __init__(self, with_block): + self.block = with_block or Block() + + def gen_data(self): + return struct.pack("H", len(self.block)) + self.block.serialize() + +SHORT_ACTIONS = {} + +# turns NextFrame into next_frame +def make_underlined(name): + return ''.join('_' + c.lower() if c.isupper() else c for c in name)[1:] + +def make_short_action(value, name, push_count=0): + + def __len__(self): + return 1 # 1 (Action ID) + + def serialize(self): + return chr(self.ACTION_ID) + + act = type(name, (Action,), dict(ACTION_ID=value, ACTION_NAME=name, push_count=push_count, + __len__=__len__, serialize=serialize)) + + SHORT_ACTIONS[name[6:].lower()] = act + SHORT_ACTIONS[make_underlined(name[6:])] = act + + return act + +ActionNextFrame = make_short_action(0x04, "ActionNextFrame") +ActionPreviousFrame = make_short_action(0x05, "ActionPreviousFrame") +ActionPlay = make_short_action(0x06, "ActionPlay") +ActionStop = make_short_action(0x07, "ActionStop") +ActionToggleQuality = make_short_action(0x08, "ActionToggleQuality") +ActionStopSounds = make_short_action(0x09, "ActionStopSounds") +ActionAdd = make_short_action(0x0a, "ActionAdd", -1) +ActionSubtract = make_short_action(0x0b, "ActionSubtract", -1) +ActionMultiply = make_short_action(0x0c, "ActionMultiply", -1) +ActionDivide = make_short_action(0x0d, "ActionDivide", -1) +ActionEquals = make_short_action(0x0e, "ActionEquals", -1) +ActionLess = make_short_action(0x0f, "ActionLess", -1) +ActionAnd = make_short_action(0x10, "ActionAnd", -1) +ActionOr = make_short_action(0x11, "ActionOr", -1) +ActionNot = make_short_action(0x12, "ActionNot") +ActionStringEquals = make_short_action(0x13, "ActionStringEquals", -1) +ActionStringLength = make_short_action(0x14, "ActionStringLength") +ActionStringExtract = make_short_action(0x15, "ActionStringExtract") +ActionPop = make_short_action(0x17, "ActionPop", -1) +ActionToInteger = make_short_action(0x18, "ActionToInteger") +ActionGetVariable = make_short_action(0x1c, "ActionGetVariable") +ActionSetVariable = make_short_action(0x1d, "ActionSetVariable", -2) +ActionSetTarget2 = make_short_action(0x20, "ActionSetTarget2") +ActionStringAdd = make_short_action(0x21, "ActionStringAdd", -1) +ActionGetProperty = make_short_action(0x22, "ActionGetProperty", -1) +ActionSetProperty = make_short_action(0x23, "ActionSetProperty", -3) +ActionCloneSprite = make_short_action(0x24, "ActionCloneSprite") +ActionRemoveSprite = make_short_action(0x25, "ActionRemoveSprite") +ActionTrace = make_short_action(0x26, "ActionTrace", -1) +ActionStartDrag = make_short_action(0x27, "ActionStartDrag") +ActionEndDrag = make_short_action(0x28, "ActionEndDrag") +ActionStringLess = make_short_action(0x29, "ActionStringLess") +ActionThrow = make_short_action(0x2a, "ActionThrow") +ActionCastOp = make_short_action(0x2b, "ActionCastOp") +ActionImplementsOp = make_short_action(0x2c, "ActionImplementsOp") +ActionRandomNumber = make_short_action(0x30, "ActionRandomNumber") +ActionMBStringLength = make_short_action(0x31, "ActionMBStringLength") +ActionCharToAscii = make_short_action(0x32, "ActionCharToAscii") +ActionAsciiToChar = make_short_action(0x33, "ActionAsciiToChar") +ActionGetTime = make_short_action(0x34, "ActionGetTime") +ActionMBStringExtract = make_short_action(0x35, "ActionMBStringExtract") +ActionMBCharToAscii = make_short_action(0x36, "ActionMBCharToAscii") +ActionMBAsciiToChar = make_short_action(0x37, "ActionMBAsciiToChar") +ActionDelVar = make_short_action(0x3a, "ActionDelVar") +ActionDelThreadVars = make_short_action(0x3b, "ActionDelThreadVars") +ActionDefineLocalVal = make_short_action(0x3c, "ActionDefineLocalVal") +ActionCallFunction = make_short_action(0x3d, "ActionCallFunction") +ActionReturn = make_short_action(0x3e, "ActionReturn") +ActionModulo = make_short_action(0x3f, "ActionModulo", -1) +ActionNewObject = make_short_action(0x40, "ActionNewObject") +ActionDefineLocal = make_short_action(0x41, "ActionDefineLocal") +ActionInitArray = make_short_action(0x42, "ActionInitArray") +ActionInitObject = make_short_action(0x43, "ActionInitObject") +ActionTypeof = make_short_action(0x44, "ActionTypeof") +ActionGetTargetPath = make_short_action(0x45, "ActionGetTargetPath") +ActionEnumerate = make_short_action(0x46, "ActionEnumerate") +ActionTypedAdd = make_short_action(0x47, "ActionTypedAdd", -1) +ActionTypedLess = make_short_action(0x48, "ActionTypedLess", -1) +ActionTypedEquals = make_short_action(0x49, "ActionTypedEquals", -1) +ActionConvertToNumber = make_short_action(0x4a, "ActionConvertToNumber") +ActionConvertToString = make_short_action(0x4b, "ActionConvertToString") +ActionDuplicate = make_short_action(0x4c, "ActionDuplicate", 1) +ActionSwap = make_short_action(0x4d, "ActionSwap") +ActionGetMember = make_short_action(0x4e, "ActionGetMember", -1) +ActionSetMember = make_short_action(0x4f, "ActionSetMember", -3) +ActionIncrement = make_short_action(0x50, "ActionIncrement") +ActionDecrement = make_short_action(0x51, "ActionDecrement") +ActionCallMethod = make_short_action(0x52, "ActionCallMethod") +ActionCallNewMethod = make_short_action(0x53, "ActionCallNewMethod") +ActionBitAnd = make_short_action(0x60, "ActionBitAnd", -1) +ActionBitOr = make_short_action(0x61, "ActionBitOr", -1) +ActionBitXor = make_short_action(0x62, "ActionBitXor", -1) +ActionShiftLeft = make_short_action(0x63, "ActionShiftLeft", -1) +ActionShiftRight = make_short_action(0x64, "ActionShiftRight", -1) +ActionShiftUnsigned = make_short_action(0x65, "ActionShiftUnsigned", -1) +ActionStrictEquals = make_short_action(0x66, "ActionStrictEquals", -1) +ActionGreater = make_short_action(0x67, "ActionGreater", -1) +ActionStringGreater = make_short_action(0x68, "ActionStringGreater", -1) +ActionExtends = make_short_action(0x69, "ActionExtends") Added: pypy/branch/avm/pypy/translator/avm1/avm1gen.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/avm1gen.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,507 @@ + +""" backend generator routines +""" + +from pypy.objspace.flow import model as flowmodel +from pypy.rpython.ootypesystem import ootype +from pypy.translator.avm1 import avm1, types_ as types +from pypy.translator.oosupport.treebuilder import SubOperation +from pypy.translator.oosupport.metavm import Generator as OOGenerator, InstructionList +from pypy.translator.oosupport.constant import push_constant +from collections import namedtuple + +ClassName = namedtuple("ClassName", "namespace classname") +Scope = namedtuple("Scope", "block parent callback islabel") + +def render_sub_op(sub_op, db, generator): + op = sub_op.op + instr_list = db.genoo.opcodes.get(op.opname, None) + assert instr_list is not None, 'Unknown opcode: %s ' % op + assert isinstance(instr_list, InstructionList) + # Don't do that, please + #assert instr_list[-1] is StoreResult, "Cannot inline an operation that doesn't store the result" + + # record that we know about the type of result and args + db.cts.lltype_to_cts(op.result.concretetype) + for v in op.args: + db.cts.lltype_to_cts(v.concretetype) + + instr_list = InstructionList(instr_list[:-1]) # leave the value on the stack if this is a sub-op + instr_list.render(generator, op) + # now the value is on the stack + +def make_variable(name): + if isinstance(name, Variable): + return name + return Variable(name) + +class StackDummy(object): + def __init__(self, name): + self.name = name + +class Variable(StackDummy): + def __str__(self): + return 'Variable(name="%s")' % self.name + __repr__ = __str__ + + def __add__(self, other): + if isinstance(other, StackDummy): + other = other.name + return Variable("ADD %r %r" % (self.name, other)) + + def __radd__(self, other): + return other + self + +class ScriptObject(StackDummy): + pass + +class Function(StackDummy): + pass + +class AVM1Gen(OOGenerator): + """ AVM1 'assembler' generator routines """ + + def __init__(self, block=None): + self.stack = [] + self.namespaces = {} + self.block = block or avm1.Block(None, True) + self.scope = Scope(self.block, None, None, False) + self.action(self.block.constants) + self.ilasm = self + + def new_label(self): + return self.fn_block.new_label() + + def push_stack(self, *values): + print "PUSHSTACK:", values + for element in values: + self.stack.append(element) + + @property + def fn_block(self): + if self.scope.islabel: + return self.scope.parent.block + return self.scope.block + + def pop_stack(self, n=1): + v = [self.stack.pop() for i in xrange(n)] + print "POPSTACK:", v + return v + + def store_register(self, name, index=-1): + index = self.fn_block.store_register(name, index) + self.action(avm1.ActionStoreRegister(index)) + return index + + def find_register(self, name): + print "FINDING REGISTER:", name + return self.fn_block.find_register(name) + + def action(self, action): + return self.scope.block.add_action(action) + + def set_label(self, label): + print "SETLABEL:", label + + if self.scope.islabel: + self.exit_scope() + + label, block = self._branch_start(label) + self.enter_scope(block, islabel=True) + + def enter_scope(self, new_code_obj, exit_callback=None, islabel=False): + print "ENTERSCOPE" + self.scope = Scope(new_code_obj, self.scope, exit_callback, islabel) + + def in_function(self): + return self.fn_block.FUNCTION_TYPE + + def exit_scope(self): + print "EXITSCOPE" + block_len = self.finalize_block(self.scope.block) + exit_callback = self.scope.callback + + # Go up to the parent scope. + self.scope = self.scope.parent + + self.scope.block.current_offset += block_len + if exit_callback is not None: + exit_callback() + + def finalize_block(self, block): + for label, branch_block in block.branch_blocks: + if not branch_block.sealed: + branch_block.seal() + + # Set the label. + block.labels[label] = block.current_offset + + print label, block.current_offset + + # Add the actions, which updates current_offset + for act in branch_block.actions: + + block.add_action(act) + return block.seal() + + # def begin_namespace(self, _namespace): + # n = _namespace.split('.') + # namespace = self.namespaces + # if n[0] not in self.namespaces: + # self.namespaces[n[0]] = {} + # self.push_const(n[0]) + # self.init_object() + # self.set_variable() + # self.push_var(n[0]) + # for ns in n[1:]: + # if not ns in namespace: + # namespace[ns] = {} + # namespace = namespace[ns] + # self.push_const(ns) + # self.init_object() + # self.set_member() + + def begin_function(self, name, arglist): + self.enter_scope(self.action(avm1.ActionDefineFunction2(self.block, name, arglist))) + + def begin_static_method(self, function_name, _class, arglist): + def exit_callback(block): + self.set_member() + self.load(_class) + self.push_const(function_name) + self.enter_scope(self.action(avm1.ActionDefineFunction2(self.block, "", arglist, 0)), exit_callback) + self.push_stack(Function(function_name)) + + def begin_method(self, function_name, _class, arglist): + def exit_callback(block): + self.set_member() + self.load(_class) + self.push_const("prototype") + self.get_member() + self.push_const(function_name) + self.enter_scope(self.action(avm1.ActionDefineFunction2(self.block, "", arglist, 0)), exit_callback) + self.push_stack(Function(function_name)) + + def set_variable(self): + value, name = self.pop_stack(2) + print "SETVARIABLE: %r = %r" % (name, value) + if isinstance(name, Variable): + name = name.name + assert isinstance(name, basestring) + if self.find_register(name) >= 0 and self.in_function() == 2: + self.store_register(name) + self.action(avm1.ActionSetVariable()) + + def get_variable(self): + name, = self.pop_stack() + print "GETVARIABLE:", name + self.action(avm1.ActionGetVariable()) + self.push_stack(make_variable(name)) + + def set_member(self): + self.action(avm1.ActionSetMember()) + value, name, obj = self.pop_stack(3) + print "SETMEMBER: %s.%s = %r" % (obj, name, value) + + def get_member(self): + self.action(avm1.ActionGetMember()) + name, obj = self.pop_stack(2) + print "GETMEMBER:", name, obj + self.push_stack(Variable("%s.%s" % (obj, name))) + + def push_reg_index(self, index): + self.action(avm1.ActionPush((index, avm1.REGISTER))) + + def push_arg(self, v): + assert self.in_function() > 0, "avm1gen::push_arg called while not in function scope." + self.push_local(v) + +# def push_value(self, v): +# self.action(avm1.ActionPush(v.name)) + + def push_var(self, v): + k = self.find_register(v) + print k + if k >= 0: + self.push_stack(Variable(v)) + self.push_reg_index(k) + else: + if self.in_function() == 2: + if v in avm1.preload: + setattr(self.scope.block, avm1.preload[v]) + self.scope.block.eval_flags() + return self.push_var(v) + self.push_const(v) + self.get_variable() + if self.in_function() == 2: + self.store_register(v) + + def push_this(self): + self.push_var("this") + + def push_local(self, v): + self.push_var(v.name) + + def push_const(self, *args): + self.push_stack(*args) + self.action(avm1.ActionPush(types.pytype_to_avm1(v) for v in args)) + + def return_stmt(self): + print "RETURNSTMT" + self.pop_stack() + self.action(avm1.ActionReturn()) + + def swap(self): + a, b = self.pop_stack(2) + self.push_stack(b, a) + self.action(avm1.ActionSwap()) + + def is_equal(self, value=None): + if value is not None: + self.push_const(value) + self.action(avm1.ActionEquals()) + self.pop_stack(2) + + def is_not_equal(self, value=None): + self.is_equal(value) + self.action(avm1.ActionNot()) + self.pop_stack(2) + + def init_object(self, members={}): + self.load(members.items()) + self.push_const(len(members)) + self.action(avm1.ActionInitObject()) + self.pop_stack(self.pop_stack()[0]) + self.push_stack(ScriptObject("object")) + + def init_array(self, members=[]): + self.load(members) + self.push_const(len(members)) + self.action(avm1.ActionInitArray()) + self.pop_stack(self.pop_stack()[0]) + self.push_stack(ScriptObject("array")) + + # Assumes the args and number of args are on the stack. + def call_function(self, func_name): + self.push_const(func_name) + self.action(avm1.ActionCallFunction()) + name, nargs = self.pop_stack() + self.pop_stack(nargs) + self.push_stack(Variable("%s_RETURN" % func_name)) + + def call_function_constargs(self, func_name, *args): + p = self.push_const(*reversed((func_name, len(args))+args)) + self.action(avm1.ActionCallFunction()) + self.pop_stack(2+len(args)) + self.push_stack(Variable("%s_RETURN" % func_name)) + + # Assumes the args and number of args and ScriptObject are on the stack. + def call_method_n(self, func_name): + self.load(func_name) + self.action(avm1.ActionCallMethod()) + name, obj, nargs = self.pop_stack(3) + self.pop_stack(nargs) + self.push_stack(Variable("%s.%s_RETURN" % (obj.name, name))) + + # Assumes the args and number of args are on the stack. + def call_method_constvar(self, _class, func_name): + self.push_var(_class) + self.call_method_n(func_name) + + # Assumes the value is on the stack. + # def set_proto_field(self, objname, member_name): + # self.push_const("prototype") + # self.push_var(objname) + # self.get_member() + # self.swap() + # self.push_const(member_name) + # self.swap() + # self.set_member() + + # Assumes the value is on the stack. + # def set_static_field(self, objname, member_name): + # self.push_var(objname) + # self.swap() + # self.push_const(member_name) + # self.swap() + # self.set_member() + + # If no args are passed then it is assumed that the args and number of args are on the stack. + def newobject_constthis(self, obj, *args): + if len(args) > 0: + self.load(args+(len(args), obj)) + else: + self.load(obj) + self.newobject() + + + def newobject(self): + self.action(avm1.ActionNewObject()) + name, nargs = self.pop_stack(2) + args = self.pop_stack(nargs) + self.push_stack(ScriptObject(name)) + + # FIXME: will refactor later + #load_str = load_const + + def begin_switch_varname(self, varname): + self.push_var(varname) + self.switch_register = self.store_register(varname) + + def write_case(self, testee): + self.push_const(avm1.RegisterByIndex(self.switch_register), testee) + self.action(avm1.ActionStrictEquals()) + self.pop_stack(2) + if len(self.case_label) < 1: + self.case_label, self.case_block = self.branch_if_true() + else: + self.branch_if_true(self.case_label) + + def write_break(self): + self.exit_scope() + self.case_flag = False + self.case_block = None + + def enter_case_branch(self): + self.enter_scope(self.case_block) + + def throw(self): # Assumes the value to be thrown is on the stack. + self.action(avm1.ActionThrow()) + self.pop_stack() + + # oosupport Generator routines + def emit(self, op): + a = avm1.SHORT_ACTIONS[op] + if a.push_count > 0: + a.push_stack(StackDummy("Generated by %r" % op)) + elif a.push_count < 0: + self.pop_stack(-a.push_count) + self.action(a()) + + def pop(self, TYPE): + self.action(avm1.ActionPop()) + self.pop_stack() + + def dup(self, TYPE): + self.action(avm1.ActionDuplicate()) + self.push_stack(*[self.pop_stack()] * 2) + + def load(self, v): + if hasattr(v, "__iter__") and not isinstance(v, basestring): + for i in v: + self.load(i) + elif isinstance(v, ClassName): + if v.namespace: + ns = v.namespace.split('.') + self.push_var(ns[0]) + for i in ns[:0:-1]: + self.push_const(i) + self.get_member() + else: + self.push_var(v.classname) + elif isinstance(v, flowmodel.Variable): + if v.concretetype is ootype.Void: + return # ignore it + elif self.load_variable_hook(v): + return + else: + self.push_local(v) + elif isinstance(v, flowmodel.Constant): + push_constant(self.db, v.concretetype, v.value, self) + elif isinstance(v, SubOperation): + render_sub_op(v, self.db, self) + else: + self.push_const(v) + #self.push_var(v) + + def load_variable_hook(self, v): + return False + + #def downcast(self, TYPE): + # pass + + #def getclassobject(self, OOINSTANCE): + # pass + + #def instantiate(self): + # pass + + #def instanceof(self, TYPE): + # pass + + def _make_label(self, label): + print "MAKE LABEL:", label + if label == "" or label is None: + label = self.new_label() + + blocks = dict(self.fn_block.branch_blocks) + + if label in self.fn_block.branch_blocks: + block = blocks[label] + else: + block = avm1.Block(self.block, False) + self.fn_block.branch_blocks.append((label, block)) + + return (label, block) + + def _branch_start(self, label): + return self._make_label(label) + + # Boolean value should be on stack when this is called + def branch_unconditionally(self, label): + print "BRANCH TO:", label + label, block = self._branch_start(label) + self.action(avm1.ActionJump(label)) + return label, block + + def branch_conditionally(self, iftrue, label): + label, block = self._branch_start(label) + if not iftrue: + self.action(avm1.ActionNot()) + self.action(avm1.ActionIf(label)) + self.pop_stack() + return label, block + + def branch_if_equal(self, label): + label, block = self._branch_start(label) + self.action(avm1.ActionEquals()) + self.action(avm1.ActionIf(label)) + self.pop_stack(2) + return label, block + + def call_graph(self, graph): + self.call_function(graph.func_name) + + def call_method(self, OOCLASS, method_name): + pass + + def call_oostring(self, OOTYPE): + self.action(avm1.ActionConvertToString()) + + call_oounicode = call_oostring + + def new(self, TYPE): + if isinstance(TYPE, ootype.List): + self.oonewarray(None) + + def oonewarray(self, TYPE, length=1): + self.newobject_constthis("Array", length) + + def push_null(self, TYPE=None): + self.action(avm1.ActionPush(avm1.NULL)) + self.push_stack(avm1.NULL) + + def push_undefined(self): + self.action(avm1.ActionPush(avm1.UNDEFINED)) + self.push_stack(avm1.UNDEFINED) + + def push_primitive_constant(self, TYPE, value): + if TYPE is ootype.Void: + self.push_null() + elif TYPE is ootype.String: + if value._str is None: + self.push_null() + else: + self.push_const(value._str) + else: + self.push_const(value) Added: pypy/branch/avm/pypy/translator/avm1/constant.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/constant.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,119 @@ + +from pypy.translator.oosupport import constant as c + +CONST_OBJNAME = "PYPY_INTERNAL_CONSTANTS" + +DEBUG_CONST_INIT = False +DEBUG_CONST_INIT_VERBOSE = False +SERIALIZE = False + +# ______________________________________________________________________ +# Constant Generators +# +# Different generators implementing different techniques for loading +# constants (Static fields, singleton fields, etc) + +class AVM1ConstGenerator(c.BaseConstantGenerator): + """ + AVM1 constant generator. It implements the oosupport + constant generator in terms of the AVM1 virtual machine. + """ + + def __init__(self, db): + c.BaseConstantGenerator.__init__(self, db) + self.cts = db.genoo.TypeSystem(db) + + def _begin_gen_constants(self, gen, all_constants): + gen.push_const(CONST_OBJNAME) + gen.init_array() + gen.store_register(CONST_OBJNAME) + gen.set_variable() + return gen + + def _end_gen_constants(self, gen, numsteps): + pass + + def _declare_const(self, gen, const): + pass + + def _close_step(self, gen, stepnum): + pass + + # _________________________________________________________________ + # OOSupport interface + + def push_constant(self, gen, const): + gen.push_var(CONST_OBJNAME) + gen.push_const(const.name) + gen.get_member() + + def _create_pointers(self, gen, all_constants): + pass + + def _initialize_data(self, gen, all_constants): + """ Iterates through each constant, initializing its data. """ + # gen.add_section("Initialize Data Phase") + for const in all_constants: + # gen.add_comment("Constant: %s" % const.name) + const.initialize_data(self, gen) + + +class AVM1ArrayListConst(c.ListConst): + + NAME = 'NOT_IMPL' + + def __init__(self, db, list, count): + c.AbstractConst.__init__(self, db, list, count) + self.name = '%s__%d' % (self.NAME, count) + + def create_pointer(self, gen): + assert False + + def initialize_data(self, constgen, gen): + assert not self.is_null() + + if self._do_not_initialize(): + return + + gen.push_var(CONST_OBJNAME) + gen.push_const(self.name) + gen.init_array(self.value._list) + gen.set_member() + + # for idx, item in enumerate(self.value._list): + # gen.push_const(CONST_OBJNAME, CONST_OBJNAME) + # gen.get_variable() + # gen.get_variable() + # gen.push_const(self.name, self.name) + # gen.get_member() + # gen.push_const(idx) + # gen.load(item) + # gen.set_member() + # gen.set_member() + +class AVM1ArrayConst(AVM1ArrayListConst): + NAME = 'ARRAY' + +class AVM1ListConst(AVM1ArrayListConst): + NAME = 'LIST' + +class AVM1DictConst(c.DictConst): + def initialize_data(self, constgen, gen): + assert not self.is_null() + + gen.push_var(CONST_OBJNAME) + gen.push_const(self.name) + gen.init_object(self.value._dict) + gen.set_member() + + # for key, value in self.value._dict.iteritems(): + # gen.push_const(CONST_OBJNAME, CONST_OBJNAME) + # gen.get_variable() + # gen.get_variable() + # gen.push_const(self.name, self.name) + # gen.get_member() + # gen.load(key) + # gen.load(value) + # gen.set_member() + # gen.set_member() + Added: pypy/branch/avm/pypy/translator/avm1/database.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/database.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,84 @@ +#from pypy.translator.avm.class_ import Class +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.support import Counter +from pypy.translator.oosupport.database import Database as OODatabase + +try: + set +except NameError: + from sets import Set as set + +class LowLevelDatabase(OODatabase): + def __init__(self, genoo): + OODatabase.__init__(self, genoo) + self.classes = {} # INSTANCE --> class_name + self.classnames = set() # (namespace, name) + self.functions = {} # graph --> function_name + self.methods = {} # graph --> method_name + self.consts = {} # value --> AbstractConst + self.delegates = {} # StaticMethod --> type_name + self.const_count = Counter() # store statistics about constants + + def next_count(self): + return self.unique() + + def _default_class_name(self, INSTANCE): + parts = INSTANCE._name.rsplit('.', 1) + if len(parts) == 2: + return parts + else: + return None, parts[0] + + def pending_function(self, graph, functype=None): + if functype is None: + function = self.genoo.Function(self, graph) + else: + function = functype(self, graph) + self.pending_node(function) + return function.get_name() + + # def pending_class(self, INSTANCE): + # try: + # return self.classes[INSTANCE] + # except KeyError: + # pass + + # if isinstance(INSTANCE, dotnet.NativeInstance): + # self.classes[INSTANCE] = INSTANCE._name + # return INSTANCE._name + # else: + # namespace, name = self._default_class_name(INSTANCE) + # name = self.get_unique_class_name(namespace, name) + # if namespace is None: + # full_name = name + # else: + # full_name = '%s.%s' % (namespace, name) + # self.classes[INSTANCE] = full_name + # cls = Class(self, INSTANCE, namespace, name) + # self.pending_node(cls) + # return full_name + + def record_function(self, graph, name): + self.functions[graph] = name + + def graph_name(self, graph): + # XXX: graph name are not guaranteed to be unique + return self.functions.get(graph, None) + + def get_unique_class_name(self, namespace, name): + base_name = name + i = 0 + while (namespace, name) in self.classnames: + name = '%s_%d' % (base_name, i) + i+= 1 + self.classnames.add((namespace, name)) + return name + + def class_name(self, INSTANCE): + #if INSTANCE is ootype.ROOT: + # return types.object.classname() + try: + NATIVE_INSTANCE = INSTANCE._hints['NATIVE_INSTANCE'] + return NATIVE_INSTANCE._name + except KeyError: + return self.classes[INSTANCE] Added: pypy/branch/avm/pypy/translator/avm1/function.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/function.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,198 @@ +from functools import partial + +from pypy.objspace.flow import model as flowmodel +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem.lltype import Void +from pypy.translator.oosupport.function import Function as OOFunction +from pypy.translator.avm1.node import Node +from pypy.translator.avm1.avm1gen import ClassName + +def load_variable_hook(self, v): + if v.name in self.argset: + selftype, selfname = self.args[0] + if self.is_method and v.name == selfname: + self.generator.push_this() # special case for 'self' + else: + self.generator.push_arg(v) + return True + return False + + +class Function(OOFunction, Node): + + auto_propagate_exceptions = True + + def __init__(self, *args, **kwargs): + OOFunction.__init__(self, *args, **kwargs) + + if hasattr(self.db.genoo, 'exceptiontransformer'): + self.auto_propagate_exceptions = False + + namespace = getattr(self.graph.func, '_namespace_', None) + if namespace: + if '.' in namespace: + self.namespace, self.classname = namespace.rsplit('.', 1) + else: + self.namespace = None + self.classname = namespace + else: + self.namespace = None + self.classname = None + + def _create_generator(self, ilasm): + ilasm.db = self.db + ilasm.load_variable_hook = partial(load_variable_hook, self) + return ilasm + + def record_ll_meta_exc(self, ll_meta_exc): + # record the type only if it doesn't belong to a native_class + ll_exc = ll_meta_exc._INSTANCE + NATIVE_INSTANCE = ll_exc._hints.get('NATIVE_INSTANCE', None) + if NATIVE_INSTANCE is None: + OOFunction.record_ll_meta_exc(self, ll_meta_exc) + + def begin_render(self): + self._set_args() + self._set_locals() + if self.args: + args = zip(*self.args)[1] + else: + args = () + if self.is_method: + self.generator.begin_method(self.name, ClassName(self.namespace, self.classname), args[1:]) + elif self.classname: + self.generator.begin_static_method(self.name, ClassName(self.namespace, self.classname), args) + else: + self.generator.begin_function(self.name, args) + + def end_render(self): + if self.generator.scope.islabel: + self.generator.exit_scope() + self.generator.exit_scope() + + def render_return_block(self, block): + print "RETURN BLOCK RENDERING" + return_var = block.inputargs[0] + if return_var.concretetype is not Void: + self.generator.load(return_var) + self.generator.return_stmt() + + def set_label(self, label): + return self.generator.set_label(label) + + # def _render_op(self, op): + # #instr_list = self.db.genoo.opcodes.get(op.opname, None) + # #instr_list.render(self.generator, op) + # super(Function, self)._render_op(op) + + def _setup_link(self, link): + target = link.target + linkvars = [] + for to_load, to_store in zip(link.args, target.inputargs): + if isinstance(to_load, flowmodel.Variable) and to_load.name == to_store.name: + continue + if to_load.concretetype is ootype.Void: + continue + linkvars.append((to_load, to_store)) + + # after SSI_to_SSA it can happen to have to_load = [a, b] and + # to_store = [b, c]. If we store each variable sequentially, + # 'b' would be overwritten before being read. To solve, we + # first load all the values on the stack, then store in the + # appropriate places. + + if self._trace_enabled(): + self._trace('link', writeline=True) + for to_load, to_store in linkvars: + self._trace_value('%s <-- %s' % (to_store, to_load), to_load) + self._trace('', writeline=True) + + for to_load, to_store in linkvars: + self.generator.load(to_store) + self.generator.load(to_load) + self.generator.set_variable() + + + # def begin_try(self, cond): + # if cond: + # self.ilasm.begin_try() + + # def end_try(self, target_label, cond): + # if cond: + # self.ilasm.leave(target_label) + # self.ilasm.end_try() + # else: + # self.ilasm.branch(target_label) + + # def begin_catch(self, llexitcase): + # ll_meta_exc = llexitcase + # ll_exc = ll_meta_exc._INSTANCE + # cts_exc = self.cts.lltype_to_cts(ll_exc) + # self.ilasm.begin_catch(cts_exc.classname()) + + # def end_catch(self, target_label): + # self.ilasm.leave(target_label) + # self.ilasm.end_catch() + + # def render_raise_block(self, block): + # exc = block.inputargs[1] + # self.load(exc) + # self.ilasm.opcode('throw') + + # def store_exception_and_link(self, link): + # if self._is_raise_block(link.target): + # # the exception value is on the stack, use it as the 2nd target arg + # assert len(link.args) == 2 + # assert len(link.target.inputargs) == 2 + # self.store(link.target.inputargs[1]) + # else: + # # the exception value is on the stack, store it in the proper place + # if isinstance(link.last_exception, flowmodel.Variable): + # self.ilasm.opcode('dup') + # self.store(link.last_exc_value) + # self.ilasm.call_method( + # 'class [mscorlib]System.Type object::GetType()', + # virtual=True) + # self.store(link.last_exception) + # else: + # self.store(link.last_exc_value) + # self._setup_link(link) + + # def render_numeric_switch(self, block): + # if block.exitswitch.concretetype in (ootype.SignedLongLong, ootype.UnsignedLongLong): + # # TODO: it could be faster to check is the values fit in + # # 32bit, and perform a cast in that case + # self.render_numeric_switch_naive(block) + # return + + # cases, min_case, max_case, default = self._collect_switch_cases(block) + # is_sparse = self._is_sparse_switch(cases, min_case, max_case) + + # naive = (min_case < 0) or is_sparse + # if naive: + # self.render_numeric_switch_naive(block) + # return + + # targets = [] + # for i in xrange(max_case+1): + # link, lbl = cases.get(i, default) + # targets.append(lbl) + # self.generator.load(block.exitswitch) + # self.ilasm.switch(targets) + # self.render_switch_case(*default) + # for link, lbl in cases.itervalues(): + # self.render_switch_case(link, lbl) + + # def call_oostring(self, ARGTYPE): + # if isinstance(ARGTYPE, ootype.Instance): + # argtype = self.cts.types.object + # else: + # argtype = self.cts.lltype_to_cts(ARGTYPE) + # self.call_signature('string [pypylib]pypy.runtime.Utils::OOString(%s, int32)' % argtype) + + # def call_oounicode(self, ARGTYPE): + # argtype = self.cts.lltype_to_cts(ARGTYPE) + # self.call_signature('string [pypylib]pypy.runtime.Utils::OOUnicode(%s)' % argtype) + + # Those parts of the generator interface that are function + # specific Added: pypy/branch/avm/pypy/translator/avm1/genavm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/genavm.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,38 @@ + +import py +from pypy.translator.oosupport.genoo import GenOO +from pypy.translator.avm1.avm1gen import AVM1Gen +from pypy.translator.avm1.constant import AVM1ConstGenerator +from pypy.translator.avm1.database import LowLevelDatabase +from pypy.translator.avm1.function import Function +from pypy.translator.avm1.opcodes import opcodes +from pypy.translator.avm1.types import AVM1TypeSystem + +class GenAVM1(GenOO): + + opcodes = opcodes + Function = Function + Database = LowLevelDatabase + TypeSystem = AVM1TypeSystem + + ConstantGenerator = AVM1ConstGenerator + + def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): + GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) + self.const_stat = str(tmpdir.join('const_stat')) + self.ilasm = None + + def create_assembler(self): + return AVM1Gen() + + def generate_source(self): + if self.ilasm is None: + self.ilasm = self.create_assembler() + self.fix_names() + self.gen_entrypoint() + self.gen_pendings() + self.db.gen_constants(self.ilasm) + + # Don't do treebuilding stuff + def stack_optimization(self): + pass Added: pypy/branch/avm/pypy/translator/avm1/metavm.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/metavm.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,97 @@ + +from pypy.rpython.ootypesystem import ootype +from pypy.translator.oosupport.metavm import MicroInstruction + +class _SetField(MicroInstruction): + def render(self, generator, op): + this, field, value = op.args + + if value.concretetype is ootype.Void: + return + + generator.load(this) + generator.load(field) + generator.load(value) + generator.set_member() + +class _GetField(MicroInstruction): + def render(self, generator, op): + + if op.result.concretetype is ootype.Void: + return + + this, field = op.args + generator.load(this) + generator.load(field) + generator.get_member() + +class _StoreResultStart(MicroInstruction): + def render(self, generator, op): + print "STORERESULT START:", op.result.name + generator.push_const(op.result.name) + +class _StoreResultEnd(MicroInstruction): + def render(self, generator, op): + print "STORERESULT END:", op.result.name + generator.set_variable() + + +class _PushArgsForFunctionCall(MicroInstruction): + def render(self, generator, op): + args = op.args + for arg in args: + print arg, type(arg) + generator.load(arg) + generator.push_const(len(args)) + +class _CallMethod(MicroInstruction): + def render(self, generator, op): + args = op.args + method_name = args[0].value + this = args[1] + + if isinstance(this.concretetype, ootype.Array): + for arg in args[1:]: + generator.load(arg) + if method_name == "ll_setitem_fast": + generator.set_member() + generator.push_const(False) # Spoof a method call result + elif method_name == "ll_getitem_fast": + generator.get_member() + elif method_name == "ll_length": + generator.load("length") + generator.get_member() + else: + assert False + else: + if len(args) > 2: + for arg in args[2:]: + generator.load(arg) + generator.push_const(len(args)-2) + else: + generator.push_const(0) + generator.load(this) + generator.call_method_n(method_name) + +class CallConstantMethod(MicroInstruction): + def __init__(self, obj, func_name): + self.obj = obj + self.func_name = func_name + + def render(self, generator, op): + generator.push_var(self.obj) + generator.call_method_n(self.func_name) + +class PushConst(MicroInstruction): + def __init__(self, *args): + self.args = args + + def render(self, generator, op): + generator.push_const(*self.args) + +PushArgsForFunctionCall = _PushArgsForFunctionCall() +StoreResultStart = _StoreResultStart() +StoreResultEnd = _StoreResultEnd() +GetField = _GetField() +SetField = _SetField() +CallMethod = _CallMethod() Added: pypy/branch/avm/pypy/translator/avm1/opcodes.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/opcodes.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,141 @@ + +from pypy.translator.oosupport import metavm as om +from pypy.translator.avm1 import metavm as am, avm1 as a + +DoNothing = [om.PushAllArgs] + +misc_ops = { + 'new': [om.New], + 'runtimenew': [om.RuntimeNew], + 'oosetfield': [am.SetField], + 'oogetfield': [am.GetField], + 'oonewarray': [om.OONewArray], + 'oosend': [am.CallMethod], +} + +unary_ops = { + 'same_as': DoNothing, + 'bool_not': 'not', + 'int_neg': 'negate', + 'int_neg_ovf': 'negate', + 'int_abs': [am.PushArgsForFunctionCall, am.CallConstantMethod("Math", "abs")], + + 'cast_int_to_char': [am.PushArgsForFunctionCall, am.CallConstantMethod("String", "fromCharCode")], + 'cast_int_to_unichar': [am.PushArgsForFunctionCall, am.CallConstantMethod("String", "fromCharCode")], + 'cast_int_to_float': DoNothing, + 'cast_int_to_longlong': DoNothing, + 'cast_int_to_uint': DoNothing, + 'cast_int_to_long': DoNothing, + 'cast_uint_to_float': DoNothing, + 'cast_uint_to_longlong': DoNothing, + 'cast_uint_to_int' : DoNothing, + 'cast_uint_to_long': DoNothing, + + 'cast_bool_to_int': 'convert_to_number', + 'cast_bool_to_uint': 'convert_to_number', + 'cast_bool_to_float': 'convert_to_number', + + +} + +binary_ops = { + 'int_add': 'typed_add', + 'int_sub': 'subtract', + 'int_mul': 'multiply', + 'int_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'int_mod': 'modulo', + 'int_lt': 'typed_less', + 'int_le': [om.PushAllArgs, 'greater', 'not'], + 'int_eq': 'typed_equals', + 'int_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'int_gt': 'greater', + 'int_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'int_and': 'bit_and', + 'int_or': 'bit_or', + 'int_lshift': 'shift_left', + 'int_rshift': 'shift_right', + 'int_xor': 'bit_xor', + + 'uint_add': 'typed_add', + 'uint_sub': 'subtract', + 'uint_mul': 'multiply', + 'uint_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'uint_mod': 'modulo', + 'uint_lt': 'typed_less', + 'uint_le': [om.PushAllArgs, 'greater', 'not'], + 'uint_eq': 'typed_equals', + 'uint_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'uint_gt': 'greater', + 'uint_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'uint_and': 'bit_and', + 'uint_or': 'bit_or', + 'uint_lshift': 'shift_left', + 'uint_rshift': 'shift_right', + 'uint_xor': 'bit_xor', + + 'float_add': 'typed_add', + 'float_sub': 'subtract', + 'float_mul': 'multiply', + 'float_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'float_mod': 'modulo', + 'float_lt': 'typed_less', + 'float_le': [om.PushAllArgs, 'greater', 'not'], + 'float_eq': 'typed_equals', + 'float_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'float_gt': 'greater', + 'float_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'float_and': 'bit_and', + 'float_or': 'bit_or', + 'float_lshift': 'shift_left', + 'float_rshift': 'shift_right', + 'float_xor': 'bit_xor', + + 'llong_add': 'typed_add', + 'llong_sub': 'subtract', + 'llong_mul': 'multiply', + 'llong_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'llong_mod': 'modulo', + 'llong_lt': 'typed_less', + 'llong_le': [om.PushAllArgs, 'greater', 'not'], + 'llong_eq': 'typed_equals', + 'llong_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'llong_gt': 'greater', + 'llong_ge': [om.PushAllArgs, 'typed_less', 'not'], + + 'llong_and': 'bit_and', + 'llong_or': 'bit_or', + 'llong_lshift': 'shift_left', + 'llong_rshift': 'shift_right', + 'llong_xor': 'bit_xor', + + 'ullong_add': 'typed_add', + 'ullong_sub': 'subtract', + 'ullong_mul': 'multiply', + 'ullong_floordiv': [om.PushAllArgs, 'divide', am.PushConst(1), am.CallConstantMethod("Math", "floor")], + 'ullong_mod': 'modulo', + 'ullong_lt': 'typed_less', + 'ullong_le': [om.PushAllArgs, 'greater', 'not'], + 'ullong_eq': 'typed_equals', + 'ullong_ne': [om.PushAllArgs, 'typed_equals', 'not'], + 'ullong_gt': 'greater', + 'ullong_ge': [om.PushAllArgs, 'typed_less', 'not'], + 'ullong_lshift': 'shift_left', + 'ullong_rshift': 'shift_right', +} + +opcodes = misc_ops.copy() +opcodes.update(unary_ops) +opcodes.update(binary_ops) + +for key, value in opcodes.iteritems(): + if isinstance(value, str): + value = [am.StoreResultStart, om.PushAllArgs, value, am.StoreResultEnd] + if am.StoreResultStart not in value: + value.insert(0, am.StoreResultStart) + if am.StoreResultEnd not in value: + value.append(am.StoreResultEnd) + value = om.InstructionList(value) + opcodes[key] = value Added: pypy/branch/avm/pypy/translator/avm1/records.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/records.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,737 @@ + +from pypy.translator.avm1.util import BitStream +from math import log, ceil, sqrt + +def serialize_style_list(lst): + bits = BitStream() + + if len(lst) <= 0xFF: + bits.write_bit_value(len(lst), 8) + else: + bits.write_bit_value(0xFF, 8) + bits.write_bit_value(len(lst), 16) + + for style in lst: + bits += style.serialize() + + return bits + +def nbits(n, *a): + return int(ceil(log(max(1, n, *a), 2))) + +def nbits_abs(n, *a): + return nbits(abs(n), *(abs(n) for n in a)) + +def style_list_bits(lst): + return nbits(len(lst)) + +def clamp(n, minimum, maximum): + return max(minimum, min(n, maximum)) + +class RecordHeader(object): + + def __init__(self, type, length): + self.type = type + self.length = length + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.type, 10) + if self.length < 0x3F: + bits.write_int_value(self.length, 6) + else: + bits.write_int_value(0x3F, 6) + bits.write_int_value(self.length, 32) + return bits + + def parse(self, bitstream): + self.type = bitstream.read_bit_value(10) + self.length = bitstream.read_bit_value(6) + if self.length >= 0x3F: + self.length = bitstream.read_bit_value(32) + +class _EndShapeRecord(object): + + def __call__(self, *a, **b): + pass + + def serialize(self): + bitstream = BitStream() + bitstream.zero_fill(6) + return bitstream + +EndShapeRecord = _EndShapeRecord() + +class Rect(object): + + def __init__(self, XMin=0, XMax=0, YMin=0, YMax=0): + self.XMin = XMin + self.XMax = XMax + self.YMin = YMin + self.YMax = YMax + + def union(self, rect, *rects): + r = Rect(min(self.XMin, rect.XMin), + max(self.XMax, rect.XMax), + min(self.YMin, rect.YMin), + max(self.YMax, rect.YMax)) + if len(rects) > 0: + return r.union(*rects) + return r + + def serialize(self): + if self.XMin > self.XMax or self.YMin > self.YMax: + raise ValueError, "Maximum values in a RECT must be larger than the minimum values." + + # Find our values in twips. + twpXMin = self.XMin * 20 + twpXMax = self.XMax * 20 + twpYMin = self.YMin * 20 + twpYMax = self.YMax * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + NBits = nbits_abs(twpXMin, twpXMax, twpYMin, twpYMax)+1 + + if NBits > 31: + raise ValueError, "Number of bits per value field cannot exceede 31." + + # And write out our bits. + bits = BitStream() + bits.write_int_value(NBits, 5) + bits.write_int_value(twpXMin, NBits) + bits.write_int_value(twpXMax, NBits) + bits.write_int_value(twpYMin, NBits) + bits.write_int_value(twpYMax, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bitstream.read_bit_value(5) + self.XMin = bitstream.read_bit_value(NBits) + self.XMax = bitstream.read_bit_value(NBits) + self.YMin = bitstream.read_bit_value(NBits) + self.YMax = bitstream.read_bit_value(NBits) + +class XY(object): + + def __init__(self, X=0, Y=0): + self.X = 0 + self.Y = 0 + + def serialize(self): + # Convert to twips plz. + twpX = self.X * 20 + twpY = self.Y * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + NBits = nbits_abs(twpX, twpY) + + bits = BitStream() + bits.write_int_value(NBits, 5) + bits.write_int_value(twpX, NBits) + bits.write_int_value(twpY, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bitstream.read_bit_value(5) + self.X = bitstream.read_bit_value(NBits) + self.Y = bitstream.read_bit_value(NBits) + +class RGB(object): + + def __init__(self, color): + self.color = color & 0xFFFFFF + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.color, 24) + return bits + + def parse(self, bitstream): + self.color = bitstream.read_bit_value(24) + +class RGBA(RGB): + + def __init__(self, color, alpha=1.0): + super(RGBA, self).__init__(color) + self.alpha = alpha + + def serialize(self): + bits = RGB.serialize(self) + + from pypy.translator.avm1.tags import DefineShape + + # If we are in a DefineShape and the version does not support + # alpha (DefineShape1 or DefineShape2), don't use alpha! + if DefineShape._current_variant not in (1, 2): + bits.write_int_value(int(self.alpha * 0xFF), 8) + + return bits + + def parse(self, bitstream): + RGB.parse(self, bitstream) + self.alpha = bitstream.read_bit_value(8) / 0xFF + +class CXForm(object): + has_alpha = False + def __init__(self, rmul=1, gmul=1, bmul=1, radd=0, gadd=0, badd=0): + self.rmul = rmul + self.gmul = gmul + self.bmul = bmul + self.amul = 1 + self.radd = radd + self.gadd = gadd + self.badd = badd + self.aadd = 0 + + def serialize(self): + has_add_terms = self.radd != 0 or self.gadd != 0 or self.badd != 0 or self.aadd != 0 + has_mul_terms = self.rmul != 1 or self.gmul != 1 or self.bmul != 1 or self.amul != 1 + + rm = abs(self.rmul * 256) + gm = abs(self.gmul * 256) + bm = abs(self.bmul * 256) + am = abs(self.amul * 256) + + ro = clamp(self.radd, -255, 255) + go = clamp(self.gadd, -255, 255) + bo = clamp(self.badd, -255, 255) + ao = clamp(self.aadd, -225, 255) + + NBits = 0 + if has_mul_terms: NBits = nbits_abs(rm, gm, bm, am) + if has_add_terms: NBits = max(NBits, nbits_abs(ro, go, bo, ao)) + + bits = BitStream() + bits.write_int_value(NBits, 4) + + if has_mul_terms: + bits.write_int_value(rm, NBits) + bits.write_int_value(gm, NBits) + bits.write_int_value(bm, NBits) + if self.has_alpha: bits.write_int_value(am, NBits) + + if has_add_terms: + bits.write_int_value(ro, NBits) + bits.write_int_value(go, NBits) + bits.write_int_value(bo, NBits) + if self.has_alpha: bits.write_int_value(ao, NBits) + + return bits + +class CXFormWithAlpha(CXForm): + has_alpha = True + def __init__(self, rmul=1, gmul=1, bmul=1, amul=1, radd=0, gadd=0, badd=0, aadd=0): + super(self, CXFormWithAlpha).__init__(rmul, gmul, bmul, radd, gadd, badd) + self.amul = amul + self.aadd = aadd + +class Matrix(object): + + def __init__(self, a=1, b=0, c=0, d=1, tx=0, ty=0): + self.a, self.b, self.c, self.d, self.tx, self.ty = a, b, c, d, tx, ty + + def serialize(self): + + def write_prefixed_values(a, b): + NBits = nbits(a, b) + bits.write_int_value(NBits, 5) + bits.write_int_value(a, NBits) + bits.write_int_value(b, NBits) + + bits = BitStream() + if self.a != 1 or self.d != 1: # HasScale + bits.write_bit(True) + write_prefixed_values(self.a, self.d) + else: + bits.write_bit(False) + + if self.b != 0 or self.c != 0: # HasRotate + bits.write_bit(True) + write_prefixed_values(self.b, self.c) + else: + bits.write_bit(False) + + write_prefixed_values(self.tx * 20, self.ty * 20) + return bits + +class Shape(object): + + def __init__(self): + self.shapes = [] + + self.edge_bounds = Rect() + self.shape_bounds = Rect() + + self.has_scaling = False + self.has_non_scaling = False + + self.bounds_calculated = False + + def add_shape_record(self, shape): + self.shapes.append(shape) + self.bounds_calculated = False + + def add_shape(self, shape): + self.shapes.expand(shape.shapes) + self.bounds_calculated = False + + def serialize(self): + if EndShapeRecord not in self.shapes: + self.shapes.append(EndShapeRecord()) + + bits = BitStream() + + bits.write_int_value(0, 8) # NumFillBits and NumLineBits + for record in self.shapes: + bits += record.serialize() + + return bits + + def calculate_bounds(self): + + if self.bounds_calculated: + return + + last = 0, 0 + style = None + for record in self.shapes: + last, (self.shape_bounds, self.edge_bounds), (has_scale, has_non_scale, style) = \ + record.calculate_bounds(last, self.shape_bounds, self.edge_bounds, style) + if has_scale: + self.has_scaling = True + if has_non_scale: + self.has_non_scaling = True + + self.bounds_calculated = True + +class ShapeWithStyle(Shape): + + def __init__(self, fills=None, strokes=None): + super(self, ShapeWithStyle).__init__(self) + self.fills = fills or [] + self.strokes = strokes or [] + + def add_fill_style(self, style): + style.parent = self.fills + self.fills.append(style) + + def add_line_style(self, style): + style.parent = self.strokes + self.strokes.append(style) + + def add_shape(self, shape): + Shape.add_shape(self, shape) + try: + self.fills += shape.fills + self.strokes += shape.strokes + except AttributeError: + pass + + def serialize(self): + bits = BitStream() + bits += serialize_style_list(self.fills) + bits += serialize_style_list(self.strokes) + bits.write_int_value(style_list_bits(self.fills), 4) + bits.write_int_value(style_list_bits(self.strokes), 4) + return bits + +class LineStyle(object): + + caps = "round" + + def __init__(self, width=1, color=0, alpha=1.0): + self.width = width + self.color = RGBA(color, alpha) + + @property + def index(self): + return self.parent.find(self) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.width * 20, 16) + bits += self.color.serialize() + return bits + +class LineStyle2(LineStyle): + + def __init__(self, width=1, fillstyle=None, pixel_hinting=False, scale_mode=None, caps="round", joints="round", miter_limit=3): + + color, alpha, self.fillstyle = 0, 1.0, None + + if isinstance(fillstyle, RGBA): + color = fillstyle.color + alpha = fillstyle.alpha + elif isinstance(fillstyle, RGB): + color = fillstyle.color + elif isinstance(fillstyle, int): + if fillstyle > 0xFFFFFF: + color = fillstyle & 0xFFFFFF + alpha = fillstyle >> 6 & 0xFF + else: + color = fillstyle + elif isinstance(fillstyle, FillStyleSolidFill): + color = fillstyle.color.color + alpha = fillstyle.color.alpha + elif isinstance(fillstyle, FillStyle): + self.fillstyle = fillstyle + + super(self, LineStyle2).__init__(self, width, color, alpha) + self.pixel_hinting = pixel_hinting + self.h_scale = (scale_mode == "normal" or scale_mode == "horizontal") + self.v_scale = (scale_mode == "normal" or scale_mode == "vertical") + + if caps == "square": self.caps = 2 + elif caps == None: self.caps = 1 + elif caps == "round": self.caps = 0 + else: + raise ValueError, "Invalid cap style '%s'." % caps + + if joints == "miter": self.joints = 2 + elif joints == "bevel": self.joints = 1 + elif joints == "round": self.joints = 0 + + self.miter_limit = miter_limit + + def serialize(self): + + bits = BitStream() + bits.write_int_value(self.width * 20, 8) + bits.write_int_value(self.width * 20 >> 8, 8) + + bits.write_int_value(self.caps, 2) + bits.write_int_value(self.joints, 2) + bits.write_bit(self.fillstyle is not None); + bits.write_bit(self.h_scale) + bits.write_bit(self.v_scale) + bits.write_bit(self.pixel_hinting) + + if self.joints == 2: + bits.write_fixed_value(self.miter_limit, 16, True) + + if self.fillstyle: + bits.write_bits(self.fillstyle.serialize()) + else: + bits.write_bits(self.color.serialize()) + + return bits + + def cap_style_logic(self, style, last, delta): + # Half thickness (radius of round cap; diameter is thickness) + off = style.width / 2.0 + dx, dy = delta + lx, ly = last + + if style.caps == "round": + r = Rect() + r.XMin = cmp(dx, 0) * off + r.YMin = cmp(dy, 0) * off + r.XMax = r.XMin + dx + r.YMax = r.XMax + dy + return r + + if style.caps == "square": + + # Account for the length of the caps. + dellen = sqrt(dx*dx + dy*dy) # Delta length + norm = (dellen+off*2)/dellen # Extra length + dx *= norm # Add the extra length + dy *= norm + sqx, sqy = delta # Square cap offset + norm = off/dellen # Offset amount + sqx *= norm # Position offsets. + sqy *= norm + + # And offset the position. + lx -= sqx + ly -= sqy + + # Right-hand normal to vector delta relative to (0, 0). + p1x, p1y = (-dy, dx) + norm = sqrt(p1x*p1x + p1y*p1y) + p1x /= norm + p1y /= norm + + # Left-hand normal to vector delta relative to (0, 0) + p2x, p2y = (-p1x, -p1y) + + # Right-hand normal to vector delta relative to delta. + p3x, p3y = (p1x + dx, p1y + dy) + + # Left-hand normal to vector delta relative to delta. + p4x, p4y = (p2x + dx, p2y + dy) + + return Rect( + min(p1x, p2x, p3x, p4x) + lx, + max(p1x, p2x, p3x, p4x) + lx, + min(p1y, p2y, p3y, p4y) + ly, + max(p1y, p2y, p3y, p4y) + ly) + +class FillStyle(object): + + TYPE = -1 + + @property + def index(self): + return self.parent.find(self) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.TYPE, 8) + bits += self.serialize_inner() + return bits + +class FillStyleSolidFill(object): + + def __init_(self, color, alpha=1.0): + self.color = RGBA(color, alpha) + + def serialize_inner(self): + return self.color.serialize() + +class GradRecord(object): + + def __init__(self, ratio, color, alpha=1.0): + self.ratio = ratio + self.color = RGBA(color, alpha) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.ratio, 8) + bits += self.color.serialize() + return bits + +class Gradient(object): + + def __init__(self, grads, spread="pad", interpolation="rgb", focalpoint=0): + import operator + grads.sort(key=operator.attrgetter("ratio")) + self.grads = grads + self.spread = spread + self.interpolation = interpolation + self.focalpoint = focalpoint + + @classmethod + def from_begin_gradient_fill(cls, colors, alphas, ratios, spread, interpolation, focalpoint): + grads = [GradRecord(*t) for t in zip(ratios, colors, alphas)] + return cls(grads, spread, interpolation, focalpoint) + +class StraightEdgeRecord(object): + + def __init__(self, delta_x, delta_y): + self.delta_x = delta_x + self.delta_y = delta_y + self.bounds_calculated = False + + def serialize(self): + + bits = BitStream() + + if self.delta_x == 0 and self.delta_y == 0: + return bits + + bits.write_bit(True) # TypeFlag + bits.write_bit(True) # StraightFlag + + X = self.delta_x * 20 + Y = self.delta_y * 20 + + NBits = nbits_abs(X, Y) + + if NBits > 15: + raise ValueError("Number of bits per value field cannot exceed 15") + + bits.write_int_value(NBits, 4) + NBits += 2 + if X == 0: + # Vertical Line + bits.write_bit(False) # GeneralLineFlag + bits.write_bit(True) # VerticalLineFlag + bits.write_int_value(Y, NBits) + elif Y == 0: + # Horizontal Line + bits.write_bit(False) # GeneralLineFlag + bits.write_bit(True) # HorizontalLineFlag + bits.write_int_value(X, NBits) + else: + # General Line + bits.write_bit(True) # GeneralLineFlag + bits.write_int_value(X, NBits) + bits.write_int_value(Y, NBits) + + return bits + + def calculate_bounds(self, last, shape_bounds, edge_bounds, style): + rect = Rect(last[0], last[1], self.delta_x, self.delta_y) + return ((self.delta_x, self.delta_y), + (shape_bounds.union(rect), + edge_bounds.union(LineStyle2.cap_style_logic(style, + last, (self.delta_x, self.delta_y)))), + (False, False, style)) + + +class CurvedEdgeRecord(object): + + def __init__(self, controlx, controly, anchorx, anchory): + self.controlx = controlx + self.controly = controly + self.anchorx = anchorx + self.anchory = anchory + + def serialize(self): + + bits = BitStream() + + if self.delta_x == 0 and self.delta_y == 0: + return bits + + bits.write_bit(True) # TypeFlag + bits.write_bit(False) # StraightFlag + + cX = self.controlx * 20 + cY = self.controly * 20 + aX = self.anchorx * 20 + aY = self.anchory * 20 + + NBits = nbits_abs(cX, cY, aX, aY) + + if NBits > 15: + raise ValueError("Number of bits per value field cannot exceed 15") + + bits.write_int_value(NBits, 4) + NBits += 2 + bits.write_int_value(cX, NBits) + bits.write_int_value(cY, NBits) + bits.write_int_value(aX, NBits) + bits.write_int_value(aY, NBits) + return bits + + def _get_x(self, t): + return self.controlx * 2 * (1-t) * t + self.anchorx * t * t; + + def _get_y(self, t): + return self.controly * 2 * (1-t) * t + self.anchory * t * t; + + def _get_p(self, t): + return (self._get_x(t), self._get_y(t)) + + def calculate_bounds(self, last, shape_bounds, edge_bounds, style): + union = Rect(0, 0, 0, 0) + # CurvedEdgeRecord Bounds + # Formulas somewhat based on + # http://code.google.com/p/bezier/source/browse/trunk/bezier/src/flash/geom/Bezier.as + # Maths here may be incorrect + + # extremumX = last.x - 2 * control.x + anchor.x + # extremumX = last.x - 2 * ( controlDeltaX - last.x ) + anchorDeltaX - last.x + # extremumX = (last.x - last.x) - 2 * ( controlDeltaX - last.x ) + anchorDeltaX + # extremumX = -2 * ( controlDeltaX - last.x ) + anchorDeltaX + + # For the case of last.[x/y] = 0, we can use the formula below. + + x = -2 * self.controlx + self.anchorx + t = -self.controlx / x + p = self._get_x(t) + + if t <= 0 or t >= 1: + union.XMin = last[0] + min(self.anchorx, 0) + union.XMax = union.XMin + max(self.anchorx, 0) + else: + union.XMin = min(p, 0, self.anchorx + last[0]) + union.XMax = union.XMin + max(p - last[0], 0, self.anchorx) + + y = -2 * self.controly + self.anchory + t = -self.controly / y + p = self._get_y(t) + + if t <= 0 or t >= 1: + union.YMin = last[1] + min(self.anchory, 0) + union.YMax = union.YMin + max(self.anchory, 0) + else: + union.YMin = min(p, 0, self.anchory + last[1]) + union.YMax = union.YMin + max(p - last[0], 0, self.anchorY) + + # CapStyle logic: + + # Assume that p0 is last (start anchor), + # p1 is control, and p2 is (end) anchor. + + # Get some small increments in the segment to + # find somewhat of a slope derivative type thing. + + # We should be able to pass these two line deltas + # into LineStyle2.cap_style_logic and union the + # results. + + slope1 = self._get_p(0.01) + slope2 = (self.anchorx - self._get_x(0.99), self.anchory - self._get_y(0.99)) + end_cap_rect = LineStyle2.cap_style_logic(style, last, slope2) + start_cap_rect = LineStyle2.cap_style_logic(style, last, slope1) + + return ((self.anchorx, self.anchory), + (shape_bounds.union(union), + edge_bounds.union(union, start_cap_rect, end_cap_rect)), + (False, False, style)) + +class StyleChangeRecord(object): + + def __init__(self, delta_x, delta_y, linestyle=None, + fillstyle0=None, fillstyle1=None, + fillstyles=None, linestyles=None): + + self.delta_x = delta_x + self.delta_y = delta_y + self.linestyle = linestyle + self.fillstyle0 = fillstyle0 + self.fillstyle1 = fillstyle1 + self.fillstyles = fillstyles + self.linestyles = linestyles + + def serialize(self): + bits = BitStream() + if self.fillstyle0 is not None and self.fillstyle1 is not None and \ + self.fillstyle0.parent != self.fillstyle1.parent: + raise ValueError("fillstyle0 and fillstyle1 do not have the same parent!") + + fsi0 = 0 if self.fillstyle0 is None else self.fillstyle0.index + fsi1 = 0 if self.fillstyle1 is None else self.fillstyle1.index + lsi = 0 if self.linestyle is None else self.linestyle.index + + fbit = 0 if self.fillstyle0 is None else style_list_bits(self.fillstyle0.parent) + lbit = 0 if self.linestyle is None else style_list_bits(self.linestyle.parent) + + from pypy.translator.avm.tags import DefineShape + + new_styles = ((DefineShape._current_variant > 1) and + ((self.linestyles != None and len(self.linestyles) > 0) or + (self.fillstyles != None and len(self.fillstyles) > 0))) + + bits.write_bit(False) # TypeFlag + bits.write_bit(new_styles) # StateNewStyles + bits.write_bit(lsi > 0) # StateLineStyle + bits.write_bit(fsi0 > 0) # StateFillStyle0 + bits.write_bit(fsi1 > 0) # StateFillStyle1 + + move_flag = self.delta_x != 0 or self.delta_y != 0 + + if move_flag: + bits += XY(self.delta_x, self.delta_y).serialize() + + if fsi0 > 0: bits.write_int_value(fsi0, fbit) # FillStyle0 + if fsi1 > 0: bits.write_int_value(fsi1, fbit) # FillStyle1 + if lsi > 0: bits.write_int_value(lsi, lbit) # LineStyle + + if new_styles: + bits += ShapeWithStyle._serialize_style_list(self.fillstyles) # FillStyles + bits += ShapeWithStyle._serialize_style_list(self.linestyles) # LineStyles + + bits.write_int_value(style_list_bits(self.fillstyles), 4) # FillBits + bits.write_int_value(style_list_bits(self.linestyles), 4) # LineBits + + return bits Added: pypy/branch/avm/pypy/translator/avm1/records_flymake.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/records_flymake.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,737 @@ + +from pypy.translator.avm1.util import BitStream +from math import log, ceil, sqrt + +def serialize_style_list(lst): + bits = BitStream() + + if len(lst) <= 0xFF: + bits.write_bit_value(len(lst), 8) + else: + bits.write_bit_value(0xFF, 8) + bits.write_bit_value(len(lst), 16) + + for style in lst: + bits += style.serialize() + + return bits + +def nbits(n, *a): + return int(ceil(log(max(1, n, *a), 2))) + +def nbits_abs(n, *a): + return nbits(abs(n), *(abs(n) for n in a)) + +def style_list_bits(lst): + return nbits(len(lst)) + +def clamp(n, minimum, maximum): + return max(minimum, min(n, maximum)) + +class RecordHeader(object): + + def __init__(self, type, length): + self.type = type + self.length = length + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.type, 10) + if self.length < 0x3F: + bits.write_int_value(self.length, 6) + else: + bits.write_int_value(0x3F, 6) + bits.write_int_value(self.length, 32) + return bits + + def parse(self, bitstream): + self.type = bitstream.read_bit_value(10) + self.length = bitstream.read_bit_value(6) + if self.length >= 0x3F: + self.length = bitstream.read_bit_value(32) + +class _EndShapeRecord(object): + + def __call__(self, *a, **b): + pass + + def serialize(self): + bitstream = BitStream() + bitstream.zero_fill(6) + return bitstream + +EndShapeRecord = _EndShapeRecord() + +class Rect(object): + + def __init__(self, XMin=0, XMax=0, YMin=0, YMax=0): + self.XMin = XMin + self.XMax = XMax + self.YMin = YMin + self.YMax = YMax + + def union(self, rect, *rects): + r = Rect(min(self.XMin, rect.XMin), + max(self.XMax, rect.XMax), + min(self.YMin, rect.YMin), + max(self.YMax, rect.YMax)) + if len(rects) > 0: + return r.union(*rects) + return r + + def serialize(self): + if self.XMin > self.XMax or self.YMin > self.YMax: + raise ValueError, "Maximum values in a RECT must be larger than the minimum values." + + # Find our values in twips. + twpXMin = self.XMin * 20 + twpXMax = self.XMax * 20 + twpYMin = self.YMin * 20 + twpYMax = self.YMax * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + NBits = nbits_abs(twpXMin, twpXMax, twpYMin, twpYMax)+1 + + if NBits > 31: + raise ValueError, "Number of bits per value field cannot exceede 31." + + # And write out our bits. + bits = BitStream() + bits.write_int_value(NBits, 5) + bits.write_int_value(twpXMin, NBits) + bits.write_int_value(twpXMax, NBits) + bits.write_int_value(twpYMin, NBits) + bits.write_int_value(twpYMax, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bitstream.read_bit_value(5) + self.XMin = bitstream.read_bit_value(NBits) + self.XMax = bitstream.read_bit_value(NBits) + self.YMin = bitstream.read_bit_value(NBits) + self.YMax = bitstream.read_bit_value(NBits) + +class XY(object): + + def __init__(self, X=0, Y=0): + self.X = 0 + self.Y = 0 + + def serialize(self): + # Convert to twips plz. + twpX = self.X * 20 + twpY = self.Y * 20 + + # Find the number of bits required to store the longest + # value, then add one to account for the sign bit. + NBits = nbits_abs(twpX, twpY) + + bits = BitStream() + bits.write_int_value(NBits, 5) + bits.write_int_value(twpX, NBits) + bits.write_int_value(twpY, NBits) + + return bits + + def parse(self, bitstream): + + NBits = bitstream.read_bit_value(5) + self.X = bitstream.read_bit_value(NBits) + self.Y = bitstream.read_bit_value(NBits) + +class RGB(object): + + def __init__(self, color): + self.color = color & 0xFFFFFF + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.color, 24) + return bits + + def parse(self, bitstream): + self.color = bitstream.read_bit_value(24) + +class RGBA(RGB): + + def __init__(self, color, alpha=1.0): + super(RGBA, self).__init__(color) + self.alpha = alpha + + def serialize(self): + bits = RGB.serialize(self) + + from pypy.translator.avm1.tags import DefineShape + + # If we are in a DefineShape and the version does not support + # alpha (DefineShape1 or DefineShape2), don't use alpha! + if DefineShape._current_variant not in (1, 2): + bits.write_int_value(int(self.alpha * 0xFF), 8) + + return bits + + def parse(self, bitstream): + RGB.parse(self, bitstream) + self.alpha = bitstream.read_bit_value(8) / 0xFF + +class CXForm(object): + has_alpha = False + def __init__(self, rmul=1, gmul=1, bmul=1, radd=0, gadd=0, badd=0): + self.rmul = rmul + self.gmul = gmul + self.bmul = bmul + self.amul = 1 + self.radd = radd + self.gadd = gadd + self.badd = badd + self.aadd = 0 + + def serialize(self): + has_add_terms = self.radd != 0 or self.gadd != 0 or self.badd != 0 or self.aadd != 0 + has_mul_terms = self.rmul != 1 or self.gmul != 1 or self.bmul != 1 or self.amul != 1 + + rm = abs(self.rmul * 256) + gm = abs(self.gmul * 256) + bm = abs(self.bmul * 256) + am = abs(self.amul * 256) + + ro = clamp(self.radd, -255, 255) + go = clamp(self.gadd, -255, 255) + bo = clamp(self.badd, -255, 255) + ao = clamp(self.aadd, -225, 255) + + NBits = 0 + if has_mul_terms: NBits = nbits_abs(rm, gm, bm, am) + if has_add_terms: NBits = max(NBits, nbits_abs(ro, go, bo, ao)) + + bits = BitStream() + bits.write_int_value(NBits, 4) + + if has_mul_terms: + bits.write_int_value(rm, NBits) + bits.write_int_value(gm, NBits) + bits.write_int_value(bm, NBits) + if self.has_alpha: bits.write_int_value(am, NBits) + + if has_add_terms: + bits.write_int_value(ro, NBits) + bits.write_int_value(go, NBits) + bits.write_int_value(bo, NBits) + if self.has_alpha: bits.write_int_value(ao, NBits) + + return bits + +class CXFormWithAlpha(CXForm): + has_alpha = True + def __init__(self, rmul=1, gmul=1, bmul=1, amul=1, radd=0, gadd=0, badd=0, aadd=0): + super(self, CXFormWithAlpha).__init__(rmul, gmul, bmul, radd, gadd, badd) + self.amul = amul + self.aadd = aadd + +class Matrix(object): + + def __init__(self, a=1, b=0, c=0, d=1, tx=0, ty=0): + self.a, self.b, self.c, self.d, self.tx, self.ty = a, b, c, d, tx, ty + + def serialize(self): + + def write_prefixed_values(a, b): + NBits = nbits(a, b) + bits.write_int_value(NBits, 5) + bits.write_int_value(a, NBits) + bits.write_int_value(b, NBits) + + bits = BitStream() + if self.a != 1 or self.d != 1: # HasScale + bits.write_bit(True) + write_prefixed_values(self.a, self.d) + else: + bits.write_bit(False) + + if self.b != 0 or self.c != 0: # HasRotate + bits.write_bit(True) + write_prefixed_values(self.b, self.c) + else: + bits.write_bit(False) + + write_prefixed_values(self.tx * 20, self.ty * 20) + return bits + +class Shape(object): + + def __init__(self): + self.shapes = [] + + self.edge_bounds = Rect() + self.shape_bounds = Rect() + + self.has_scaling = False + self.has_non_scaling = False + + self.bounds_calculated = False + + def add_shape_record(self, shape): + self.shapes.append(shape) + self.bounds_calculated = False + + def add_shape(self, shape): + self.shapes.expand(shape.shapes) + self.bounds_calculated = False + + def serialize(self): + if EndShapeRecord not in self.shapes: + self.shapes.append(EndShapeRecord()) + + bits = BitStream() + + bits.write_int_value(0, 8) # NumFillBits and NumLineBits + for record in self.shapes: + bits += record.serialize() + + return bits + + def calculate_bounds(self): + + if self.bounds_calculated: + return + + last = 0, 0 + style = None + for record in self.shapes: + last, (self.shape_bounds, self.edge_bounds), (has_scale, has_non_scale, style) = \ + record.calculate_bounds(last, self.shape_bounds, self.edge_bounds, style) + if has_scale: + self.has_scaling = True + if has_non_scale: + self.has_non_scaling = True + + self.bounds_calculated = True + +class ShapeWithStyle(Shape): + + def __init__(self, fills=None, strokes=None): + super(self, ShapeWithStyle).__init__(self) + self.fills = fills or [] + self.strokes = strokes or [] + + def add_fill_style(self, style): + style.parent = self.fills + self.fills.append(style) + + def add_line_style(self, style): + style.parent = self.strokes + self.strokes.append(style) + + def add_shape(self, shape): + Shape.add_shape(self, shape) + try: + self.fills += shape.fills + self.strokes += shape.strokes + except AttributeError: + pass + + def serialize(self): + bits = BitStream() + bits += serialize_style_list(self.fills) + bits += serialize_style_list(self.strokes) + bits.write_int_value(style_list_bits(self.fills), 4) + bits.write_int_value(style_list_bits(self.strokes), 4) + return bits + +class LineStyle(object): + + caps = "round" + + def __init__(self, width=1, color=0, alpha=1.0): + self.width = width + self.color = RGBA(color, alpha) + + @property + def index(self): + return self.parent.find(self) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.width * 20, 16) + bits += self.color.serialize() + return bits + +class LineStyle2(LineStyle): + + def __init__(self, width=1, fillstyle=None, pixel_hinting=False, scale_mode=None, caps="round", joints="round", miter_limit=3): + + color, alpha, self.fillstyle = 0, 1.0, None + + if isinstance(fillstyle, RGBA): + color = fillstyle.color + alpha = fillstyle.alpha + elif isinstance(fillstyle, RGB): + color = fillstyle.color + elif isinstance(fillstyle, int): + if fillstyle > 0xFFFFFF: + color = fillstyle & 0xFFFFFF + alpha = fillstyle >> 6 & 0xFF + else: + color = fillstyle + elif isinstance(fillstyle, FillStyleSolidFill): + color = fillstyle.color.color + alpha = fillstyle.color.alpha + elif isinstance(fillstyle, FillStyle): + self.fillstyle = fillstyle + + super(self, LineStyle2).__init__(self, width, color, alpha) + self.pixel_hinting = pixel_hinting + self.h_scale = (scale_mode == "normal" or scale_mode == "horizontal") + self.v_scale = (scale_mode == "normal" or scale_mode == "vertical") + + if caps == "square": self.caps = 2 + elif caps == None: self.caps = 1 + elif caps == "round": self.caps = 0 + else: + raise ValueError, "Invalid cap style '%s'." % caps + + if joints == "miter": self.joints = 2 + elif joints == "bevel": self.joints = 1 + elif joints == "round": self.joints = 0 + + self.miter_limit = miter_limit + + def serialize(self): + + bits = BitStream() + bits.write_int_value(self.width * 20, 8) + bits.write_int_value(self.width * 20 >> 8, 8) + + bits.write_int_value(self.caps, 2) + bits.write_int_value(self.joints, 2) + bits.write_bit(self.fillstyle is not None); + bits.write_bit(self.h_scale) + bits.write_bit(self.v_scale) + bits.write_bit(self.pixel_hinting) + + if self.joints == 2: + bits.write_fixed_value(self.miter_limit, 16, True) + + if self.fillstyle: + bits.write_bits(self.fillstyle.serialize()) + else: + bits.write_bits(self.color.serialize()) + + return bits + + def cap_style_logic(self, style, last, delta): + # Half thickness (radius of round cap; diameter is thickness) + off = style.width / 2.0 + dx, dy = delta + lx, ly = last + + if style.caps == "round": + r = Rect() + r.XMin = cmp(dx, 0) * off + r.YMin = cmp(dy, 0) * off + r.XMax = r.XMin + dx + r.YMax = r.XMax + dy + return r + + if style.caps == "square": + + # Account for the length of the caps. + dellen = sqrt(dx*dx + dy*dy) # Delta length + norm = (dellen+off*2)/dellen # Extra length + dx *= norm # Add the extra length + dy *= norm + sqx, sqy = delta # Square cap offset + norm = off/dellen # Offset amount + sqx *= norm # Position offsets. + sqy *= norm + + # And offset the position. + lx -= sqx + ly -= sqy + + # Right-hand normal to vector delta relative to (0, 0). + p1x, p1y = (-dy, dx) + norm = sqrt(p1x*p1x + p1y*p1y) + p1x /= norm + p1y /= norm + + # Left-hand normal to vector delta relative to (0, 0) + p2x, p2y = (-p1x, -p1y) + + # Right-hand normal to vector delta relative to delta. + p3x, p3y = (p1x + dx, p1y + dy) + + # Left-hand normal to vector delta relative to delta. + p4x, p4y = (p2x + dx, p2y + dy) + + return Rect( + min(p1x, p2x, p3x, p4x) + lx, + max(p1x, p2x, p3x, p4x) + lx, + min(p1y, p2y, p3y, p4y) + ly, + max(p1y, p2y, p3y, p4y) + ly) + +class FillStyle(object): + + TYPE = -1 + + @property + def index(self): + return self.parent.find(self) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.TYPE, 8) + bits += self.serialize_inner() + return bits + +class FillStyleSolidFill(object): + + def __init_(self, color, alpha=1.0): + self.color = RGBA(color, alpha) + + def serialize_inner(self): + return self.color.serialize() + +class GradRecord(object): + + def __init__(self, ratio, color, alpha=1.0): + self.ratio = ratio + self.color = RGBA(color, alpha) + + def serialize(self): + bits = BitStream() + bits.write_int_value(self.ratio, 8) + bits += self.color.serialize() + return bits + +class Gradient(object): + + def __init__(self, grads, spread="pad", interpolation="rgb", focalpoint=0): + import operator + grads.sort(key=operator.attrgetter("ratio")) + self.grads = grads + self.spread = spread + self.interpolation = interpolation + self.focalpoint = focalpoint + + @classmethod + def from_begin_gradient_fill(cls, colors, alphas, ratios, spread, interpolation, focalpoint): + grads = [GradRecord(*t) for t in zip(ratios, colors, alphas)] + return cls(grads, spread, interpolation, focalpoint) + +class StraightEdgeRecord(object): + + def __init__(self, delta_x, delta_y): + self.delta_x = delta_x + self.delta_y = delta_y + self.bounds_calculated = False + + def serialize(self): + + bits = BitStream() + + if self.delta_x == 0 and self.delta_y == 0: + return bits + + bits.write_bit(True) # TypeFlag + bits.write_bit(True) # StraightFlag + + X = self.delta_x * 20 + Y = self.delta_y * 20 + + NBits = nbits_abs(X, Y) + + if NBits > 15: + raise ValueError("Number of bits per value field cannot exceed 15") + + bits.write_int_value(NBits, 4) + NBits += 2 + if X == 0: + # Vertical Line + bits.write_bit(False) # GeneralLineFlag + bits.write_bit(True) # VerticalLineFlag + bits.write_int_value(Y, NBits) + elif Y == 0: + # Horizontal Line + bits.write_bit(False) # GeneralLineFlag + bits.write_bit(True) # HorizontalLineFlag + bits.write_int_value(X, NBits) + else: + # General Line + bits.write_bit(True) # GeneralLineFlag + bits.write_int_value(X, NBits) + bits.write_int_value(Y, NBits) + + return bits + + def calculate_bounds(self, last, shape_bounds, edge_bounds, style): + rect = Rect(last[0], last[1], self.delta_x, self.delta_y) + return ((self.delta_x, self.delta_y), + (shape_bounds.union(rect), + edge_bounds.union(LineStyle2.cap_style_logic(style, + last, (self.delta_x, self.delta_y)))), + (False, False, style)) + + +class CurvedEdgeRecord(object): + + def __init__(self, controlx, controly, anchorx, anchory): + self.controlx = controlx + self.controly = controly + self.anchorx = anchorx + self.anchory = anchory + + def serialize(self): + + bits = BitStream() + + if self.delta_x == 0 and self.delta_y == 0: + return bits + + bits.write_bit(True) # TypeFlag + bits.write_bit(False) # StraightFlag + + cX = self.controlx * 20 + cY = self.controly * 20 + aX = self.anchorx * 20 + aY = self.anchory * 20 + + NBits = nbits_abs(cX, cY, aX, aY) + + if NBits > 15: + raise ValueError("Number of bits per value field cannot exceed 15") + + bits.write_int_value(NBits, 4) + NBits += 2 + bits.write_int_value(cX, NBits) + bits.write_int_value(cY, NBits) + bits.write_int_value(aX, NBits) + bits.write_int_value(aY, NBits) + return bits + + def _get_x(self, t): + return self.controlx * 2 * (1-t) * t + self.anchorx * t * t; + + def _get_y(self, t): + return self.controly * 2 * (1-t) * t + self.anchory * t * t; + + def _get_p(self, t): + return (self._get_x(t), self._get_y(t)) + + def calculate_bounds(self, last, shape_bounds, edge_bounds, style): + union = Rect(0, 0, 0, 0) + # CurvedEdgeRecord Bounds + # Formulas somewhat based on + # http://code.google.com/p/bezier/source/browse/trunk/bezier/src/flash/geom/Bezier.as + # Maths here may be incorrect + + # extremumX = last.x - 2 * control.x + anchor.x + # extremumX = last.x - 2 * ( controlDeltaX - last.x ) + anchorDeltaX - last.x + # extremumX = (last.x - last.x) - 2 * ( controlDeltaX - last.x ) + anchorDeltaX + # extremumX = -2 * ( controlDeltaX - last.x ) + anchorDeltaX + + # For the case of last.[x/y] = 0, we can use the formula below. + + x = -2 * self.controlx + self.anchorx + t = -self.controlx / x + p = self._get_x(t) + + if t <= 0 or t >= 1: + union.XMin = last[0] + min(self.anchorx, 0) + union.XMax = union.XMin + max(self.anchorx, 0) + else: + union.XMin = min(p, 0, self.anchorx + last[0]) + union.XMax = union.XMin + max(p - last[0], 0, self.anchorx) + + y = -2 * self.controly + self.anchory + t = -self.controly / y + p = self._get_y(t) + + if t <= 0 or t >= 1: + union.YMin = last[1] + min(self.anchory, 0) + union.YMax = union.YMin + max(self.anchory, 0) + else: + union.YMin = min(p, 0, self.anchory + last[1]) + union.YMax = union.YMin + max(p - last[0], 0, self.anchorY) + + # CapStyle logic: + + # Assume that p0 is last (start anchor), + # p1 is control, and p2 is (end) anchor. + + # Get some small increments in the segment to + # find somewhat of a slope derivative type thing. + + # We should be able to pass these two line deltas + # into LineStyle2.cap_style_logic and union the + # results. + + slope1 = self._get_p(0.01) + slope2 = (self.anchorx - self._get_x(0.99), self.anchory - self._get_y(0.99)) + end_cap_rect = LineStyle2.cap_style_logic(style, last, slope2) + start_cap_rect = LineStyle2.cap_style_logic(style, last, slope1) + + return ((self.anchorx, self.anchory), + (shape_bounds.union(union), + edge_bounds.union(union, start_cap_rect, end_cap_rect)), + (False, False, style)) + +class StyleChangeRecord(object): + + def __init__(self, delta_x, delta_y, linestyle=None, + fillstyle0=None, fillstyle1=None, + fillstyles=None, linestyles=None): + + self.delta_x = delta_x + self.delta_y = delta_y + self.linestyle = linestyle + self.fillstyle0 = fillstyle0 + self.fillstyle1 = fillstyle1 + self.fillstyles = fillstyles + self.linestyles = linestyles + + def serialize(self): + bits = BitStream() + if self.fillstyle0 is not None and self.fillstyle1 is not None and \ + self.fillstyle0.parent != self.fillstyle1.parent: + raise ValueError("fillstyle0 and fillstyle1 do not have the same parent!") + + fsi0 = 0 if self.fillstyle0 is None else self.fillstyle0.index + fsi1 = 0 if self.fillstyle1 is None else self.fillstyle1.index + lsi = 0 if self.linestyle is None else self.linestyle.index + + fbit = 0 if self.fillstyle0 is None else style_list_bits(self.fillstyle0.parent) + lbit = 0 if self.linestyle is None else style_list_bits(self.linestyle.parent) + + from pypy.translator.avm.tags import DefineShape + + new_styles = ((DefineShape._current_variant > 1) and + ((self.linestyles != None and len(self.linestyles) > 0) or + (self.fillstyles != None and len(self.fillstyles) > 0))) + + bits.write_bit(False) # TypeFlag + bits.write_bit(new_styles) # StateNewStyles + bits.write_bit(lsi > 0) # StateLineStyle + bits.write_bit(fsi0 > 0) # StateFillStyle0 + bits.write_bit(fsi1 > 0) # StateFillStyle1 + + move_flag = self.delta_x != 0 or self.delta_y != 0 + + if move_flag: + bits += XY(self.delta_x, self.delta_y).serialize() + + if fsi0 > 0: bits.write_int_value(fsi0, fbit) # FillStyle0 + if fsi1 > 0: bits.write_int_value(fsi1, fbit) # FillStyle1 + if lsi > 0: bits.write_int_value(lsi, lbit) # LineStyle + + if new_styles: + bits += ShapeWithStyle._serialize_style_list(self.fillstyles) # FillStyles + bits += ShapeWithStyle._serialize_style_list(self.linestyles) # LineStyles + + bits.write_int_value(style_list_bits(self.fillstyles), 4) # FillBits + bits.write_int_value(style_list_bits(self.linestyles), 4) # LineBits + + return bits Added: pypy/branch/avm/pypy/translator/avm1/swf.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/swf.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,61 @@ + +import struct + +from pypy.translator.avm1.util import BitStream +from pypy.translator.avm1.records import Rect + +class SwfData(object): + def __init__(self, width=600, height=400, fps=24, compress=False, version=10): + self.next_character_id = 1 + self.width = width + self.height = height + self.fps = fps + self.compress = compress + self.version = version + self.frame_count = 1 + + self.tags = [] + + def __getitem__(self, i): + return self.tags.__getitem__(i) + + def __iadd__(self, other): + if hasattr(other, "TAG_TYPE"): + self.add_tag(other) + else: + self.add_tags(other) + return self + + def add_tag(self, tag): + if self.version > tag.TAG_MIN_VERSION: + if hasattr(tag, "characterid") and tag.characterid == None: + tag.characterid = self.next_character_id + self.next_character_id += 1 + self.tags.append(tag) + + def add_tags(self, tag_container): + if hasattr(tag_container, "tags"): + self.tags += tag_container.tags + else: + self.tags += tag_container + + def serialize(self): + + header = self._gen_header() + data = self._gen_data_stub() + data += ''.join(tag.serialize() for tag in self.tags) + + header[2] = struct.pack("> 8) & 0xFFFF, self.color & 0xFF) + +class DoAction(SwfTag, Block): + + TAG_TYPE = 12 + TAG_MIN_VERSION = 3 + + def __init__(self): + Block.__init__(self, None, True) + + def serialize_data(self): + return Block.serialize(self) + +class DoABC(SwfTag, AbcFile): + + TAG_TYPE = 82 + TAG_MIN_VERSION = 9 + + def __init__(self, name="PyPy", flags=0): + AbcFile.__init__(self) + self.name = name + self.flags = flags + + def serialize_data(self): + return struct.pack(" + +PyPy AVM Backend Test Case: %s + + + + + + + +""" + + crossdomain_xml=""" + +""" + +def parse_result(string): + if string == "true": + return True + elif string == "false": + return False + elif string == "undefined" or string == "null": + return None + elif all(c in "123456789-" for c in string): + return int(string) + elif "," in string: + return [parse_result(s) for s in string.split(",")] + return string + +class TestCase(object): + def __init__(self, name, swfdata): + self.name = name + self.swfdata = swfdata + self.result = None + +class TestHandler(BaseHTTPRequestHandler): + """The HTTP handler class that provides the tests and handles results""" + + def do_GET(self): + testcase = self.server.testcase + if self.path == "/test.html": + data = config.html_page % (testcase.name, testcase.swfdata.width, testcase.swfdata.height) + mime = 'text/html' + elif self.path == "/test.swf": + data = testcase.swfdata.serialize() + mime = 'application/x-shockwave-flash' + elif self.path == "/crossdomain.xml": + data = config.crossdomain_xml + mime = 'text/xml' + self.serve_data(mime, data) + + def do_POST(self): + if self.path == "/test.result": + form = parse_qs(self.rfile.read(int(self.headers['content-length']))) + self.server.testcase.result = form['result'][0] + + def serve_data(self, content_type, data): + self.send_response(200) + self.send_header("Content-type", content_type) + self.send_header("Content-length", len(data)) + self.end_headers() + self.wfile.write(data) + +class BrowserTest(object): + """The browser driver""" + + def start_server(self, port, testcase): + server_address = ('', port) + self.httpd = HTTPServer(server_address, TestHandler) + self.httpd.testcase = testcase + + def get_result(self): + testcase = self.httpd.testcase + start_time = time.time() + while testcase.result is None: + if time.time() - start_time > 10: + assert False + self.httpd.handle_request() + return testcase.result + +def browsertest(name, swfdata): + testcase = TestCase(str(name), swfdata) + driver = BrowserTest() + driver.start_server(config.http_port, testcase) + webbrowser.open('http://localhost:%d/test.html' % config.http_port) + + return parse_result(driver.get_result()) Added: pypy/branch/avm/pypy/translator/avm1/test/harness.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/harness.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,67 @@ + +from pypy.translator.avm1.test import browsertest as b +from pypy.translator.avm1 import avm1 as a, avm1gen as g, swf as s, tags as t, records as r + +class TestHarness(object): + def __init__(self, name): + self.testname = name + self.swf = s.SwfData() + self.swf.add_tag(t.SetBackgroundColor(0x333333)) + self.swf.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "txt", + "Testing %s." % (name,), color=r.RGBA(0xFFFFFF))) + self.swf.add_tag(t.PlaceObject2(1, 2)) + self.actions = g.AVM1Gen(t.DoAction()) + self.swf.add_tag(self.actions.block) + self.swf.add_tag(t.ShowFrame()) + self.swf.add_tag(t.End()) + self.start_test() + + def print_text(self, text): + self.actions.push_const("txt", "\n" + text) + self.actions.push_var("txt") + self.actions.swap() + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_var(self, prefix, varname): + self.actions.push_const("txt") + self.actions.push_var("txt") + self.actions.push_const("\n" + prefix) + self.actions.push_var(varname) + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_stack(self, prefix): + self.actions.push_const("txt") + self.actions.swap() + self.actions.push_var("txt") + self.actions.swap() + self.actions.push_const("\n" + prefix) + self.actions.swap() + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def start_test(self): + self.print_text("Running test %s." % self.testname) + self.actions.push_const("result") + + def finish_test(self): + # result value is on the stack, + # followed by the string "result" + self.actions.set_variable() + self.print_var("Got: ", "result") + self.actions.push_const("/test.result", "") # URL, target + self.actions.action(a.ActionGetURL2("POST", True, True)) + self.actions.push_const("javascript:window.close()", "") # Close the window. + self.actions.action(a.ActionGetURL2("")) + + def do_test(self, debug=False): + self.finish_test() + self.actions.scope.block.seal() + if debug: + f = open("test.swf", "w") + f.write(self.swf.serialize()) + f.close() + return b.browsertest(self.testname, self.swf) Added: pypy/branch/avm/pypy/translator/avm1/test/harness_flymake.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/harness_flymake.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,67 @@ + +from pypy.translator.avm1.test import browsertest as b +from pypy.translator.avm1 import avm1 as a, avm1gen as g, swf as s, tags as t, records as r + +class TestHarness(object): + def __init__(self, name): + self.testname = name + self.swf = s.SwfData() + self.swf.add_tag(t.SetBackgroundColor(0x333333)) + self.swf.add_tag(t.DefineEditText(r.Rect(0, 0, 0, 0), "txt", + "Testing %s." % (name,), color=r.RGBA(0xFFFFFF))) + self.swf.add_tag(t.PlaceObject2(1, 2)) + self.actions = g.AVM1Gen(t.DoAction()) + self.swf.add_tag(self.actions.block) + self.swf.add_tag(t.ShowFrame()) + self.swf.add_tag(t.End()) + self.start_test() + + def print_text(self, text): + self.actions.push_const("txt", "\n" + text) + self.actions.push_var("txt") + self.actions.swap() + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_var(self, prefix, varname): + self.actions.push_const("txt") + self.actions.push_var("txt") + self.actions.push_const("\n" + prefix) + self.actions.push_var(varname) + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def print_stack(self, prefix): + self.actions.push_const("txt") + self.actions.swap() + self.actions.push_var("txt") + self.actions.swap() + self.actions.push_const("\n" + prefix) + self.actions.swap() + self.actions.emit('typed_add') + self.actions.emit('typed_add') + self.actions.set_variable() + + def start_test(self): + self.print_text("Running test %s." % self.testname) + self.actions.push_const("result") + + def finish_test(self): + # result value is on the stack, + # followed by the string "result" + self.actions.set_variable() + self.print_var("Got: ", "result") + self.actions.push_const("/test.result", "") # URL, target + self.actions.action(a.ActionGetURL2("POST", True, True)) + self.actions.push_const("javascript:window.close()", "") # Close the window. + self.actions.action(a.ActionGetURL2("")) + + def do_test(self, debug=False): + self.finish_test() + self.actions.scope.block.seal() + if debug: + f = open("test.swf", "w") + f.write(self.swf.serialize()) + f.close() + return b.browsertest(self.testname, self.swf) Added: pypy/branch/avm/pypy/translator/avm1/test/mylib.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/mylib.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.carbonpython import export + + at export(int, int) +def sum(a, b): + return a+b Added: pypy/branch/avm/pypy/translator/avm1/test/runtest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/runtest.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,138 @@ +import platform + +import py +from pypy.translator.translator import TranslationContext +from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin +from pypy.rpython.lltypesystem.lltype import typeOf +from pypy.rpython.ootypesystem import ootype +from pypy.annotation.model import lltype_to_annotation +from pypy.translator.backendopt.all import backend_optimizations +from pypy.translator.backendopt.checkvirtual import check_virtual_methods +from pypy.translator.oosupport.support import patch_os, unpatch_os +from pypy.translator.avm1.test.harness import TestHarness +from pypy.translator.avm1.genavm import GenAVM1 + +# def translate_space_op(gen, op): +# if op.opname == "cast_int_to_char": +# gen.push_arg(op.args[0]) +# gen.push_const(1) +# gen.push_var("String") +# gen.call_method("fromCharCode") + +def compile_function(func, name, annotation=[], graph=None, backendopt=True, + auto_raise_exc=False, exctrans=False, + annotatorpolicy=None, nowrap=False): + olddefs = patch_os() + gen = _build_gen(func, annotation, name, graph, backendopt, + exctrans, annotatorpolicy, nowrap) + harness = TestHarness(name) + gen.ilasm = harness.actions + gen.generate_source() + unpatch_os(olddefs) # restore original values + return gen, harness + +def _build_gen(func, annotation, name, graph=None, backendopt=True, exctrans=False, + annotatorpolicy=None, nowrap=False): + try: + func = func.im_func + except AttributeError: + pass + t = TranslationContext() + if graph is not None: + graph.func = func + ann = t.buildannotator(policy=annotatorpolicy) + inputcells = [ann.typeannotation(an) for an in annotation] + ann.build_graph_types(graph, inputcells) + t.graphs.insert(0, graph) + else: + ann = t.buildannotator(policy=annotatorpolicy) + ann.build_types(func, annotation) + + t.buildrtyper(type_system="ootype").specialize() + if backendopt: + check_virtual_methods(ootype.ROOT) + backend_optimizations(t) + + main_graph = t.graphs[0] + tmpdir = py.path.local('.') + + return GenAVM1(tmpdir, t, None, exctrans) + +class AVM1Test(BaseRtypingTest, OORtypeMixin): + def __init__(self): + self._func = None + self._ann = None + self._genoo = None + self._harness = None + + def _compile(self, fn, args, ann=None, backendopt=True, auto_raise_exc=False, exctrans=False): + if ann is None: + ann = [lltype_to_annotation(typeOf(x)) for x in args] + self._genoo, self._harness = compile_function(fn, + "%s.%s" % (self.__class__.__name__, fn.func_name), + ann, + backendopt=backendopt, + auto_raise_exc=auto_raise_exc, + exctrans=exctrans) + self._func = fn + self._ann = ann + return self._harness + + def _skip_win(self, reason): + if platform.system() == 'Windows': + py.test.skip('Windows --> %s' % reason) + + def _skip_powerpc(self, reason): + if platform.processor() == 'powerpc': + py.test.skip('PowerPC --> %s' % reason) + + def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): + pass + + def _get_backendopt(self, backendopt): + if backendopt is None: + backendopt = getattr(self, 'backendopt', True) # enable it by default + return backendopt + + def interpret(self, fn, args, annotation=None, backendopt=None, exctrans=False): + backendopt = self._get_backendopt(backendopt) + harness = self._compile(fn, args, annotation, backendopt=backendopt, exctrans=exctrans) + harness.actions.call_function_constargs(fn.func_name, *args) + result = harness.do_test(True) + return result + + def interpret_raises(self, exception, fn, args, backendopt=None, exctrans=False): + import exceptions # needed by eval + backendopt = self._get_backendopt(backendopt) + try: + self.interpret(fn, args, backendopt=backendopt, exctrans=exctrans) + except ExceptionWrapper, ex: + assert issubclass(eval(ex.class_name), exception) + else: + assert False, 'function did raise no exception at all' + + float_eq = BaseRtypingTest.float_eq_approx + + def is_of_type(self, x, type_): + return True # we can't really test the type + + def ll_to_string(self, s): + return s + + def ll_to_unicode(self, s): + return s + + def ll_to_list(self, l): + return l + + def ll_to_tuple(self, t): + return t + + def class_name(self, value): + return value.class_name.split(".")[-1] + + def is_of_instance_type(self, val): + return isinstance(val, InstanceWrapper) + + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on gencli tests') Added: pypy/branch/avm/pypy/translator/avm1/test/runtest2.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/runtest2.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,213 @@ + +import py, re, subprocess +from pypy.translator.translator import TranslationContext +from pypy.translator.avm.test.browsertest import browsertest +# from pypy.translator.backendopt.all import backend_optimizations +# from pypy.translator.js.js import JS +# from pypy.translator.js.test.browsertest import jstest +# from pypy.translator.js import conftest +#from pypy.translator.js.log import log +from pypy.conftest import option +from pypy.rpython.test.tool import BaseRtypingTest, OORtypeMixin +# from pypy.rlib.nonconst import NonConstant +# from pypy.rpython.ootypesystem import ootype + +from pypy.rpython.llinterp import LLException + +log = log.runtest + +port = 8080 + +class AVMException(LLException): + pass + +#def _CLI_is_on_path(): +# if py.path.local.sysfind('js') is None: #we recommend Spidermonkey +# return False +# return True + +class compile_function(object): + def __init__(self, function, annotations, stackless=False, view=False, html=None, is_interactive=False, root = None, run_browser = True, policy = None): + + self.html = html + self.is_interactive = is_interactive + t = TranslationContext() + + if policy is None: + from pypy.annotation.policy import AnnotatorPolicy + policy = AnnotatorPolicy() + policy.allow_someobjects = False + + ann = t.buildannotator(policy=policy) + ann.build_types(function, annotations) + if view or option.view: + t.view() + t.buildrtyper(type_system="ootype").specialize() + + if view or option.view: + t.view() + #self.js = JS(t, [function, callback_function], stackless) + self.js = JS(t, function, stackless) + self.js.write_source() + if root is None and use_tg: + from pypy.translator.js.demo.jsdemo.controllers import Root + self.root = Root + else: + self.root = root + self.run_browser = run_browser + self.function_calls = [] + + def source(self): + return self.js.tmpfile.open().read() + + def _conv(self, v): + if isinstance(v, str): + return repr(v) + return str(v).lower() + + def __call__(self, *kwds): + return self.call(None, kwds) + + def call(self, entry_function, kwds): + args = ', '.join([self._conv(kw) for kw in kwds]) #lowerstr for (py)False->(js)false, etc. + + if entry_function is None: + entry_function = self.js.translator.graphs[0].name + else: + entry_function = self.js.translator.annotator.bookkeeper.getdesc(entry_function).cached_graph(None) + function_call = "%s(%s)" % (entry_function, args) + self.function_calls.append(function_call) + #if self.js.stackless: + # function_call = "slp_entry_point('%s')" % function_call + + if use_browsertest: + if not use_tg: + log("Used html: %r" % self.html) + output = browsertest(self.js.filename, function_call) + else: + global port + from pypy.translator.js.test.tgtest import run_tgtest + out = run_tgtest(self, tg_root = self.root, port=port, run_browser=self.run_browser).results + assert out[1] == 'undefined' or out[1] == "" + output = out[0] + port += 1 + return self.reinterpret(output) + else: +# cmd = 'echo "load(\'%s\'); print(%s)" | js 2>&1' % (self.js.filename, function_call) +# log(cmd) +# output = os.popen(cmd).read().strip() + js = subprocess.Popen(["js"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + input = "load(%r);\n" % self.js.filename.strpath + for call in self.function_calls[:-1]: + input += "%s;\n" % call + input += "print(\"'\" + %s + \"'\");\n" % self.function_calls[-1] + js.stdin.write(input) + stdout, stderr = js.communicate() + output = (stderr + stdout).strip() + for s in output.split('\n'): + log(s) + + m = re.match("'(.*)'", output, re.DOTALL) + if not m: + log("Error: %s" % output) + raise JSException(output) + return self.reinterpret(m.group(1)) + + def reinterpret(cls, s): + #while s.startswith(" "): + # s = s[1:] # :-) quite inneficient, but who cares + if s == 'false': + res = False + elif s == 'true': + res = True + elif s == 'undefined': + res = None + elif s == 'inf': + res = 1e300 * 1e300 + elif s == 'NaN': + res = (1e300 * 1e300) / (1e300 * 1e300) + elif s.startswith('[') or s.startswith('('): + l = s[1:-1].split(',') + res = [cls.reinterpret(i) for i in l] + else: + try: + res = float(s) + if float(int(res)) == res: + return int(res) + except ValueError: + res = str(s) + return res + reinterpret = classmethod(reinterpret) + +class JsTest(BaseRtypingTest, OORtypeMixin): + def _compile(self, _fn, args, policy=None): + argnames = _fn.func_code.co_varnames[:_fn.func_code.co_argcount] + func_name = _fn.func_name + if func_name == '': + func_name = 'func' + source = py.code.Source(""" + def %s(): + from pypy.rlib.nonconst import NonConstant + res = _fn(%s) + if isinstance(res, type(None)): + return None + else: + return str(res)""" + % (func_name, ",".join(["%s=NonConstant(%r)" % (name, i) for + name, i in zip(argnames, args)]))) + exec source.compile() in locals() + return compile_function(locals()[func_name], [], policy=policy) + + def string_to_ll(self, s): + return s + + def interpret(self, fn, args, policy=None): + f = self._compile(fn, args, policy) + res = f(*args) + return res + + def interpret_raises(self, exception, fn, args): + #import exceptions # needed by eval + #try: + #import pdb; pdb.set_trace() + try: + res = self.interpret(fn, args) + except JSException, e: + s = e.args[0] + assert s.startswith('uncaught exception:') + assert re.search(exception.__name__, s) + else: + raise AssertionError("Did not raise, returned %s" % res) + #except ExceptionWrapper, ex: + # assert issubclass(eval(ex.class_name), exception) + #else: + # assert False, 'function did raise no exception at all' + + def ll_to_string(self, s): + return str(s) + + def ll_to_list(self, l): + return l + + def ll_unpack_tuple(self, t, length): + assert len(t) == length + return tuple(t) + + def class_name(self, value): + return value[:-8].split('.')[-1] + + def is_of_instance_type(self, val): + m = re.match("^<.* object>$", val) + return bool(m) + + def read_attr(self, obj, name): + py.test.skip('read_attr not supported on genjs tests') + +def check_source_contains(compiled_function, pattern): + import re + + source = compiled_function.js.tmpfile.open().read() + return re.search(pattern, source) Added: pypy/branch/avm/pypy/translator/avm1/test/simpletest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/simpletest.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,17 @@ + +from pypy.translator.avm.swf import SwfData +from pypy.translator.avm.tags import SetBackgroundColor, DefineShape, End +from pypy.translator.avm.records import ShapeWithStyle, LineStyle, StraightEdgeRecord, StyleChangeRecord + +linestyle = LineStyle(3, 0x000000) + +shape = ShapeWithStyle() +shape.add_line_style(linestyle) +shape.add_shape_record(StyleChangeRecord(20, 20, linestyle)) +shape.add_shape_record(StraightEdgeRecord(100, 100)) + +swf = SwfData(400, 300) +swf.add_tag(SetBackgroundColor(0x333333)) +swf.add_tag(DefineShape(shape)) +swf.add_tag(End()) +print swf.serialize() Added: pypy/branch/avm/pypy/translator/avm1/test/test_backendopt.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_backendopt.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import compile_function +from pypy.translator.oosupport.test_template.backendopt import BaseTestOptimizedSwitch + +class TestOptimizedSwitch(BaseTestOptimizedSwitch): + def getcompiled(self, fn, annotation): + return compile_function(fn, annotation, backendopt=True) Added: pypy/branch/avm/pypy/translator/avm1/test/test_bool.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_bool.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rbool import BaseTestRbool + +class TestCliBool(CliTest, BaseTestRbool): + pass + Added: pypy/branch/avm/pypy/translator/avm1/test/test_builtin.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_builtin.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,29 @@ +import platform +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.builtin import BaseTestBuiltin, BaseTestTime + + +def skip_os(self): + py.test.skip("CLI doesn't support the os module, yet") + +def skip_win(): + if platform.system() == 'Windows': + py.test.skip("Doesn't work on Windows, yet") + +class TestCliBuiltin(CliTest, BaseTestBuiltin): + test_os_path_exists = skip_os + test_os_isdir = skip_os + test_os_dup_oo = skip_os + test_os_access = skip_os + + def test_builtin_math_frexp(self): + self._skip_powerpc("Mono math floating point problem") + BaseTestBuiltin.test_builtin_math_frexp(self) + + def test_debug_llinterpcall(self): + py.test.skip("so far, debug_llinterpcall is only used on lltypesystem") + + +class TestCliTime(CliTest, BaseTestTime): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_carbonpython.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_carbonpython.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,175 @@ +import py +py.test.skip("it passes usually, but fails on buildbot, no clue why") + +import os +import os.path +from pypy.tool import udir +from pypy.translator.cli.rte import Target +from pypy.translator.cli.carbonpython import DllDef, export, collect_entrypoints,\ + collect_class_entrypoints, compile_dll +from pypy.translator.cli.test.runtest import CliFunctionWrapper, CliTest + +TEMPLATE = """ +using System; +using System.Collections; +class CarbonPytonTest { + public static void Main() { + %s + } +} +""" + +class TestCarbonPython(CliTest): + + def _csharp(self, source, references=[], netmodules=[]): + tmpfile = udir.udir.join('tmp.cs') + tmpfile.write(TEMPLATE % source) + flags = ['/r:%s' % ref for ref in references] + flags += ['/addmodule:%s' % mod for mod in netmodules] + + class MyTarget(Target): + SOURCES = [str(tmpfile)] + FLAGS = flags + OUTPUT = 'tmp.exe' + SRC_DIR = str(udir.udir) + + func = CliFunctionWrapper(MyTarget.get()) + return func() + + def test_compilation(self): + res = self._csharp('Console.WriteLine(42);') + assert res == 42 + + def test_func_namespace(self): + def foo(x): + return x+1 + def bar(x): + return foo(x) + foo._namespace_ = 'MyNamespace.MyClass' + bar._namespace_ = 'MyClass' + res = self.interpret(bar, [41], backendopt=False) + assert res == 42 + + def test_simple_functions(self): + def foo(x): + return x+1 + def bar(x): + return x*2 + dll = DllDef('test', 'Test', [(foo, [int]), + (bar, [int])]) + dll.compile() + res = self._csharp('Console.WriteLine("{0}, {1}", Test.foo(42), Test.bar(42));', ['test']) + assert res == (43, 84) + + def test_export(self): + @export(int, float) + def foo(x, y): + pass + @export(int, float, namespace='test') + def bar(x, y): + pass + @export + def baz(): + pass + + assert foo._inputtypes_ == (int, float) + assert not hasattr(foo, '_namespace_') + assert bar._inputtypes_ == (int, float) + assert bar._namespace_ == 'test' + assert baz._inputtypes_ == () + + def test_collect_entrypoints(self): + @export(int, float) + def foo(x, y): + pass + def bar(x, y): + pass + mydict = dict(foo=foo, bar=bar, x=42) + entrypoints = collect_entrypoints(mydict) + assert entrypoints == [(foo, (int, float))] + + def test_collect_class_entrypoints(self): + class NotExported: + def __init__(self): + pass + + class MyClass: + @export + def __init__(self): + pass + @export(int) + def foo(self, x): + return x + + assert collect_class_entrypoints(NotExported) == [] + entrypoints = collect_class_entrypoints(MyClass) + assert len(entrypoints) == 2 + assert entrypoints[0][1] == () # __init__ inputtypes + assert entrypoints[1][1] == (MyClass, int) # foo inputtypes + + def test_compile_class(self): + py.test.skip('This test fails every other day. No clue why :-(') + class MyClass: + @export(int) + def __init__(self, x): + self.x = x + @export(int, int) + def add(self, y, z): + return self.x + y + z + MyClass.__module__ = 'Test' # put the class in the Test namespace + + entrypoints = collect_entrypoints({'MyClass': MyClass}) + dll = DllDef('test', 'Test', entrypoints) + dll.compile() + res = self._csharp(""" + Test.MyClass obj = new Test.MyClass(); + obj.__init__(39); + Console.WriteLine(obj.add(1, 2)); + """, ['test']) + assert res == 42 + + def test_export_cliclass(self): + py.test.skip('it fails every other day on builbot, no clue why') + from pypy.translator.cli.dotnet import CLR + + @export(CLR.System.Collections.ArrayList, int) + def getitem(obj, i): + return obj.get_Item(i) + + entrypoints = collect_entrypoints({'getitem': getitem}) + dll = DllDef('test', 'Test', entrypoints) + dll.compile() + res = self._csharp(""" + ArrayList obj = new ArrayList(); + obj.Add(42); + Console.WriteLine(Test.getitem(obj, 0)); + """, ['test']) + assert res == 42 + + def test_compile_dll(self): + py.test.skip('This test fails every other day. No clue why :-(') + cwd, _ = os.path.split(__file__) + mylib_py = os.path.join(cwd, 'mylib.py') + compile_dll(mylib_py, copy_dll=False) + res = self._csharp(""" + Console.WriteLine(mylib.sum(20, 22)); + """, ['mylib']) + assert res == 42 + + def test_compile_dll_alternative_name(self): + cwd, _ = os.path.split(__file__) + mylib_py = os.path.join(cwd, 'mylib.py') + compile_dll(mylib_py, 'mylibxxx.dll', copy_dll=False) + res = self._csharp(""" + Console.WriteLine(mylibxxx.sum(20, 22)); + """, ['mylibxxx']) + assert res == 42 + + def test_compile_netmodule(self): + def foo(x): + return x+1 + dll = DllDef('mymodule', 'Test', [(foo, [int])], isnetmodule=True) + dll.compile() + res = self._csharp('Console.WriteLine("{0}", Test.foo(41));', + netmodules = ['mymodule']) + Added: pypy/branch/avm/pypy/translator/avm1/test/test_cast.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_cast.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.cast import BaseTestCast + +class TestCast(BaseTestCast, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_class.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_class.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,11 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.class_ import BaseTestClass, BaseTestSpecialcase + +# ====> ../../oosupport/test_template/class_.py + +class TestCliClass(CliTest, BaseTestClass): + pass + +class TestCliSpecialCase(CliTest, BaseTestSpecialcase): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_constant.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_constant.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,5 @@ +from pypy.translator.oosupport.test_template.constant import BaseTestConstant +from pypy.translator.cli.test.runtest import CliTest + +class TestConstant(BaseTestConstant, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_cts.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_cts.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,24 @@ +from pypy.translator.cli import cts + +def test_primitive(): + void = cts.CliPrimitiveType('void') + assert str(void) == void.typename() == 'void' + assert void == cts.CliPrimitiveType('void') + +def test_class(): + Math = cts.CliClassType('mscorlib', 'System.Math') + assert str(Math) == Math.typename() == 'class [mscorlib]System.Math' + assert Math.classname() == '[mscorlib]System.Math' + assert Math == cts.CliClassType('mscorlib', 'System.Math') + +def test_generic(): + Dict = cts.CliGenericType('mscorlib', 'System.Dict', 2) + assert str(Dict) == Dict.typename() == 'class [mscorlib]System.Dict`2' + + int32 = cts.CliPrimitiveType('int32') + Math = cts.CliClassType('mscorlib', 'System.Math') + MyDict = Dict.specialize(int32, Math) + assert isinstance(MyDict, cts.CliSpecializedType) + classname = '[mscorlib]System.Dict`2' + assert str(MyDict) == MyDict.typename() == 'class ' + classname + assert MyDict.classname() == classname Added: pypy/branch/avm/pypy/translator/avm1/test/test_dict.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_dict.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,24 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +import pypy.translator.oosupport.test_template.dict as oodict + +class TestCliDict(CliTest, oodict.BaseTestDict): + def test_dict_of_dict(self): + py.test.skip("CLI doesn't support recursive dicts") + + def test_recursive(self): + py.test.skip("CLI doesn't support recursive dicts") + + def test_dict_of_void_special_case(self): + def fn(n): + d = {} + for i in xrange(n): + d[i] = None + return d[0] + assert self.interpret(fn, [2]) is None + +class TestCliEmptyDict(CliTest, oodict.BaseTestEmptyDict): + pass + +class TestCliConstantDict(CliTest, oodict.BaseTestConstantDict): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_dotnet.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_dotnet.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,711 @@ +import py +from pypy.annotation.annrpython import RPythonAnnotator +from pypy.annotation import model as annmodel +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.ootypesystem.ootype import meth, Meth, Char, Signed, Float, String,\ + ROOT, overload, Instance, new +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.cli.dotnet import SomeCliClass, SomeCliStaticMethod,\ + NativeInstance, CLR, box, unbox, OverloadingResolver, NativeException,\ + native_exc, new_array, init_array, typeof, eventhandler, clidowncast,\ + fieldinfo_for_const, classof, cast_record_to_object, cast_object_to_record + +System = CLR.System +ArrayList = CLR.System.Collections.ArrayList +OpCodes = System.Reflection.Emit.OpCodes +DynamicMethod = System.Reflection.Emit.DynamicMethod +Utils = CLR.pypy.runtime.Utils +FUNCTYPE = ootype.StaticMethod([ootype.Signed, ootype.Signed], ootype.Signed) + +class TestDotnetAnnotation(object): + + def test_overloaded_meth_string(self): + C = Instance("test", ROOT, {}, + {'foo': overload(meth(Meth([Char], Signed)), + meth(Meth([String], Float)), + resolver=OverloadingResolver), + 'bar': overload(meth(Meth([Signed], Char)), + meth(Meth([Float], String)), + resolver=OverloadingResolver)}) + def fn1(): + return new(C).foo('a') + def fn2(): + return new(C).foo('aa') + def fn3(x): + return new(C).bar(x) + a = RPythonAnnotator() + assert isinstance(a.build_types(fn1, []), annmodel.SomeInteger) + assert isinstance(a.build_types(fn2, []), annmodel.SomeFloat) + assert isinstance(a.build_types(fn3, [int]), annmodel.SomeChar) + assert isinstance(a.build_types(fn3, [float]), annmodel.SomeString) + + def test_class(self): + def fn(): + return System.Math + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliClass) + assert s.const is System.Math + + def test_fullname(self): + def fn(): + return CLR.System.Math + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliClass) + assert s.const is System.Math + + def test_staticmeth(self): + def fn(): + return System.Math.Abs + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, SomeCliStaticMethod) + assert s.cli_class is System.Math + assert s.meth_name == 'Abs' + + def test_staticmeth_call(self): + def fn1(): + return System.Math.Abs(42) + def fn2(): + return System.Math.Abs(42.5) + a = RPythonAnnotator() + assert type(a.build_types(fn1, [])) is annmodel.SomeInteger + assert type(a.build_types(fn2, [])) is annmodel.SomeFloat + + def test_new_instance(self): + def fn(): + return ArrayList() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert isinstance(s.ootype, NativeInstance) + assert s.ootype._name == '[mscorlib]System.Collections.ArrayList' + + def test_box(self): + def fn(): + return box(42) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + assert not s.can_be_None + + def test_box_can_be_None(self): + def fn(flag): + if flag: + return box(42) + else: + return box(None) + a = RPythonAnnotator() + s = a.build_types(fn, [bool]) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + assert s.can_be_None + + def test_unbox(self): + def fn(): + x = box(42) + return unbox(x, ootype.Signed) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInteger) + + def test_unbox_can_be_None(self): + class Foo: + pass + def fn(): + x = box(42) + return unbox(x, Foo) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInstance) + assert s.can_be_None + + def test_array(self): + def fn(): + x = ArrayList() + return x.ToArray() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._isArray + assert s.ootype._ELEMENT._name == '[mscorlib]System.Object' + + def test_array_getitem(self): + def fn(): + x = ArrayList().ToArray() + return x[0] + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + + def test_mix_None_and_instance(self): + def fn(x): + if x: + return None + else: + return box(42) + a = RPythonAnnotator() + s = a.build_types(fn, [bool]) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.can_be_None == True + + def test_box_instance(self): + class Foo: + pass + def fn(): + return box(Foo()) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeOOInstance) + assert s.ootype._name == '[mscorlib]System.Object' + + def test_unbox_instance(self): + class Foo: + pass + def fn(): + boxed = box(Foo()) + return unbox(boxed, Foo) + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeInstance) + assert s.classdef.name.endswith('Foo') + + def test_can_be_None(self): + def fn(): + ttype = System.Type.GetType('foo') + return ttype.get_Namespace() + a = RPythonAnnotator() + s = a.build_types(fn, []) + assert isinstance(s, annmodel.SomeString) + assert s.can_be_None + + +class TestDotnetRtyping(CliTest): + def _skip_pythonnet(self, msg): + pass + + def test_staticmeth_call(self): + def fn(x): + return System.Math.Abs(x) + assert self.interpret(fn, [-42]) == 42 + + def test_staticmeth_overload(self): + self._skip_pythonnet('Pythonnet bug!') + def fn(x, y): + return System.Math.Abs(x), System.Math.Abs(y) + res = self.interpret(fn, [-42, -42.5]) + item0, item1 = self.ll_to_tuple(res) + assert item0 == 42 + assert item1 == 42.5 + + def test_tostring(self): + StringBuilder = CLR.System.Text.StringBuilder + def fn(): + x = StringBuilder() + x.Append(box("foo")).Append(box("bar")) + return x.ToString() + res = self.ll_to_string(self.interpret(fn, [])) + assert res == 'foobar' + + def test_box(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + x.Add(box('Foo')) + return x.get_Count() + assert self.interpret(fn, []) == 2 + + def test_whitout_box(self): + def fn(): + x = ArrayList() + x.Add(42) # note we have forgot box() + py.test.raises(TypeError, self.interpret, fn, []) + + def test_unbox(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + return unbox(x.get_Item(0), ootype.Signed) + assert self.interpret(fn, []) == 42 + + def test_unbox_string(self): + def fn(): + x = ArrayList() + x.Add(box('foo')) + return unbox(x.get_Item(0), ootype.String) + assert self.interpret(fn, []) == 'foo' + + def test_box_method(self): + def fn(): + x = box(42) + t = x.GetType() + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'Int32' + + def test_box_object(self): + def fn(): + return box(System.Object()).ToString() + res = self.interpret(fn, []) + assert res == 'System.Object' + + def test_array(self): + def fn(): + x = ArrayList() + x.Add(box(42)) + array = x.ToArray() + return unbox(array[0], ootype.Signed) + assert self.interpret(fn, []) == 42 + + def test_new_array(self): + def fn(): + x = new_array(System.Object, 2) + x[0] = box(42) + x[1] = box(43) + return unbox(x[0], ootype.Signed) + unbox(x[1], ootype.Signed) + assert self.interpret(fn, []) == 42+43 + + def test_init_array(self): + def fn(): + x = init_array(System.Object, box(42), box(43)) + return unbox(x[0], ootype.Signed) + unbox(x[1], ootype.Signed) + assert self.interpret(fn, []) == 42+43 + + def test_array_setitem_None(self): + py.test.skip('Mono bug :-(') + def fn(): + x = init_array(System.Object, box(42), box(43)) + x[0] = None + return x[0] + assert self.interpret(fn, []) is None + + def test_array_length(self): + def fn(): + x = init_array(System.Object, box(42), box(43)) + return len(x) + assert self.interpret(fn, []) == 2 + + def test_null(self): + def fn(): + return System.Object.Equals(None, None) + assert self.interpret(fn, []) == True + + def test_null_bound_method(self): + def fn(): + x = ArrayList() + x.Add(None) + return x.get_Item(0) + assert self.interpret(fn, []) is None + + def test_native_exception_precise(self): + ArgumentOutOfRangeException = NativeException(CLR.System.ArgumentOutOfRangeException) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return False + except ArgumentOutOfRangeException: + return True + assert self.interpret(fn, []) == True + + def test_native_exception_superclass(self): + SystemException = NativeException(CLR.System.Exception) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return False + except SystemException: + return True + assert self.interpret(fn, []) == True + + def test_native_exception_object(self): + SystemException = NativeException(CLR.System.Exception) + def fn(): + x = ArrayList() + try: + x.get_Item(0) + return "Impossible!" + except SystemException, e: + ex = native_exc(e) + return ex.get_Message() + res = self.ll_to_string(self.interpret(fn, [])) + assert res.startswith("Index is less than 0") + + def test_native_exception_invoke(self): + TargetInvocationException = NativeException(CLR.System.Reflection.TargetInvocationException) + def fn(): + x = ArrayList() + t = x.GetType() + meth = t.GetMethod('get_Item') + args = init_array(System.Object, box(0)) + try: + meth.Invoke(x, args) + return "Impossible!" + except TargetInvocationException, e: + inner = native_exc(e).get_InnerException() + message = str(inner.get_Message()) + return message + res = self.ll_to_string(self.interpret(fn, [])) + assert res.startswith("Index is less than 0") + + def test_typeof(self): + def fn(): + x = box(42) + return x.GetType() == typeof(System.Int32) + res = self.interpret(fn, []) + assert res is True + + def test_typeof_pypylib(self): + DelegateType = CLR.pypy.test.DelegateType_int__int_2 + def fn(): + return typeof(DelegateType) is not None + res = self.interpret(fn, []) + assert res is True + + def test_typeof_functype(self): + # this test is overridden in TestPythonnet + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + + def test_clidowncast(self): + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(c, ArrayList) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + + def test_clidowncast_lltype(self): + ARRAY_LIST = ArrayList._INSTANCE + def fn(): + a = ArrayList() + b = ArrayList() + a.Add(b) + c = a.get_Item(0) # type of c is Object + c = clidowncast(c, ARRAY_LIST) + c.Add(None) + return c.get_Item(0) + res = self.interpret(fn, []) + assert res is None + + def test_mix_None_and_instance(self): + def g(x): + return x + def fn(flag): + if flag: + x = None + else: + x = box(42) + return g(x) + res = self.interpret(fn, [1]) + assert res is None + + def test_box_unbox_instance(self): + class Foo: + pass + def fn(): + obj = Foo() + b_obj = box(obj) + obj2 = unbox(b_obj, Foo) + return obj is obj2 + res = self.interpret(fn, []) + assert res is True + + def test_unbox_instance_fail(self): + class Foo: + pass + def fn(): + b_obj = box(42) + return unbox(b_obj, Foo) + res = self.interpret(fn, []) + assert res is None + + def test_box_unbox_ooinstance(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + + def test_box_unbox_ooinstance_fail(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + def fn(flag): + b_obj = System.Object() + a2 = unbox(b_obj, A) + return a2 + res = self.interpret(fn, [True]) + assert res is None + + def test_box_unbox_oorecord(self): + A = ootype.Record({'xx': ootype.Signed}) + def fn(flag): + a = ootype.new(A) + a.xx = 42 + b_obj = box(a) + a2 = unbox(b_obj, A) + return a2.xx + res = self.interpret(fn, [True]) + assert res == 42 + + def test_instance_wrapping(self): + class Foo: + pass + def fn(): + obj = Foo() + x = ArrayList() + x.Add(box(obj)) + obj2 = unbox(x.get_Item(0), Foo) + return obj is obj2 + res = self.interpret(fn, []) + assert res is True + + def test_compare_string_None(self): + from pypy.rlib.nonconst import NonConstant + def null(): + if NonConstant(True): + return None + else: + return "" + + def fn(): + ttype = System.Type.GetType('Consts, mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089') + namespace = ttype.get_Namespace() + if namespace is not None: + return False + else: + return True + res = self.interpret(fn, [], backendopt=False) + assert res is True + + def test_delegate(self): + class Foo: + def __init__(self): + self.x = 0 + def m(self, sender, args): + self.x = 42 + + def fn(flag): + f = Foo() + if flag: + f.m(None, None) + delegate = eventhandler(f.m) + delegate.Invoke(None, None) + return f.x + res = self.interpret(fn, [False]) + assert res == 42 + + def test_static_fields(self): + DummyClass = CLR.pypy.test.DummyClass + def fn(): + obj = System.Object() + DummyClass.myfield = obj + return DummyClass.myfield is obj + res = self.interpret(fn, []) + assert res + + def test_pypylib(self): + def fn(): + return CLR.pypy.runtime.Utils.OOString(42, -1) + res = self.interpret(fn, []) + assert self.ll_to_string(res) == '42' + + def test_call_delegate(self): + def build_fn(): + tInt = typeof(System.Int32) + args = init_array(System.Type, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_0) + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE)) + return myfunc + + def fn(x, y): + myfunc = unbox(build_fn(), FUNCTYPE) + a = myfunc(x, y) + mytuple = (x, y) + b = myfunc(*mytuple) + return a+b + res = self.interpret(fn, [10, 11]) + assert res == 42 + + def test_bound_delegate(self): + def build_fn(): + tObjArray = System.Type.GetType("System.Object[]") + tInt = typeof(System.Int32) + args = init_array(System.Type, tObjArray, tInt, tInt) + meth = Utils.CreateDynamicMethod("add", tInt, args) + il = meth.GetILGenerator() + il.Emit(OpCodes.Ldarg_1) + il.Emit(OpCodes.Ldarg_2) + il.Emit(OpCodes.Add) + il.Emit(OpCodes.Ret) + array = new_array(System.Object, 0) + myfunc = meth.CreateDelegate(typeof(FUNCTYPE), array) + return myfunc + + def fn(): + myfunc = unbox(build_fn(), FUNCTYPE) + return myfunc(30, 12) + res = self.interpret(fn, []) + assert res == 42 + + def test_valuetype_field(self): + class Foo: + def __init__(self, x): + self.x = x + + def fn(): + f = Foo(OpCodes.Add) + return f + self.interpret(fn, []) + + def test_fieldinfo_for_const(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + const.xx = 42 + def fn(): + fieldinfo = fieldinfo_for_const(const) + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 + + def test_fieldinfo_for_const_pbc(self): + A = ootype.Instance('A', ootype.ROOT, {'xx': ootype.Signed}) + const = ootype.new(A) + fieldinfo = fieldinfo_for_const(const) + def fn(): + const.xx = 42 + obj = fieldinfo.GetValue(None) + # get the 'xx' field by using reflection + t = obj.GetType() + x_info = t.GetField('xx') + x_value = x_info.GetValue(obj) + return unbox(x_value, ootype.Signed) + res = self.interpret(fn, []) + assert res == 42 + + def test_classof(self): + int32_class = classof(System.Int32) + def fn(): + int32_obj = box(int32_class) + int32_type = clidowncast(int32_obj, System.Type) + return int32_type.get_Name() + assert self.interpret(fn, []) == 'Int32' + + def test_classof_compare(self): + int32_a = classof(System.Int32) + int32_b = classof(System.Int32) + def fn(): + return int32_a is int32_b + assert self.interpret(fn, []) + + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res.startswith('StaticMethod__') + + def test_mix_classof(self): + a = classof(System.Int32) + b = classof(FUNCTYPE) + def fn(flag): + if flag: + x = a + else: + x = b + return clidowncast(box(x), System.Type).get_Name() + res = self.interpret(fn, [True]) + assert res == 'Int32' + + def test_cast_record(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + def fn(flag): + if flag: + obj = cast_record_to_object(record) + else: + obj = System.Object() + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, [True]) + assert res + + def test_cast_record_pbc(self): + T = ootype.Record({'x': ootype.Signed}) + record = ootype.new(T) + record.x = 42 + obj = cast_record_to_object(record) + def fn(): + record2 = cast_object_to_record(T, obj) + return record is record2 + res = self.interpret(fn, []) + assert res + + def test_cast_record_mix_object(self): + T = ootype.Record({'x': ootype.Signed}) + NULL = ootype.null(System.Object._INSTANCE) + record = cast_record_to_object(ootype.new(T)) + assert record != NULL + assert NULL != record + + +class TestPythonnet(TestDotnetRtyping): + # don't interpreter functions but execute them directly through pythonnet + def interpret(self, f, args, backendopt='ignored'): + return f(*args) + + def _skip_pythonnet(self, msg): + py.test.skip(msg) + + def test_whitout_box(self): + pass # it makes sense only during translation + + def test_typeof_functype(self): + def fn(): + t = typeof(FUNCTYPE) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' + + def test_classof_functype(self): + # this test is overridden in TestPythonnet + c = classof(FUNCTYPE) + def fn(): + obj = box(c) + t = clidowncast(obj, System.Type) + return t.get_Name() + res = self.interpret(fn, []) + assert res == 'DelegateType_int__int_2' + + def test_fieldinfo_for_const(self): + pass # it makes sense only during translation + + def test_fieldinfo_for_const_pbc(self): + pass # it makes sense only during translation Added: pypy/branch/avm/pypy/translator/avm1/test/test_exception.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_exception.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,22 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.exception import BaseTestException + +class TestCliException(CliTest, BaseTestException): + use_exception_transformer = False + backendopt = False + + def interpret(self, *args, **kwds): + kwds['exctrans'] = self.use_exception_transformer + return CliTest.interpret(self, *args, **kwds) + + def test_raise_and_catch_other(self): + pass + + def test_raise_prebuilt_and_catch_other(self): + pass + + +class TestCliExceptionTransformer(TestCliException): + use_exception_transformer = True + backendopt = False Added: pypy/branch/avm/pypy/translator/avm1/test/test_float.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_float.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,26 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rfloat import BaseTestRfloat + +class TestCliFloat(CliTest, BaseTestRfloat): + + inf = 'Infinity' + minus_inf = '-Infinity' + nan = 'NaN' + + def test_parse_float(self): + ex = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] + def fn(i): + s = ex[i] + try: + return float(s) + except ValueError: + return -999.0 + + for i in range(len(ex)): + expected = fn(i) + res = self.interpret(fn, [i]) + assert res == expected + + def test_r_singlefloat(self): + py.test.skip("not implemented: single-precision floats") Added: pypy/branch/avm/pypy/translator/avm1/test/test_harness.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_harness.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,14 @@ +import autopath + +from pypy.translator.avm.test import browsertest, harness as h +from pypy.translator.avm import avm1 as a + +def test_harness(): + harness = h.TestHarness("harness") + harness.start_test("harness") + harness.actions.add_action(a.ActionPush(True)) + harness.finish_test(True) + harness.do_test() + +if __name__ == "__main__": + test_harness() Added: pypy/branch/avm/pypy/translator/avm1/test/test_int.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_int.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,50 @@ +import autopath +import py +from pypy.translator.avm1.test.runtest import AVM1Test +from pypy.rpython.test.test_rint import BaseTestRint +from pypy.rlib.rarithmetic import r_longlong + +class TestAVM1Int(AVM1Test, BaseTestRint): + def test_char_constant(self): + def dummyfn(i): + return chr(i) + _ = self.interpret(dummyfn, [ord(' ')]) + assert _ == ' ' + _ = self.interpret(dummyfn, [ord('a')]) + assert _ == 'a' + + def test_rarithmetic(self): + pass # it doesn't make sense here + + div_mod_iteration_count = 20 + + def test_div_mod(self): + import random + + for inttype in (int, r_longlong): + + # def d(x, y): + # return x/y + + # for i in range(self.div_mod_iteration_count): + # x = inttype(random.randint(-100000, 100000)) + # y = inttype(random.randint(-100000, 100000)) + # if not y: continue + # res = self.interpret(d, [x, y]) + # print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", d(x, y) + # assert res == d(x, y) + + def m(x, y): + return x%y + + for i in range(self.div_mod_iteration_count): + x = inttype(random.randint(-100000, 100000)) + y = inttype(random.randint(-100000, 100000)) + if not y: continue + res = self.interpret(m, [x, y]) + print "x:", x, "y:", y, "result in Flash:", res, "result in Python:", m(x, y) + assert res == m(x, y) + + +if __name__=="__main__": + TestAVM1Int().test_div_mod() Added: pypy/branch/avm/pypy/translator/avm1/test/test_list.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_list.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,27 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rlist import BaseTestRlist +from pypy.rlib.rarithmetic import r_uint + +class TestCliList(CliTest, BaseTestRlist): + def test_recursive(self): + py.test.skip("CLI doesn't support recursive lists") + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_list_unsigned(self): + def fn(x): + lst = [r_uint(0), r_uint(1)] + lst[0] = r_uint(x) + return lst[0] + res = self.interpret(fn, [42]) + assert res == 42 + + def test_list_bool(self): + def fn(x): + lst = [True, False] + lst[0] = x + return lst[0] + res = self.interpret(fn, [False]) + assert res == False Added: pypy/branch/avm/pypy/translator/avm1/test/test_objectmodel.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_objectmodel.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,10 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.objectmodel import \ + BaseTestObjectModel + +def skip_r_dict(self): + py.test.skip('r_dict support is still incomplete') + +class TestCliObjectModel(CliTest, BaseTestObjectModel): + test_rtype_r_dict_bm = skip_r_dict Added: pypy/branch/avm/pypy/translator/avm1/test/test_oo.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_oo.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,125 @@ +from pypy.translator.cli.test.runtest import CliTest + +class MyClass: + INCREMENT = 1 + + def __init__(self, x, y): + self.x = x + self.y = y + + def compute(self): + return self.x + self.y + + def compute_and_multiply(self, factor): + return self.compute() * factor + + def static_meth(x, y): + return x*y + static_meth = staticmethod(static_meth) + + def class_attribute(self): + return self.x + self.INCREMENT + +class MyDerivedClass(MyClass): + INCREMENT = 2 + + def __init__(self, x, y): + MyClass.__init__(self, x+12, y+34) + + def compute(self): + return self.x - self.y + + +# helper functions +def call_method(obj): + return obj.compute() + +def init_and_compute(cls, x, y): + return cls(x, y).compute() + +def nonnull_helper(lst): + if lst is None: + return 1 + else: + return 2 + + +class TestOO(CliTest): + def test_indirect_call(self): + def f(): + return 1 + def g(): + return 2 + def fn(flag): + if flag: + x = f + else: + x = g + return x() + assert self.interpret(fn, [True]) == 1 + assert self.interpret(fn, [False]) == 2 + + def test_indirect_call_arguments(self): + def f(x): + return x+1 + def g(x): + return x+2 + def fn(flag, n): + if flag: + x = f + else: + x = g + return x(n) + assert self.interpret(fn, [True, 42]) == 43 + + + def test_compute(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_compute_multiply(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_inheritance(self): + def fn(x, y): + obj = MyDerivedClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_liskov(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return call_method(base) + call_method(derived) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_static_method(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.static_meth(x,y) + derived.static_meth(x, y)\ + + MyClass.static_meth(x, y) + MyDerivedClass.static_meth(x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_class_attribute(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.class_attribute() + derived.class_attribute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_runtimenew(self): + def fn(x, y): + return init_and_compute(MyClass, x, y) + init_and_compute(MyDerivedClass, x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_nonnull(self): + def fn(x, y): + return nonnull_helper([]) + nonnull_helper(None) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + Added: pypy/branch/avm/pypy/translator/avm1/test/test_op.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_op.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,19 @@ +import sys +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.operations import BaseTestOperations +from pypy.rlib.rarithmetic import ovfcheck + +# ====> ../../oosupport/test_template/operations.py + +class TestOperations(CliTest, BaseTestOperations): + def test_int_div_overflow(self): + import py + py.test.skip('fixme!') + def fn(x, y): + try: + return x//y + except OverflowError: + return 42 + res = self.interpret(fn, [-sys.maxint-1, -1]) + assert res == 42 + Added: pypy/branch/avm/pypy/translator/avm1/test/test_overflow.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_overflow.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,5 @@ +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.overflow import BaseTestOverflow + +class TestOverflow(BaseTestOverflow, CliTest): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_pbc.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_pbc.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,6 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rpbc import BaseTestRPBC + +class TestCliPBC(CliTest, BaseTestRPBC): + pass Added: pypy/branch/avm/pypy/translator/avm1/test/test_primitive.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_primitive.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,13 @@ +import os +import time + +from pypy.translator.cli.test.runtest import CliTest + +class TestPrimitive(CliTest): + + def test_time_time(self): + def fn(): + return time.time() + t1 = self.interpret(fn, []) + t2 = self.interpret(fn, []) + assert t1 <= t2 Added: pypy/branch/avm/pypy/translator/avm1/test/test_query.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_query.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,39 @@ +import py +from pypy.translator.cli import query +from pypy.translator.cli.dotnet import CLR, CliNamespace + +def setup_module(module): + from pypy.translator.cli.query import load_assembly, mscorlib + load_assembly(mscorlib) + +def test_load_assembly(): + query.load_assembly(query.mscorlib) + assert 'System.Math' in query.Types + assert 'System.Collections.ArrayList' in query.Types + +def test_namespaces(): + assert CLR.System._name == 'System' + assert CLR.System.Collections._name == 'System.Collections' + py.test.raises(AttributeError, getattr, CLR, 'Foo') + py.test.raises(AttributeError, getattr, CLR.System, 'Foo') + +def test_CLR_getattr(): + System = CLR.System + assert isinstance(System, CliNamespace) + assert System._name == 'System' + assert hasattr(CLR, 'System') + +def test_static_fields(): + desc = query.get_class_desc('System.Reflection.Emit.OpCodes') + assert ('Add', 'System.Reflection.Emit.OpCode') in desc.StaticFields + +def test_System_Object(): + Object = CLR.System.Object + assert Object._name == '[mscorlib]System.Object' + assert 'Equals' in Object._static_methods + assert 'ToString' in Object._INSTANCE._methods + +def test_array(): + cls = query.get_cli_class('System.Object[]') + assert cls._INSTANCE._isArray + assert cls._INSTANCE._ELEMENT is CLR.System.Object._INSTANCE Added: pypy/branch/avm/pypy/translator/avm1/test/test_range.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_range.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,7 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rrange import BaseTestRrange + +class TestCliRange(CliTest, BaseTestRrange): + def test_rlist_range(self): + pass # it doesn't make sense here Added: pypy/branch/avm/pypy/translator/avm1/test/test_runtest.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_runtest.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,18 @@ +import autopath +import py +from pypy.translator.oosupport.test_template.runtest import BaseTestRunTest +from pypy.translator.avm1.test.runtest import AVM1Test + +class TestRunTest(BaseTestRunTest, AVM1Test): + + def test_auto_raise_exc(self): + def fn(): + raise ValueError + f = self._compile(fn, [], auto_raise_exc=True) + py.test.raises(ValueError, f) + + def test_big_arglist(self): + def fn(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9): + return a0 + res = self.interpret(fn, [42]*10) + assert res == 42 Added: pypy/branch/avm/pypy/translator/avm1/test/test_snippet.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_snippet.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,47 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.translator.oosupport.test_template.snippets import BaseTestSnippets + +class TestSnippets(BaseTestSnippets, CliTest): + def test_llshl(self): + py.test.skip('llshl currently broken on CLI') + + def test_link_SSA(self): + def fn(): + lst = [42, 43, 44] + for i in range(len(lst)): + item = lst[i] + if i < 10: + lst[i] = item+10 + return lst + res = self.ll_to_list(self.interpret(fn, [])) + assert res == [52, 53, 54] + + def test_mangle(self): + class Foo: + def le(self): + return 42 + + def fn(): + f = Foo() + return f.le() + res = self.interpret(fn, [], backendopt=False) + + def test_link_vars_overlapping(self): + from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift + def fn(maxofs): + lastofs = 0 + ofs = 1 + while ofs < maxofs: + lastofs = ofs + try: + ofs = ovfcheck_lshift(ofs, 1) + except OverflowError: + ofs = maxofs + else: + ofs = ofs + 1 + return lastofs + res = self.interpret(fn, [64]) + expected = fn(64) + assert res == expected + Added: pypy/branch/avm/pypy/translator/avm1/test/test_streamio.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_streamio.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,28 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rlib.test.test_streamio import BaseTestBufferingInputStreamTests,\ + BaseTestBufferingOutputStream, BaseTestLineBufferingOutputStream,\ + BaseTestCRLFFilter, BaseTestBufferingInputOutputStreamTests,\ + BaseTestTextInputFilter, BaseTestTextOutputFilter + +class TestBufferingInputStreamTests(CliTest, BaseTestBufferingInputStreamTests): + pass + +class TestBufferingOutputStream(CliTest, BaseTestBufferingOutputStream): + pass + +class TestLineBufferingOutputStream(CliTest, BaseTestLineBufferingOutputStream): + pass + +class TestCRLFFilter(CliTest, BaseTestCRLFFilter): + pass + +class TestBufferingInputOutputStreamTests(CliTest, BaseTestBufferingInputOutputStreamTests): + pass + +class TestTextInputFilter(CliTest, BaseTestTextInputFilter): + pass + +class TestTextOutputFilter(CliTest, BaseTestTextOutputFilter): + pass + Added: pypy/branch/avm/pypy/translator/avm1/test/test_string.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_string.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,40 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +import pypy.translator.oosupport.test_template.string as oostring + +class TestCliString(CliTest, oostring.BaseTestString): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("CLI interpret doesn't support unicode for input arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_upper(self): + py.test.skip("CLI doens't support backquotes inside string literals") + test_lower = test_upper + + def test_hlstr(self): + py.test.skip("CLI tests can't have string as input arguments") + + test_inplace_add = test_hlstr + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_compare(self): + strings = ['aa', 'ZZ'] + def fn(i, j): + return strings[i] < strings[j] + assert self.interpret(fn, [0, 1], backendopt=False) == fn(0, 1) + + def test_literal_length(self): + strings = ['aa', 'a\x01', 'a\x00'] + def fn(): + for s in strings: + assert len(s) == 2 + self.interpret(fn, [], backendopt=False) Added: pypy/branch/avm/pypy/translator/avm1/test/test_tuple.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_tuple.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,12 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rtuple import BaseTestRtuple + +class TestCliTuple(CliTest, BaseTestRtuple): + def test_builtin_records(self): + def fn(x, y): + return x, y + res = self.interpret(fn, [1.0, 1]) + assert res.item0 == 1.0 and res.item1 == 1 + res = self.interpret(fn, [1.0, 1.0]) + assert res.item0 == 1.0 and res.item1 == 1.0 Added: pypy/branch/avm/pypy/translator/avm1/test/test_unicode.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_unicode.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,23 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_runicode import BaseTestRUnicode + +# ====> ../../../rpython/test/test_runicode.py + +class TestCliUnicode(CliTest, BaseTestRUnicode): + + EMPTY_STRING_HASH = 0 + + def test_unichar_const(self): + py.test.skip("CLI interpret doesn't support unicode for input arguments") + test_unichar_eq = test_unichar_const + test_unichar_ord = test_unichar_const + test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const + + def test_getitem_exc(self): + py.test.skip('fixme!') + + def test_inplace_add(self): + py.test.skip("CLI tests can't have string as input arguments") Added: pypy/branch/avm/pypy/translator/avm1/test/test_weakref.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/test/test_weakref.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,6 @@ +import py +from pypy.translator.cli.test.runtest import CliTest +from pypy.rpython.test.test_rweakref import BaseTestRweakref + +class TestCliWeakRef(CliTest, BaseTestRweakref): + pass Added: pypy/branch/avm/pypy/translator/avm1/types_.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/types_.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,36 @@ + +from pypy.translator.avm import avm1 +from pypy.rlib.rarithmetic import r_longlong, r_ulonglong +from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import lltype + +_pytype_to_avm1 = { + str: avm1.STRING, + unicode: avm1.STRING, + int: avm1.INTEGER, + long: avm1.INTEGER, + r_longlong: avm1.INTEGER, + r_ulonglong: avm1.INTEGER, + bool: avm1.BOOLEAN, + float: avm1.DOUBLE, +} + +def pytype_to_avm1(value): + return (value, _pytype_to_avm1[type(value)]) + +def lltype_to_avm1(value): + return None + #return _lltype_to_avm1[value] + +class AVM1TypeSystem(object): + def __init__(self, db): + self.db = db + + def escape_name(self, name): + return name + + def lltype_to_cts(self, TYPE): + return lltype_to_avm1(TYPE) + + def llvar_to_cts(self, var): + return self.lltype_to_cts(var.concretetype), var.name Added: pypy/branch/avm/pypy/translator/avm1/util.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm1/util.py Wed Nov 11 00:58:15 2009 @@ -0,0 +1,256 @@ + +import struct, os, math + +ALIGN_LEFT = "left" +ALIGN_RIGHT = "right" + +class BitStream(object): + + """ BitStream is a class for taking care of data structures that are bit-packed, like SWF.""" + + def __init__(self, bits=[]): + """ + Constructor. + """ + self.bits = [bool(b) and b != "0" for b in bits] + self.cursor = 0 + self.chunks = set((0,)) + + def read_bit(self): + """Reads a bit from the bit stream and returns it as either True or False. IndexError is thrown if reading past the end of the stream.""" + self.cursor += 1 + return self.bits[self.cursor-1] + + def read_bits(self, length): + """Reads length bits and return them in their own bit stream.""" + self.cursor += length + return BitStream(self.bits[self.cursor-length:self.cursor]) + + def write_bit(self, value): + """Writes the boolean value to the bit stream.""" + if self.cursor < len(self.bits): + self.bits[self.cursor] = bool(value) + else: + self.bits.append(bool(value)) + self.cursor += 1 + + def write_bits(self, bits, offset=0, length=0): + """Writes length bits from bits to this bit stream, starting reading at offset. If length + is 0, the entire stream is used.""" + if length < 1: + length = len(bits) + + if length > self.bits_available(): + for i in range(length - self.bits_available()): + self.bits.append(False) + + self.bits[self.cursor:self.cursor+length] = (bool(x) for x in bits[offset:offset+length]) + self.cursor += length + + def read_int_value(self, length): + """Read length bits and return a number containing those bits with the last bit read being + the least significant bit.""" + n = 0 + for i in reversed(xrange(length)): + n |= self.read_bit() << i + return n + + def write_int_value(self, value, length=-1): + """Writes an int to the specified number of bits in the stream, the most significant bit + first. If length is not specified or negative, the log base 2 of value is taken.""" + + if length < 0: + try: + length = int(math.ceil(math.log(value, 2))) # Get the log base 2, or number of bits value will fit in. + except ValueError: + length = 1 + self.chunk() + for i in reversed(xrange(length)): + self.write_bit(value & (1 << i)) + + + def read_fixed_value(self, eight_bit): + """Reads a fixed point number, either 8.8 or 16.16. If eight_bit is True, an + 8.8 format is used, otherwise 16.16.""" + return self.read_int_value(length) / float(0x100 if eight_bit else 0x10000) + + def write_fixed_value(self, value, eight_bit): + """Writes a fixed point number of length, decimal part first. If eight_bit is True, + an 8.8 format is used instead of a 16.16 format.""" + self.write_bit_value(value * float(0x100 if eight_bit else 0x10000), 8 if eight_bit else 16) + + # Precalculated, see the Wikipedia links below. + _EXPN_BIAS = {16: 16, 32: 127, 64: 1023} + _N_EXPN_BITS = {16: 5, 32: 8, 64: 8} + _N_FRAC_BITS = {16: 10, 32: 23, 64: 52} + _FLOAT_NAME = {16: "float16", 32: "float", 64: "double"} + + def read_float_value(self, length): + """Reads a floating point number of length, which must be 16 (float16), 32 (float), + or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" + + if length not in BitStream._FLOAT_NAME: + raise ValueError, "length is not 16, 32 or 64." + + sign = self.read_bit() + expn = self.read_int_value(BitStream._N_EXPN_BITS[length]) + frac = self.read_int_value(BitStream._N_FRAC_BITS[length]) + + frac_total = float(1 << BitStream._N_FRAC_BITS[length]) + + if expn == 0: + if frac == 0: + return 0 + else: + return ~frac + 1 if sign else frac + elif expn == frac_total - 1: + if frac == 0: + return float("-inf") if sign else float("inf") + else: + return float("nan") + + return (-1 if sign else 1) * 2**(expn - BitStream._EXPN_BIAS[length]) * (1 + frac / frac_total) + + def write_float_value(self, value, length): + """Writes a floating point number of length, which must be 16 (float16), + 32 (float), or 64 (double). See: http://en.wikipedia.org/wiki/IEEE_floating-point_standard""" + + if length not in BitStream._FLOAT_NAME: + raise ValueError, "length is not 16, 32 or 64." + + if value == 0: # value is zero, so we don't care about length + self.write_int_value(0, length) + + if math.isnan(value): + self.one_fill(length) + return + elif value == float("-inf"): # negative infinity + self.one_fill(BitStream._N_EXPN_BITS[length] + 1) # sign merged + self.zero_fill(BitStream._N_FRAC_BITS[length]) + return + elif value == float("inf"): # positive infinity + self.write_bit(False) + self.one_fill(BitStream._N_EXPN_BITS[length]) + self.zero_fill(BitStream._N_FRAC_BITS[length]) + return + + if value < 0: + self.write_bit(True) + value = ~value + 1 + else: + self.write_bit(False) + + exp = BitStream._EXPN_BIAS[length] + if value < 1: + while int(value) != 1: + value *= 2 + exp -= 1 + else: + while int(value) != 1: + value /= 2 + exp += 1 + + if exp < 0 or exp > ( 1 << BitStream._N_EXPN_BITS[length] ): + raise ValueError, "Exponent out of range in %s [%d]." % (BitStream._FLOAT_NAME[length], length) + + frac_total = 1 << BitStream._N_FRAC_BITS[length] + self.write_int_value(exp, BitStream._N_EXPN_BITS[length]) + self.write_int_value(int((value-1)*frac_total) & (frac_total - 1), BitStream._N_FRAC_BITS[length]) + + + def one_fill(self, amount): + """Fills amount bits with one. The equivalent of calling + self.write_boolean(True) amount times, but more efficient.""" + + if amount > self.bits_available(): + for i in range(amount - self.bits_available()): + self.bits.append(True) + + self.bits[self.cursor:self.cursor+amount] = [True] * amount + self.cursor += amount + + def zero_fill(self, amount): + """Fills amount bits with zero. The equivalent of calling + self.write_boolean(False) amount times, but more efficient.""" + + if amount > self.bits_available(): + for i in range(amount - self.bits_available()): + self.bits.append(False) + + self.bits[self.cursor:self.cursor+amount] = [False] * amount + self.cursor += amount + + def seek(self, offset, whence=os.SEEK_SET): + if whence == os.SEEK_SET: + self.cursor = offset + elif whence == os.SEEK_CUR: + self.cursor += offset + elif whence == os.SEEK_END: + self.cursor = len(self.bits) - abs(offset) + + def rewind(self): + self.seek(0, os.SEEK_SET) + + def skip_to_end(self): + self.seek(0, os.SEEK_END) + + def bits_available(self): + return len(self.bits) - self.cursor + + def flush(self): + self.zero_fill(8 - (len(self) % 8)) + + def chunk(self): + self.chunks.add(int(math.ceil(self.cursor / 8.))) + + def __len__(self): + return len(self.bits) + + def __getitem__(self, i): + return self.bits.__getitem__(i) + + def __setitem__(self, i, v): + return self.bits.__setitem__(i, v) + + def __str__(self): + return "".join("1" if b else "0" for b in self.bits) + + def __add__(self, bits): + b = BitStream() + b.write_bits(self) + b.write_bits(bits) + return b + + def __iadd__(self, bits): + self.write_bits(bits) + return self + + def serialize(self, align=ALIGN_LEFT, endianness=None): + """Serialize bit array into a byte string, aligning either + on the right (ALIGN_RIGHT) or left (ALIGN_LEFT). Endianness + can also be "<" for little-endian modes.""" + lst = self.bits[:] + leftover = len(lst) % 8 + if leftover > 0: + if align == ALIGN_RIGHT: + lst[:0] = [False] * (8-leftover) # Insert some False values to pad the list so it is aligned to the right. + else: + lst += [False] * (8-leftover) + + lst = BitStream(lst) + tmp = [lst.read_int_value(8) for i in xrange(int(math.ceil(len(lst)/8.0)))] + + bytes = [None] * len(tmp) + if endianness == "<": + m = sorted(self.chunks) + [len(tmp)] + for start, end in zip(m, m[1::]): + bytes[start:end] = tmp[end-1:None if start == 0 else start-1:-1] + else: + bytes = tmp + return ''.join(chr(b) for b in bytes) + + def parse(self, string, endianness="<"): + """Parse a bit array from a byte string into this BitStream.""" + for char in string: + self.write_int_value(ord(char), 8) + Modified: pypy/branch/avm/pypy/translator/avm2/abc_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/abc_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/abc_.py Wed Nov 11 00:58:15 2009 @@ -29,7 +29,6 @@ def serialize(self): def write_pool(pool, prefix_count=True): - print len(pool) code = "" if prefix_count: code += u32(len(pool)) Modified: pypy/branch/avm/pypy/translator/avm2/assembler.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/assembler.py (original) +++ pypy/branch/avm/pypy/translator/avm2/assembler.py Wed Nov 11 00:58:15 2009 @@ -32,6 +32,7 @@ def add_instruction(self, instruction): instruction.set_assembler_props(self) + print instruction self.code += instruction.serialize() def add_instructions(self, instructions): @@ -87,6 +88,7 @@ raise BackpatchNotSealed(b) v = b.lbl.address - b.base l = b.location + print "sealing backpatch:", v, l self.code = replace_substr(self.code, s24(v), l, l+3) self.backpatches = [] Modified: pypy/branch/avm/pypy/translator/avm2/avm2gen.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/avm2gen.py (original) +++ pypy/branch/avm/pypy/translator/avm2/avm2gen.py Wed Nov 11 00:58:15 2009 @@ -3,8 +3,9 @@ """ from pypy.objspace.flow import model as flowmodel +from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong from pypy.rpython.ootypesystem import ootype -from pypy.translator.avm2 import assembler, constants, instructions, abc_ as abc, types_ as types, traits +from pypy.translator.avm2 import assembler, constants, instructions, abc_ as abc, types_ as types, traits, util from pypy.translator.oosupport.treebuilder import SubOperation from pypy.translator.oosupport.metavm import Generator from pypy.translator.oosupport.constant import push_constant @@ -26,7 +27,19 @@ self.gen.enter_context(ctx) return ctx -class ScriptContext(object): +class _MethodContextMixin(object): + def new_method(self, name, params, rettype, static=False): + meth = abc.AbcMethodInfo(name, [t.multiname() for t, n in params], rettype.multiname(), param_names=[n for t, n in params]) + trait = traits.AbcMethodTrait(constants.QName(name), meth) + if static: + self.add_static_trait(trait) + else: + self.add_instance_trait(trait) + ctx = MethodContext(self.gen, meth, self, params) + self.gen.enter_context(ctx) + return ctx + +class ScriptContext(_MethodContextMixin): CONTEXT_TYPE = "script" def __init__(self, gen, parent): @@ -35,7 +48,7 @@ self.pending_classes = {} def make_init(self): - self.init = abc.AbcMethodInfo("", [], constants.QName("void")) + self.init = abc.AbcMethodInfo("", [], constants.ANY_NAME) ctx = MethodContext(self.gen, self.init, self, []) self.gen.enter_context(ctx) return ctx @@ -46,14 +59,20 @@ self.pending_classes[name] = (ctx, bases) self.gen.enter_context(ctx) return ctx - + + def add_trait(self, trait): + self.traits.append(trait) + + add_static_trait = add_trait + add_instance_trait = add_trait + def exit(self): assert self.parent.CONTEXT_TYPE == "global" self.make_init() for context, parents in self.pending_classes.itervalues(): - parent = context.super_name + parent = self.pending_classes.get(context.super_name, None) if parents is None: parents = [] @@ -63,10 +82,12 @@ parents.append(constants.QName("Object")) + self.gen.I(instructions.getscopeobject(0)) for parent in reversed(parents): self.gen.I(instructions.getlex(parent), instructions.pushscope()) self.traits.append(traits.AbcClassTrait(context.name, context.classobj)) + self.gen.I(instructions.getlex(context.super_name)) self.gen.I(instructions.newclass(context.index)) self.gen.I(*[instructions.popscope()]*len(parents)) self.gen.I(instructions.initproperty(context.name)) @@ -75,7 +96,7 @@ self.gen.exit_context() return self.parent -class ClassContext(object): +class ClassContext(_MethodContextMixin): CONTEXT_TYPE = "class" def __init__(self, gen, name, super_name, parent): @@ -83,16 +104,17 @@ self.instance_traits = [] self.static_traits = [] self.cinit = None + self.iinit = None def make_cinit(self): - self.cinit = abc.AbcMethodInfo("", [], constants.QName("*")) + self.cinit = abc.AbcMethodInfo("", [], constants.ANY_NAME) ctx = MethodContext(self.gen, self.cinit, self, []) self.gen.enter_context(ctx) return ctx def make_iinit(self, params=None): params = params or () - self.iinit = abc.AbcMethodInfo("", [t.multiname() for t, n in params], constants.QName("*"), param_names=[n for t, n in params]) + self.iinit = abc.AbcMethodInfo("", [t.multiname() for t, n in params], constants.QName("void"), param_names=[n for t, n in params]) ctx = MethodContext(self.gen, self.iinit, self, params) self.gen.enter_context(ctx) @@ -100,16 +122,6 @@ self.gen.emit('constructsuper', 0) return ctx - def new_method(self, name, params, rettype, static=False): - meth = abc.AbcMethodInfo(name, [t.multiname() for t, n in params], rettype.multiname(), param_names=[n for t, n in params]) - if static: - self.add_static_trait(traits.AbcMethodTrait(constants.QName(name), meth)) - else: - self.add_instance_trait(traits.AbcMethodTrait(constants.QName(name), meth)) - ctx = MethodContext(self.gen, meth, self, params) - self.gen.enter_context(ctx) - return ctx - def add_instance_trait(self, trait): self.instance_traits.append(trait) return len(self.instance_traits) @@ -120,6 +132,9 @@ def exit(self): assert self.parent.CONTEXT_TYPE == "script" + if self.iinit is None: + self.make_iinit() + self.gen.exit_context() if self.cinit is None: self.make_cinit() self.gen.exit_context() @@ -204,7 +219,6 @@ return self.context.has_local(name) def begin_class(self, name, super_name=None, bases=None): - print name.name return self.context.new_class(name, super_name, bases) def begin_method(self, name, arglist, returntype, static=False): @@ -252,18 +266,22 @@ self.I(instructions.INSTRUCTIONS[instr](*args, **kwargs)) def set_label(self, label): - print "set label :", label self.emit('label', label) def branch_unconditionally(self, label): - print "jump label:", label self.emit('jump', label) - + + def branch_conditionally(self, iftrue, label): + if iftrue: + self.emit('iftrue', label) + else: + self.emit('iffalse', label) + def call_function_constargs(self, name, *args): - for i in args: - self.load(i) - self.emit('getlocal', 0) - self.emit('callproperty', constants.QName(name), len(args)) + if args: + self.load(*args) + self.emit('getglobalscope') + self.emit('callproplex', constants.QName(name), len(args)) def load(self, v, *args): if isinstance(v, flowmodel.Variable): @@ -273,7 +291,8 @@ self.push_local(v) elif isinstance(v, flowmodel.Constant): push_constant(self.db, v.concretetype, v.value, self) - pass + elif hasattr(v, "multiname"): + self.I(instructions.getlex(v.multiname())) else: self.push_const(v) @@ -284,33 +303,36 @@ self.SL(name) def store(self, v): - self.store_var(v.name) + if v.concretetype is not ootype.Void: + self.store_var(v.name) - def prepare_call_oostring(self, OOTYPE): - self.I(instructions.findpropstrict(types._str_qname)) + # def prepare_call_oostring(self, OOTYPE): + # self.I(instructions.findpropstrict(types._str_qname)) - def call_oostring(self, OOTYPE): - self.pop() - self.I(instructions.callproperty(types._str_qname, 1)) + # def call_oostring(self, OOTYPE): + # self.I(instructions.callproperty(types._str_qname, 1)) - call_oounicode = call_oostring - prepare_call_oounicode = prepare_call_oostring + # call_oounicode = call_oostring + # prepare_call_oounicode = prepare_call_oostring def newarray(self, TYPE, length=1): - self.I(instructions.getlex(types._arr_qname)) + self.load(types._arr_qname) self.push_const(length) self.I(instructions.construct(1)) def oonewarray(self, TYPE, length=1): - self.I(instructions.getlex(types._vec_qname)) - self.load(TYPE) + self.load(types._vec_qname) + self.load(self.cts.lltype_to_cts(TYPE.ITEM)) self.I(instructions.applytype(1)) - self.push_const(length) - self.I(instructions.coerce(constants.TypeName(types._vec_qname, self.cts.lltype_to_cts(TYPE).multiname()))) + self.load(length) self.I(instructions.construct(1)) + self.I(instructions.coerce(self.cts.lltype_to_cts(TYPE).multiname())) def array_setitem(self, ARRAY): self.I(instructions.setproperty(constants.MultinameL(constants.PROP_NAMESPACE_SET))) + + def array_getitem(self, ARRAY): + self.I(instructions.getproperty(constants.MultinameL(constants.PROP_NAMESPACE_SET))) def push_this(self): self.GL("this") @@ -324,22 +346,23 @@ self.GL(v) def push_const(self, v): - if isinstance(v, int): - if 0 <= v < 256: + if isinstance(v, (long, int, r_int, r_uint, r_longlong, r_ulonglong, float)): + if isinstance(v, float) or v > util.U32_MAX or v < -util.S32_MAX: + self.I(instructions.pushdouble(self.constants.double_pool.index_for(v))) + elif 0 <= v < 256: self.I(instructions.pushbyte(v)) elif v >= 0: self.I(instructions.pushuint(self.constants.uint_pool.index_for(v))) else: self.I(instructions.pushint(self.constants.int_pool.index_for(v))) - elif isinstance(v, float): - self.I(instructions.pushdouble(self.constants.double_pool.index_for(v))) elif isinstance(v, basestring): - print "PUSHSTRING" self.I(instructions.pushstring(self.constants.utf8_pool.index_for(v))) elif v is True: self.I(instructions.pushtrue()) elif v is False: self.I(instructions.pushfalse()) + else: + assert False, "value for push_const not a literal value" def push_undefined(self): self.I(instructions.pushundefined()) @@ -372,3 +395,6 @@ if members: self.load(*members) self.oonewarray(TYPE, len(members)) + + def new(self, TYPE): + self.emit('constructprop', self.cts.lltype_to_cts(TYPE).multiname()) Modified: pypy/branch/avm/pypy/translator/avm2/class_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/class_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/class_.py Wed Nov 11 00:58:15 2009 @@ -1,6 +1,6 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.node import Node -from pypy.translator.avm2 import types_ as types, constants as c +from pypy.translator.avm2 import types_ as types, constants as c, traits from pypy.translator.oosupport.constant import push_constant from pypy.translator.cli.ilgenerator import CLIBaseGenerator @@ -82,7 +82,7 @@ cts_type = self.cts.lltype_to_cts(f_type) f_name = self.cts.escape_name(f_name) if cts_type != types.types.void: - ilasm.field(f_name, cts_type) + ilasm.context.add_instance_trait(traits.AbcSlotTrait(c.QName(f_name), cts_type.multiname())) self._ctor() self._toString() @@ -106,7 +106,7 @@ for i, ARG in enumerate(METH.ARGS) if ARG is not ootype.Void] returntype = self.cts.lltype_to_cts(METH.RESULT) - ilasm.begin_function(m_name, arglist, returntype) + ilasm.begin_method(m_name, arglist, returntype) ilasm.emit('findpropstrict', c.QName("Error")) ilasm.push_const("Abstract method %s::%s called" % (self.name, m_name)) ilasm.emit('constructprop', c.QName("Error"), 1) @@ -116,31 +116,28 @@ ilasm.exit_context() def _ctor(self): - # self.ilasm.begin_function('.ctor', [], 'void', False, 'specialname', 'rtspecialname', 'instance') - # self.ilasm.opcode('ldarg.0') - # self.ilasm.call('instance void %s::.ctor()' % self.get_base_class()) + self.ilasm.context.make_iinit() # set default values for fields default_values = self.INSTANCE._fields.copy() default_values.update(self.INSTANCE._overridden_defaults) for f_name, (F_TYPE, f_default) in default_values.iteritems(): if getattr(F_TYPE, '_is_value_type', False): continue # we can't set it to null - INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) + # INSTANCE_DEF, _ = self.INSTANCE._lookup_field(f_name) cts_type = self.cts.lltype_to_cts(F_TYPE) f_name = self.cts.escape_name(f_name) - if cts_type != CTS.types.void: - self.ilasm.opcode('ldarg.0') + if cts_type != types.types.void: + self.ilasm.push_this() push_constant(self.db, F_TYPE, f_default, self.gen) - class_name = self.db.class_name(INSTANCE_DEF) - self.ilasm.set_field((cts_type, class_name, f_name)) + # class_name = self.db.class_name(INSTANCE_DEF) + self.ilasm.emit('setproperty', c.packagedQName(f_name)) - self.ilasm.opcode('ret') - self.ilasm.end_function() + self.ilasm.exit_context() def _toString(self): - self.ilasm.begin_function('ToString', [], 'string', False, 'virtual', 'instance', 'default') - self.ilasm.opcode('ldarg.0') - self.ilasm.call('string class [pypylib]pypy.test.Result::InstanceToPython(object)') - self.ilasm.ret() - self.ilasm.end_function() + self.ilasm.begin_method('toString', [], types.types.string) + self.ilasm.push_this() + self.ilasm.load("InstanceWrapper('%s')" % (self.name)) + self.ilasm.emit('returnvalue') + self.ilasm.exit_context() Modified: pypy/branch/avm/pypy/translator/avm2/constant.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/constant.py (original) +++ pypy/branch/avm/pypy/translator/avm2/constant.py Wed Nov 11 00:58:15 2009 @@ -34,7 +34,7 @@ from pypy.translator.avm2 import constants, types_ as types, traits from pypy.rpython.lltypesystem import lltype -CONST_CLASS = constants.QName("__constants__") +CONST_CLASS = constants.packagedQName("pypy.runtime", "Constants") DEBUG_CONST_INIT = False DEBUG_CONST_INIT_VERBOSE = False @@ -53,29 +53,38 @@ self.cts = db.genoo.TypeSystem(db) def _begin_gen_constants(self, gen, all_constants): - pass + self.ilasm = ilasm + self.begin_class() + return gen def _end_gen_constants(self, gen, numsteps): assert gen.ilasm is self.ilasm self.end_class() def begin_class(self): - self.ilasm.begin_class(CONST_CLASSNAME) + self.ctx = self.ilasm.begin_class(CONST_CLASS) + self.ctx.make_cinit() def end_class(self): - self.ilasm.end_class() + self.ilasm.exit_context() def _declare_const(self, gen, const): self.ilasm.context.add_static_trait(traits.AbcConstTrait(const.name, const.get_type())) def downcast_constant(self, gen, const, EXPECTED_TYPE): type = self.cts.lltype_to_cts(EXPECTED_TYPE) - gen.ilasm.emit('coerce', type) + gen.emit('coerce', type.multiname()) def _get_key_for_const(self, value): if isinstance(value, ootype._view) and isinstance(value._inst, ootype._record): return value._inst return BaseConstantGenerator._get_key_for_const(self, value) + + def push_constant(self, gen, const): + type_ = const.get_type() + gen.emit('getlex', CONST_CLASS) + gen.emit('getproperty', constants.QName(const.name)) + # def _create_complex_const(self, value): # if isinstance(value, _fieldinfo): @@ -96,7 +105,7 @@ # AbstractConst or DictConst), I created a mixin, and then mixed it in # to each sub-class of that base-class. Kind of awkward. -class CLIBaseConstMixin(object): +class Avm2BaseConstMixin(object): """ A mix-in with a few extra methods the CLI backend uses """ def get_type(self): @@ -107,9 +116,9 @@ """ Overload the oosupport version so that we use the CLI opcode for pushing NULL """ assert self.is_null() - gen.ilasm.opcode('ldnull') + gen.ilasm.opcode('pushnull') -class CLIDictMixin(CLIBaseConstMixin): +class Avm2DictMixin(Avm2BaseConstMixin): def _check_for_void_dict(self, gen): KEYTYPE = self.value._TYPE._KEYTYPE keytype = self.cts.lltype_to_cts(KEYTYPE) @@ -148,34 +157,33 @@ # routines. In order to get rid of them, we would need to implement # the generator interface in the CLI. -# class CLIRecordConst(CLIBaseConstMixin, RecordConst): -# def create_pointer(self, gen): -# self.db.const_count.inc('Record') -# super(CLIRecordConst, self).create_pointer(gen) - -# class CLIInstanceConst(CLIBaseConstMixin, InstanceConst): -# def create_pointer(self, gen): -# self.db.const_count.inc('Instance') -# self.db.const_count.inc('Instance', self.OOTYPE()) -# super(CLIInstanceConst, self).create_pointer(gen) - - -# class CLIClassConst(CLIBaseConstMixin, ClassConst): -# def is_inline(self): -# return True - -# def push_inline(self, gen, EXPECTED_TYPE): -# if not self.is_null(): -# if hasattr(self.value, '_FUNC'): -# FUNC = self.value._FUNC -# classname = self.db.record_delegate(FUNC) -# else: -# INSTANCE = self.value._INSTANCE -# classname = self.db.class_name(INSTANCE) -# gen.ilasm.opcode('ldtoken', classname) -# gen.ilasm.call('class [mscorlib]System.Type class [mscorlib]System.Type::GetTypeFromHandle(valuetype [mscorlib]System.RuntimeTypeHandle)') -# return -# super(CLIClassConst, self).push_inline(gen, EXPECTED_TYPE) +class Avm2RecordConst(Avm2BaseConstMixin, RecordConst): + def create_pointer(self, gen): + self.db.const_count.inc('Record') + super(CLIRecordConst, self).create_pointer(gen) + +class Avm2InstanceConst(Avm2BaseConstMixin, InstanceConst): + def create_pointer(self, gen): + self.db.const_count.inc('Instance') + self.db.const_count.inc('Instance', self.OOTYPE()) + super(Avm2InstanceConst, self).create_pointer(gen) + + +class Avm2ClassConst(Avm2BaseConstMixin, ClassConst): + def is_inline(self): + return True + + def push_inline(self, gen, EXPECTED_TYPE): + if not self.is_null(): + if hasattr(self.value, '_FUNC'): + FUNC = self.value._FUNC + classname = self.db.record_delegate(FUNC) + else: + INSTANCE = self.value._INSTANCE + classname = self.db.class_name(INSTANCE) + gen.emit('getlex', constants.QName(classname)) + return + super(Avm2ClassConst, self).push_inline(gen, EXPECTED_TYPE) # class CLIListConst(CLIBaseConstMixin, ListConst): Modified: pypy/branch/avm/pypy/translator/avm2/constants.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/constants.py (original) +++ pypy/branch/avm/pypy/translator/avm2/constants.py Wed Nov 11 00:58:15 2009 @@ -123,8 +123,8 @@ return value.kind, pool.namespace_pool.index_for(value) if isinstance(value, NamespaceSet): return TYPE_NAMESPACE_SET_NamespaceSet, pool.nsset_pool.index_for(value) - if hasattr(value, "KIND"): - return value.KIND, pool.multiname_pool.index_for(value) + if hasattr(value, "multiname"): + return value.multiname().KIND, pool.multiname_pool.index_for(value.multiname()) raise ValueError, "This is not an ABC-compatible type." # ====================================== @@ -185,12 +185,14 @@ PACKAGE_NAMESPACE = Namespace(TYPE_NAMESPACE_PackageNamespace, "") PACKAGE_I_NAMESPACE = Namespace(TYPE_NAMESPACE_PackageInternalNs, "") -PRIVATE_NAMESPACE = Namespace(TYPE_NAMESPACE_PrivateNamespace, "*") +PRIVATE_NAMESPACE = Namespace(TYPE_NAMESPACE_PrivateNamespace, "") AS3_NAMESPACE = Namespace(TYPE_NAMESPACE_Namespace, "http://adobe.com/AS3/2006/builtin") NO_NAMESPACE_SET = NamespaceSet() PROP_NAMESPACE_SET = NamespaceSet(PRIVATE_NAMESPACE, PACKAGE_NAMESPACE, PACKAGE_I_NAMESPACE, AS3_NAMESPACE) +ANY_NAME = object() + def packagedQName(ns, name): return QName(name, Namespace(TYPE_NAMESPACE_PackageNamespace, ns)) @@ -221,6 +223,9 @@ assert self._ns_set_index is not None, "Please call write_to_pool before serializing" return chr(self.KIND) + u32(self._ns_set_index) + def multiname(self): + return self + class MultinameLA(MultinameL): KIND = TYPE_MULTINAME_MultinameLA @@ -290,6 +295,9 @@ assert self._ns_index is not None, "Please call write_to_pool before serializing" return chr(self.KIND) + u32(self._ns_index) + u32(self._name_index) + def multiname(self): + return self + class QNameA(QName): KIND = TYPE_MULTINAME_QNameA @@ -329,16 +337,19 @@ def write_to_pool(self, pool): assert self.name != "" - if self.name == "*": - self._name_index = 0 - else: - self._name_index = pool.utf8_pool.index_for(self.name) + # if self.name == "*": + # self._name_index = 0 + # else: + self._name_index = pool.utf8_pool.index_for(self.name) def serialize(self): assert self._name_index is not None, "Please call write_to_pool before serializing" return chr(self.KIND) + u32(self._name_index) -class RtqNameA(object): + def multiname(self): + return self + +class RtqNameA(RtqName): KIND = TYPE_MULTINAME_RtqNameA class TypeName(object): @@ -369,6 +380,9 @@ assert self._types_indices is not None, "Please call write_to_pool before serializing" return ''.join([chr(self.KIND), u32(self._name_index), u32(len(self._types_indices))] + [u32(a) for a in self._types_indices]) + def multiname(self): + return self + # ====================================== # Constant Pool # ====================================== @@ -381,10 +395,10 @@ self.int_pool = ValuePool(0, self) self.uint_pool = ValuePool(0, self) self.double_pool = ValuePool(float("nan"), self) - self.utf8_pool = ValuePool("", self) + self.utf8_pool = ValuePool(object(), self) # don't use "" because multinames expect "*" self.namespace_pool = ValuePool(ANY_NAMESPACE, self) self.nsset_pool = ValuePool(NO_NAMESPACE_SET, self) - self.multiname_pool = ValuePool(object(), self, debug=True) + self.multiname_pool = ValuePool(ANY_NAME, self, debug=True) def write(self, value): if hasattr(value, "write_to_pool"): Modified: pypy/branch/avm/pypy/translator/avm2/database.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/database.py (original) +++ pypy/branch/avm/pypy/translator/avm2/database.py Wed Nov 11 00:58:15 2009 @@ -2,7 +2,7 @@ #from pypy.translator.avm.class_ import Class from pypy.rpython.ootypesystem import ootype from pypy.translator.cli.support import Counter -from pypy.translator.avm2 import runtime, types_ as types, class_ as c +from pypy.translator.avm2 import runtime, types_ as types, class_ as c, record from pypy.translator.oosupport.database import Database as OODatabase try: @@ -59,7 +59,7 @@ name = self._default_record_name(RECORD) name = self.get_unique_class_name(None, name) self.recordnames[RECORD] = name - r = Record(self, RECORD, name) + r = record.Record(self, RECORD, name) self.pending_node(r) return name @@ -102,9 +102,18 @@ def class_name(self, INSTANCE): if INSTANCE is ootype.ROOT: - return types.types.object.classname() + return "Object" try: NATIVE_INSTANCE = INSTANCE._hints['NATIVE_INSTANCE'] return NATIVE_INSTANCE._name except KeyError: return self.classes[INSTANCE] + + def record_delegate(self, TYPE): + try: + return self.delegates[TYPE] + except KeyError: + name = 'StaticMethod__%d' % len(self.delegates) + self.delegates[TYPE] = name + self.pending_node(Delegate(self, TYPE, name)) + return name Modified: pypy/branch/avm/pypy/translator/avm2/function.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/function.py (original) +++ pypy/branch/avm/pypy/translator/avm2/function.py Wed Nov 11 00:58:15 2009 @@ -125,10 +125,10 @@ # self.ilasm.leave(target_label) # self.ilasm.end_catch() - # def render_raise_block(self, block): - # exc = block.inputargs[1] - # self.load(exc) - # self.ilasm.opcode('throw') + def render_raise_block(self, block): + exc = block.inputargs[1] + self.generator.load(exc) + self.generator.emit('throw') # def store_exception_and_link(self, link): # if self._is_raise_block(link.target): Modified: pypy/branch/avm/pypy/translator/avm2/genavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/genavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/genavm.py Wed Nov 11 00:58:15 2009 @@ -2,7 +2,7 @@ import py from pypy.translator.oosupport.genoo import GenOO from pypy.translator.avm2.avm2gen import Avm2ilasm -from pypy.translator.avm2.constant import Avm2ConstGenerator +from pypy.translator.avm2.constant import Avm2ConstGenerator, Avm2ClassConst, Avm2InstanceConst, Avm2RecordConst from pypy.translator.avm2.database import LowLevelDatabase from pypy.translator.avm2.function import Function from pypy.translator.avm2.opcodes import opcodes @@ -16,6 +16,10 @@ TypeSystem = Avm2TypeSystem ConstantGenerator = Avm2ConstGenerator + + ClassConst = Avm2ClassConst + InstanceConst = Avm2InstanceConst + RecordConst = Avm2RecordConst def __init__(self, tmpdir, translator, entrypoint, config=None, exctrans=False): GenOO.__init__(self, tmpdir, translator, entrypoint, config, exctrans) Modified: pypy/branch/avm/pypy/translator/avm2/instructions.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/instructions.py (original) +++ pypy/branch/avm/pypy/translator/avm2/instructions.py Wed Nov 11 00:58:15 2009 @@ -32,6 +32,8 @@ INSTRUCTIONS[name] = self def _repr(self): + if self.opcode is None: + return self.name return "%s (0x%#X)" % (self.name, self.opcode) def _set_assembler_props(self, asm): @@ -255,7 +257,7 @@ #{ Instructions that push one value to the stack and take no arguments. dup = _Avm2ShortInstruction(0x2A, "dup", 1) -getglobalscope = _Avm2ShortInstruction(0x6A, "getglobalscope", 1) +getglobalscope = _Avm2ShortInstruction(0x64, "getglobalscope", 1) getlocal_0 = _Avm2ShortInstruction(0xD0, "getlocal_0", 1) getlocal_1 = _Avm2ShortInstruction(0xD1, 'getlocal_1', 1) getlocal_2 = _Avm2ShortInstruction(0xD2, 'getlocal_2', 1) @@ -427,7 +429,7 @@ #{ Special Instructions debug = _Avm2DebugInstruction(0xEF, 'debug') -label_internal = _Avm2ShortInstruction(0x09, 'label')() +label_internal = _Avm2ShortInstruction(0x09, 'label_interal')() label = _Avm2LabelInstruction(None, 'label') lookupswitch = _Avm2LookupSwitchInstruction(0x1B, 'lookupswitch') Modified: pypy/branch/avm/pypy/translator/avm2/metavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/metavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/metavm.py Wed Nov 11 00:58:15 2009 @@ -1,10 +1,11 @@ from pypy.translator.cli import oopspec from pypy.rpython.ootypesystem import ootype -from pypy.translator.oosupport.metavm import InstructionList, MicroInstruction, \ +from pypy.translator.oosupport.metavm import MicroInstruction, \ PushAllArgs, StoreResult, GetField, SetField, DownCast from pypy.translator.oosupport.metavm import _Call as _OOCall # from pypy.translator.cli.comparer import EqualityComparer from pypy.translator.avm2.runtime import _static_meth, NativeInstance +from pypy.translator.avm2 import types_ as types, constants as c STRING_HELPER_CLASS = '[pypylib]pypy.runtime.String' @@ -83,8 +84,17 @@ generator.call_signature('object [pypylib]pypy.runtime.Utils::RuntimeNew(class [mscorlib]System.Type)') generator.cast_to(op.result.concretetype) -class CallConstantMethod(MicroInstruction): - pass +class _OOString(MicroInstruction): + def render(self, generator, op): + if op.args[1] == -1: + generator.emit('findpropstrict', types._str_qname) + generator.load(op.args[0]) + generator.emit('callproperty', 1) + else: + generator.emit('findpropstrict', c.QName("parseInt")) + generator.load(op.args[0]) + generator.load(op.args[1]) + generator.emit('callproperty', 2) # class _NewCustomDict(MicroInstruction): # def render(self, generator, op): @@ -240,22 +250,22 @@ generator.ilasm.call_method('class [mscorlib]System.Reflection.FieldInfo class [mscorlib]System.Type::GetField(string)', virtual=True) -OOTYPE_TO_MNEMONIC = { - ootype.Bool: 'i1', - ootype.Char: 'i2', - ootype.UniChar: 'i2', - ootype.Signed: 'i4', - ootype.SignedLongLong: 'i8', - ootype.Unsigned: 'u4', - ootype.UnsignedLongLong: 'u8', - ootype.Float: 'r8', - } - -class _CastPrimitive(MicroInstruction): - def render(self, generator, op): - TO = op.result.concretetype - mnemonic = OOTYPE_TO_MNEMONIC[TO] - generator.ilasm.opcode('conv.%s' % mnemonic) +# OOTYPE_TO_MNEMONIC = { +# ootype.Bool: 'i1', +# ootype.Char: 'i2', +# ootype.UniChar: 'i2', +# ootype.Signed: 'i4', +# ootype.SignedLongLong: 'i8', +# ootype.Unsigned: 'u4', +# ootype.UnsignedLongLong: 'u8', +# ootype.Float: 'r8', +# } + +# class _CastPrimitive(MicroInstruction): +# def render(self, generator, op): +# TO = op.result.concretetype +# mnemonic = OOTYPE_TO_MNEMONIC[TO] +# generator.ilasm.opcode('conv.%s' % mnemonic) Call = _Call() CallMethod = _CallMethod() @@ -270,4 +280,5 @@ GetStaticField = _GetStaticField() SetStaticField = _SetStaticField() FieldInfoForConst = _FieldInfoForConst() -CastPrimitive = _CastPrimitive() +OOString = _OOString() +# CastPrimitive = _CastPrimitive() Modified: pypy/branch/avm/pypy/translator/avm2/opcodes.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/opcodes.py (original) +++ pypy/branch/avm/pypy/translator/avm2/opcodes.py Wed Nov 11 00:58:15 2009 @@ -1,10 +1,10 @@ from pypy.translator.avm2.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, \ NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField,\ - FieldInfoForConst + TypeOf, EventHandler, GetStaticField, SetStaticField,\ + FieldInfoForConst, OOString from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ - New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray + New, RuntimeNew, CastTo, PushPrimitive, OONewArray from pypy.translator.cli.cts import WEAKREF from pypy.rpython.ootypesystem import ootype @@ -52,7 +52,7 @@ 'ooidentityhash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oohash': [PushAllArgs, 'callvirt instance int32 object::GetHashCode()'], 'oostring': [OOString], - 'oounicode': [OOUnicode], + 'oounicode': [OOString], 'ooparse_int': [PushAllArgs, 'call int32 [pypylib]pypy.runtime.Utils::OOParseInt(string, int32)'], 'ooparse_float': [PushAllArgs, 'call float64 [pypylib]pypy.runtime.Utils::OOParseFloat(string)'], # 'oonewcustomdict': [NewCustomDict], @@ -122,7 +122,7 @@ 'cast_float_to_uint': 'conv.u4', 'cast_longlong_to_float': 'conv.r8', 'cast_float_to_longlong': 'conv.i8', - 'cast_primitive': [PushAllArgs, CastPrimitive], +# 'cast_primitive': [PushAllArgs, CastPrimitive], 'truncate_longlong_to_int': 'conv.i4', } Modified: pypy/branch/avm/pypy/translator/avm2/traits.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/traits.py (original) +++ pypy/branch/avm/pypy/translator/avm2/traits.py Wed Nov 11 00:58:15 2009 @@ -93,7 +93,9 @@ def write_to_pool(self, pool): super(AbcSlotTrait, self).write_to_pool(pool) - if self.value is not None: + if self.value is None: + self._value_index = 0 + else: self._value_kind, self._value_index = py_to_abc(self.value) if self._value_index is None: self._value_index = self._value_kind Modified: pypy/branch/avm/pypy/translator/avm2/types_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/types_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/types_.py Wed Nov 11 00:58:15 2009 @@ -9,6 +9,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.translator.avm2 import constants from pypy.translator.cli import oopspec +from pypy.translator.cli.option import getoption from pypy.tool.ansi_print import ansi_log @@ -72,7 +73,7 @@ self.itemtype = itemtype def multiname(self): - return constants.TypeName(_vec_qname, itemtype) + return constants.TypeName(_vec_qname, self.itemtype.multiname()) T = Avm2PrimitiveType N = Avm2PackagedType @@ -87,7 +88,8 @@ # weakref = CliClassType('pypylib', 'pypy.runtime.WeakReference') type = T('Class') object = T('Object') - list = N('List', 'pypy.lib') + # list = N('List', 'pypy.lib') + list = Avm2ArrayType dict = N('Dict', 'pypy.lib') sb = N('StringBuilder', 'pypy.lib') del T @@ -149,7 +151,7 @@ return Avm2NamespacedType(delegate) elif isinstance(t, (ootype.Array, ootype.List)): item_type = self.lltype_to_cts(t.ITEM) - return types.list + return types.list(item_type) elif isinstance(t, ootype.Dict): key_type = self.lltype_to_cts(t._KEYTYPE) value_type = self.lltype_to_cts(t._VALUETYPE) Modified: pypy/branch/avm/pypy/translator/avm2/util.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/util.py (original) +++ pypy/branch/avm/pypy/translator/avm2/util.py Wed Nov 11 00:58:15 2009 @@ -2,23 +2,27 @@ import struct, re from collections import namedtuple +# 5 bytes with the 7 low bits contributing to the number +U32_MAX = 2**(7*5) - 1 +S32_MAX = 2**((7*5) - 2) + def serialize_u32(value): s = "" i = 0 while True: - i += 1 if i == 5: - raise ValueError, "value does not fit in a u32" + raise ValueError("value does not fit in a u32: %r" % s) bits = value & 0b01111111 # low 7 bits value >>= 7 if not value: s += chr(bits) break s += chr(0b10000000 | bits) + i += 1 return s def replace_substr(string, replace, start, stop): - return string[:start] + replace + string[stop+1:] + return string[:start] + replace + string[stop:] def serialize_s24(value): m = struct.pack(" Author: magcius Date: Wed Nov 11 01:01:36 2009 New Revision: 69139 Modified: pypy/branch/avm/pypy/translator/avm2/instructions.py Log: Remove a dependency on "decorator" library. Modified: pypy/branch/avm/pypy/translator/avm2/instructions.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/instructions.py (original) +++ pypy/branch/avm/pypy/translator/avm2/instructions.py Wed Nov 11 01:01:36 2009 @@ -1,7 +1,6 @@ from pypy.translator.avm2.util import serialize_u32 as u32, Avm2Label from pypy.translator.avm2.constants import METHODFLAG_Activation, METHODFLAG_SetsDxns -from decorator import decorator INSTRUCTIONS = {} From magcius at codespeak.net Wed Nov 11 01:02:46 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 11 Nov 2009 01:02:46 +0100 (CET) Subject: [pypy-svn] r69140 - pypy/branch/avm/pypy/translator/avm2 Message-ID: <20091111000246.C4F4E168104@codespeak.net> Author: magcius Date: Wed Nov 11 01:02:45 2009 New Revision: 69140 Added: pypy/branch/avm/pypy/translator/avm2/delegate.py pypy/branch/avm/pypy/translator/avm2/record.py Log: Add some files that I forgot.. Added: pypy/branch/avm/pypy/translator/avm2/delegate.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/delegate.py Wed Nov 11 01:02:45 2009 @@ -0,0 +1,47 @@ +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.cts import CTS +from pypy.translator.cli.node import Node + +class Delegate(Node): + def __init__(self, db, TYPE, name): + self.cts = CTS(db) + self.TYPE = TYPE + self.name = name + + def __eq__(self, other): + return self.TYPE == other.TYPE + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.TYPE) + + def get_name(self): + return self.name + + def dependencies(self): + # record we know about result and argument types + self.cts.lltype_to_cts(self.TYPE.RESULT) + for ARG in self.TYPE.ARGS: + self.cts.lltype_to_cts(ARG) + + + def render(self, ilasm): + TYPE = self.TYPE + ilasm.begin_class(self.name) + # ilasm.begin_function('.ctor', + # [('object', "'object'"), ('native int', "'method'")], + # 'void', + # False, + # 'hidebysig', 'specialname', 'rtspecialname', 'instance', 'default', + # runtime=True) + # ilasm.end_function() + + resulttype = self.cts.lltype_to_cts(TYPE.RESULT) + arglist = [(self.cts.lltype_to_cts(ARG), '') for ARG in TYPE.ARGS if ARG is not ootype.Void] + ilasm.begin_function('Invoke', arglist, resulttype, False, + 'virtual', 'hidebysig', 'instance', 'default', + runtime=True) + ilasm.end_function() + ilasm.end_class() Added: pypy/branch/avm/pypy/translator/avm2/record.py ============================================================================== --- (empty file) +++ pypy/branch/avm/pypy/translator/avm2/record.py Wed Nov 11 01:02:45 2009 @@ -0,0 +1,115 @@ +from pypy.rpython.ootypesystem import ootype +from pypy.translator.cli.node import Node +from pypy.translator.avm2 import constants as c, types_ as types, traits + +class Record(Node): + def __init__(self, db, record, name): + self.db = db + self.cts = db.genoo.TypeSystem(db) + self.record = record + self.name = name + + def __hash__(self): + return hash(self.record) + + def __eq__(self, other): + return self.record == other.record + + def __ne__(self, other): + return not self == other + + def get_name(self): + return self.name + + def get_base_class(self): + return c.QName("Object") + + def render(self, ilasm): + self.ilasm = ilasm + ilasm.begin_class(c.QName(self.name)) + for f_name, (FIELD_TYPE, f_default) in self.record._fields.iteritems(): + f_name = self.cts.escape_name(f_name) + cts_type = self.cts.lltype_to_cts(FIELD_TYPE) + if cts_type != types.types.void: + ilasm.context.add_instance_trait(traits.AbcSlotTrait(c.QName(f_name), cts_type.multiname())) + self._toString() + #self._getHashCode() + ilasm.exit_context() + + def _toString(self): + # only for testing purposes, and only if the Record represents a tuple + from pypy.translator.cli.test.runtest import format_object + + for f_name in self.record._fields: + if not f_name.startswith('item'): + return # it's not a tuple + + self.ilasm.begin_method('toString', [], types._str_qname) + self.ilasm.push_const("(") + for i in xrange(len(self.record._fields)): + f_name = 'item%d' % i + FIELD_TYPE, f_default = self.record._fields[f_name] + if FIELD_TYPE is ootype.Void: + continue + self.ilasm.push_this() + #self.ilasm.get_field((f_type, self.name, f_name)) + self.ilasm.emit('getproperty', c.QName(f_name)) + self.ilasm.emit('callproperty', c.Multiname("toString", c.PROP_NAMESPACE_SET), 0) + self.ilasm.emit('add') + self.ilasm.push_const(", ") + self.ilasm.emit('add') + self.ilasm.push_const(")") + self.ilasm.emit('add') + self.ilasm.emit('returnvalue') + self.ilasm.exit_context() + + # def _equals(self): + # # field by field comparison + # record_type = self.cts.lltype_to_cts(self.record) + # self.ilasm.begin_function('Equals', [('object', 'obj')], 'bool', + # False, 'virtual', 'instance', 'default') + # self.ilasm.locals([(record_type, 'self')]) + # self.ilasm.opcode('ldarg.1') + # self.ilasm.opcode('castclass', record_type.classname()) + # self.ilasm.opcode('stloc.0') + + # equal = 'bool [pypylib]pypy.runtime.Utils::Equal<%s>(!!0, !!0)' + # self.ilasm.opcode('ldc.i4', '1') + # for f_name, (FIELD_TYPE, default) in self.record._fields.iteritems(): + # if FIELD_TYPE is ootype.Void: + # continue + # f_type = self.cts.lltype_to_cts(FIELD_TYPE) + # f_name = self.cts.escape_name(f_name) + # self.ilasm.opcode('ldarg.0') + # self.ilasm.get_field((f_type, record_type.classname(), f_name)) + # self.ilasm.opcode('ldloc.0') + # self.ilasm.get_field((f_type, record_type.classname(), f_name)) + # self.ilasm.call(equal % f_type) + # self.ilasm.opcode('and') + + # self.ilasm.opcode('ret') + # self.ilasm.end_function() + + # def _getHashCode(self): + # record_type = self.cts.lltype_to_cts(self.record) + # self.ilasm.begin_function('GetHashCode', [], 'int32', False, 'virtual', 'instance', 'default') + # gethash = 'int32 [pypylib]pypy.runtime.Utils::GetHashCode<%s>(!!0)' + + # self.ilasm.opcode('ldc.i4.0') # initial hash + # if self.record._fields: + # for f_name, (FIELD_TYPE, default) in self.record._fields.iteritems(): + # if FIELD_TYPE is ootype.Void: + # continue + # else: + # # compute the hash for this field + # f_name = self.cts.escape_name(f_name) + # f_type = self.cts.lltype_to_cts(FIELD_TYPE) + # self.ilasm.opcode('ldarg.0') + # self.ilasm.get_field((f_type, record_type.classname(), f_name)) + # self.ilasm.call(gethash % f_type) + + # # xor with the previous value + # self.ilasm.opcode('xor') + + # self.ilasm.opcode('ret') + # self.ilasm.end_function() From magcius at codespeak.net Wed Nov 11 03:26:25 2009 From: magcius at codespeak.net (magcius at codespeak.net) Date: Wed, 11 Nov 2009 03:26:25 +0100 (CET) Subject: [pypy-svn] r69141 - in pypy/branch/avm/pypy/translator/avm2: . test Message-ID: <20091111022625.74C8E168105@codespeak.net> Author: magcius Date: Wed Nov 11 03:26:24 2009 New Revision: 69141 Modified: pypy/branch/avm/pypy/translator/avm2/abc_.py pypy/branch/avm/pypy/translator/avm2/avm2gen.py pypy/branch/avm/pypy/translator/avm2/constant.py pypy/branch/avm/pypy/translator/avm2/constants.py pypy/branch/avm/pypy/translator/avm2/genavm.py pypy/branch/avm/pypy/translator/avm2/test/browsertest.py pypy/branch/avm/pypy/translator/avm2/test/test.abc pypy/branch/avm/pypy/translator/avm2/test/test.swf pypy/branch/avm/pypy/translator/avm2/types_.py Log: More tests are working now. Modified: pypy/branch/avm/pypy/translator/avm2/abc_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/abc_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/abc_.py Wed Nov 11 03:26:24 2009 @@ -76,6 +76,8 @@ code += u32(self._return_type_index) code += ''.join(u32(index) for index in self._param_types_indices) + print self._return_type_index + print self._param_types_indices code += u32(self._name_index) if self.options: @@ -98,9 +100,10 @@ return code def write_to_pool(self, pool): + print "Emitting function %s(%s):%s" % (self.name, self.param_types, self.return_type) self._name_index = pool.utf8_pool.index_for(self.name) self._return_type_index = pool.multiname_pool.index_for(self.return_type) - + if self.param_types: self._param_types_indices = [pool.multiname_pool.index_for(i) for i in self.param_types] else: @@ -281,7 +284,7 @@ def serialize(self): code = "" code += u32(self._method_info_index) - code += u32(self.code._stack_depth_max) + code += u32(self.code._stack_depth_max+1) # just to be safe. code += u32(len(self.code.temporaries)) code += u32(0) # FIXME: For now, init_scope_depth is always 0. code += u32(self.code._scope_depth_max) Modified: pypy/branch/avm/pypy/translator/avm2/avm2gen.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/avm2gen.py (original) +++ pypy/branch/avm/pypy/translator/avm2/avm2gen.py Wed Nov 11 03:26:24 2009 @@ -278,10 +278,10 @@ self.emit('iffalse', label) def call_function_constargs(self, name, *args): + self.emit('getglobalscope') if args: self.load(*args) - self.emit('getglobalscope') - self.emit('callproplex', constants.QName(name), len(args)) + self.emit('callproperty', constants.QName(name), len(args)) def load(self, v, *args): if isinstance(v, flowmodel.Variable): @@ -396,5 +396,11 @@ self.load(*members) self.oonewarray(TYPE, len(members)) + def set_field(self, TYPE, fieldname): + self.emit('setproperty', constants.QName(fieldname)) + def new(self, TYPE): - self.emit('constructprop', self.cts.lltype_to_cts(TYPE).multiname()) + # XXX: assume no args for now + t = self.cts.lltype_to_cts(TYPE).multiname() + self.emit('findpropstrict', t) + self.emit('constructprop', t, 0) Modified: pypy/branch/avm/pypy/translator/avm2/constant.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/constant.py (original) +++ pypy/branch/avm/pypy/translator/avm2/constant.py Wed Nov 11 03:26:24 2009 @@ -53,12 +53,12 @@ self.cts = db.genoo.TypeSystem(db) def _begin_gen_constants(self, gen, all_constants): - self.ilasm = ilasm + self.ilasm = gen self.begin_class() return gen def _end_gen_constants(self, gen, numsteps): - assert gen.ilasm is self.ilasm + assert gen is self.ilasm self.end_class() def begin_class(self): @@ -67,9 +67,10 @@ def end_class(self): self.ilasm.exit_context() - + self.ilasm.exit_context() + def _declare_const(self, gen, const): - self.ilasm.context.add_static_trait(traits.AbcConstTrait(const.name, const.get_type())) + self.ctx.add_static_trait(traits.AbcConstTrait(constants.QName(const.name), const.get_type().multiname())) def downcast_constant(self, gen, const, EXPECTED_TYPE): type = self.cts.lltype_to_cts(EXPECTED_TYPE) @@ -85,6 +86,20 @@ gen.emit('getlex', CONST_CLASS) gen.emit('getproperty', constants.QName(const.name)) + #def _push_constant_during_init(self, gen, const): + # self.push_constant(gen, const) + # gen.store_var('current_constant') + + def _store_constant(self, gen, const): + type_ = const.get_type() + gen.emit('getlex', CONST_CLASS) + gen.emit('setproperty', constants.QName(const.name)) + + def _declare_step(self, gen, stepnum): + pass + + def _close_step(self, gen, stepnum): + pass # def _create_complex_const(self, value): # if isinstance(value, _fieldinfo): @@ -118,30 +133,30 @@ assert self.is_null() gen.ilasm.opcode('pushnull') -class Avm2DictMixin(Avm2BaseConstMixin): - def _check_for_void_dict(self, gen): - KEYTYPE = self.value._TYPE._KEYTYPE - keytype = self.cts.lltype_to_cts(KEYTYPE) - keytype_T = self.cts.lltype_to_cts(self.value._TYPE.KEYTYPE_T) - VALUETYPE = self.value._TYPE._VALUETYPE - valuetype = self.cts.lltype_to_cts(VALUETYPE) - valuetype_T = self.cts.lltype_to_cts(self.value._TYPE.VALUETYPE_T) - if VALUETYPE is ootype.Void: - gen.add_comment(' CLI Dictionary w/ void value') - class_name = PYPY_DICT_OF_VOID % keytype - for key in self.value._dict: - gen.ilasm.opcode('dup') - push_constant(self.db, KEYTYPE, key, gen) - meth = 'void class %s::ll_set(%s)' % (class_name, keytype_T) - gen.ilasm.call_method(meth, False) - return True - return False - - def initialize_data(self, constgen, gen): - # special case: dict of void, ignore the values - if self._check_for_void_dict(gen): - return - return super(CLIDictMixin, self).initialize_data(constgen, gen) +# class Avm2DictMixin(Avm2BaseConstMixin): +# def _check_for_void_dict(self, gen): +# KEYTYPE = self.value._TYPE._KEYTYPE +# keytype = self.cts.lltype_to_cts(KEYTYPE) +# keytype_T = self.cts.lltype_to_cts(self.value._TYPE.KEYTYPE_T) +# VALUETYPE = self.value._TYPE._VALUETYPE +# valuetype = self.cts.lltype_to_cts(VALUETYPE) +# valuetype_T = self.cts.lltype_to_cts(self.value._TYPE.VALUETYPE_T) +# if VALUETYPE is ootype.Void: +# gen.add_comment(' CLI Dictionary w/ void value') +# class_name = PYPY_DICT_OF_VOID % keytype +# for key in self.value._dict: +# gen.ilasm.opcode('dup') +# push_constant(self.db, KEYTYPE, key, gen) +# meth = 'void class %s::ll_set(%s)' % (class_name, keytype_T) +# gen.ilasm.call_method(meth, False) +# return True +# return False + +# def initialize_data(self, constgen, gen): +# # special case: dict of void, ignore the values +# if self._check_for_void_dict(gen): +# return +# return super(Avm2DictMixin, self).initialize_data(constgen, gen) # ______________________________________________________________________ # Constant Classes @@ -160,7 +175,7 @@ class Avm2RecordConst(Avm2BaseConstMixin, RecordConst): def create_pointer(self, gen): self.db.const_count.inc('Record') - super(CLIRecordConst, self).create_pointer(gen) + super(Avm2RecordConst, self).create_pointer(gen) class Avm2InstanceConst(Avm2BaseConstMixin, InstanceConst): def create_pointer(self, gen): Modified: pypy/branch/avm/pypy/translator/avm2/constants.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/constants.py (original) +++ pypy/branch/avm/pypy/translator/avm2/constants.py Wed Nov 11 03:26:24 2009 @@ -154,6 +154,9 @@ assert self._name_index is not None, "Please call write_to_pool before serializing" return chr(self.kind) + u32(self._name_index) + def __repr__(self): + return self.name or 'global' + class NamespaceSet(object): def __init__(self, *namespaces): @@ -298,6 +301,9 @@ def multiname(self): return self + def __repr__(self): + return "QName: %s::%s" % (self.ns, self.name) + class QNameA(QName): KIND = TYPE_MULTINAME_QNameA Modified: pypy/branch/avm/pypy/translator/avm2/genavm.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/genavm.py (original) +++ pypy/branch/avm/pypy/translator/avm2/genavm.py Wed Nov 11 03:26:24 2009 @@ -35,7 +35,7 @@ self.fix_names() self.gen_entrypoint() self.gen_pendings() - # self.db.gen_constants(self.ilasm) + self.db.gen_constants(self.ilasm) # Don't do treebuilding stuff def stack_optimization(self): Modified: pypy/branch/avm/pypy/translator/avm2/test/browsertest.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/test/browsertest.py (original) +++ pypy/branch/avm/pypy/translator/avm2/test/browsertest.py Wed Nov 11 03:26:24 2009 @@ -37,6 +37,11 @@ return int(string) elif "," in string: return [parse_result(s) for s in string.split(",")] + else: + try: + return float(string) + except ValueError: + pass return string class TestCase(object): Modified: pypy/branch/avm/pypy/translator/avm2/test/test.abc ============================================================================== Binary files. No diff available. Modified: pypy/branch/avm/pypy/translator/avm2/test/test.swf ============================================================================== Binary files. No diff available. Modified: pypy/branch/avm/pypy/translator/avm2/types_.py ============================================================================== --- pypy/branch/avm/pypy/translator/avm2/types_.py (original) +++ pypy/branch/avm/pypy/translator/avm2/types_.py Wed Nov 11 03:26:24 2009 @@ -28,7 +28,7 @@ return hash(self.typename()) def __eq__(self, other): - return self.typename() == other.typename() + return isinstance(other, self.__class__) and self.typename() == other.typename() def __ne__(self, other): return self.typename() != other.typename() @@ -46,7 +46,7 @@ class Avm2NamespacedType(Avm2Type): - nstype = constants.TYPE_NAMESPACE_Namespace + nstype = constants.TYPE_NAMESPACE_PackageNamespace def __init__(self, name, namespace=''): if '::' in name and namespace == '': @@ -64,8 +64,6 @@ def multiname(self): return constants.QName(self.name, constants.Namespace(self.nstype, self.ns)) -class Avm2PackagedType(Avm2NamespacedType): - nstype = constants.TYPE_NAMESPACE_PackageNamespace class Avm2ArrayType(Avm2Type): @@ -76,7 +74,7 @@ return constants.TypeName(_vec_qname, self.itemtype.multiname()) T = Avm2PrimitiveType -N = Avm2PackagedType +N = Avm2NamespacedType class types: void = T('void') int = T('int') From arigo at codespeak.net Wed Nov 11 10:31:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 10:31:10 +0100 (CET) Subject: [pypy-svn] r69142 - in pypy/trunk/pypy/jit: backend/llgraph backend/test backend/x86 metainterp metainterp/test Message-ID: <20091111093110.7EBE5318138@codespeak.net> Author: arigo Date: Wed Nov 11 10:31:09 2009 New Revision: 69142 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/test/test_ll_random.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/resoperation.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Merge the 'merge-guards-2' branch: merge guard_nonnull followed by guard_class and/or guard_value. Needs a new guard, guard_nonnull_class. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Nov 11 10:31:09 2009 @@ -134,6 +134,7 @@ 'guard_overflow' : ((), None), 'guard_nonnull' : (('ref',), None), 'guard_isnull' : (('ref',), None), + 'guard_nonnull_class' : (('ref', 'ref'), None), 'newstr' : (('int',), 'ref'), 'strlen' : (('ref',), 'int'), 'strgetitem' : (('ref', 'int'), 'int'), @@ -562,6 +563,11 @@ if value.typeptr != expected_class: raise GuardFailed + def op_guard_nonnull_class(self, _, value, expected_class): + if not value: + raise GuardFailed + self.op_guard_class(_, value, expected_class) + def op_guard_value(self, _, value, expected_value): if value != expected_value: raise GuardFailed Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Wed Nov 11 10:31:09 2009 @@ -466,8 +466,8 @@ #null_box = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(T))) self.execute_operation(rop.GUARD_CLASS, [t_box, T_box], 'void') assert not self.guard_failed - #self.execute_operation(rop.GUARD_CLASS_INVERSE, [t_box, null_box], - # 'void') + self.execute_operation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], 'void') + assert not self.guard_failed def test_failing_guards(self): t_box, T_box = self.alloc_instance(self.T) @@ -489,10 +489,12 @@ def test_failing_guard_class(self): t_box, T_box = self.alloc_instance(self.T) u_box, U_box = self.alloc_instance(self.U) - #null_box = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.nullptr(T))) + null_box = self.null_instance() for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]), (rop.GUARD_CLASS, [u_box, T_box]), - #(rop.GUARD_VALUE_INVERSE, [BoxInt(10), BoxInt(10)]), + (rop.GUARD_NONNULL_CLASS, [t_box, U_box]), + (rop.GUARD_NONNULL_CLASS, [u_box, T_box]), + (rop.GUARD_NONNULL_CLASS, [null_box, T_box]), ]: assert self.execute_operation(opname, args, 'void') == None assert self.guard_failed Modified: pypy/trunk/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_ll_random.py Wed Nov 11 10:31:09 2009 @@ -203,6 +203,21 @@ op = ResOperation(self.opnum, [v, c_vtable2], None) return op, (vtable == vtable2) +class GuardNonNullClassOperation(GuardClassOperation): + def gen_guard(self, builder, r): + if r.random() < 0.5: + return GuardClassOperation.gen_guard(self, builder, r) + else: + v = BoxPtr(lltype.nullptr(llmemory.GCREF.TO)) + op = ResOperation(rop.SAME_AS, [ConstPtr(v.value)], v) + builder.loop.operations.append(op) + v2, S2 = builder.get_structptr_var(r, must_have_vtable=True) + vtable2 = S2._hints['vtable']._as_ptr() + c_vtable2 = ConstAddr(llmemory.cast_ptr_to_adr(vtable2), + builder.cpu) + op = ResOperation(self.opnum, [v, c_vtable2], None) + return op, False + class GetFieldOperation(test_random.AbstractOperation): def field_descr(self, builder, r): v, S = builder.get_structptr_var(r) @@ -577,6 +592,7 @@ OPERATIONS.append(RaisingCallOperationGuardNoException(rop.CALL)) OPERATIONS.append(RaisingCallOperationWrongGuardException(rop.CALL)) OPERATIONS.append(CallOperationException(rop.CALL)) +OPERATIONS.append(GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) LLtypeOperationBuilder.OPERATIONS = OPERATIONS Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Nov 11 10:31:09 2009 @@ -696,10 +696,10 @@ self.mc.CMP(locs[0], locs[1]) return self.implement_guard(addr, self.mc.JNE) - def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): + def _cmp_guard_class(self, mc, locs): offset = self.cpu.vtable_offset if offset is not None: - self.mc.CMP(mem(locs[0], offset), locs[1]) + mc.CMP(mem(locs[0], offset), locs[1]) else: # XXX hard-coded assumption: to go from an object to its class # we use the following algorithm: @@ -714,8 +714,24 @@ type_info_group = llop.gc_get_type_info_group(llmemory.Address) type_info_group = rffi.cast(lltype.Signed, type_info_group) expected_typeid = (classptr - type_info_group) >> 2 - self.mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) - # + mc.CMP16(mem(locs[0], 0), imm32(expected_typeid)) + + def genop_guard_guard_class(self, ign_1, guard_op, addr, locs, ign_2): + self._cmp_guard_class(self.mc._mc, locs) + return self.implement_guard(addr, self.mc.JNE) + + def genop_guard_guard_nonnull_class(self, ign_1, guard_op, + addr, locs, ign_2): + mc = self.mc._mc + mc.CMP(locs[0], imm8(1)) + mc.write(constlistofchars('\x72\x00')) # JB later + jb_location = mc.get_relative_pos() + self._cmp_guard_class(mc, locs) + # patch the JB above + offset = mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + mc.overwrite(jb_location-1, [chr(offset)]) + # return self.implement_guard(addr, self.mc.JNE) def _no_const_locs(self, args): @@ -880,8 +896,9 @@ print msg raise NotImplementedError(msg) - def not_implemented_op_guard(self, op, regalloc, arglocs, resloc, descr): - msg = "not implemented operation (guard): %s" % op.getopname() + def not_implemented_op_guard(self, op, guard_op, + failaddr, arglocs, resloc): + msg = "not implemented operation (guard): %s" % guard_op.getopname() print msg raise NotImplementedError(msg) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Nov 11 10:31:09 2009 @@ -424,7 +424,9 @@ y = self.loc(op.args[1]) self.perform_guard(op, [x, y], None) self.rm.possibly_free_vars(op.args) - + + consider_guard_nonnull_class = consider_guard_class + def _consider_binop_part(self, op, ignored): x = op.args[0] argloc = self.loc(op.args[1]) Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Nov 11 10:31:09 2009 @@ -42,8 +42,8 @@ class OptValue(object): - _attrs_ = ('box', 'known_class', 'guard_class_index', 'level') - guard_class_index = -1 + _attrs_ = ('box', 'known_class', 'last_guard_index', 'level') + last_guard_index = -1 level = LEVEL_UNKNOWN @@ -88,10 +88,15 @@ return None def make_constant_class(self, classbox, opindex): - if self.level < LEVEL_KNOWNCLASS: - self.known_class = classbox - self.level = LEVEL_KNOWNCLASS - self.guard_class_index = opindex + assert self.level < LEVEL_KNOWNCLASS + self.known_class = classbox + self.level = LEVEL_KNOWNCLASS + self.last_guard_index = opindex + + def make_nonnull(self, opindex): + assert self.level < LEVEL_NONNULL + self.level = LEVEL_NONNULL + self.last_guard_index = opindex def is_nonnull(self): level = self.level @@ -104,7 +109,7 @@ else: return False - def make_nonnull(self): + def ensure_nonnull(self): if self.level < LEVEL_NONNULL: self.level = LEVEL_NONNULL @@ -577,17 +582,26 @@ elif value.is_null(): raise InvalidLoop self.emit_operation(op) - value.make_nonnull() + value.make_nonnull(len(self.newoperations) - 1) def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.args[0]) emit_operation = True - if value.guard_class_index != -1: - # there already has been a guard_class on this value, which is - # rather silly. replace the original guard_class with a guard_value - guard_class_op = self.newoperations[value.guard_class_index] - guard_class_op.opnum = op.opnum - guard_class_op.args[1] = op.args[1] + if value.last_guard_index != -1: + # there already has been a guard_nonnull or guard_class or + # guard_nonnull_class on this value, which is rather silly. + # replace the original guard with a guard_value + old_guard_op = self.newoperations[value.last_guard_index] + old_opnum = old_guard_op.opnum + old_guard_op.opnum = op.opnum + old_guard_op.args = [old_guard_op.args[0], op.args[1]] + if old_opnum == rop.GUARD_NONNULL: + # hack hack hack. Change the guard_opnum on + # old_guard_op.descr so that when resuming, + # the operation is not skipped by pyjitpl.py. + descr = old_guard_op.descr + assert isinstance(descr, compile.ResumeGuardDescr) + descr.guard_opnum = rop.GUARD_NONNULL_CLASS emit_operation = False constbox = op.args[1] assert isinstance(constbox, Const) @@ -610,8 +624,29 @@ # earlier, in optimizefindnode.py. assert realclassbox.same_constant(expectedclassbox) return - self.emit_operation(op) - value.make_constant_class(expectedclassbox, len(self.newoperations) - 1) + emit_operation = True + if value.last_guard_index != -1: + # there already has been a guard_nonnull or guard_class or + # guard_nonnull_class on this value. + old_guard_op = self.newoperations[value.last_guard_index] + if old_guard_op.opnum == rop.GUARD_NONNULL: + # it was a guard_nonnull, which we replace with a + # guard_nonnull_class. + old_guard_op.opnum = rop.GUARD_NONNULL_CLASS + old_guard_op.args = [old_guard_op.args[0], op.args[1]] + # hack hack hack. Change the guard_opnum on + # old_guard_op.descr so that when resuming, + # the operation is not skipped by pyjitpl.py. + descr = old_guard_op.descr + assert isinstance(descr, compile.ResumeGuardDescr) + descr.guard_opnum = rop.GUARD_NONNULL_CLASS + emit_operation = False + if emit_operation: + self.emit_operation(op) + last_guard_index = len(self.newoperations) - 1 + else: + last_guard_index = value.last_guard_index + value.make_constant_class(expectedclassbox, last_guard_index) def optimize_GUARD_NO_EXCEPTION(self, op): if not self.exception_might_have_happened: @@ -669,7 +704,7 @@ assert fieldvalue is not None self.make_equal_to(op.result, fieldvalue) else: - value.make_nonnull() + value.ensure_nonnull() self.heap_op_optimizer.optimize_GETFIELD_GC(op, value) # note: the following line does not mean that the two operations are @@ -681,7 +716,7 @@ if value.is_virtual(): value.setfield(op.descr, self.getvalue(op.args[1])) else: - value.make_nonnull() + value.ensure_nonnull() fieldvalue = self.getvalue(op.args[1]) self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue) @@ -708,7 +743,7 @@ if value.is_virtual(): self.make_constant_int(op.result, value.getlength()) else: - value.make_nonnull() + value.ensure_nonnull() self.optimize_default(op) def optimize_GETARRAYITEM_GC(self, op): @@ -719,7 +754,7 @@ itemvalue = value.getitem(indexbox.getint()) self.make_equal_to(op.result, itemvalue) return - value.make_nonnull() + value.ensure_nonnull() self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value) # note: the following line does not mean that the two operations are @@ -733,7 +768,7 @@ if indexbox is not None: value.setitem(indexbox.getint(), self.getvalue(op.args[2])) return - value.make_nonnull() + value.ensure_nonnull() fieldvalue = self.getvalue(op.args[2]) self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue) @@ -873,7 +908,7 @@ self.optimizer.make_equal_to(op.result, fieldvalue) return # default case: produce the operation - value.make_nonnull() + value.ensure_nonnull() self.optimizer.optimize_default(op) # then remember the result of reading the field fieldvalue = self.optimizer.getvalue(op.result) Modified: pypy/trunk/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/pypy/jit/metainterp/resoperation.py Wed Nov 11 10:31:09 2009 @@ -119,6 +119,7 @@ 'GUARD_CLASS', 'GUARD_NONNULL', 'GUARD_ISNULL', + 'GUARD_NONNULL_CLASS', '_GUARD_FOLDABLE_LAST', 'GUARD_NO_EXCEPTION', 'GUARD_EXCEPTION', Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Wed Nov 11 10:31:09 2009 @@ -1003,10 +1003,10 @@ class A(object): def g(self, x): - return x - 1 + return x - 5 class B(A): def g(self, y): - return y - 2 + return y - 3 a1 = A() a2 = A() @@ -1021,8 +1021,125 @@ hint(a, promote=True) return x res = self.meta_interp(f, [299], listops=True) + assert res == f(299) self.check_loops(guard_class=0, guard_value=3) + def test_merge_guardnonnull_guardclass(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 3 + class B(A): + def g(self, y): + return y - 5 + + a1 = A() + b1 = B() + def f(x): + l = [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x = a.g(x) + else: + x -= 7 + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, + guard_nonnull_class=2, guard_isnull=1) + + def test_merge_guardnonnull_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + pass + class B(A): + pass + + a1 = A() + b1 = B() + def f(x): + l = [b1] * 100 + [None] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x -= 5 + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + + def test_merge_guardnonnull_guardvalue_2(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + pass + class B(A): + pass + + a1 = A() + b1 = B() + def f(x): + l = [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x -= 5 + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [299], listops=True) + assert res == f(299) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + + def test_merge_guardnonnull_guardclass_guardvalue(self): + from pypy.rlib.objectmodel import instantiate + myjitdriver = JitDriver(greens = [], reds = ['x', 'l']) + + class A(object): + def g(self, x): + return x - 3 + class B(A): + def g(self, y): + return y - 5 + + a1 = A() + a2 = A() + b1 = B() + def f(x): + l = [a2] * 100 + [None] * 100 + [b1] * 100 + [a1] * 100 + while x > 0: + myjitdriver.can_enter_jit(x=x, l=l) + myjitdriver.jit_merge_point(x=x, l=l) + a = l[x] + if a: + x = a.g(x) + else: + x -= 7 + hint(a, promote=True) + return x + res = self.meta_interp(f, [399], listops=True) + assert res == f(399) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=3, + guard_nonnull_class=0, guard_isnull=1) + def test_residual_call_doesnt_lose_info(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'l']) Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Nov 11 10:31:09 2009 @@ -1472,6 +1472,56 @@ """ self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + def test_merge_guard_nonnull_guard_class(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_class(p1, ConstClass(node_vtable)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_nonnull_class(p1, ConstClass(node_vtable)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + def test_merge_guard_nonnull_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i3, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + jump(p2, i0, i1, i3, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + + def test_merge_guard_nonnull_guard_class_guard_value(self): + ops = """ + [p1, i0, i1, i2, p2] + guard_nonnull(p1) [i0] + i3 = int_add(i1, i2) + guard_class(p1, ConstClass(node_vtable)) [i2] + i4 = int_sub(i3, 1) + guard_value(p1, ConstPtr(myptr)) [i1] + jump(p2, i0, i1, i4, p2) + """ + expected = """ + [p1, i0, i1, i2, p2] + guard_value(p1, ConstPtr(myptr)) [i0] + i3 = int_add(i1, i2) + i4 = int_sub(i3, 1) + jump(p2, i0, i1, i4, p2) + """ + self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) # ---------- From arigo at codespeak.net Wed Nov 11 10:32:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 10:32:28 +0100 (CET) Subject: [pypy-svn] r69143 - pypy/branch/merge-guards-2 Message-ID: <20091111093228.2E4DD318138@codespeak.net> Author: arigo Date: Wed Nov 11 10:32:27 2009 New Revision: 69143 Removed: pypy/branch/merge-guards-2/ Log: Remove merged branch. From cfbolz at codespeak.net Wed Nov 11 10:35:40 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 11 Nov 2009 10:35:40 +0100 (CET) Subject: [pypy-svn] r69144 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091111093540.D8366318138@codespeak.net> Author: cfbolz Date: Wed Nov 11 10:35:40 2009 New Revision: 69144 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Wed Nov 11 10:35:40 2009 @@ -1,9 +1,10 @@ people present: - - Anto - Armin - Samuele - Carl Friedrich + - Maciek + - Holger TASKS @@ -14,19 +15,22 @@ - directly call assembler for residual portal calls - making the CLI backend working with logging - compress the virtuals part of resume data more (Samuele, Carl Friedrich) - - understand the memory behaviour of pystone with the JIT DONE - - fix the guard_value(bool, 1) -> guard_true hack DONE - try to do something non-insane about Python-level exceptions (Maciek, Armin) - make the assembler produced by generate_failure smaller - - lose less information across residual calls DONE - - merge guard_nonnull(x, ...), guard_class(x, ...) (Armin, Maciek) - - merge guard_class(x, ...), guard_value(x, ...) DONE + - merge guard_nonnull(x, ...), guard_class(x, ...) DONE, cli needs fixing - put the class into the structure to get only one promote when using an instance - look into failing pypy-c-jit apptests - - port writeanalyze to ootype (Anto, Carl Friedrich around) - + - port writeanalyze to ootype DONE + - fix buildbot on py.test 1.1 branch (Holger, Samuele around) - at the end of the sprint, migrate tasks to jit.txt + +Talks +----- + + - JIT introduction Monday 17.00 + - py.test update Thursday 15.00 + - discuss benchmarking a bit From arigo at codespeak.net Wed Nov 11 10:44:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 10:44:49 +0100 (CET) Subject: [pypy-svn] r69145 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091111094449.7D488318138@codespeak.net> Author: arigo Date: Wed Nov 11 10:44:48 2009 New Revision: 69145 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix two of the three failures here. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Wed Nov 11 10:44:48 2009 @@ -158,7 +158,7 @@ ops = self.get_by_bytecode("LOAD_GLOBAL") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "getfield_gc", "guard_nonnull"] + "getfield_gc", "guard_nonnull_class"] assert not ops[1] # second LOAD_GLOBAL folded away ops = self.get_by_bytecode("CALL_FUNCTION") assert len(ops) == 2 @@ -202,7 +202,7 @@ ops = self.get_by_bytecode("LOAD_ATTR") assert len(ops) == 2 assert ops[0].get_opnames() == ["getfield_gc", "getarrayitem_gc", - "guard_nonnull"] + "guard_nonnull_class"] assert not ops[1] # second LOAD_ATTR folded away def test_default_and_kw(self): From hpk at codespeak.net Wed Nov 11 10:53:08 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 10:53:08 +0100 (CET) Subject: [pypy-svn] r69146 - pypy/branch/py11 Message-ID: <20091111095308.6F213318138@codespeak.net> Author: hpk Date: Wed Nov 11 10:53:07 2009 New Revision: 69146 Removed: pypy/branch/py11/pytest_resultlog.py Log: removing pytest_resultlog plugin which should be fine with py.test-1.1 From hpk at codespeak.net Wed Nov 11 11:01:37 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 11:01:37 +0100 (CET) Subject: [pypy-svn] r69147 - pypy/branch/py11/pypy/config Message-ID: <20091111100137.9A55716810A@codespeak.net> Author: hpk Date: Wed Nov 11 11:01:37 2009 New Revision: 69147 Modified: pypy/branch/py11/pypy/config/makerestdoc.py Log: fix import Modified: pypy/branch/py11/pypy/config/makerestdoc.py ============================================================================== --- pypy/branch/py11/pypy/config/makerestdoc.py (original) +++ pypy/branch/py11/pypy/config/makerestdoc.py Wed Nov 11 11:01:37 2009 @@ -212,7 +212,7 @@ """ register a :config: ReST link role for use in documentation. """ try: from docutils.parsers.rst import directives, states, roles - from py.impl.rest.directive import register_linkrole + from pypy.tool.rest.directive import register_linkrole except ImportError: return # enable :config: link role From hpk at codespeak.net Wed Nov 11 11:28:15 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 11:28:15 +0100 (CET) Subject: [pypy-svn] r69148 - in pypy/branch/py11/py/bin: . win32 Message-ID: <20091111102815.B4D5D1683B6@codespeak.net> Author: hpk Date: Wed Nov 11 11:28:15 2009 New Revision: 69148 Added: pypy/branch/py11/py/bin/ pypy/branch/py11/py/bin/_findpy.py (contents, props changed) pypy/branch/py11/py/bin/env.cmd pypy/branch/py11/py/bin/env.py (contents, props changed) pypy/branch/py11/py/bin/py.cleanup (contents, props changed) pypy/branch/py11/py/bin/py.convert_unittest (contents, props changed) pypy/branch/py11/py/bin/py.countloc (contents, props changed) pypy/branch/py11/py/bin/py.lookup (contents, props changed) pypy/branch/py11/py/bin/py.svnwcrevert (contents, props changed) pypy/branch/py11/py/bin/py.test (contents, props changed) pypy/branch/py11/py/bin/py.which (contents, props changed) pypy/branch/py11/py/bin/win32/ pypy/branch/py11/py/bin/win32/py.cleanup.cmd pypy/branch/py11/py/bin/win32/py.convert_unittest.cmd pypy/branch/py11/py/bin/win32/py.countloc.cmd pypy/branch/py11/py/bin/win32/py.lookup.cmd pypy/branch/py11/py/bin/win32/py.svnwcrevert.cmd pypy/branch/py11/py/bin/win32/py.test.cmd pypy/branch/py11/py/bin/win32/py.which.cmd Log: adding bin directory of pytest-1.1.0 checkout to pypy's inlined py directory Added: pypy/branch/py11/py/bin/_findpy.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/_findpy.py Wed Nov 11 11:28:15 2009 @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# +# find and import a version of 'py' +# +import sys +import os +from os.path import dirname as opd, exists, join, basename, abspath + +def searchpy(current): + while 1: + last = current + initpy = join(current, '__init__.py') + if not exists(initpy): + pydir = join(current, 'py') + # recognize py-package and ensure it is importable + if exists(pydir) and exists(join(pydir, '__init__.py')): + #for p in sys.path: + # if p == current: + # return True + if current != sys.path[0]: # if we are already first, then ok + sys.stderr.write("inserting into sys.path: %s\n" % current) + sys.path.insert(0, current) + return True + current = opd(current) + if last == current: + return False + +if not searchpy(abspath(os.curdir)): + if not searchpy(opd(abspath(sys.argv[0]))): + if not searchpy(opd(__file__)): + pass # let's hope it is just on sys.path + +import py + +if __name__ == '__main__': + print ("py lib is at %s" % py.__file__) Added: pypy/branch/py11/py/bin/env.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/env.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +for /F "usebackq delims=" %%i in (`python "%~dp0\env.py"`) do %%i Added: pypy/branch/py11/py/bin/env.py ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/env.py Wed Nov 11 11:28:15 2009 @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import sys, os, os.path + +progpath = sys.argv[0] +packagedir = os.path.dirname(os.path.dirname(os.path.abspath(progpath))) +packagename = os.path.basename(packagedir) +bindir = os.path.join(packagedir, 'bin') +if sys.platform == 'win32': + bindir = os.path.join(bindir, 'win32') +rootdir = os.path.dirname(packagedir) + +def prepend_path(name, value): + sep = os.path.pathsep + curpath = os.environ.get(name, '') + newpath = [value] + [ x for x in curpath.split(sep) if x and x != value ] + return setenv(name, sep.join(newpath)) + +def setenv(name, value): + shell = os.environ.get('SHELL', '') + comspec = os.environ.get('COMSPEC', '') + if shell.endswith('csh'): + cmd = 'setenv %s "%s"' % (name, value) + elif shell.endswith('sh'): + cmd = '%s="%s"; export %s' % (name, value, name) + elif comspec.endswith('cmd.exe'): + cmd = 'set %s=%s' % (name, value) + else: + assert False, 'Shell not supported.' + return cmd + +print(prepend_path('PATH', bindir)) +print(prepend_path('PYTHONPATH', rootdir)) Added: pypy/branch/py11/py/bin/py.cleanup ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.cleanup Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pycleanup() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.convert_unittest ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.convert_unittest Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pyconvert_unittest() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.countloc ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.countloc Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pycountloc() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.lookup ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.lookup Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pylookup() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.svnwcrevert ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.svnwcrevert Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pysvnwcrevert() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.test ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.test Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pytest() \ No newline at end of file Added: pypy/branch/py11/py/bin/py.which ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/py.which Wed Nov 11 11:28:15 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pywhich() \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.cleanup.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.cleanup.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.cleanup" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.convert_unittest.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.convert_unittest.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.convert_unittest" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.countloc.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.countloc.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.countloc" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.lookup.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.lookup.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.lookup" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.svnwcrevert.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.svnwcrevert.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.svnwcrevert" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.test.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.test.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.test" %* \ No newline at end of file Added: pypy/branch/py11/py/bin/win32/py.which.cmd ============================================================================== --- (empty file) +++ pypy/branch/py11/py/bin/win32/py.which.cmd Wed Nov 11 11:28:15 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.which" %* \ No newline at end of file From hpk at codespeak.net Wed Nov 11 11:40:26 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 11:40:26 +0100 (CET) Subject: [pypy-svn] r69150 - pypy/branch/py11/pypy/doc Message-ID: <20091111104026.6BF081683B6@codespeak.net> Author: hpk Date: Wed Nov 11 11:40:25 2009 New Revision: 69150 Modified: pypy/branch/py11/pypy/doc/confrest_oldpy.py Log: write doc html to a raw file Modified: pypy/branch/py11/pypy/doc/confrest_oldpy.py ============================================================================== --- pypy/branch/py11/pypy/doc/confrest_oldpy.py (original) +++ pypy/branch/py11/pypy/doc/confrest_oldpy.py Wed Nov 11 11:40:25 2009 @@ -186,7 +186,9 @@ id = 'docinfoline')) page.contentspace.append(py.xml.raw(content)) - outputpath.ensure().write(page.unicode().encode(encoding)) + f = outputpath.open('w') + f.write(page.unicode().encode(encoding)) + f.close() # XXX this function comes from apigen/linker.py, put it # somewhere in py lib From arigo at codespeak.net Wed Nov 11 13:08:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:08:07 +0100 (CET) Subject: [pypy-svn] r69153 - pypy/branch/jit-log-class-func Message-ID: <20091111120807.63995318138@codespeak.net> Author: arigo Date: Wed Nov 11 13:08:07 2009 New Revision: 69153 Added: pypy/branch/jit-log-class-func/ - copied from r69152, pypy/trunk/ Log: A branch. From arigo at codespeak.net Wed Nov 11 13:09:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:09:09 +0100 (CET) Subject: [pypy-svn] r69154 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091111120909.96D5C318138@codespeak.net> Author: arigo Date: Wed Nov 11 13:09:09 2009 New Revision: 69154 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Log: (fijal, arigo) Introduce a list that maps the address of functions and classes to their name. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Wed Nov 11 13:09:09 2009 @@ -65,6 +65,8 @@ self.counter = 0 self.class_sizes = [] self._class_sizes_seen = {} + self.list_of_addr2name = [] + self._functions_addr_seen = {} # set later with .start() self.metainterp_sd = None @@ -172,7 +174,8 @@ portal_code = self.make_portal_bytecode(portal_graph) self.metainterp_sd.info_from_codewriter(portal_code, leave_code, - self.class_sizes) + self.class_sizes, + self.list_of_addr2name) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -306,6 +309,8 @@ methdescr.setup(jitcodes) def getcalldescr(self, v_func, args, result, consider_effects_of=None): + if isinstance(v_func, Constant): + self.register_known_function(v_func.value) non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -330,6 +335,8 @@ self._class_sizes_seen[key] = True sizedescr = self.cpu.sizeof(STRUCT) self.class_sizes.append((vtable, sizedescr)) + vtable_addr = llmemory.cast_ptr_to_adr(vtable) + self.list_of_addr2name.append((vtable_addr, STRUCT.__name__)) def register_known_ooclass(self, cls, CLASS): # ootype only @@ -338,6 +345,12 @@ typedescr = self.cpu.typedescrof(CLASS) self.class_sizes.append((cls, typedescr)) + def register_known_function(self, func): + if self.rtyper.type_system.name == 'lltypesystem': + if func._obj not in self._functions_addr_seen: + self._functions_addr_seen[func._obj] = True + func_addr = llmemory.cast_ptr_to_adr(func) + self.list_of_addr2name.append((func_addr, func._obj._name)) class BytecodeMaker(object): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:09:09 2009 @@ -1022,15 +1022,18 @@ self.portal_code = None self.leave_code = None - self._class_sizes = None + self._class_sizes = None + self._list_of_addr2name = None def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes): + def info_from_codewriter(self, portal_code, leave_code, class_sizes, + list_of_addr2name=[]): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes + self._list_of_addr2name = list_of_addr2name def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1063,6 +1066,10 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) + def get_addr2name_dict(self): + # for debugging + pass #... + def bytecode_for_address(self, fnaddress): if we_are_translated(): d = self.globaldata.indirectcall_dict Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Wed Nov 11 13:09:09 2009 @@ -121,12 +121,14 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): return FakeMethDescr(CLASS, methname) + def sizeof(self, STRUCT): + return ('sizeof', STRUCT) if type_system == 'lltype': FakeCPU.ts = typesystem.llhelper @@ -366,6 +368,24 @@ assert jitcode._source.count('oononnull') == 2 assert jitcode._source.count('ooisnull') == 2 + def test_list_of_addr2name(self): + class A1: + def g(self): + self.x = 123 + return 5 + def f(): + a = A1() + a.y = a.g() + return a + graphs = self.make_graphs(f, []) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert len(cw.list_of_addr2name) == 2 + assert cw.list_of_addr2name[0][1].endswith('.A1') + assert cw.list_of_addr2name[1][1] == 'A1.g' + class ImmutableFieldsTests: def test_fields(self): From arigo at codespeak.net Wed Nov 11 13:10:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:10:16 +0100 (CET) Subject: [pypy-svn] r69155 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091111121016.7A928318138@codespeak.net> Author: arigo Date: Wed Nov 11 13:10:15 2009 New Revision: 69155 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Log: Bah. Revert. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Wed Nov 11 13:10:15 2009 @@ -65,8 +65,6 @@ self.counter = 0 self.class_sizes = [] self._class_sizes_seen = {} - self.list_of_addr2name = [] - self._functions_addr_seen = {} # set later with .start() self.metainterp_sd = None @@ -174,8 +172,7 @@ portal_code = self.make_portal_bytecode(portal_graph) self.metainterp_sd.info_from_codewriter(portal_code, leave_code, - self.class_sizes, - self.list_of_addr2name) + self.class_sizes) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -309,8 +306,6 @@ methdescr.setup(jitcodes) def getcalldescr(self, v_func, args, result, consider_effects_of=None): - if isinstance(v_func, Constant): - self.register_known_function(v_func.value) non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -335,8 +330,6 @@ self._class_sizes_seen[key] = True sizedescr = self.cpu.sizeof(STRUCT) self.class_sizes.append((vtable, sizedescr)) - vtable_addr = llmemory.cast_ptr_to_adr(vtable) - self.list_of_addr2name.append((vtable_addr, STRUCT.__name__)) def register_known_ooclass(self, cls, CLASS): # ootype only @@ -345,12 +338,6 @@ typedescr = self.cpu.typedescrof(CLASS) self.class_sizes.append((cls, typedescr)) - def register_known_function(self, func): - if self.rtyper.type_system.name == 'lltypesystem': - if func._obj not in self._functions_addr_seen: - self._functions_addr_seen[func._obj] = True - func_addr = llmemory.cast_ptr_to_adr(func) - self.list_of_addr2name.append((func_addr, func._obj._name)) class BytecodeMaker(object): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:10:15 2009 @@ -1022,18 +1022,15 @@ self.portal_code = None self.leave_code = None - self._class_sizes = None - self._list_of_addr2name = None + self._class_sizes = None def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes, - list_of_addr2name=[]): + def info_from_codewriter(self, portal_code, leave_code, class_sizes): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes - self._list_of_addr2name = list_of_addr2name def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1066,10 +1063,6 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) - def get_addr2name_dict(self): - # for debugging - pass #... - def bytecode_for_address(self, fnaddress): if we_are_translated(): d = self.globaldata.indirectcall_dict Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Wed Nov 11 13:10:15 2009 @@ -121,14 +121,12 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): return FakeMethDescr(CLASS, methname) - def sizeof(self, STRUCT): - return ('sizeof', STRUCT) if type_system == 'lltype': FakeCPU.ts = typesystem.llhelper @@ -368,24 +366,6 @@ assert jitcode._source.count('oononnull') == 2 assert jitcode._source.count('ooisnull') == 2 - def test_list_of_addr2name(self): - class A1: - def g(self): - self.x = 123 - return 5 - def f(): - a = A1() - a.y = a.g() - return a - graphs = self.make_graphs(f, []) - cw = CodeWriter(self.rtyper) - cw.candidate_graphs = [graphs[0]] - cw._start(self.metainterp_sd, None) - jitcode = cw.make_one_bytecode((graphs[0], None), False) - assert len(cw.list_of_addr2name) == 2 - assert cw.list_of_addr2name[0][1].endswith('.A1') - assert cw.list_of_addr2name[1][1] == 'A1.g' - class ImmutableFieldsTests: def test_fields(self): From arigo at codespeak.net Wed Nov 11 13:12:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:12:00 +0100 (CET) Subject: [pypy-svn] r69156 - in pypy/branch/jit-log-class-func/pypy/jit/metainterp: . test Message-ID: <20091111121200.297B7318138@codespeak.net> Author: arigo Date: Wed Nov 11 13:11:59 2009 New Revision: 69156 Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_codewriter.py Log: (fijal, arigo) Introduce a list that maps the address of functions and classes to their name. Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py Wed Nov 11 13:11:59 2009 @@ -65,6 +65,8 @@ self.counter = 0 self.class_sizes = [] self._class_sizes_seen = {} + self.list_of_addr2name = [] + self._functions_addr_seen = {} # set later with .start() self.metainterp_sd = None @@ -172,7 +174,8 @@ portal_code = self.make_portal_bytecode(portal_graph) self.metainterp_sd.info_from_codewriter(portal_code, leave_code, - self.class_sizes) + self.class_sizes, + self.list_of_addr2name) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -306,6 +309,8 @@ methdescr.setup(jitcodes) def getcalldescr(self, v_func, args, result, consider_effects_of=None): + if isinstance(v_func, Constant): + self.register_known_function(v_func.value) non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -330,6 +335,8 @@ self._class_sizes_seen[key] = True sizedescr = self.cpu.sizeof(STRUCT) self.class_sizes.append((vtable, sizedescr)) + vtable_addr = llmemory.cast_ptr_to_adr(vtable) + self.list_of_addr2name.append((vtable_addr, STRUCT.__name__)) def register_known_ooclass(self, cls, CLASS): # ootype only @@ -338,6 +345,12 @@ typedescr = self.cpu.typedescrof(CLASS) self.class_sizes.append((cls, typedescr)) + def register_known_function(self, func): + if self.rtyper.type_system.name == 'lltypesystem': + if func._obj not in self._functions_addr_seen: + self._functions_addr_seen[func._obj] = True + func_addr = llmemory.cast_ptr_to_adr(func) + self.list_of_addr2name.append((func_addr, func._obj._name)) class BytecodeMaker(object): Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:11:59 2009 @@ -1022,15 +1022,18 @@ self.portal_code = None self.leave_code = None - self._class_sizes = None + self._class_sizes = None + self._list_of_addr2name = None def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes): + def info_from_codewriter(self, portal_code, leave_code, class_sizes, + list_of_addr2name=[]): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes + self._list_of_addr2name = list_of_addr2name def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1063,6 +1066,10 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) + def get_addr2name_dict(self): + # for debugging + pass #... + def bytecode_for_address(self, fnaddress): if we_are_translated(): d = self.globaldata.indirectcall_dict Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_codewriter.py Wed Nov 11 13:11:59 2009 @@ -121,12 +121,14 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): return FakeMethDescr(CLASS, methname) + def sizeof(self, STRUCT): + return ('sizeof', STRUCT) if type_system == 'lltype': FakeCPU.ts = typesystem.llhelper @@ -366,6 +368,24 @@ assert jitcode._source.count('oononnull') == 2 assert jitcode._source.count('ooisnull') == 2 + def test_list_of_addr2name(self): + class A1: + def g(self): + self.x = 123 + return 5 + def f(): + a = A1() + a.y = a.g() + return a + graphs = self.make_graphs(f, []) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert len(cw.list_of_addr2name) == 2 + assert cw.list_of_addr2name[0][1].endswith('.A1') + assert cw.list_of_addr2name[1][1] == 'A1.g' + class ImmutableFieldsTests: def test_fields(self): From hpk at codespeak.net Wed Nov 11 13:33:41 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 13:33:41 +0100 (CET) Subject: [pypy-svn] r69157 - in pypy/branch/py11/pypy: . module/__builtin__/test module/_file/test module/bz2/test Message-ID: <20091111123341.E2E52318139@codespeak.net> Author: hpk Date: Wed Nov 11 13:33:41 2009 New Revision: 69157 Modified: pypy/branch/py11/pypy/conftest.py pypy/branch/py11/pypy/module/__builtin__/test/test_import.py pypy/branch/py11/pypy/module/_file/test/test_file_extra.py pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py Log: fixing usages of path.write() to specify 'wb' for binary writes setting collect_ignore variable for lib/py Modified: pypy/branch/py11/pypy/conftest.py ============================================================================== --- pypy/branch/py11/pypy/conftest.py (original) +++ pypy/branch/py11/pypy/conftest.py Wed Nov 11 13:33:41 2009 @@ -16,11 +16,7 @@ rsyncdirs = ['.', '../lib-python', '../demo'] rsyncignore = ['_cache'] -# XXX workaround for a py.test bug clashing with lib/py symlink -# do we really need the latter? -empty_conftest = type(sys)('conftest') -empty_conftest.__file__ = "?" -sys.modules['pypy.lib.py.conftest'] = empty_conftest +collect_ignore = ['./lib/py'] # PyPy's command line extra options (these are added # to py.test's standard options) Modified: pypy/branch/py11/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/branch/py11/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/branch/py11/pypy/module/__builtin__/test/test_import.py Wed Nov 11 13:33:41 2009 @@ -92,7 +92,7 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read()) + p.join('lone.pyc').write(p.join('x.pyc').read(), mode='wb') return str(root) Modified: pypy/branch/py11/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/py11/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/py11/pypy/module/_file/test/test_file_extra.py Wed Nov 11 13:33:41 2009 @@ -19,7 +19,7 @@ def setup_module(mod): - udir.join('sample').write(SAMPLE) + udir.join('sample').write(SAMPLE, 'wb') class BaseROTests: Modified: pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py Wed Nov 11 13:33:41 2009 @@ -15,7 +15,7 @@ def create_temp_file(crlf=False): f = py.test.ensuretemp("bz2").join("foo") data = (DATA, DATA_CRLF)[crlf] - f.write(data) + f.write(data, 'wb') def create_broken_temp_file(): f = py.test.ensuretemp("bz2").join("foo") From antocuni at codespeak.net Wed Nov 11 13:37:18 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 13:37:18 +0100 (CET) Subject: [pypy-svn] r69158 - pypy/trunk/pypy/jit/backend/test Message-ID: <20091111123718.06A81318139@codespeak.net> Author: antocuni Date: Wed Nov 11 13:37:18 2009 New Revision: 69158 Modified: pypy/trunk/pypy/jit/backend/test/support.py Log: don't crash if len(args) == 1 Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Wed Nov 11 13:37:18 2009 @@ -31,24 +31,28 @@ if listcomp: t.config.translation.list_comprehension_operations = True + arglist = ", ".join(['int(argv[%d])' % (i + 1) for i in range(len(args))]) + if len(args) == 1: + arglist += ',' + arglist = '(%s)' % arglist if repeat != 1: src = py.code.Source(""" def entry_point(argv): - args = (%s,) + args = %s res = function(*args) for k in range(%d - 1): res = function(*args) print res return 0 - """ % (", ".join(['int(argv[%d])' % (i + 1) for i in range(len(args))]), repeat)) + """ % (arglist, repeat)) else: src = py.code.Source(""" def entry_point(argv): - args = (%s,) + args = %s res = function(*args) print res return 0 - """ % (", ".join(['int(argv[%d])' % (i + 1) for i in range(len(args))]),)) + """ % (arglist,)) exec src.compile() in locals() t.buildannotator().build_types(function, [int] * len(args)) From arigo at codespeak.net Wed Nov 11 13:37:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:37:58 +0100 (CET) Subject: [pypy-svn] r69159 - in pypy/branch/jit-log-class-func/pypy/jit/metainterp: . test Message-ID: <20091111123758.555D7318139@codespeak.net> Author: arigo Date: Wed Nov 11 13:37:57 2009 New Revision: 69159 Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/logger.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_logger.py Log: Have the logger use the information. Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/logger.py Wed Nov 11 13:37:57 2009 @@ -7,9 +7,10 @@ class Logger(object): - def __init__(self, ts, guard_number=False): - self.ts = ts - self.guard_number=guard_number + def __init__(self, metainterp_sd, guard_number=False): + self.metainterp_sd = metainterp_sd + self.ts = metainterp_sd.cpu.ts + self.guard_number = guard_number def log_loop(self, inputargs, operations, number=0, type=None): if type is None: @@ -57,7 +58,11 @@ elif isinstance(arg, BoxFloat): return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): - return 'ConstClass(cls' + str(mv) + ')' + addr = arg.getaddr(self.metainterp_sd.cpu) + name = self.metainterp_sd.get_name_from_address(addr) + if not name: + name = 'cls' + str(mv) + return 'ConstClass(' + name + ')' else: return '?' Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:37:57 2009 @@ -996,8 +996,8 @@ self.cpu = cpu self.stats = stats self.options = options - self.logger_noopt = Logger(cpu.ts) - self.logger_ops = Logger(cpu.ts, guard_number=True) + self.logger_noopt = Logger(self) + self.logger_ops = Logger(self, guard_number=True) RESULT = portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) @@ -1023,7 +1023,8 @@ self.portal_code = None self.leave_code = None self._class_sizes = None - self._list_of_addr2name = None + self._addr2name_keys = None + self._addr2name_values = None def _freeze_(self): return True @@ -1033,7 +1034,8 @@ self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes - self._list_of_addr2name = list_of_addr2name + self._addr2name_keys = [key for key, value in list_of_addr2name] + self._addr2name_values = [value for key, value in list_of_addr2name] def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1066,9 +1068,26 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) - def get_addr2name_dict(self): - # for debugging - pass #... + def get_name_from_address(self, addr): + # for debugging only + if we_are_translated(): + d = self.globaldata.addr2name + if d is None: + # Build the dictionary at run-time. This is needed + # because the keys are function/class addresses, so they + # can change from run to run. + d = {} + keys = self._addr2name_keys + values = self._addr2name_values + for i in range(len(keys)): + d[keys[i]] = values[i] + self.globaldata.addr2name = d + return d.get(addr, '') + else: + for i in range(len(self._addr2name_keys)): + if fnaddress == self._addr2name_keys[i]: + return self._addr2name_values[i] + return '' def bytecode_for_address(self, fnaddress): if we_are_translated(): @@ -1122,6 +1141,7 @@ def __init__(self, staticdata, prebuilt_fail_descr_list): self.initialized = False self.indirectcall_dict = None + self.addr2name = None self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_logger.py Wed Nov 11 13:37:57 2009 @@ -34,6 +34,14 @@ class TestLogger(object): ts = llhelper + def make_metainterp_sd(self): + class FakeMetaInterpSd: + class cpu: + ts = self.ts + def get_name_from_address(self, addr): + return 'Name' + return FakeMetaInterpSd() + def reparse(self, inp, namespace=None, check_equal=True): """ parse loop once, then log it and parse again. Checks that we get the same thing. @@ -41,7 +49,7 @@ if namespace is None: namespace = {} loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop, namespace) oloop = pure_parse(output, namespace=namespace) if check_equal: @@ -99,7 +107,7 @@ jump(i0, descr=target) ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop) assert output.splitlines()[-1] == "jump(i0, descr=)" pure_parse(output) @@ -111,7 +119,7 @@ guard_true(i0, descr=fdescr) [i0] ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts, guard_number=True) + logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) @@ -119,18 +127,34 @@ def boom(): raise Exception namespace['fdescr'].get_index = boom - logger = Logger(self.ts, guard_number=False) + logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + def test_class_name(self): + from pypy.rpython.lltypesystem import lltype + AbcVTable = lltype.Struct('AbcVTable') + abcvtable = lltype.malloc(AbcVTable, immortal=True) + namespace = {'Name': abcvtable} + inp = ''' + [i0] + p = new_with_vtable(ConstClass(Name)) + ''' + loop = pure_parse(inp, namespace=namespace) + logger = Logger(self.make_metainterp_sd()) + output = logger.log_loop(loop) + assert output.splitlines()[-1].endswith( + " = new_with_vtable(ConstClass(Name))") + pure_parse(output, namespace=namespace) + def test_intro_loop(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_loop, [], [], 1, "foo") assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_bridge, [], [], 3) assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) From arigo at codespeak.net Wed Nov 11 13:41:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:41:59 +0100 (CET) Subject: [pypy-svn] r69160 - in pypy/branch/jit-log-class-func/pypy/jit/metainterp: . test Message-ID: <20091111124159.8321C318139@codespeak.net> Author: arigo Date: Wed Nov 11 13:41:59 2009 New Revision: 69160 Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_pyjitpl.py Log: Test and fix for get_name_from_address(). Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:41:59 2009 @@ -1085,7 +1085,7 @@ return d.get(addr, '') else: for i in range(len(self._addr2name_keys)): - if fnaddress == self._addr2name_keys[i]: + if addr == self._addr2name_keys[i]: return self._addr2name_values[i] return '' Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/test/test_pyjitpl.py Wed Nov 11 13:41:59 2009 @@ -142,3 +142,14 @@ assert equaloplists(metainterp.history.operations, [ ResOperation(rop.SAME_AS, [b2], boxes[1]), ]) + +def test_get_name_from_address(): + class FakeMetaInterpSd(pyjitpl.MetaInterpStaticData): + def __init__(self): + pass + metainterp_sd = FakeMetaInterpSd() + metainterp_sd.info_from_codewriter(None, None, None, + [(123, "a"), (456, "b")]) + assert metainterp_sd.get_name_from_address(123) == 'a' + assert metainterp_sd.get_name_from_address(456) == 'b' + assert metainterp_sd.get_name_from_address(789) == '' From arigo at codespeak.net Wed Nov 11 13:48:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 13:48:27 +0100 (CET) Subject: [pypy-svn] r69161 - pypy/branch/jit-log-class-func/pypy/jit/metainterp Message-ID: <20091111124827.679BC318138@codespeak.net> Author: arigo Date: Wed Nov 11 13:48:26 2009 New Revision: 69161 Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Log: Fix tests. Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 13:48:26 2009 @@ -1023,8 +1023,8 @@ self.portal_code = None self.leave_code = None self._class_sizes = None - self._addr2name_keys = None - self._addr2name_values = None + self._addr2name_keys = [] + self._addr2name_values = [] def _freeze_(self): return True From fijal at codespeak.net Wed Nov 11 13:54:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 13:54:59 +0100 (CET) Subject: [pypy-svn] r69162 - pypy/trunk/pypy/interpreter Message-ID: <20091111125459.7447B49843C@codespeak.net> Author: fijal Date: Wed Nov 11 13:54:59 2009 New Revision: 69162 Modified: pypy/trunk/pypy/interpreter/error.py Log: make normalize_exception unroll_safe Modified: pypy/trunk/pypy/interpreter/error.py ============================================================================== --- pypy/trunk/pypy/interpreter/error.py (original) +++ pypy/trunk/pypy/interpreter/error.py Wed Nov 11 13:54:59 2009 @@ -1,5 +1,6 @@ import os, sys from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib import jit AUTO_DEBUG = os.getenv('PYPY_DEBUG') RECORD_INTERPLEVEL_TRACEBACK = True @@ -141,6 +142,7 @@ import debug debug.fire(self) + @jit.unroll_safe def normalize_exception(self, space): """Normalize the OperationError. In other words, fix w_type and/or w_value to make sure that the __class__ of w_value is exactly w_type. From antocuni at codespeak.net Wed Nov 11 14:17:16 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 14:17:16 +0100 (CET) Subject: [pypy-svn] r69164 - in pypy/trunk/pypy/jit/backend: cli test Message-ID: <20091111131716.0A8BF318138@codespeak.net> Author: antocuni Date: Wed Nov 11 14:17:16 2009 New Revision: 69164 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/test/support.py Log: use the full optimizer also for the translated tests, and implement guard_nonnull_class for cli Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Wed Nov 11 14:17:16 2009 @@ -492,6 +492,19 @@ self.push_arg(op, 1) self.il.Emit(OpCodes.Bne_Un, il_label) + def emit_op_guard_nonnull_class(self, op): + assert len(op.args) == 2 + il_label = self.newbranch(op) + # nonnull check + self.push_arg(op, 0) + self.il.Emit(OpCodes.Brfalse, il_label) + # class check + self.push_arg(op, 0) + meth = dotnet.typeof(System.Object).GetMethod("GetType") + self.il.Emit(OpCodes.Callvirt, meth) + self.push_arg(op, 1) + self.il.Emit(OpCodes.Bne_Un, il_label) + def emit_op_guard_no_exception(self, op): il_label = self.newbranch(op) self.av_inputargs.load(self) Modified: pypy/trunk/pypy/jit/backend/test/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/support.py (original) +++ pypy/trunk/pypy/jit/backend/test/support.py Wed Nov 11 14:17:16 2009 @@ -1,6 +1,7 @@ import py import sys from pypy.rlib.debug import debug_print +from pypy.rlib.jit import OPTIMIZER_FULL from pypy.translator.translator import TranslationContext class BaseCompiledMixin(object): @@ -64,6 +65,7 @@ warmrunnerdesc.state.set_param_trace_eagerness(2) # for tests warmrunnerdesc.state.set_param_trace_limit(trace_limit) warmrunnerdesc.state.set_param_inlining(inline) + warmrunnerdesc.state.set_param_optimizer(OPTIMIZER_FULL) mixlevelann = warmrunnerdesc.annhelper entry_point_graph = mixlevelann.getgraph(entry_point, [s_list_of_strings], annmodel.SomeInteger()) From antocuni at codespeak.net Wed Nov 11 14:27:19 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 14:27:19 +0100 (CET) Subject: [pypy-svn] r69165 - pypy/trunk/pypy/translator/cli/test Message-ID: <20091111132719.82E99318138@codespeak.net> Author: antocuni Date: Wed Nov 11 14:27:19 2009 New Revision: 69165 Modified: pypy/trunk/pypy/translator/cli/test/test_runtest.py Log: kill this failing tests, as debug_print is already tested in a better way in test_standalone Modified: pypy/trunk/pypy/translator/cli/test/test_runtest.py ============================================================================== --- pypy/trunk/pypy/translator/cli/test/test_runtest.py (original) +++ pypy/trunk/pypy/translator/cli/test/test_runtest.py Wed Nov 11 14:27:19 2009 @@ -21,24 +21,3 @@ return len(s) res = self.interpret(fn, ["hello"]) assert res == 5 - - def test_debug_print(self): - from pypy.rlib.debug import debug_print - def fn(s): - debug_print('Hello world', 42) - return s - func = self._compile(fn, [42]) - stdout, stderr, retval = func.run(42) - assert retval == 0 - assert stdout == '42\n' - assert stderr == 'Hello world 42\n' - - def fn(s): - # too many arguments, ignore it - debug_print('Hello world', 42, 43, 44, 45, 46, 47, 48) - return s - func = self._compile(fn, [42]) - stdout, stderr, retval = func.run(42) - assert retval == 0 - assert stdout == '42\n' - assert stderr == '' From antocuni at codespeak.net Wed Nov 11 14:47:28 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 14:47:28 +0100 (CET) Subject: [pypy-svn] r69166 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091111134728.0E4D8318138@codespeak.net> Author: antocuni Date: Wed Nov 11 14:47:28 2009 New Revision: 69166 Modified: pypy/trunk/pypy/jit/backend/cli/runner.py Log: one more type to support Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Wed Nov 11 14:47:28 2009 @@ -301,6 +301,8 @@ return ootype.nullruntimeclass elif T is ootype.Signed: return dotnet.classof(System.Int32) + elif T is ootype.Unsigned: + return dotnet.classof(System.UInt32) elif T is ootype.Bool: return dotnet.classof(System.Boolean) elif T is ootype.Float: From antocuni at codespeak.net Wed Nov 11 14:48:08 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 14:48:08 +0100 (CET) Subject: [pypy-svn] r69167 - pypy/trunk/pypy/tool Message-ID: <20091111134808.3710F318138@codespeak.net> Author: antocuni Date: Wed Nov 11 14:48:07 2009 New Revision: 69167 Modified: pypy/trunk/pypy/tool/logparser.py Log: (antocuni, arigo) support also uppercase hex numbers Modified: pypy/trunk/pypy/tool/logparser.py ============================================================================== --- pypy/trunk/pypy/tool/logparser.py (original) +++ pypy/trunk/pypy/tool/logparser.py Wed Nov 11 14:48:07 2009 @@ -12,8 +12,8 @@ def parse_log_file(filename): - r_start = re.compile(r"\[([0-9a-f]+)\] \{([\w-]+)$") - r_stop = re.compile(r"\[([0-9a-f]+)\] ([\w-]+)\}$") + r_start = re.compile(r"\[([0-9a-fA-F]+)\] \{([\w-]+)$") + r_stop = re.compile(r"\[([0-9a-fA-F]+)\] ([\w-]+)\}$") lasttime = 0 log = DebugLog() time_decrase = False From antocuni at codespeak.net Wed Nov 11 14:49:29 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 14:49:29 +0100 (CET) Subject: [pypy-svn] r69168 - pypy/trunk/pypy/translator/backendopt Message-ID: <20091111134929.297A3318139@codespeak.net> Author: antocuni Date: Wed Nov 11 14:49:28 2009 New Revision: 69168 Modified: pypy/trunk/pypy/translator/backendopt/canraise.py Log: be slightly more relaxed, as there are (officially supported) cases in which lltype function are seen by ootype Modified: pypy/trunk/pypy/translator/backendopt/canraise.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/canraise.py (original) +++ pypy/trunk/pypy/translator/backendopt/canraise.py Wed Nov 11 14:49:28 2009 @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem.lloperation import llop, LL_OPERATIONS from pypy.rpython.lltypesystem import lltype from pypy.translator.backendopt import graphanalyze +from pypy.translator.simplify import get_funcobj import py from pypy.tool.ansi_print import ansi_log @@ -17,8 +18,7 @@ return True def analyze_external_call(self, op): - deref = self.translator.rtyper.type_system_deref - fnobj = deref(op.args[0].value) + fnobj = get_funcobj(op.args[0].value) return getattr(fnobj, 'canraise', True) def analyze_external_method(self, op, TYPE, meth): From fijal at codespeak.net Wed Nov 11 15:09:52 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 15:09:52 +0100 (CET) Subject: [pypy-svn] r69169 - pypy/branch/faster-raise Message-ID: <20091111140952.B8E45318138@codespeak.net> Author: fijal Date: Wed Nov 11 15:09:52 2009 New Revision: 69169 Added: pypy/branch/faster-raise/ - copied from r69168, pypy/trunk/ Log: Another branch. From antocuni at codespeak.net Wed Nov 11 15:19:03 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 15:19:03 +0100 (CET) Subject: [pypy-svn] r69170 - pypy/trunk/pypy/module/gc Message-ID: <20091111141903.C4DF8318138@codespeak.net> Author: antocuni Date: Wed Nov 11 15:19:02 2009 New Revision: 69170 Modified: pypy/trunk/pypy/module/gc/__init__.py Log: remove dump_heap_stats in ootype translations Modified: pypy/trunk/pypy/module/gc/__init__.py ============================================================================== --- pypy/trunk/pypy/module/gc/__init__.py (original) +++ pypy/trunk/pypy/module/gc/__init__.py Wed Nov 11 15:19:02 2009 @@ -14,3 +14,9 @@ 'garbage' : 'space.newlist([])', 'dump_heap_stats': 'interp_gc.dump_heap_stats', } + + def __init__(self, space, w_name): + ts = space.config.translation.type_system + if ts == 'ootype': + del self.interpleveldefs['dump_heap_stats'] + MixedModule.__init__(self, space, w_name) From hpk at codespeak.net Wed Nov 11 15:22:20 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 15:22:20 +0100 (CET) Subject: [pypy-svn] r69171 - in pypy/branch/py11/pypy: doc/statistic module/bz2/test Message-ID: <20091111142220.0ABBA318138@codespeak.net> Author: hpk Date: Wed Nov 11 15:22:19 2009 New Revision: 69171 Modified: pypy/branch/py11/pypy/doc/statistic/confrest.py pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py Log: fix two more little issues discovered through the test run Modified: pypy/branch/py11/pypy/doc/statistic/confrest.py ============================================================================== --- pypy/branch/py11/pypy/doc/statistic/confrest.py (original) +++ pypy/branch/py11/pypy/doc/statistic/confrest.py Wed Nov 11 15:22:19 2009 @@ -1,5 +1,5 @@ import py -from py.impl.doc.confrest import * +from pypy.doc.confrest import * class PyPyPage(Page): def fill_menubar(self): Modified: pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/branch/py11/pypy/module/bz2/test/test_bz2_file.py Wed Nov 11 15:22:19 2009 @@ -20,7 +20,7 @@ def create_broken_temp_file(): f = py.test.ensuretemp("bz2").join("foo") data = DATA[:100] - f.write(data) + f.write(data, 'wb') def decompress(data): import popen2 From pedronis at codespeak.net Wed Nov 11 15:32:34 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 11 Nov 2009 15:32:34 +0100 (CET) Subject: [pypy-svn] r69172 - pypy/build/bot2/pypybuildbot Message-ID: <20091111143234.C1C12318138@codespeak.net> Author: pedronis Date: Wed Nov 11 15:32:34 2009 New Revision: 69172 Modified: pypy/build/bot2/pypybuildbot/master.py Log: (cfbolz, pedronis) adding a mac os x slave Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Nov 11 15:32:34 2009 @@ -58,6 +58,7 @@ pypyJITBenchmarkFactory = pypybuilds.PyPyJITBenchmarkFactory() LINUX32 = "own-linux-x86-32" +MACOSX32 = "own-macosx-x86-32" CPYLINUX32 = "pypy-c-lib-python-linux-x86-32" CPYWIN32 = "pypy-c-lib-python-win-32" CPYLINUX32_VM = 'pypy-c-lib-python-linux-x86-32vm' @@ -93,6 +94,12 @@ "factory": pypyOwnTestFactory, "category": 'own' }, + {"name": MACOSX32, + "slavenames": ["minime"], + "builddir": MACOSX32, + "factory": pypyOwnTestFactory, + "category": 'own' + }, {"name": CPYLINUX32, "slavenames": ["wyvern", "cobra"], "builddir": CPYLINUX32, From fijal at codespeak.net Wed Nov 11 15:39:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 15:39:36 +0100 (CET) Subject: [pypy-svn] r69173 - in pypy/branch/faster-raise/pypy: interpreter interpreter/test module/__builtin__ Message-ID: <20091111143936.F0BBF49843C@codespeak.net> Author: fijal Date: Wed Nov 11 15:39:36 2009 New Revision: 69173 Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/interpreter/error.py pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py Log: (arigo, fijal) Implement helpers for checking types valid for exeptions. Change a world a bit. Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Wed Nov 11 15:39:36 2009 @@ -802,34 +802,44 @@ w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + # The code below only works + # for the simple case (new-style instance). + # These methods are patched with the full logic by the __builtin__ + # module when it is loaded + def abstract_issubclass_w(self, w_cls1, w_cls2): - # Equivalent to 'issubclass(cls1, cls2)'. The code below only works - # for the simple case (new-style class, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'issubclass(cls1, cls2)'. return self.is_true(self.issubtype(w_cls1, w_cls2)) def abstract_isinstance_w(self, w_obj, w_cls): - # Equivalent to 'isinstance(obj, cls)'. The code below only works - # for the simple case (new-style instance, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, cls)'. return self.is_true(self.isinstance(w_obj, w_cls)) def abstract_isclass_w(self, w_obj): - # Equivalent to 'isinstance(obj, type)'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, type)'. return self.is_true(self.isinstance(w_obj, self.w_type)) def abstract_getclass(self, w_obj): - # Equivalent to 'obj.__class__'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'obj.__class__'. + return self.type(w_obj) + + # CPython rules allows old style classes or subclasses + # of BaseExceptions to be exceptions. + # This is slightly less general than the case above, so we prefix + # it with exception_ + + def exception_is_valid_class_w(self, w_obj): + return (self.is_true(self.isinstance(w_obj, self.w_type)) and + self.is_true(self.issubtype(w_obj, self.w_BaseException))) + + def exception_getclass(self, w_obj): return self.type(w_obj) + def exception_issubclass_w(self, w_cls1, w_cls2): + return self.is_true(self.issubtype(w_cls1, w_cls2)) + + # end of special support code + def eval(self, expression, w_globals, w_locals, hidden_applevel=False): "NOT_RPYTHON: For internal debugging." import types Modified: pypy/branch/faster-raise/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/error.py Wed Nov 11 15:39:36 2009 @@ -177,16 +177,15 @@ while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if (space.abstract_isclass_w(w_type) and - is_valid_exception_class(space, w_type)): + if space.exception_is_valid_class_w(w_type): # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): # raise Type: we assume we have to instantiate Type w_value = space.call_function(w_type) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) else: - w_valuetype = space.abstract_getclass(w_value) - if space.abstract_issubclass_w(w_valuetype, w_type): + w_valuetype = space.exception_getclass(w_value) + if space.exception_issubclass_w(w_valuetype, w_type): # raise Type, Instance: let etype be the exact type of value w_type = w_valuetype else: @@ -198,7 +197,7 @@ else: # raise Type, X: assume X is the constructor argument w_value = space.call_function(w_type, w_value) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) elif space.full_exceptions and space.is_w(space.type(w_type), space.w_str): @@ -208,8 +207,8 @@ else: # the only case left here is (inst, None), from a 'raise inst'. w_inst = w_type - w_instclass = space.abstract_getclass(w_inst) - if not is_valid_exception_class(space, w_instclass): + w_instclass = space.exception_getclass(w_inst) + if not space.exception_is_valid_class_w(w_instclass): instclassname = w_instclass.getname(space, '?') msg = ("exceptions must be classes, or instances," "or strings (deprecated) not %s" % (instclassname,)) @@ -240,23 +239,6 @@ except OperationError: pass # ignored - -def is_valid_exception_class(space, w_type): - """Assuming that 'w_type' is a new-style or old-style class, is it - correct to use it as the class of an exception? The answer is no - if it is a new-style class that doesn't inherit from BaseException. - """ - if not space.full_exceptions: - return True # always, for the flow space - try: - return space.is_true( - space.issubtype(w_type, space.w_BaseException)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - return True # assuming w_type is an old-style class - - # Utilities from pypy.tool.ansi_print import ansi_print Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py Wed Nov 11 15:39:36 2009 @@ -244,3 +244,14 @@ assert 0 except KeyError: pass + + def test_obscure_bases(self): + # this test checks bug-to-bug cpython compatibility + e = ValueError() + e.__bases__ = (5,) + try: + raise e + except ValueError: + pass + + Modified: pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py Wed Nov 11 15:39:36 2009 @@ -148,6 +148,9 @@ space.abstract_issubclass_w = ab.abstract_issubclass_w.__get__(space) space.abstract_isclass_w = ab.abstract_isclass_w.__get__(space) space.abstract_getclass = ab.abstract_getclass.__get__(space) + space.exception_is_valid_class_w = ab.exception_is_valid_class_w.__get__(space) + space.exception_getclass = ab.exception_getclass.__get__(space) + space.exception_issubclass_w = ab.exception_issubclass_w.__get__(space) def startup(self, space): # install zipimport hook if --withmod-zipimport is used Modified: pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py Wed Nov 11 15:39:36 2009 @@ -11,6 +11,7 @@ from pypy.interpreter.error import OperationError from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.__builtin__.interp_classobj import W_InstanceObject +from pypy.interpreter.baseobjspace import ObjSpace as BaseObjSpace def _get_bases(space, w_cls): """Returns 'cls.__bases__'. Returns None if there is @@ -152,6 +153,31 @@ " or tuple of classes and types") return _issubclass_recurse(space, w_derived, w_klass_or_tuple) +# ------------------------------------------------------------ +# Exception helpers + +def exception_is_valid_class_w(space, w_obj): + obj = space.interpclass_w(w_obj) + if isinstance(obj, W_ClassObject): + return True + return BaseObjSpace.exception_is_valid_class_w(space, w_obj) + +def exception_getclass(space, w_obj): + obj = space.interpclass_w(w_obj) + if isinstance(obj, W_InstanceObject): + return obj.w_class + return BaseObjSpace.exception_getclass(space, w_obj) + +def exception_issubclass_w(space, w_cls1, w_cls2): + cls1 = space.interpclass_w(w_cls1) + cls2 = space.interpclass_w(w_cls2) + if isinstance(cls1, W_ClassObject): + if isinstance(cls2, W_ClassObject): + return cls1.is_subclass_of(cls2) + return False + if isinstance(cls2, W_ClassObject): + return False + return BaseObjSpace.exception_issubclass_w(space, w_cls1, w_cls2) # ____________________________________________________________ # App-level interface From fijal at codespeak.net Wed Nov 11 16:11:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 16:11:54 +0100 (CET) Subject: [pypy-svn] r69174 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091111151154.CDE9316810C@codespeak.net> Author: fijal Date: Wed Nov 11 16:11:54 2009 New Revision: 69174 Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py Log: (arigo, fijal) Improve testing and fix exception_match Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Wed Nov 11 16:11:54 2009 @@ -9,7 +9,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized from pypy.rlib.timer import DummyTimer, Timer -from pypy.rlib.jit import we_are_jitted, dont_look_inside +from pypy.rlib.jit import we_are_jitted, dont_look_inside, unroll_safe import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -675,12 +675,19 @@ return make_sure_not_resized(self.unpackiterable(w_iterable, expected_length)[:]) + @unroll_safe def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): return True # fast path (also here to handle string exceptions) try: - return self.abstract_issubclass_w(w_exc_type, w_check_class) + if self.is_true(self.isinstance(w_check_class, self.w_tuple)): + for w_t in self.viewiterable(w_check_class): + if self.exception_match(w_exc_type, w_t): + return True + else: + return False + return self.exception_issubclass_w(w_exc_type, w_check_class) except OperationError, e: if e.match(self, self.w_TypeError): # string exceptions maybe return False Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_raise.py Wed Nov 11 16:11:54 2009 @@ -245,6 +245,24 @@ except KeyError: pass + def test_catch_tuple(self): + class A: + pass + + try: + raise ValueError + except (ValueError, A): + pass + else: + fail("Did not raise") + + try: + raise A() + except (ValueError, A): + pass + else: + fail("Did not raise") + def test_obscure_bases(self): # this test checks bug-to-bug cpython compatibility e = ValueError() @@ -254,4 +272,22 @@ except ValueError: pass - + # explodes on CPython and py.test, not sure why + + flag = False + class A(BaseException): + class __metaclass__(type): + def __getattribute__(self, name): + if flag and name == '__bases__': + fail("someone read bases attr") + else: + return type.__getattribute__(self, name) + + try: + a = A() + flag = True + raise a + except 42: + pass + except A: + pass From arigo at codespeak.net Wed Nov 11 16:33:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 16:33:44 +0100 (CET) Subject: [pypy-svn] r69178 - pypy/branch/faster-raise/pypy/module/_file Message-ID: <20091111153344.9E0F41680FE@codespeak.net> Author: arigo Date: Wed Nov 11 16:33:43 2009 New Revision: 69178 Modified: pypy/branch/faster-raise/pypy/module/_file/interp_file.py Log: Remove an obscure usage of abstract_isinstance_w. Modified: pypy/branch/faster-raise/pypy/module/_file/interp_file.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_file/interp_file.py (original) +++ pypy/branch/faster-raise/pypy/module/_file/interp_file.py Wed Nov 11 16:33:43 2009 @@ -387,7 +387,8 @@ head = "closed" else: head = "open" - if self.space.abstract_isinstance_w(self.w_name, self.space.w_str): + if self.space.is_true(self.space.isinstance(self.w_name, + self.space.w_str)): info = "%s file '%s', mode '%s'" % ( head, self.space.str_w(self.w_name), From fijal at codespeak.net Wed Nov 11 16:33:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 16:33:59 +0100 (CET) Subject: [pypy-svn] r69179 - in pypy/branch/faster-raise/pypy: interpreter module/__builtin__ Message-ID: <20091111153359.BE57F1680FE@codespeak.net> Author: fijal Date: Wed Nov 11 16:33:59 2009 New Revision: 69179 Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/interpreter/error.py pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py Log: (arigo, fijal) Make flow objspace happy, also be slightly more efficient Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Wed Nov 11 16:33:59 2009 @@ -835,9 +835,17 @@ # This is slightly less general than the case above, so we prefix # it with exception_ - def exception_is_valid_class_w(self, w_obj): - return (self.is_true(self.isinstance(w_obj, self.w_type)) and - self.is_true(self.issubtype(w_obj, self.w_BaseException))) + def exception_is_valid_obj_as_class_w(self, w_obj): + if not self.is_true(self.isinstance(w_obj, self.w_type)): + return False + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_obj, self.w_BaseException)) + + def exception_is_valid_class_w(self, w_cls): + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_cls, self.w_BaseException)) def exception_getclass(self, w_obj): return self.type(w_obj) Modified: pypy/branch/faster-raise/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/error.py Wed Nov 11 16:33:59 2009 @@ -177,7 +177,7 @@ while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if space.exception_is_valid_class_w(w_type): + if space.exception_is_valid_obj_as_class_w(w_type): # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): # raise Type: we assume we have to instantiate Type Modified: pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/__init__.py Wed Nov 11 16:33:59 2009 @@ -149,6 +149,7 @@ space.abstract_isclass_w = ab.abstract_isclass_w.__get__(space) space.abstract_getclass = ab.abstract_getclass.__get__(space) space.exception_is_valid_class_w = ab.exception_is_valid_class_w.__get__(space) + space.exception_is_valid_obj_as_class_w = ab.exception_is_valid_obj_as_class_w.__get__(space) space.exception_getclass = ab.exception_getclass.__get__(space) space.exception_issubclass_w = ab.exception_issubclass_w.__get__(space) Modified: pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/abstractinst.py Wed Nov 11 16:33:59 2009 @@ -156,11 +156,17 @@ # ------------------------------------------------------------ # Exception helpers -def exception_is_valid_class_w(space, w_obj): +def exception_is_valid_obj_as_class_w(space, w_obj): obj = space.interpclass_w(w_obj) if isinstance(obj, W_ClassObject): return True - return BaseObjSpace.exception_is_valid_class_w(space, w_obj) + return BaseObjSpace.exception_is_valid_obj_as_class_w(space, w_obj) + +def exception_is_valid_class_w(space, w_cls): + cls = space.interpclass_w(w_cls) + if isinstance(cls, W_ClassObject): + return True + return BaseObjSpace.exception_is_valid_class_w(space, w_cls) def exception_getclass(space, w_obj): obj = space.interpclass_w(w_obj) From arigo at codespeak.net Wed Nov 11 17:09:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 17:09:25 +0100 (CET) Subject: [pypy-svn] r69181 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091111160925.22BA8168102@codespeak.net> Author: arigo Date: Wed Nov 11 17:09:24 2009 New Revision: 69181 Modified: pypy/branch/faster-raise/pypy/interpreter/error.py Log: Kill unused method. Modified: pypy/branch/faster-raise/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/error.py Wed Nov 11 17:09:24 2009 @@ -73,13 +73,6 @@ else: return '%s: %s' % (exc_typename, exc_value) - def getframe(self): - "The frame this exception was raised in, or None." - if self.application_traceback: - return self.application_traceback.frame - else: - return None - def record_interpreter_traceback(self): """Records the current traceback inside the interpreter. This traceback is only useful to debug the interpreter, not the From arigo at codespeak.net Wed Nov 11 17:09:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 17:09:42 +0100 (CET) Subject: [pypy-svn] r69182 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091111160942.2137A168102@codespeak.net> Author: arigo Date: Wed Nov 11 17:09:41 2009 New Revision: 69182 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py Log: Kill this, which only makes sense for refcounting. Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Wed Nov 11 17:09:41 2009 @@ -162,9 +162,6 @@ raise if not we_are_jitted(): executioncontext.return_trace(self, w_exitvalue) - # on exit, we try to release self.last_exception -- breaks an - # obvious reference cycle, so it helps refcounting implementations - self.last_exception = None finally: executioncontext.leave(self) return w_exitvalue From arigo at codespeak.net Wed Nov 11 17:15:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 17:15:51 +0100 (CET) Subject: [pypy-svn] r69183 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091111161551.230C2168102@codespeak.net> Author: arigo Date: Wed Nov 11 17:15:50 2009 New Revision: 69183 Modified: pypy/branch/faster-raise/pypy/interpreter/pytraceback.py pypy/branch/faster-raise/pypy/interpreter/typedef.py Log: Compute lazily tb_lineno, instead of computing it ALL THE TIME WHEN GOING THROUGH ALL THE FRAMES OF ALL THE EXCEPTIONS Modified: pypy/branch/faster-raise/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pytraceback.py Wed Nov 11 17:15:50 2009 @@ -5,20 +5,23 @@ class PyTraceback(baseobjspace.Wrappable): """Traceback object - Public fields: + Public app-level fields: * 'tb_frame' * 'tb_lasti' * 'tb_lineno' * 'tb_next' """ - def __init__(self, space, frame, lasti, lineno, next): + def __init__(self, space, frame, lasti, next): self.space = space self.frame = frame self.lasti = lasti - self.lineno = lineno self.next = next + def descr_tb_lineno(space, self): + lineno = offset2lineno(self.frame.pycode, self.lasti) + return space.wrap(lineno) + def descr__reduce__(self, space): from pypy.interpreter.mixedmodule import MixedModule w_mod = space.getbuiltinmodule('_pickle_support') @@ -49,9 +52,8 @@ frame.force_f_back() if frame.pycode.hidden_applevel: return - lineno = offset2lineno(frame.pycode, last_instruction) tb = operror.application_traceback - tb = PyTraceback(space, frame, last_instruction, lineno, tb) + tb = PyTraceback(space, frame, last_instruction, tb) operror.application_traceback = tb def offset2lineno(c, stopat): Modified: pypy/branch/faster-raise/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/typedef.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/typedef.py Wed Nov 11 17:15:50 2009 @@ -825,7 +825,7 @@ unwrap_spec=['self', ObjSpace, W_Root]), tb_frame = interp_attrproperty('frame', cls=PyTraceback), tb_lasti = interp_attrproperty('lasti', cls=PyTraceback), - tb_lineno = interp_attrproperty('lineno', cls=PyTraceback), + tb_lineno = GetSetProperty(PyTraceback.descr_tb_lineno), tb_next = interp_attrproperty('next', cls=PyTraceback), ) PyTraceback.typedef.acceptable_as_base_class = False From arigo at codespeak.net Wed Nov 11 17:30:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 17:30:34 +0100 (CET) Subject: [pypy-svn] r69184 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091111163034.C19D9168105@codespeak.net> Author: arigo Date: Wed Nov 11 17:30:34 2009 New Revision: 69184 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py pypy/branch/faster-raise/pypy/interpreter/pyopcode.py Log: (antocuni around) Get rid of 'blockcount'. Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Wed Nov 11 17:30:34 2009 @@ -56,7 +56,6 @@ self.valuestack_w = [None] * code.co_stacksize self.valuestackdepth = 0 self.lastblock = None - self.blockcount = 0 if space.config.objspace.honor__builtins__: self.builtin = space.builtin.pick_builtin(w_globals) # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. @@ -70,31 +69,29 @@ def append_block(self, block): block.previous = self.lastblock self.lastblock = block - self.blockcount += 1 def pop_block(self): block = self.lastblock self.lastblock = block.previous - self.blockcount -= 1 return block + def blockstack_non_empty(self): + return self.lastblock is not None + def get_blocklist(self): """Returns a list containing all the blocks in the frame""" - lst = [None] * self.blockcount + lst = [] block = self.lastblock - i = 0 while block is not None: - lst[i] = block - i += 1 + lst.append(block) block = block.previous return lst def set_blocklist(self, lst): self.lastblock = None - self.blockcount = 0 - i = len(lst) - while i > 0: - block = lst[i-1] + i = len(lst) - 1 + while i >= 0: + block = lst[i] i -= 1 self.append_block(block) Modified: pypy/branch/faster-raise/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyopcode.py Wed Nov 11 17:30:34 2009 @@ -274,11 +274,8 @@ @jit.unroll_safe def unrollstack(self, unroller_kind): - n = self.blockcount - n = jit.hint(n, promote=True) - while n > 0: + while self.blockstack_non_empty(): block = self.pop_block() - n -= 1 if (block.handling_mask & unroller_kind) != 0: return block block.cleanupstack(self) From arigo at codespeak.net Wed Nov 11 17:43:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 17:43:26 +0100 (CET) Subject: [pypy-svn] r69185 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091111164326.4C4EF168105@codespeak.net> Author: arigo Date: Wed Nov 11 17:43:25 2009 New Revision: 69185 Modified: pypy/branch/faster-raise/pypy/interpreter/error.py pypy/branch/faster-raise/pypy/interpreter/pytraceback.py Log: Oups. Forgot some '.lineno'. Modified: pypy/branch/faster-raise/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/error.py Wed Nov 11 17:43:25 2009 @@ -95,7 +95,7 @@ print >> file, "Traceback (application-level):" while tb is not None: co = tb.frame.pycode - lineno = tb.lineno + lineno = tb.get_lineno() fname = co.co_filename if fname.startswith('\n'): lines = fname.split('\n') Modified: pypy/branch/faster-raise/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pytraceback.py Wed Nov 11 17:43:25 2009 @@ -18,9 +18,11 @@ self.lasti = lasti self.next = next + def get_lineno(self): + return offset2lineno(self.frame.pycode, self.lasti) + def descr_tb_lineno(space, self): - lineno = offset2lineno(self.frame.pycode, self.lasti) - return space.wrap(lineno) + return space.wrap(self.get_lineno()) def descr__reduce__(self, space): from pypy.interpreter.mixedmodule import MixedModule @@ -33,7 +35,6 @@ tup_state = [ w(self.frame), w(self.lasti), - w(self.lineno), w(self.next), ] nt = space.newtuple @@ -42,10 +43,9 @@ def descr__setstate__(self, space, w_args): from pypy.interpreter.pyframe import PyFrame args_w = space.unpackiterable(w_args) - w_frame, w_lasti, w_lineno, w_next = args_w + w_frame, w_lasti, w_next = args_w self.frame = space.interp_w(PyFrame, w_frame) self.lasti = space.int_w(w_lasti) - self.lineno = space.int_w(w_lineno) self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): From fijal at codespeak.net Wed Nov 11 17:52:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 17:52:28 +0100 (CET) Subject: [pypy-svn] r69186 - pypy/branch/faster-raise/pypy/module/pypyjit Message-ID: <20091111165228.7575B168106@codespeak.net> Author: fijal Date: Wed Nov 11 17:52:27 2009 New Revision: 69186 Modified: pypy/branch/faster-raise/pypy/module/pypyjit/interp_jit.py Log: (arigo, fijal) Make last_exception a virtualizable field Modified: pypy/branch/faster-raise/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/interp_jit.py Wed Nov 11 17:52:27 2009 @@ -22,6 +22,7 @@ PyFrame._virtualizable2_ = ['last_instr', 'pycode', 'valuestackdepth', 'valuestack_w[*]', 'fastlocals_w[*]', 'f_forward', + 'last_exception', ] JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] From antocuni at codespeak.net Wed Nov 11 18:05:05 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 18:05:05 +0100 (CET) Subject: [pypy-svn] r69187 - pypy/trunk/pypy/translator Message-ID: <20091111170505.550D5168102@codespeak.net> Author: antocuni Date: Wed Nov 11 18:05:04 2009 New Revision: 69187 Modified: pypy/trunk/pypy/translator/driver.py Log: automatically use the workaround if the mono version is known to be buggy Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Wed Nov 11 18:05:04 2009 @@ -606,7 +606,18 @@ then EXE=$0 fi -if uname -s | grep -iq Cygwin ; then MONO=; else MONO=mono; fi +if uname -s | grep -iq Cygwin +then + MONO= +else + MONO=mono + # workaround for known mono buggy versions + VER=`mono -V | head -1 | sed s/'Mono JIT compiler version \(.*\) (.*'/'\1/'` + if [[ 2.1 < "$VER" && "$VER" < 2.4.3 ]] + then + MONO="mono -O=-branch" + fi +fi $LEDIT $MONO "$(dirname $EXE)/$(basename $EXE)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() From arigo at codespeak.net Wed Nov 11 18:08:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 18:08:29 +0100 (CET) Subject: [pypy-svn] r69188 - pypy/branch/jit-log-class-func/pypy/jit/metainterp Message-ID: <20091111170829.1C567168102@codespeak.net> Author: arigo Date: Wed Nov 11 18:08:28 2009 New Revision: 69188 Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Log: Fix an issue that shows up translating pypy-c-jit. This would be tested by a recursive "z" test, which we don't have (and don't really want) any more... Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/codewriter.py Wed Nov 11 18:08:28 2009 @@ -175,7 +175,8 @@ self.metainterp_sd.info_from_codewriter(portal_code, leave_code, self.class_sizes, - self.list_of_addr2name) + self.list_of_addr2name, + portal_runner_ptr) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -347,10 +348,14 @@ def register_known_function(self, func): if self.rtyper.type_system.name == 'lltypesystem': - if func._obj not in self._functions_addr_seen: - self._functions_addr_seen[func._obj] = True + try: + obj = func._obj + except lltype.DelayedPointer: # probably ll_portal_runner + return + if obj not in self._functions_addr_seen: + self._functions_addr_seen[obj] = True func_addr = llmemory.cast_ptr_to_adr(func) - self.list_of_addr2name.append((func_addr, func._obj._name)) + self.list_of_addr2name.append((func_addr, obj._name)) class BytecodeMaker(object): Modified: pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-log-class-func/pypy/jit/metainterp/pyjitpl.py Wed Nov 11 18:08:28 2009 @@ -1030,12 +1030,13 @@ return True def info_from_codewriter(self, portal_code, leave_code, class_sizes, - list_of_addr2name=[]): + list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes self._addr2name_keys = [key for key, value in list_of_addr2name] self._addr2name_values = [value for key, value in list_of_addr2name] + self._portal_runner_ptr = portal_runner_ptr def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1076,7 +1077,8 @@ # Build the dictionary at run-time. This is needed # because the keys are function/class addresses, so they # can change from run to run. - d = {} + k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr) + d = {k: 'recursive call'} keys = self._addr2name_keys values = self._addr2name_values for i in range(len(keys)): From fijal at codespeak.net Wed Nov 11 18:22:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 18:22:18 +0100 (CET) Subject: [pypy-svn] r69189 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091111172218.1EE04168106@codespeak.net> Author: fijal Date: Wed Nov 11 18:22:17 2009 New Revision: 69189 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py pypy/branch/faster-raise/pypy/interpreter/test/test_pyframe.py Log: (arigo, fijal) A test and a fix for some obscure code. Fix typo Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Wed Nov 11 18:22:17 2009 @@ -458,7 +458,7 @@ if self.w_f_trace is None: raise OperationError(space.w_ValueError, - space.wrap("f_lineo can only be set by a trace function.")) + space.wrap("f_lineno can only be set by a trace function.")) if new_lineno < self.pycode.co_firstlineno: raise OperationError(space.w_ValueError, @@ -553,7 +553,11 @@ else: addr += 1 - f_iblock = self.blockcount + f_iblock = 0 + block = self.lastblock + while block: + f_iblock += 1 + block = block.previous min_iblock = f_iblock + min_delta_iblock if new_lasti > self.last_instr: new_iblock = f_iblock + delta_iblock Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_pyframe.py Wed Nov 11 18:22:17 2009 @@ -42,6 +42,25 @@ origin = g.func_code.co_firstlineno assert g() == [origin+3, origin+4, origin+5] + def test_f_lineno_set(self): + def tracer(f, *args): + def x(f, *args): + if f.f_lineno == origin + 1: + f.f_lineno = origin + 2 + return x + + def function(): + xyz + return 3 + + def g(): + import sys + sys.settrace(tracer) + function() + sys.settrace(None) + origin = function.func_code.co_firstlineno + g() # assert did not crash + def test_f_back(self): import sys def f(): From antocuni at codespeak.net Wed Nov 11 18:22:43 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 18:22:43 +0100 (CET) Subject: [pypy-svn] r69190 - pypy/trunk/pypy/module/sys Message-ID: <20091111172243.2A61D168106@codespeak.net> Author: antocuni Date: Wed Nov 11 18:22:42 2009 New Revision: 69190 Modified: pypy/trunk/pypy/module/sys/version.py Log: it seems that my svn version (1.6.5) uses the new format for .svn, but it does not store a 'format' file with the version inside. Make the code assume the new format is 'format' is not there. Modified: pypy/trunk/pypy/module/sys/version.py ============================================================================== --- pypy/trunk/pypy/module/sys/version.py (original) +++ pypy/trunk/pypy/module/sys/version.py Wed Nov 11 18:22:42 2009 @@ -87,10 +87,15 @@ # to depend on an external 'svn' executable in the path. rev = int(REV) try: - f = open(os.path.join(pypydir, '.svn', 'format'), 'r') - format = int(f.readline().strip()) - f.close() - if format <= 6: # Old XML-format + formatfile = os.path.join(pypydir, '.svn', 'format') + if os.path.exists(formatfile): + f = open(formatfile, 'r') + format = int(f.readline().strip()) + f.close() + oldformat = (format <= 6) # Old XML-format + else: + oldformat = False + if oldformat: f = open(os.path.join(pypydir, '.svn', 'entries'), 'r') for line in f: line = line.strip() From hpk at codespeak.net Wed Nov 11 18:26:21 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 18:26:21 +0100 (CET) Subject: [pypy-svn] r69191 - pypy/branch/py11/pypy/module/__builtin__/test Message-ID: <20091111172621.D2579168106@codespeak.net> Author: hpk Date: Wed Nov 11 18:26:20 2009 New Revision: 69191 Modified: pypy/branch/py11/pypy/module/__builtin__/test/test_import.py Log: fixing another write place triggered by appdirect Modified: pypy/branch/py11/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/branch/py11/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/branch/py11/pypy/module/__builtin__/test/test_import.py Wed Nov 11 18:26:20 2009 @@ -76,7 +76,7 @@ code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) s2 = struct.pack("i", os.stat(str(p.join("x.py")))[stat.ST_MTIME]) - p.join("x.pyc").write(imp.get_magic() + s2 + s3) + p.join("x.pyc").write(imp.get_magic() + s2 + s3, mode='wb') else: w = space.wrap w_modname = w("compiled.x") From cfbolz at codespeak.net Wed Nov 11 18:40:33 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 11 Nov 2009 18:40:33 +0100 (CET) Subject: [pypy-svn] r69192 - pypy/trunk/pypy/translator/backendopt/test Message-ID: <20091111174033.A5101168029@codespeak.net> Author: cfbolz Date: Wed Nov 11 18:40:32 2009 New Revision: 69192 Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Log: a test to check that contains cannot write to anything Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Wed Nov 11 18:40:32 2009 @@ -163,6 +163,21 @@ assert name == "length" assert S1 is S2 + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + assert not result + class TestOOtype(BaseTestCanRaise): type_system = 'ootype' From antocuni at codespeak.net Wed Nov 11 18:43:50 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 18:43:50 +0100 (CET) Subject: [pypy-svn] r69193 - pypy/trunk/pypy/translator/backendopt/test Message-ID: <20091111174350.A1E28168029@codespeak.net> Author: antocuni Date: Wed Nov 11 18:43:49 2009 New Revision: 69193 Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Log: move this test in the base class, as it passes on ootype out of the box Modified: pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/trunk/pypy/translator/backendopt/test/test_writeanalyze.py Wed Nov 11 18:43:49 2009 @@ -130,6 +130,21 @@ result = wa.analyze(fgraph.startblock.operations[0]) assert not result + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + assert not result + class TestLLtype(BaseTestCanRaise): type_system = 'lltype' @@ -163,20 +178,6 @@ assert name == "length" assert S1 is S2 - def test_contains(self): - def g(x, y, z): - l = [x] - return f(l, y, z) - def f(x, y, z): - return y in x - - - t, wa = self.translate(g, [int, int, int]) - ggraph = graphof(t, g) - assert ggraph.startblock.operations[-1].opname == 'direct_call' - - result = wa.analyze(ggraph.startblock.operations[-1]) - assert not result class TestOOtype(BaseTestCanRaise): From hpk at codespeak.net Wed Nov 11 18:54:56 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 18:54:56 +0100 (CET) Subject: [pypy-svn] r69194 - in pypy/trunk: . dotviewer lib-python py py/bin py/bin/win32 py/impl py/impl/cmdline py/impl/code py/impl/compat py/impl/io py/impl/log py/impl/path py/impl/path/gateway py/impl/process py/impl/test py/impl/test/dist py/impl/test/looponfail py/plugin pypy pypy/annotation/test pypy/bin pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/doc/tool pypy/jit pypy/jit/backend pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/tl pypy/jit/tl/spli pypy/jit/tl/test pypy/jit/tl/tla pypy/jit/tool pypy/lang/gameboy/debug pypy/lang/gameboy/profiling pypy/lang/gameboy/profiling/evaluation pypy/lang/gameboy/test pypy/lang/gameboy/tool pypy/lang/js pypy/lang/js/test pypy/lang/js/test/ecma pypy/lang/prolog/interpreter pypy/lang/scheme pypy/lang/scheme/test pypy/lang/smalltalk/test pypy/lang/smalltalk/tool pypy/lib/app_test/ctypes_tests pypy/lib/distributed pypy/lib/test2 pypy/module/__builtin__/test pypy/module/_codecs/test pypy/module/_file/test pypy/module/_sre/test pypy/module/bz2/test pypy/module/pypyjit/test pypy/module/sys/test pypy/objspace/std/test pypy/rlib/parsing pypy/rlib/parsing/test pypy/rlib/rsdl pypy/rlib/rsdl/test pypy/rlib/test pypy/rpython/microbench pypy/rpython/module/test pypy/rpython/test pypy/tool pypy/tool/algo/test pypy/tool/bench pypy/tool/pytest pypy/tool/pytest/test pypy/tool/rest pypy/tool/test pypy/translator pypy/translator/benchmark pypy/translator/c pypy/translator/c/test pypy/translator/cli pypy/translator/cli/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/jvm pypy/translator/microbench/pybench pypy/translator/platform pypy/translator/platform/test pypy/translator/sandbox pypy/translator/sandbox/test pypy/translator/test pypy/translator/tool Message-ID: <20091111175456.B022A16802D@codespeak.net> Author: hpk Date: Wed Nov 11 18:54:49 2009 New Revision: 69194 Added: pypy/trunk/py/ pypy/trunk/py/__init__.py pypy/trunk/py/apipkg.py pypy/trunk/py/bin/ pypy/trunk/py/bin/_findpy.py pypy/trunk/py/bin/env.cmd pypy/trunk/py/bin/env.py pypy/trunk/py/bin/py.cleanup pypy/trunk/py/bin/py.convert_unittest pypy/trunk/py/bin/py.countloc pypy/trunk/py/bin/py.lookup pypy/trunk/py/bin/py.svnwcrevert pypy/trunk/py/bin/py.test pypy/trunk/py/bin/py.which pypy/trunk/py/bin/win32/ pypy/trunk/py/bin/win32/py.cleanup.cmd pypy/trunk/py/bin/win32/py.convert_unittest.cmd pypy/trunk/py/bin/win32/py.countloc.cmd pypy/trunk/py/bin/win32/py.lookup.cmd pypy/trunk/py/bin/win32/py.svnwcrevert.cmd pypy/trunk/py/bin/win32/py.test.cmd pypy/trunk/py/bin/win32/py.which.cmd pypy/trunk/py/impl/ pypy/trunk/py/impl/__init__.py pypy/trunk/py/impl/_com.py pypy/trunk/py/impl/_metainfo.py pypy/trunk/py/impl/builtin.py pypy/trunk/py/impl/cmdline/ pypy/trunk/py/impl/cmdline/__init__.py pypy/trunk/py/impl/cmdline/pycleanup.py pypy/trunk/py/impl/cmdline/pyconvert_unittest.py pypy/trunk/py/impl/cmdline/pycountloc.py pypy/trunk/py/impl/cmdline/pylookup.py pypy/trunk/py/impl/cmdline/pysvnwcrevert.py pypy/trunk/py/impl/cmdline/pytest.py pypy/trunk/py/impl/cmdline/pywhich.py pypy/trunk/py/impl/code/ pypy/trunk/py/impl/code/__init__.py pypy/trunk/py/impl/code/_assertionnew.py pypy/trunk/py/impl/code/_assertionold.py pypy/trunk/py/impl/code/assertion.py pypy/trunk/py/impl/code/code.py pypy/trunk/py/impl/code/oldmagic.py pypy/trunk/py/impl/code/oldmagic2.py pypy/trunk/py/impl/code/source.py pypy/trunk/py/impl/compat/ pypy/trunk/py/impl/compat/__init__.py pypy/trunk/py/impl/compat/dep_doctest.py pypy/trunk/py/impl/compat/dep_optparse.py pypy/trunk/py/impl/compat/dep_subprocess.py pypy/trunk/py/impl/compat/dep_textwrap.py pypy/trunk/py/impl/error.py pypy/trunk/py/impl/io/ pypy/trunk/py/impl/io/__init__.py pypy/trunk/py/impl/io/capture.py pypy/trunk/py/impl/io/terminalwriter.py pypy/trunk/py/impl/log/ pypy/trunk/py/impl/log/__init__.py pypy/trunk/py/impl/log/log.py pypy/trunk/py/impl/log/warning.py pypy/trunk/py/impl/path/ pypy/trunk/py/impl/path/__init__.py pypy/trunk/py/impl/path/cacheutil.py pypy/trunk/py/impl/path/common.py pypy/trunk/py/impl/path/gateway/ pypy/trunk/py/impl/path/gateway/__init__.py pypy/trunk/py/impl/path/gateway/channeltest.py pypy/trunk/py/impl/path/gateway/channeltest2.py pypy/trunk/py/impl/path/gateway/remotepath.py pypy/trunk/py/impl/path/local.py pypy/trunk/py/impl/path/local.py.orig pypy/trunk/py/impl/path/svnurl.py pypy/trunk/py/impl/path/svnwc.py pypy/trunk/py/impl/process/ pypy/trunk/py/impl/process/__init__.py pypy/trunk/py/impl/process/cmdexec.py pypy/trunk/py/impl/process/forkedfunc.py pypy/trunk/py/impl/process/killproc.py pypy/trunk/py/impl/std.py pypy/trunk/py/impl/test/ pypy/trunk/py/impl/test/__init__.py pypy/trunk/py/impl/test/cmdline.py pypy/trunk/py/impl/test/collect.py pypy/trunk/py/impl/test/compat.py pypy/trunk/py/impl/test/config.py pypy/trunk/py/impl/test/conftesthandle.py pypy/trunk/py/impl/test/defaultconftest.py pypy/trunk/py/impl/test/dist/ pypy/trunk/py/impl/test/dist/__init__.py pypy/trunk/py/impl/test/dist/dsession.py pypy/trunk/py/impl/test/dist/gwmanage.py pypy/trunk/py/impl/test/dist/mypickle.py pypy/trunk/py/impl/test/dist/nodemanage.py pypy/trunk/py/impl/test/dist/txnode.py pypy/trunk/py/impl/test/funcargs.py pypy/trunk/py/impl/test/looponfail/ pypy/trunk/py/impl/test/looponfail/__init__.py pypy/trunk/py/impl/test/looponfail/remote.py pypy/trunk/py/impl/test/looponfail/util.py pypy/trunk/py/impl/test/outcome.py pypy/trunk/py/impl/test/parseopt.py pypy/trunk/py/impl/test/pluginmanager.py pypy/trunk/py/impl/test/pycollect.py pypy/trunk/py/impl/test/session.py pypy/trunk/py/impl/xmlgen.py pypy/trunk/py/plugin/ pypy/trunk/py/plugin/__init__.py pypy/trunk/py/plugin/hookspec.py pypy/trunk/py/plugin/pytest__pytest.py pypy/trunk/py/plugin/pytest_assertion.py pypy/trunk/py/plugin/pytest_capture.py pypy/trunk/py/plugin/pytest_default.py pypy/trunk/py/plugin/pytest_doctest.py pypy/trunk/py/plugin/pytest_figleaf.py pypy/trunk/py/plugin/pytest_helpconfig.py pypy/trunk/py/plugin/pytest_hooklog.py pypy/trunk/py/plugin/pytest_mark.py pypy/trunk/py/plugin/pytest_monkeypatch.py pypy/trunk/py/plugin/pytest_nose.py pypy/trunk/py/plugin/pytest_pastebin.py pypy/trunk/py/plugin/pytest_pdb.py pypy/trunk/py/plugin/pytest_pylint.py pypy/trunk/py/plugin/pytest_pytester.py pypy/trunk/py/plugin/pytest_recwarn.py pypy/trunk/py/plugin/pytest_restdoc.py pypy/trunk/py/plugin/pytest_resultlog.py pypy/trunk/py/plugin/pytest_runner.py pypy/trunk/py/plugin/pytest_skipping.py pypy/trunk/py/plugin/pytest_terminal.py pypy/trunk/py/plugin/pytest_tmpdir.py pypy/trunk/py/plugin/pytest_unittest.py pypy/trunk/pypy/tool/difftime.py pypy/trunk/pypy/tool/rest/ pypy/trunk/pypy/tool/rest/__init__.py pypy/trunk/pypy/tool/rest/convert.py pypy/trunk/pypy/tool/rest/directive.py pypy/trunk/pypy/tool/rest/rest.py pypy/trunk/pypy/tool/rest/rst.py Removed: pypy/trunk/pytest_resultlog.py Modified: pypy/trunk/ (props changed) pypy/trunk/dotviewer/conftest.py pypy/trunk/lib-python/conftest.py pypy/trunk/pypy/annotation/test/autopath.py pypy/trunk/pypy/bin/autopath.py pypy/trunk/pypy/bin/py.py pypy/trunk/pypy/config/autopath.py pypy/trunk/pypy/config/config.py pypy/trunk/pypy/config/makerestdoc.py pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/config/test/test_makerestdoc.py pypy/trunk/pypy/config/test/test_pypyoption.py pypy/trunk/pypy/conftest.py pypy/trunk/pypy/doc/config/autopath.py pypy/trunk/pypy/doc/config/generate.py pypy/trunk/pypy/doc/config/makemodules.py pypy/trunk/pypy/doc/confrest.py pypy/trunk/pypy/doc/confrest_oldpy.py pypy/trunk/pypy/doc/conftest.py pypy/trunk/pypy/doc/statistic/confrest.py pypy/trunk/pypy/doc/test_redirections.py pypy/trunk/pypy/doc/tool/makeref.py pypy/trunk/pypy/doc/tool/mydot.py pypy/trunk/pypy/jit/backend/autopath.py pypy/trunk/pypy/jit/backend/test/conftest.py pypy/trunk/pypy/jit/backend/x86/autopath.py pypy/trunk/pypy/jit/conftest.py pypy/trunk/pypy/jit/tl/autopath.py pypy/trunk/pypy/jit/tl/conftest.py pypy/trunk/pypy/jit/tl/spli/autopath.py pypy/trunk/pypy/jit/tl/targettlc.py pypy/trunk/pypy/jit/tl/targettlr.py pypy/trunk/pypy/jit/tl/test/test_pypyjit.py pypy/trunk/pypy/jit/tl/tla/targettla.py pypy/trunk/pypy/jit/tl/tla/tla_assembler.py pypy/trunk/pypy/jit/tool/autopath.py pypy/trunk/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py pypy/trunk/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py pypy/trunk/pypy/lang/gameboy/profiling/gameboyTest.py pypy/trunk/pypy/lang/gameboy/test/test_cartridge.py pypy/trunk/pypy/lang/gameboy/test/test_rom.py pypy/trunk/pypy/lang/gameboy/tool/autopath.py pypy/trunk/pypy/lang/js/autopath.py pypy/trunk/pypy/lang/js/jsparser.py pypy/trunk/pypy/lang/js/test/ecma/conftest.py pypy/trunk/pypy/lang/js/test/test_interactive.py pypy/trunk/pypy/lang/js/test/test_parser.py pypy/trunk/pypy/lang/prolog/interpreter/autopath.py pypy/trunk/pypy/lang/prolog/interpreter/conftest.py pypy/trunk/pypy/lang/prolog/interpreter/interactive.py pypy/trunk/pypy/lang/prolog/interpreter/parsing.py pypy/trunk/pypy/lang/scheme/autopath.py pypy/trunk/pypy/lang/scheme/execution.py pypy/trunk/pypy/lang/scheme/test/test_interactive.py pypy/trunk/pypy/lang/smalltalk/test/test_miniimage.py pypy/trunk/pypy/lang/smalltalk/tool/analyseimage.py pypy/trunk/pypy/lang/smalltalk/tool/autopath.py pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py pypy/trunk/pypy/lib/distributed/socklayer.py pypy/trunk/pypy/lib/test2/autopath.py pypy/trunk/pypy/module/__builtin__/test/autopath.py pypy/trunk/pypy/module/__builtin__/test/test_import.py pypy/trunk/pypy/module/_codecs/test/autopath.py pypy/trunk/pypy/module/_file/test/test_file_extra.py pypy/trunk/pypy/module/_sre/test/autopath.py pypy/trunk/pypy/module/_sre/test/test_app_sre.py pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/trunk/pypy/module/bz2/test/test_bz2_file.py pypy/trunk/pypy/module/pypyjit/test/conftest.py pypy/trunk/pypy/module/sys/test/autopath.py pypy/trunk/pypy/objspace/std/test/test_complexobject.py pypy/trunk/pypy/rlib/parsing/ebnfparse.py pypy/trunk/pypy/rlib/parsing/makepackrat.py pypy/trunk/pypy/rlib/parsing/regexparse.py pypy/trunk/pypy/rlib/parsing/test/autopath.py pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py pypy/trunk/pypy/rlib/parsing/test/test_pythonparse.py pypy/trunk/pypy/rlib/rsdl/eci.py pypy/trunk/pypy/rlib/rsdl/test/autopath.py pypy/trunk/pypy/rlib/test/test_listsort.py pypy/trunk/pypy/rpython/microbench/autopath.py pypy/trunk/pypy/rpython/module/test/test_ll_os_path.py pypy/trunk/pypy/rpython/test/test_rbuiltin.py pypy/trunk/pypy/test_all.py pypy/trunk/pypy/tool/algo/test/autopath.py pypy/trunk/pypy/tool/ansi_mandelbrot.py pypy/trunk/pypy/tool/ansi_print.py pypy/trunk/pypy/tool/autopath.py pypy/trunk/pypy/tool/bench/pypyresult.py pypy/trunk/pypy/tool/genstatistic.py pypy/trunk/pypy/tool/option.py pypy/trunk/pypy/tool/pytest/appsupport.py pypy/trunk/pypy/tool/pytest/autopath.py pypy/trunk/pypy/tool/pytest/genreportdata.py pypy/trunk/pypy/tool/pytest/htmlreport.py pypy/trunk/pypy/tool/pytest/test/test_new_count.py pypy/trunk/pypy/tool/statistic_over_time.py pypy/trunk/pypy/tool/test/autopath.py pypy/trunk/pypy/tool/test/test_conftest1.py pypy/trunk/pypy/tool/test/test_pytestsupport.py pypy/trunk/pypy/tool/udir.py pypy/trunk/pypy/translator/autopath.py pypy/trunk/pypy/translator/benchmark/autopath.py pypy/trunk/pypy/translator/benchmark/benchmarks.py pypy/trunk/pypy/translator/benchmark/jitbench.py pypy/trunk/pypy/translator/c/autopath.py pypy/trunk/pypy/translator/c/test/autopath.py pypy/trunk/pypy/translator/c/test/test_extfunc.py pypy/trunk/pypy/translator/cli/conftest.py pypy/trunk/pypy/translator/cli/test/autopath.py pypy/trunk/pypy/translator/driver.py pypy/trunk/pypy/translator/goal/autopath.py pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py pypy/trunk/pypy/translator/goal/targetgbimplementation.py pypy/trunk/pypy/translator/goal/targetgbrom4.py pypy/trunk/pypy/translator/goal/targetpreimportedpypy.py pypy/trunk/pypy/translator/goal/targetpypystandalone.py pypy/trunk/pypy/translator/goal/test2/autopath.py pypy/trunk/pypy/translator/goal/translate.py pypy/trunk/pypy/translator/interactive.py pypy/trunk/pypy/translator/jvm/conftest.py pypy/trunk/pypy/translator/jvm/genjvm.py pypy/trunk/pypy/translator/microbench/pybench/autopath.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/test/test_darwin.py pypy/trunk/pypy/translator/platform/test/test_maemo.py pypy/trunk/pypy/translator/sandbox/autopath.py pypy/trunk/pypy/translator/sandbox/test/autopath.py pypy/trunk/pypy/translator/test/autopath.py pypy/trunk/pypy/translator/test/test_driver.py pypy/trunk/pypy/translator/tool/autopath.py Log: merging py11 branch that integrates the py-1.1.0 release verbatim and a py/bin/ directory containing the command line scripts. Modified: pypy/trunk/dotviewer/conftest.py ============================================================================== --- pypy/trunk/dotviewer/conftest.py (original) +++ pypy/trunk/dotviewer/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ import py def pytest_addoption(parser): - group = parser.addgroup("dotviever") + group = parser.getgroup("dotviever") group.addoption('--pygame', action="store_true", dest="pygame", default=False, help="allow interactive tests using Pygame") Modified: pypy/trunk/lib-python/conftest.py ============================================================================== --- pypy/trunk/lib-python/conftest.py (original) +++ pypy/trunk/lib-python/conftest.py Wed Nov 11 18:54:49 2009 @@ -28,7 +28,7 @@ # def pytest_addoption(parser): - group = parser.addgroup("complicance testing options") + group = parser.getgroup("complicance testing options") group.addoption('-T', '--timeout', action="store", type="string", default="1000", dest="timeout", help="fail a test module after the given timeout. " Added: pypy/trunk/py/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +""" +py.test and pylib: rapid testing and development utils + +this module uses apipkg.py for lazy-loading sub modules +and classes. The initpkg-dictionary below specifies +name->value mappings where value can be another namespace +dictionary or an import path. + +(c) Holger Krekel and others, 2009 +""" +version = "1.1.0" + +__version__ = version = version or "1.1.x" +import py.apipkg + +py.apipkg.initpkg(__name__, dict( + # access to all standard lib modules + std = '.impl.std:std', + # access to all posix errno's as classes + error = '.impl.error:error', + + _impldir = '.impl._metainfo:impldir', + _dir = '.impl._metainfo:pydir', + _pydirs = '.impl._metainfo:pydirs', + version = 'py:__version__', # backward compatibility + + _com = { + 'Registry': '.impl._com:Registry', + 'MultiCall': '.impl._com:MultiCall', + 'comregistry': '.impl._com:comregistry', + 'HookRelay': '.impl._com:HookRelay', + }, + cmdline = { + 'pytest': '.impl.cmdline.pytest:main', + 'pylookup': '.impl.cmdline.pylookup:main', + 'pycountloc': '.impl.cmdline.pycountlog:main', + 'pytest': '.impl.test.cmdline:main', + 'pylookup': '.impl.cmdline.pylookup:main', + 'pycountloc': '.impl.cmdline.pycountloc:main', + 'pycleanup': '.impl.cmdline.pycleanup:main', + 'pywhich' : '.impl.cmdline.pywhich:main', + 'pysvnwcrevert' : '.impl.cmdline.pysvnwcrevert:main', + 'pyconvert_unittest' : '.impl.cmdline.pyconvert_unittest:main', + }, + + test = { + # helpers for use from test functions or collectors + '__doc__' : '.impl.test:__doc__', + '_PluginManager' : '.impl.test.pluginmanager:PluginManager', + 'raises' : '.impl.test.outcome:raises', + 'skip' : '.impl.test.outcome:skip', + 'importorskip' : '.impl.test.outcome:importorskip', + 'fail' : '.impl.test.outcome:fail', + 'exit' : '.impl.test.outcome:exit', + # configuration/initialization related test api + 'config' : '.impl.test.config:config_per_process', + 'ensuretemp' : '.impl.test.config:ensuretemp', + 'collect': { + 'Collector' : '.impl.test.collect:Collector', + 'Directory' : '.impl.test.collect:Directory', + 'File' : '.impl.test.collect:File', + 'Item' : '.impl.test.collect:Item', + 'Module' : '.impl.test.pycollect:Module', + 'Class' : '.impl.test.pycollect:Class', + 'Instance' : '.impl.test.pycollect:Instance', + 'Generator' : '.impl.test.pycollect:Generator', + 'Function' : '.impl.test.pycollect:Function', + '_fillfuncargs' : '.impl.test.funcargs:fillfuncargs', + }, + }, + + # hook into the top-level standard library + process = { + '__doc__' : '.impl.process:__doc__', + 'cmdexec' : '.impl.process.cmdexec:cmdexec', + 'kill' : '.impl.process.killproc:kill', + 'ForkedFunc' : '.impl.process.forkedfunc:ForkedFunc', + }, + + path = { + '__doc__' : '.impl.path:__doc__', + 'svnwc' : '.impl.path.svnwc:SvnWCCommandPath', + 'svnurl' : '.impl.path.svnurl:SvnCommandPath', + 'local' : '.impl.path.local:LocalPath', + 'SvnAuth' : '.impl.path.svnwc:SvnAuth', + }, + + # some nice slightly magic APIs + magic = { + 'invoke' : '.impl.code.oldmagic:invoke', + 'revoke' : '.impl.code.oldmagic:revoke', + 'patch' : '.impl.code.oldmagic:patch', + 'revert' : '.impl.code.oldmagic:revert', + 'autopath' : '.impl.path.local:autopath', + 'AssertionError' : '.impl.code.oldmagic2:AssertionError', + }, + + # python inspection/code-generation API + code = { + '__doc__' : '.impl.code:__doc__', + 'compile' : '.impl.code.source:compile_', + 'Source' : '.impl.code.source:Source', + 'Code' : '.impl.code.code:Code', + 'Frame' : '.impl.code.code:Frame', + 'ExceptionInfo' : '.impl.code.code:ExceptionInfo', + 'Traceback' : '.impl.code.code:Traceback', + 'getfslineno' : '.impl.code.source:getfslineno', + 'getrawcode' : '.impl.code.code:getrawcode', + 'patch_builtins' : '.impl.code.code:patch_builtins', + 'unpatch_builtins' : '.impl.code.code:unpatch_builtins', + '_AssertionError' : '.impl.code.assertion:AssertionError', + }, + + # backports and additions of builtins + builtin = { + '__doc__' : '.impl.builtin:__doc__', + 'enumerate' : '.impl.builtin:enumerate', + 'reversed' : '.impl.builtin:reversed', + 'sorted' : '.impl.builtin:sorted', + 'set' : '.impl.builtin:set', + 'frozenset' : '.impl.builtin:frozenset', + 'BaseException' : '.impl.builtin:BaseException', + 'GeneratorExit' : '.impl.builtin:GeneratorExit', + 'print_' : '.impl.builtin:print_', + '_reraise' : '.impl.builtin:_reraise', + '_tryimport' : '.impl.builtin:_tryimport', + 'exec_' : '.impl.builtin:exec_', + '_basestring' : '.impl.builtin:_basestring', + '_totext' : '.impl.builtin:_totext', + '_isbytes' : '.impl.builtin:_isbytes', + '_istext' : '.impl.builtin:_istext', + '_getimself' : '.impl.builtin:_getimself', + '_getfuncdict' : '.impl.builtin:_getfuncdict', + 'builtins' : '.impl.builtin:builtins', + 'execfile' : '.impl.builtin:execfile', + 'callable' : '.impl.builtin:callable', + }, + + # input-output helping + io = { + '__doc__' : '.impl.io:__doc__', + 'dupfile' : '.impl.io.capture:dupfile', + 'TextIO' : '.impl.io.capture:TextIO', + 'BytesIO' : '.impl.io.capture:BytesIO', + 'FDCapture' : '.impl.io.capture:FDCapture', + 'StdCapture' : '.impl.io.capture:StdCapture', + 'StdCaptureFD' : '.impl.io.capture:StdCaptureFD', + 'TerminalWriter' : '.impl.io.terminalwriter:TerminalWriter', + }, + + # small and mean xml/html generation + xml = { + '__doc__' : '.impl.xmlgen:__doc__', + 'html' : '.impl.xmlgen:html', + 'Tag' : '.impl.xmlgen:Tag', + 'raw' : '.impl.xmlgen:raw', + 'Namespace' : '.impl.xmlgen:Namespace', + 'escape' : '.impl.xmlgen:escape', + }, + + log = { + # logging API ('producers' and 'consumers' connected via keywords) + '__doc__' : '.impl.log:__doc__', + '_apiwarn' : '.impl.log.warning:_apiwarn', + 'Producer' : '.impl.log.log:Producer', + 'setconsumer' : '.impl.log.log:setconsumer', + '_setstate' : '.impl.log.log:setstate', + '_getstate' : '.impl.log.log:getstate', + 'Path' : '.impl.log.log:Path', + 'STDOUT' : '.impl.log.log:STDOUT', + 'STDERR' : '.impl.log.log:STDERR', + 'Syslog' : '.impl.log.log:Syslog', + }, + + # compatibility modules (deprecated) + compat = { + '__doc__' : '.impl.compat:__doc__', + 'doctest' : '.impl.compat.dep_doctest:doctest', + 'optparse' : '.impl.compat.dep_optparse:optparse', + 'textwrap' : '.impl.compat.dep_textwrap:textwrap', + 'subprocess' : '.impl.compat.dep_subprocess:subprocess', + }, +)) Added: pypy/trunk/py/apipkg.py ============================================================================== --- (empty file) +++ pypy/trunk/py/apipkg.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,69 @@ +""" +apipkg: control the exported namespace of a python package. + +see http://pypi.python.org/pypi/apipkg + +(c) holger krekel, 2009 - MIT license +""" +import sys +from types import ModuleType + +__version__ = "1.0b2" + +def initpkg(pkgname, exportdefs): + """ initialize given package from the export definitions. """ + mod = ApiModule(pkgname, exportdefs, implprefix=pkgname) + oldmod = sys.modules[pkgname] + mod.__file__ = getattr(oldmod, '__file__', None) + mod.__version__ = getattr(oldmod, '__version__', None) + mod.__path__ = getattr(oldmod, '__path__', None) + sys.modules[pkgname] = mod + +def importobj(modpath, attrname): + module = __import__(modpath, None, None, ['__doc__']) + return getattr(module, attrname) + +class ApiModule(ModuleType): + def __init__(self, name, importspec, implprefix=None): + self.__name__ = name + self.__all__ = list(importspec) + self.__map__ = {} + self.__implprefix__ = implprefix or name + for name, importspec in importspec.items(): + if isinstance(importspec, dict): + subname = '%s.%s'%(self.__name__, name) + apimod = ApiModule(subname, importspec, implprefix) + sys.modules[subname] = apimod + setattr(self, name, apimod) + else: + modpath, attrname = importspec.split(':') + if modpath[0] == '.': + modpath = implprefix + modpath + if name == '__doc__': + self.__doc__ = importobj(modpath, attrname) + else: + self.__map__[name] = (modpath, attrname) + + def __repr__(self): + return '' % (self.__name__,) + + def __getattr__(self, name): + try: + modpath, attrname = self.__map__[name] + except KeyError: + raise AttributeError(name) + else: + result = importobj(modpath, attrname) + setattr(self, name, result) + del self.__map__[name] + return result + + def __dict__(self): + # force all the content of the module to be loaded when __dict__ is read + dictdescr = ModuleType.__dict__['__dict__'] + dict = dictdescr.__get__(self) + if dict is not None: + for name in self.__all__: + hasattr(self, name) # force attribute load, ignore errors + return dict + __dict__ = property(__dict__) Added: pypy/trunk/py/bin/_findpy.py ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/_findpy.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# +# find and import a version of 'py' +# +import sys +import os +from os.path import dirname as opd, exists, join, basename, abspath + +def searchpy(current): + while 1: + last = current + initpy = join(current, '__init__.py') + if not exists(initpy): + pydir = join(current, 'py') + # recognize py-package and ensure it is importable + if exists(pydir) and exists(join(pydir, '__init__.py')): + #for p in sys.path: + # if p == current: + # return True + if current != sys.path[0]: # if we are already first, then ok + sys.stderr.write("inserting into sys.path: %s\n" % current) + sys.path.insert(0, current) + return True + current = opd(current) + if last == current: + return False + +if not searchpy(abspath(os.curdir)): + if not searchpy(opd(abspath(sys.argv[0]))): + if not searchpy(opd(__file__)): + pass # let's hope it is just on sys.path + +import py + +if __name__ == '__main__': + print ("py lib is at %s" % py.__file__) Added: pypy/trunk/py/bin/env.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/env.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +for /F "usebackq delims=" %%i in (`python "%~dp0\env.py"`) do %%i Added: pypy/trunk/py/bin/env.py ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/env.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import sys, os, os.path + +progpath = sys.argv[0] +packagedir = os.path.dirname(os.path.dirname(os.path.abspath(progpath))) +packagename = os.path.basename(packagedir) +bindir = os.path.join(packagedir, 'bin') +if sys.platform == 'win32': + bindir = os.path.join(bindir, 'win32') +rootdir = os.path.dirname(packagedir) + +def prepend_path(name, value): + sep = os.path.pathsep + curpath = os.environ.get(name, '') + newpath = [value] + [ x for x in curpath.split(sep) if x and x != value ] + return setenv(name, sep.join(newpath)) + +def setenv(name, value): + shell = os.environ.get('SHELL', '') + comspec = os.environ.get('COMSPEC', '') + if shell.endswith('csh'): + cmd = 'setenv %s "%s"' % (name, value) + elif shell.endswith('sh'): + cmd = '%s="%s"; export %s' % (name, value, name) + elif comspec.endswith('cmd.exe'): + cmd = 'set %s=%s' % (name, value) + else: + assert False, 'Shell not supported.' + return cmd + +print(prepend_path('PATH', bindir)) +print(prepend_path('PYTHONPATH', rootdir)) Added: pypy/trunk/py/bin/py.cleanup ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.cleanup Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pycleanup() \ No newline at end of file Added: pypy/trunk/py/bin/py.convert_unittest ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.convert_unittest Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pyconvert_unittest() \ No newline at end of file Added: pypy/trunk/py/bin/py.countloc ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.countloc Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pycountloc() \ No newline at end of file Added: pypy/trunk/py/bin/py.lookup ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.lookup Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pylookup() \ No newline at end of file Added: pypy/trunk/py/bin/py.svnwcrevert ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.svnwcrevert Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pysvnwcrevert() \ No newline at end of file Added: pypy/trunk/py/bin/py.test ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.test Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pytest() \ No newline at end of file Added: pypy/trunk/py/bin/py.which ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/py.which Wed Nov 11 18:54:49 2009 @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from _findpy import py +py.cmdline.pywhich() \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.cleanup.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.cleanup.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.cleanup" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.convert_unittest.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.convert_unittest.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.convert_unittest" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.countloc.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.countloc.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.countloc" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.lookup.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.lookup.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.lookup" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.svnwcrevert.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.svnwcrevert.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.svnwcrevert" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.test.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.test.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.test" %* \ No newline at end of file Added: pypy/trunk/py/bin/win32/py.which.cmd ============================================================================== --- (empty file) +++ pypy/trunk/py/bin/win32/py.which.cmd Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ + at echo off +python "%~dp0\..\py.which" %* \ No newline at end of file Added: pypy/trunk/py/impl/__init__.py ============================================================================== Added: pypy/trunk/py/impl/_com.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/_com.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,125 @@ +""" +py lib plugins and plugin call management +""" + +import py +import inspect + +__all__ = ['Registry', 'MultiCall', 'comregistry', 'HookRelay'] + +class MultiCall: + """ execute a call into multiple python functions/methods. """ + + def __init__(self, methods, kwargs, firstresult=False): + self.methods = methods[:] + self.kwargs = kwargs.copy() + self.kwargs['__multicall__'] = self + self.results = [] + self.firstresult = firstresult + + def __repr__(self): + status = "%d results, %d meths" % (len(self.results), len(self.methods)) + return "" %(status, self.kwargs) + + def execute(self): + while self.methods: + method = self.methods.pop() + kwargs = self.getkwargs(method) + res = method(**kwargs) + if res is not None: + self.results.append(res) + if self.firstresult: + return res + if not self.firstresult: + return self.results + + def getkwargs(self, method): + kwargs = {} + for argname in varnames(method): + try: + kwargs[argname] = self.kwargs[argname] + except KeyError: + pass # might be optional param + return kwargs + +def varnames(func): + ismethod = inspect.ismethod(func) + rawcode = py.code.getrawcode(func) + try: + return rawcode.co_varnames[ismethod:] + except AttributeError: + return () + +class Registry: + """ + Manage Plugins: register/unregister call calls to plugins. + """ + def __init__(self, plugins=None): + if plugins is None: + plugins = [] + self._plugins = plugins + + def register(self, plugin): + assert not isinstance(plugin, str) + assert not plugin in self._plugins + self._plugins.append(plugin) + + def unregister(self, plugin): + self._plugins.remove(plugin) + + def isregistered(self, plugin): + return plugin in self._plugins + + def __iter__(self): + return iter(self._plugins) + + def listattr(self, attrname, plugins=None, extra=(), reverse=False): + l = [] + if plugins is None: + plugins = self._plugins + candidates = list(plugins) + list(extra) + for plugin in candidates: + try: + l.append(getattr(plugin, attrname)) + except AttributeError: + continue + if reverse: + l.reverse() + return l + +class HookRelay: + def __init__(self, hookspecs, registry): + self._hookspecs = hookspecs + self._registry = registry + for name, method in vars(hookspecs).items(): + if name[:1] != "_": + setattr(self, name, self._makecall(name)) + + def _makecall(self, name, extralookup=None): + hookspecmethod = getattr(self._hookspecs, name) + firstresult = getattr(hookspecmethod, 'firstresult', False) + return HookCaller(self, name, firstresult=firstresult, + extralookup=extralookup) + + def _getmethods(self, name, extralookup=()): + return self._registry.listattr(name, extra=extralookup) + + def _performcall(self, name, multicall): + return multicall.execute() + +class HookCaller: + def __init__(self, hookrelay, name, firstresult, extralookup=None): + self.hookrelay = hookrelay + self.name = name + self.firstresult = firstresult + self.extralookup = extralookup and [extralookup] or () + + def __repr__(self): + return "" %(self.name,) + + def __call__(self, **kwargs): + methods = self.hookrelay._getmethods(self.name, self.extralookup) + mc = MultiCall(methods, kwargs, firstresult=self.firstresult) + return self.hookrelay._performcall(self.name, mc) + +comregistry = Registry([]) Added: pypy/trunk/py/impl/_metainfo.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/_metainfo.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,9 @@ + +import py + +pydir = py.path.local(py.__file__).dirpath() +impldir = pydir.join("impl") + +# list of all directories beloging to py +assert impldir.relto(pydir) +pydirs = [pydir] Added: pypy/trunk/py/impl/builtin.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/builtin.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,203 @@ +import sys + +try: + reversed = reversed +except NameError: + def reversed(sequence): + """reversed(sequence) -> reverse iterator over values of the sequence + + Return a reverse iterator + """ + if hasattr(sequence, '__reversed__'): + return sequence.__reversed__() + if not hasattr(sequence, '__getitem__'): + raise TypeError("argument to reversed() must be a sequence") + return reversed_iterator(sequence) + + class reversed_iterator(object): + + def __init__(self, seq): + self.seq = seq + self.remaining = len(seq) + + def __iter__(self): + return self + + def next(self): + i = self.remaining + if i > 0: + i -= 1 + item = self.seq[i] + self.remaining = i + return item + raise StopIteration + + def __length_hint__(self): + return self.remaining + +try: + sorted = sorted +except NameError: + builtin_cmp = cmp # need to use cmp as keyword arg + + def sorted(iterable, cmp=None, key=None, reverse=0): + use_cmp = None + if key is not None: + if cmp is None: + def use_cmp(x, y): + return builtin_cmp(x[0], y[0]) + else: + def use_cmp(x, y): + return cmp(x[0], y[0]) + l = [(key(element), element) for element in iterable] + else: + if cmp is not None: + use_cmp = cmp + l = list(iterable) + if use_cmp is not None: + l.sort(use_cmp) + else: + l.sort() + if reverse: + l.reverse() + if key is not None: + return [element for (_, element) in l] + return l + +try: + set, frozenset = set, frozenset +except NameError: + from sets import set, frozenset + +# pass through +enumerate = enumerate + +try: + BaseException = BaseException +except NameError: + BaseException = Exception + +try: + GeneratorExit = GeneratorExit +except NameError: + class GeneratorExit(Exception): + """ This exception is never raised, it is there to make it possible to + write code compatible with CPython 2.5 even in lower CPython + versions.""" + pass + GeneratorExit.__module__ = 'exceptions' + +if sys.version_info >= (3, 0): + exec ("print_ = print ; exec_=exec") + import builtins + + # some backward compatibility helpers + _basestring = str + def _totext(obj, encoding): + if isinstance(obj, bytes): + obj = obj.decode(encoding) + elif not isinstance(obj, str): + obj = str(obj) + return obj + + def _isbytes(x): + return isinstance(x, bytes) + def _istext(x): + return isinstance(x, str) + + def _getimself(function): + return getattr(function, '__self__', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def execfile(fn, globs=None, locs=None): + if globs is None: + back = sys._getframe(1) + globs = back.f_globals + locs = back.f_locals + del back + elif locs is None: + locs = globs + fp = open(fn, "rb") + try: + source = fp.read() + finally: + fp.close() + co = compile(source, fn, "exec", dont_inherit=True) + exec_(co, globs, locs) + + def callable(obj): + return hasattr(obj, "__call__") + +else: + import __builtin__ as builtins + _totext = unicode + _basestring = basestring + execfile = execfile + callable = callable + def _isbytes(x): + return isinstance(x, str) + def _istext(x): + return isinstance(x, unicode) + + def _getimself(function): + return getattr(function, 'im_self', None) + + def _getfuncdict(function): + return getattr(function, "__dict__", None) + + def print_(*args, **kwargs): + """ minimal backport of py3k print statement. """ + sep = ' ' + if 'sep' in kwargs: + sep = kwargs.pop('sep') + end = '\n' + if 'end' in kwargs: + end = kwargs.pop('end') + file = 'file' in kwargs and kwargs.pop('file') or sys.stdout + if kwargs: + args = ", ".join([str(x) for x in kwargs]) + raise TypeError("invalid keyword arguments: %s" % args) + at_start = True + for x in args: + if not at_start: + file.write(sep) + file.write(str(x)) + at_start = False + file.write(end) + + def exec_(obj, globals=None, locals=None): + """ minimal backport of py3k exec statement. """ + if globals is None: + frame = sys._getframe(1) + globals = frame.f_globals + if locals is None: + locals = frame.f_locals + elif locals is None: + locals = globals + exec2(obj, globals, locals) + +if sys.version_info >= (3,0): + exec (""" +def _reraise(cls, val, tb): + assert hasattr(val, '__traceback__') + raise val +""") +else: + exec (""" +def _reraise(cls, val, tb): + raise cls, val, tb +def exec2(obj, globals, locals): + exec obj in globals, locals +""") + +def _tryimport(*names): + """ return the first successfully imported module. """ + assert names + for name in names: + try: + return __import__(name, None, None, '__doc__') + except ImportError: + excinfo = sys.exc_info() + _reraise(*excinfo) Added: pypy/trunk/py/impl/cmdline/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +# Added: pypy/trunk/py/impl/cmdline/pycleanup.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pycleanup.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +"""\ +py.cleanup [PATH] + +Delete pyc file recursively, starting from PATH (which defaults to the current +working directory). Don't follow links and don't recurse into directories with +a ".". +""" +import py + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + parser.add_option("-e", "--remove", dest="ext", default=".pyc", action="store", + help="remove files with the given comma-separated list of extensions" + ) + parser.add_option("-n", "--dryrun", dest="dryrun", default=False, + action="store_true", + help="display would-be-removed filenames" + ) + parser.add_option("-d", action="store_true", dest="removedir", + help="remove empty directories") + (options, args) = parser.parse_args() + if not args: + args = ["."] + ext = options.ext.split(",") + def shouldremove(p): + return p.ext in ext + + for arg in args: + path = py.path.local(arg) + py.builtin.print_("cleaning path", path, "of extensions", ext) + for x in path.visit(shouldremove, lambda x: x.check(dotfile=0, link=0)): + remove(x, options) + if options.removedir: + for x in path.visit(lambda x: x.check(dir=1), + lambda x: x.check(dotfile=0, link=0)): + if not x.listdir(): + remove(x, options) + +def remove(path, options): + if options.dryrun: + py.builtin.print_("would remove", path) + else: + py.builtin.print_("removing", path) + path.remove() + Added: pypy/trunk/py/impl/cmdline/pyconvert_unittest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pyconvert_unittest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,249 @@ +import re +import sys +import parser + +d={} +# d is the dictionary of unittest changes, keyed to the old name +# used by unittest. +# d[old][0] is the new replacement function. +# d[old][1] is the operator you will substitute, or '' if there is none. +# d[old][2] is the possible number of arguments to the unittest +# function. + +# Old Unittest Name new name operator # of args +d['assertRaises'] = ('raises', '', ['Any']) +d['fail'] = ('raise AssertionError', '', [0,1]) +d['assert_'] = ('assert', '', [1,2]) +d['failIf'] = ('assert not', '', [1,2]) +d['assertEqual'] = ('assert', ' ==', [2,3]) +d['failIfEqual'] = ('assert not', ' ==', [2,3]) +d['assertIn'] = ('assert', ' in', [2,3]) +d['assertNotIn'] = ('assert', ' not in', [2,3]) +d['assertNotEqual'] = ('assert', ' !=', [2,3]) +d['failUnlessEqual'] = ('assert', ' ==', [2,3]) +d['assertAlmostEqual'] = ('assert round', ' ==', [2,3,4]) +d['failIfAlmostEqual'] = ('assert not round', ' ==', [2,3,4]) +d['assertNotAlmostEqual'] = ('assert round', ' !=', [2,3,4]) +d['failUnlessAlmostEquals'] = ('assert round', ' ==', [2,3,4]) + +# the list of synonyms +d['failUnlessRaises'] = d['assertRaises'] +d['failUnless'] = d['assert_'] +d['assertEquals'] = d['assertEqual'] +d['assertNotEquals'] = d['assertNotEqual'] +d['assertAlmostEquals'] = d['assertAlmostEqual'] +d['assertNotAlmostEquals'] = d['assertNotAlmostEqual'] + +# set up the regular expressions we will need +leading_spaces = re.compile(r'^(\s*)') # this never fails + +pat = '' +for k in d.keys(): # this complicated pattern to match all unittests + pat += '|' + r'^(\s*)' + 'self.' + k + r'\(' # \tself.whatever( + +old_names = re.compile(pat[1:]) +linesep='\n' # nobody will really try to convert files not read + # in text mode, will they? + + +def blocksplitter(fp): + '''split a file into blocks that are headed by functions to rename''' + + blocklist = [] + blockstring = '' + + for line in fp: + interesting = old_names.match(line) + if interesting : + if blockstring: + blocklist.append(blockstring) + blockstring = line # reset the block + else: + blockstring += line + + blocklist.append(blockstring) + return blocklist + +def rewrite_utest(block): + '''rewrite every block to use the new utest functions''' + + '''returns the rewritten unittest, unless it ran into problems, + in which case it just returns the block unchanged. + ''' + utest = old_names.match(block) + + if not utest: + return block + + old = utest.group(0).lstrip()[5:-1] # the name we want to replace + new = d[old][0] # the name of the replacement function + op = d[old][1] # the operator you will use , or '' if there is none. + possible_args = d[old][2] # a list of the number of arguments the + # unittest function could possibly take. + + if possible_args == ['Any']: # just rename assertRaises & friends + return re.sub('self.'+old, new, block) + + message_pos = possible_args[-1] + # the remaining unittests can have an optional message to print + # when they fail. It is always the last argument to the function. + + try: + indent, argl, trailer = decompose_unittest(old, block) + + except SyntaxError: # but we couldn't parse it! + return block + + argnum = len(argl) + if argnum not in possible_args: + # sanity check - this one isn't real either + return block + + elif argnum == message_pos: + message = argl[-1] + argl = argl[:-1] + else: + message = None + + if argnum is 0 or (argnum is 1 and argnum is message_pos): #unittest fail() + string = '' + if message: + message = ' ' + message + + elif message_pos is 4: # assertAlmostEqual & friends + try: + pos = argl[2].lstrip() + except IndexError: + pos = '7' # default if none is specified + string = '(%s -%s, %s)%s 0' % (argl[0], argl[1], pos, op ) + + else: # assert_, assertEquals and all the rest + string = ' ' + op.join(argl) + + if message: + string = string + ',' + message + + return indent + new + string + trailer + +def decompose_unittest(old, block): + '''decompose the block into its component parts''' + + ''' returns indent, arglist, trailer + indent -- the indentation + arglist -- the arguments to the unittest function + trailer -- any extra junk after the closing paren, such as #commment + ''' + + indent = re.match(r'(\s*)', block).group() + pat = re.search('self.' + old + r'\(', block) + + args, trailer = get_expr(block[pat.end():], ')') + arglist = break_args(args, []) + + if arglist == ['']: # there weren't any + return indent, [], trailer + + for i in range(len(arglist)): + try: + parser.expr(arglist[i].lstrip('\t ')) + except SyntaxError: + if i == 0: + arglist[i] = '(' + arglist[i] + ')' + else: + arglist[i] = ' (' + arglist[i] + ')' + + return indent, arglist, trailer + +def break_args(args, arglist): + '''recursively break a string into a list of arguments''' + try: + first, rest = get_expr(args, ',') + if not rest: + return arglist + [first] + else: + return [first] + break_args(rest, arglist) + except SyntaxError: + return arglist + [args] + +def get_expr(s, char): + '''split a string into an expression, and the rest of the string''' + + pos=[] + for i in range(len(s)): + if s[i] == char: + pos.append(i) + if pos == []: + raise SyntaxError # we didn't find the expected char. Ick. + + for p in pos: + # make the python parser do the hard work of deciding which comma + # splits the string into two expressions + try: + parser.expr('(' + s[:p] + ')') + return s[:p], s[p+1:] + except SyntaxError: # It's not an expression yet + pass + raise SyntaxError # We never found anything that worked. + + +def main(): + import sys + import py + + usage = "usage: %prog [-s [filename ...] | [-i | -c filename ...]]" + optparser = py.std.optparse.OptionParser(usage) + + def select_output (option, opt, value, optparser, **kw): + if hasattr(optparser, 'output'): + optparser.error( + 'Cannot combine -s -i and -c options. Use one only.') + else: + optparser.output = kw['output'] + + optparser.add_option("-s", "--stdout", action="callback", + callback=select_output, + callback_kwargs={'output':'stdout'}, + help="send your output to stdout") + + optparser.add_option("-i", "--inplace", action="callback", + callback=select_output, + callback_kwargs={'output':'inplace'}, + help="overwrite files in place") + + optparser.add_option("-c", "--copy", action="callback", + callback=select_output, + callback_kwargs={'output':'copy'}, + help="copy files ... fn.py --> fn_cp.py") + + options, args = optparser.parse_args() + + output = getattr(optparser, 'output', 'stdout') + + if output in ['inplace', 'copy'] and not args: + optparser.error( + '-i and -c option require at least one filename') + + if not args: + s = '' + for block in blocksplitter(sys.stdin): + s += rewrite_utest(block) + sys.stdout.write(s) + + else: + for infilename in args: # no error checking to see if we can open, etc. + infile = file(infilename) + s = '' + for block in blocksplitter(infile): + s += rewrite_utest(block) + if output == 'inplace': + outfile = file(infilename, 'w+') + elif output == 'copy': # yes, just go clobber any existing .cp + outfile = file (infilename[:-3]+ '_cp.py', 'w+') + else: + outfile = sys.stdout + + outfile.write(s) + + +if __name__ == '__main__': + main() Added: pypy/trunk/py/impl/cmdline/pycountloc.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pycountloc.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# hands on script to compute the non-empty Lines of Code +# for tests and non-test code + +"""\ +py.countloc [PATHS] + +Count (non-empty) lines of python code and number of python files recursively +starting from a list of paths given on the command line (starting from the +current working directory). Distinguish between test files and normal ones and +report them separately. +""" +import py + +def main(): + parser = py.std.optparse.OptionParser(usage=__doc__) + (options, args) = parser.parse_args() + countloc(args) + +def nodot(p): + return p.check(dotfile=0) + +class FileCounter(object): + def __init__(self): + self.file2numlines = {} + self.numlines = 0 + self.numfiles = 0 + + def addrecursive(self, directory, fil="*.py", rec=nodot): + for x in directory.visit(fil, rec): + self.addfile(x) + + def addfile(self, fn, emptylines=False): + if emptylines: + s = len(p.readlines()) + else: + s = 0 + for i in fn.readlines(): + if i.strip(): + s += 1 + self.file2numlines[fn] = s + self.numfiles += 1 + self.numlines += s + + def getnumlines(self, fil): + numlines = 0 + for path, value in self.file2numlines.items(): + if fil(path): + numlines += value + return numlines + + def getnumfiles(self, fil): + numfiles = 0 + for path in self.file2numlines: + if fil(path): + numfiles += 1 + return numfiles + +def get_loccount(locations=None): + if locations is None: + localtions = [py.path.local()] + counter = FileCounter() + for loc in locations: + counter.addrecursive(loc, '*.py', rec=nodot) + + def istestfile(p): + return p.check(fnmatch='test_*.py') + isnottestfile = lambda x: not istestfile(x) + + numfiles = counter.getnumfiles(isnottestfile) + numlines = counter.getnumlines(isnottestfile) + numtestfiles = counter.getnumfiles(istestfile) + numtestlines = counter.getnumlines(istestfile) + + return counter, numfiles, numlines, numtestfiles, numtestlines + +def countloc(paths=None): + if not paths: + paths = ['.'] + locations = [py.path.local(x) for x in paths] + (counter, numfiles, numlines, numtestfiles, + numtestlines) = get_loccount(locations) + + items = counter.file2numlines.items() + items.sort(lambda x,y: cmp(x[1], y[1])) + for x, y in items: + print("%3d %30s" % (y,x)) + + print("%30s %3d" %("number of testfiles", numtestfiles)) + print("%30s %3d" %("number of non-empty testlines", numtestlines)) + print("%30s %3d" %("number of files", numfiles)) + print("%30s %3d" %("number of non-empty lines", numlines)) + Added: pypy/trunk/py/impl/cmdline/pylookup.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pylookup.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +"""\ +py.lookup [search_directory] SEARCH_STRING [options] + +Looks recursively at Python files for a SEARCH_STRING, starting from the +present working directory. Prints the line, with the filename and line-number +prepended.""" + +import sys, os +import py +from py.impl.io.terminalwriter import ansi_print, terminal_width +import re + +def rec(p): + return p.check(dotfile=0) + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-i", "--ignore-case", action="store_true", dest="ignorecase", + help="ignore case distinctions") +parser.add_option("-C", "--context", action="store", type="int", dest="context", + default=0, help="How many lines of output to show") + +def find_indexes(search_line, string): + indexes = [] + before = 0 + while 1: + i = search_line.find(string, before) + if i == -1: + break + indexes.append(i) + before = i + len(string) + return indexes + +def main(): + (options, args) = parser.parse_args() + if len(args) == 2: + search_dir, string = args + search_dir = py.path.local(search_dir) + else: + search_dir = py.path.local() + string = args[0] + if options.ignorecase: + string = string.lower() + for x in search_dir.visit('*.py', rec): + # match filename directly + s = x.relto(search_dir) + if options.ignorecase: + s = s.lower() + if s.find(string) != -1: + sys.stdout.write("%s: filename matches %r" %(x, string) + "\n") + + try: + s = x.read() + except py.error.ENOENT: + pass # whatever, probably broken link (ie emacs lock) + searchs = s + if options.ignorecase: + searchs = s.lower() + if s.find(string) != -1: + lines = s.splitlines() + if options.ignorecase: + searchlines = s.lower().splitlines() + else: + searchlines = lines + for i, (line, searchline) in enumerate(zip(lines, searchlines)): + indexes = find_indexes(searchline, string) + if not indexes: + continue + if not options.context: + sys.stdout.write("%s:%d: " %(x.relto(search_dir), i+1)) + last_index = 0 + for index in indexes: + sys.stdout.write(line[last_index: index]) + ansi_print(line[index: index+len(string)], + file=sys.stdout, esc=31, newline=False) + last_index = index + len(string) + sys.stdout.write(line[last_index:] + "\n") + else: + context = (options.context)/2 + for count in range(max(0, i-context), min(len(lines) - 1, i+context+1)): + print("%s:%d: %s" %(x.relto(search_dir), count+1, lines[count].rstrip())) + print("-" * terminal_width) Added: pypy/trunk/py/impl/cmdline/pysvnwcrevert.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pysvnwcrevert.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,55 @@ +#! /usr/bin/env python +"""\ +py.svnwcrevert [options] WCPATH + +Running this script and then 'svn up' puts the working copy WCPATH in a state +as clean as a fresh check-out. + +WARNING: you'll loose all local changes, obviously! + +This script deletes all files that have been modified +or that svn doesn't explicitly know about, including svn:ignored files +(like .pyc files, hint hint). + +The goal of this script is to leave the working copy with some files and +directories possibly missing, but - most importantly - in a state where +the following 'svn up' won't just crash. +""" + +import sys, py + +def kill(p, root): + print('< %s' % (p.relto(root),)) + p.remove(rec=1) + +def svnwcrevert(path, root=None, precious=[]): + if root is None: + root = path + wcpath = py.path.svnwc(path) + try: + st = wcpath.status() + except ValueError: # typically, "bad char in wcpath" + kill(path, root) + return + for p in path.listdir(): + if p.basename == '.svn' or p.basename in precious: + continue + wcp = py.path.svnwc(p) + if wcp not in st.unchanged and wcp not in st.external: + kill(p, root) + elif p.check(dir=1): + svnwcrevert(p, root) + +# XXX add a functional test + +parser = py.std.optparse.OptionParser(usage=__doc__) +parser.add_option("-p", "--precious", + action="append", dest="precious", default=[], + help="preserve files with this name") + +def main(): + opts, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + svnwcrevert(py.path.local(args[0]), precious=opts.precious) Added: pypy/trunk/py/impl/cmdline/pytest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pytest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,5 @@ +#!/usr/bin/env python +import py + +def main(): + py.test.cmdline.main() Added: pypy/trunk/py/impl/cmdline/pywhich.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/cmdline/pywhich.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +"""\ +py.which [name] + +print the location of the given python module or package name +""" + +import sys + +def main(): + name = sys.argv[1] + try: + mod = __import__(name) + except ImportError: + sys.stderr.write("could not import: " + name + "\n") + else: + try: + location = mod.__file__ + except AttributeError: + sys.stderr.write("module (has no __file__): " + str(mod)) + else: + print(location) Added: pypy/trunk/py/impl/code/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +""" python inspection/code generation API """ Added: pypy/trunk/py/impl/code/_assertionnew.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/_assertionnew.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,337 @@ +""" +Like _assertion.py but using builtin AST. It should replace _assertion.py +eventually. +""" + +import sys +import ast + +import py +from py.impl.code.assertion import _format_explanation, BuiltinAssertionError + + +if sys.platform.startswith("java") and sys.version_info < (2, 5, 2): + # See http://bugs.jython.org/issue1497 + _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict", + "ListComp", "GeneratorExp", "Yield", "Compare", "Call", + "Repr", "Num", "Str", "Attribute", "Subscript", "Name", + "List", "Tuple") + _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign", + "AugAssign", "Print", "For", "While", "If", "With", "Raise", + "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom", + "Exec", "Global", "Expr", "Pass", "Break", "Continue") + _expr_nodes = set(getattr(ast, name) for name in _exprs) + _stmt_nodes = set(getattr(ast, name) for name in _stmts) + def _is_ast_expr(node): + return node.__class__ in _expr_nodes + def _is_ast_stmt(node): + return node.__class__ in _stmt_nodes +else: + def _is_ast_expr(node): + return isinstance(node, ast.expr) + def _is_ast_stmt(node): + return isinstance(node, ast.stmt) + + +class Failure(Exception): + """Error found while interpreting AST.""" + + def __init__(self, explanation=""): + self.cause = sys.exc_info() + self.explanation = explanation + + +def interpret(source, frame, should_fail=False): + mod = ast.parse(source) + visitor = DebugInterpreter(frame) + try: + visitor.visit(mod) + except Failure: + failure = sys.exc_info()[1] + return getfailure(failure) + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --nomagic)") + +def run(offending_line, frame=None): + if frame is None: + frame = py.code.Frame(sys._getframe(1)) + return interpret(offending_line, frame) + +def getfailure(failure): + explanation = _format_explanation(failure.explanation) + value = failure.cause[1] + if str(value): + lines = explanation.splitlines() + if not lines: + lines.append("") + lines[0] += " << %s" % (value,) + explanation = "\n".join(lines) + text = "%s: %s" % (failure.cause[0].__name__, explanation) + if text.startswith("AssertionError: assert "): + text = text[16:] + return text + + +operator_map = { + ast.BitOr : "|", + ast.BitXor : "^", + ast.BitAnd : "&", + ast.LShift : "<<", + ast.RShift : ">>", + ast.Add : "+", + ast.Sub : "-", + ast.Mult : "*", + ast.Div : "/", + ast.FloorDiv : "//", + ast.Mod : "%", + ast.Eq : "==", + ast.NotEq : "!=", + ast.Lt : "<", + ast.LtE : "<=", + ast.Gt : ">", + ast.GtE : ">=", + ast.Is : "is", + ast.IsNot : "is not", + ast.In : "in", + ast.NotIn : "not in" +} + +unary_map = { + ast.Not : "not %s", + ast.Invert : "~%s", + ast.USub : "-%s", + ast.UAdd : "+%s" +} + + +class DebugInterpreter(ast.NodeVisitor): + """Interpret AST nodes to gleam useful debugging information.""" + + def __init__(self, frame): + self.frame = frame + + def generic_visit(self, node): + # Fallback when we don't have a special implementation. + if _is_ast_expr(node): + mod = ast.Expression(node) + co = self._compile(mod) + try: + result = self.frame.eval(co) + except Exception: + raise Failure() + explanation = self.frame.repr(result) + return explanation, result + elif _is_ast_stmt(node): + mod = ast.Module([node]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co) + except Exception: + raise Failure() + return None, None + else: + raise AssertionError("can't handle %s" %(node,)) + + def _compile(self, source, mode="eval"): + return compile(source, "", mode) + + def visit_Expr(self, expr): + return self.visit(expr.value) + + def visit_Module(self, mod): + for stmt in mod.body: + self.visit(stmt) + + def visit_Name(self, name): + explanation, result = self.generic_visit(name) + # See if the name is local. + source = "%r in locals() is not globals()" % (name.id,) + co = self._compile(source) + try: + local = self.frame.eval(co) + except Exception: + # have to assume it isn't + local = False + if not local: + return name.id, result + return explanation, result + + def visit_Compare(self, comp): + left = comp.left + left_explanation, left_result = self.visit(left) + got_result = False + for op, next_op in zip(comp.ops, comp.comparators): + if got_result and not result: + break + next_explanation, next_result = self.visit(next_op) + op_symbol = operator_map[op.__class__] + explanation = "%s %s %s" % (left_explanation, op_symbol, + next_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=next_result) + except Exception: + raise Failure(explanation) + else: + got_result = True + left_explanation, left_result = next_explanation, next_result + return explanation, result + + def visit_BoolOp(self, boolop): + is_or = isinstance(boolop.op, ast.Or) + explanations = [] + for operand in boolop.values: + explanation, result = self.visit(operand) + explanations.append(explanation) + if result == is_or: + break + name = is_or and " or " or " and " + explanation = "(" + name.join(explanations) + ")" + return explanation, result + + def visit_UnaryOp(self, unary): + pattern = unary_map[unary.op.__class__] + operand_explanation, operand_result = self.visit(unary.operand) + explanation = pattern % (operand_explanation,) + co = self._compile(pattern % ("__exprinfo_expr",)) + try: + result = self.frame.eval(co, __exprinfo_expr=operand_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_BinOp(self, binop): + left_explanation, left_result = self.visit(binop.left) + right_explanation, right_result = self.visit(binop.right) + symbol = operator_map[binop.op.__class__] + explanation = "(%s %s %s)" % (left_explanation, symbol, + right_explanation) + source = "__exprinfo_left %s __exprinfo_right" % (symbol,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_left=left_result, + __exprinfo_right=right_result) + except Exception: + raise Failure(explanation) + return explanation, result + + def visit_Call(self, call): + func_explanation, func = self.visit(call.func) + arg_explanations = [] + ns = {"__exprinfo_func" : func} + arguments = [] + for arg in call.args: + arg_explanation, arg_result = self.visit(arg) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + arguments.append(arg_name) + arg_explanations.append(arg_explanation) + for keyword in call.keywords: + arg_explanation, arg_result = self.visit(keyword.value) + arg_name = "__exprinfo_%s" % (len(ns),) + ns[arg_name] = arg_result + keyword_source = "%s=%%s" % (keyword.id) + arguments.append(keyword_source % (arg_name,)) + arg_explanations.append(keyword_source % (arg_explanation,)) + if call.starargs: + arg_explanation, arg_result = self.visit(call.starargs) + arg_name = "__exprinfo_star" + ns[arg_name] = arg_result + arguments.append("*%s" % (arg_name,)) + arg_explanations.append("*%s" % (arg_explanation,)) + if call.kwargs: + arg_explanation, arg_result = self.visit(call.kwargs) + arg_name = "__exprinfo_kwds" + ns[arg_name] = arg_result + arguments.append("**%s" % (arg_name,)) + arg_explanations.append("**%s" % (arg_explanation,)) + args_explained = ", ".join(arg_explanations) + explanation = "%s(%s)" % (func_explanation, args_explained) + args = ", ".join(arguments) + source = "__exprinfo_func(%s)" % (args,) + co = self._compile(source) + try: + result = self.frame.eval(co, **ns) + except Exception: + raise Failure(explanation) + # Only show result explanation if it's not a builtin call or returns a + # bool. + if not isinstance(call.func, ast.Name) or \ + not self._is_builtin_name(call.func): + source = "isinstance(__exprinfo_value, bool)" + co = self._compile(source) + try: + is_bool = self.frame.eval(co, __exprinfo_value=result) + except Exception: + is_bool = False + if not is_bool: + pattern = "%s\n{%s = %s\n}" + rep = self.frame.repr(result) + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def _is_builtin_name(self, name): + pattern = "%r not in globals() and %r not in locals()" + source = pattern % (name.id, name.id) + co = self._compile(source) + try: + return self.frame.eval(co) + except Exception: + return False + + def visit_Attribute(self, attr): + if not isinstance(attr.ctx, ast.Load): + return self.generic_visit(attr) + source_explanation, source_result = self.visit(attr.value) + explanation = "%s.%s" % (source_explanation, attr.attr) + source = "__exprinfo_expr.%s" % (attr.attr,) + co = self._compile(source) + try: + result = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + raise Failure(explanation) + # Check if the attr is from an instance. + source = "%r in getattr(__exprinfo_expr, '__dict__', {})" + source = source % (attr.attr,) + co = self._compile(source) + try: + from_instance = self.frame.eval(co, __exprinfo_expr=source_result) + except Exception: + from_instance = True + if from_instance: + rep = self.frame.repr(result) + pattern = "%s\n{%s = %s\n}" + explanation = pattern % (rep, rep, explanation) + return explanation, result + + def visit_Assert(self, assrt): + test_explanation, test_result = self.visit(assrt.test) + if test_explanation.startswith("False\n{False =") and \ + test_explanation.endswith("\n"): + test_explanation = test_explanation[15:-2] + explanation = "assert %s" % (test_explanation,) + if not test_result: + try: + raise BuiltinAssertionError + except Exception: + raise Failure(explanation) + return explanation, test_result + + def visit_Assign(self, assign): + value_explanation, value_result = self.visit(assign.value) + explanation = "... = %s" % (value_explanation,) + name = ast.Name("__exprinfo_expr", ast.Load(), assign.value.lineno, + assign.value.col_offset) + new_assign = ast.Assign(assign.targets, name, assign.lineno, + assign.col_offset) + mod = ast.Module([new_assign]) + co = self._compile(mod, "exec") + try: + self.frame.exec_(co, __exprinfo_expr=value_result) + except Exception: + raise Failure(explanation) + return explanation, value_result Added: pypy/trunk/py/impl/code/_assertionold.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/_assertionold.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,558 @@ +import py +import sys, inspect +from compiler import parse, ast, pycodegen +from py.impl.code.assertion import BuiltinAssertionError, _format_explanation + +passthroughex = (KeyboardInterrupt, SystemExit, MemoryError) + +class Failure: + def __init__(self, node): + self.exc, self.value, self.tb = sys.exc_info() + self.node = node + +class View(object): + """View base class. + + If C is a subclass of View, then C(x) creates a proxy object around + the object x. The actual class of the proxy is not C in general, + but a *subclass* of C determined by the rules below. To avoid confusion + we call view class the class of the proxy (a subclass of C, so of View) + and object class the class of x. + + Attributes and methods not found in the proxy are automatically read on x. + Other operations like setting attributes are performed on the proxy, as + determined by its view class. The object x is available from the proxy + as its __obj__ attribute. + + The view class selection is determined by the __view__ tuples and the + optional __viewkey__ method. By default, the selected view class is the + most specific subclass of C whose __view__ mentions the class of x. + If no such subclass is found, the search proceeds with the parent + object classes. For example, C(True) will first look for a subclass + of C with __view__ = (..., bool, ...) and only if it doesn't find any + look for one with __view__ = (..., int, ...), and then ..., object,... + If everything fails the class C itself is considered to be the default. + + Alternatively, the view class selection can be driven by another aspect + of the object x, instead of the class of x, by overriding __viewkey__. + See last example at the end of this module. + """ + + _viewcache = {} + __view__ = () + + def __new__(rootclass, obj, *args, **kwds): + self = object.__new__(rootclass) + self.__obj__ = obj + self.__rootclass__ = rootclass + key = self.__viewkey__() + try: + self.__class__ = self._viewcache[key] + except KeyError: + self.__class__ = self._selectsubclass(key) + return self + + def __getattr__(self, attr): + # attributes not found in the normal hierarchy rooted on View + # are looked up in the object's real class + return getattr(self.__obj__, attr) + + def __viewkey__(self): + return self.__obj__.__class__ + + def __matchkey__(self, key, subclasses): + if inspect.isclass(key): + keys = inspect.getmro(key) + else: + keys = [key] + for key in keys: + result = [C for C in subclasses if key in C.__view__] + if result: + return result + return [] + + def _selectsubclass(self, key): + subclasses = list(enumsubclasses(self.__rootclass__)) + for C in subclasses: + if not isinstance(C.__view__, tuple): + C.__view__ = (C.__view__,) + choices = self.__matchkey__(key, subclasses) + if not choices: + return self.__rootclass__ + elif len(choices) == 1: + return choices[0] + else: + # combine the multiple choices + return type('?', tuple(choices), {}) + + def __repr__(self): + return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__) + + +def enumsubclasses(cls): + for subcls in cls.__subclasses__(): + for subsubclass in enumsubclasses(subcls): + yield subsubclass + yield cls + + +class Interpretable(View): + """A parse tree node with a few extra methods.""" + explanation = None + + def is_builtin(self, frame): + return False + + def eval(self, frame): + # fall-back for unknown expression nodes + try: + expr = ast.Expression(self.__obj__) + expr.filename = '' + self.__obj__.filename = '' + co = pycodegen.ExpressionCodeGenerator(expr).getCode() + result = frame.eval(co) + except passthroughex: + raise + except: + raise Failure(self) + self.result = result + self.explanation = self.explanation or frame.repr(self.result) + + def run(self, frame): + # fall-back for unknown statement nodes + try: + expr = ast.Module(None, ast.Stmt([self.__obj__])) + expr.filename = '' + co = pycodegen.ModuleCodeGenerator(expr).getCode() + frame.exec_(co) + except passthroughex: + raise + except: + raise Failure(self) + + def nice_explanation(self): + return _format_explanation(self.explanation) + + +class Name(Interpretable): + __view__ = ast.Name + + def is_local(self, frame): + co = compile('%r in locals() is not globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_global(self, frame): + co = compile('%r in globals()' % self.name, '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def is_builtin(self, frame): + co = compile('%r not in locals() and %r not in globals()' % ( + self.name, self.name), '?', 'eval') + try: + return frame.is_true(frame.eval(co)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + super(Name, self).eval(frame) + if not self.is_local(frame): + self.explanation = self.name + +class Compare(Interpretable): + __view__ = ast.Compare + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + for operation, expr2 in self.ops: + if hasattr(self, 'result'): + # shortcutting in chained expressions + if not frame.is_true(self.result): + break + expr2 = Interpretable(expr2) + expr2.eval(frame) + self.explanation = "%s %s %s" % ( + expr.explanation, operation, expr2.explanation) + co = compile("__exprinfo_left %s __exprinfo_right" % operation, + '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_left=expr.result, + __exprinfo_right=expr2.result) + except passthroughex: + raise + except: + raise Failure(self) + expr = expr2 + +class And(Interpretable): + __view__ = ast.And + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if not frame.is_true(expr.result): + break + self.explanation = '(' + ' and '.join(explanations) + ')' + +class Or(Interpretable): + __view__ = ast.Or + + def eval(self, frame): + explanations = [] + for expr in self.nodes: + expr = Interpretable(expr) + expr.eval(frame) + explanations.append(expr.explanation) + self.result = expr.result + if frame.is_true(expr.result): + break + self.explanation = '(' + ' or '.join(explanations) + ')' + + +# == Unary operations == +keepalive = [] +for astclass, astpattern in { + ast.Not : 'not __exprinfo_expr', + ast.Invert : '(~__exprinfo_expr)', + }.items(): + + class UnaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + expr = Interpretable(self.expr) + expr.eval(frame) + self.explanation = astpattern.replace('__exprinfo_expr', + expr.explanation) + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(UnaryArith) + +# == Binary operations == +for astclass, astpattern in { + ast.Add : '(__exprinfo_left + __exprinfo_right)', + ast.Sub : '(__exprinfo_left - __exprinfo_right)', + ast.Mul : '(__exprinfo_left * __exprinfo_right)', + ast.Div : '(__exprinfo_left / __exprinfo_right)', + ast.Mod : '(__exprinfo_left % __exprinfo_right)', + ast.Power : '(__exprinfo_left ** __exprinfo_right)', + }.items(): + + class BinaryArith(Interpretable): + __view__ = astclass + + def eval(self, frame, astpattern=astpattern, + co=compile(astpattern, '?', 'eval')): + left = Interpretable(self.left) + left.eval(frame) + right = Interpretable(self.right) + right.eval(frame) + self.explanation = (astpattern + .replace('__exprinfo_left', left .explanation) + .replace('__exprinfo_right', right.explanation)) + try: + self.result = frame.eval(co, __exprinfo_left=left.result, + __exprinfo_right=right.result) + except passthroughex: + raise + except: + raise Failure(self) + + keepalive.append(BinaryArith) + + +class CallFunc(Interpretable): + __view__ = ast.CallFunc + + def is_bool(self, frame): + co = compile('isinstance(__exprinfo_value, bool)', '?', 'eval') + try: + return frame.is_true(frame.eval(co, __exprinfo_value=self.result)) + except passthroughex: + raise + except: + return False + + def eval(self, frame): + node = Interpretable(self.node) + node.eval(frame) + explanations = [] + vars = {'__exprinfo_fn': node.result} + source = '__exprinfo_fn(' + for a in self.args: + if isinstance(a, ast.Keyword): + keyword = a.name + a = a.expr + else: + keyword = None + a = Interpretable(a) + a.eval(frame) + argname = '__exprinfo_%d' % len(vars) + vars[argname] = a.result + if keyword is None: + source += argname + ',' + explanations.append(a.explanation) + else: + source += '%s=%s,' % (keyword, argname) + explanations.append('%s=%s' % (keyword, a.explanation)) + if self.star_args: + star_args = Interpretable(self.star_args) + star_args.eval(frame) + argname = '__exprinfo_star' + vars[argname] = star_args.result + source += '*' + argname + ',' + explanations.append('*' + star_args.explanation) + if self.dstar_args: + dstar_args = Interpretable(self.dstar_args) + dstar_args.eval(frame) + argname = '__exprinfo_kwds' + vars[argname] = dstar_args.result + source += '**' + argname + ',' + explanations.append('**' + dstar_args.explanation) + self.explanation = "%s(%s)" % ( + node.explanation, ', '.join(explanations)) + if source.endswith(','): + source = source[:-1] + source += ')' + co = compile(source, '?', 'eval') + try: + self.result = frame.eval(co, **vars) + except passthroughex: + raise + except: + raise Failure(self) + if not node.is_builtin(frame) or not self.is_bool(frame): + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +class Getattr(Interpretable): + __view__ = ast.Getattr + + def eval(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + co = compile('__exprinfo_expr.%s' % self.attrname, '?', 'eval') + try: + self.result = frame.eval(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + self.explanation = '%s.%s' % (expr.explanation, self.attrname) + # if the attribute comes from the instance, its value is interesting + co = compile('hasattr(__exprinfo_expr, "__dict__") and ' + '%r in __exprinfo_expr.__dict__' % self.attrname, + '?', 'eval') + try: + from_instance = frame.is_true( + frame.eval(co, __exprinfo_expr=expr.result)) + except passthroughex: + raise + except: + from_instance = True + if from_instance: + r = frame.repr(self.result) + self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation) + +# == Re-interpretation of full statements == + +class Assert(Interpretable): + __view__ = ast.Assert + + def run(self, frame): + test = Interpretable(self.test) + test.eval(frame) + # simplify 'assert False where False = ...' + if (test.explanation.startswith('False\n{False = ') and + test.explanation.endswith('\n}')): + test.explanation = test.explanation[15:-2] + # print the result as 'assert ' + self.result = test.result + self.explanation = 'assert ' + test.explanation + if not frame.is_true(test.result): + try: + raise BuiltinAssertionError + except passthroughex: + raise + except: + raise Failure(self) + +class Assign(Interpretable): + __view__ = ast.Assign + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = '... = ' + expr.explanation + # fall-back-run the rest of the assignment + ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr')) + mod = ast.Module(None, ast.Stmt([ass])) + mod.filename = '' + co = pycodegen.ModuleCodeGenerator(mod).getCode() + try: + frame.exec_(co, __exprinfo_expr=expr.result) + except passthroughex: + raise + except: + raise Failure(self) + +class Discard(Interpretable): + __view__ = ast.Discard + + def run(self, frame): + expr = Interpretable(self.expr) + expr.eval(frame) + self.result = expr.result + self.explanation = expr.explanation + +class Stmt(Interpretable): + __view__ = ast.Stmt + + def run(self, frame): + for stmt in self.nodes: + stmt = Interpretable(stmt) + stmt.run(frame) + + +def report_failure(e): + explanation = e.node.nice_explanation() + if explanation: + explanation = ", in: " + explanation + else: + explanation = "" + sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation)) + +def check(s, frame=None): + if frame is None: + import sys + frame = sys._getframe(1) + frame = py.code.Frame(frame) + expr = parse(s, 'eval') + assert isinstance(expr, ast.Expression) + node = Interpretable(expr.node) + try: + node.eval(frame) + except passthroughex: + raise + except Failure: + e = sys.exc_info()[1] + report_failure(e) + else: + if not frame.is_true(node.result): + sys.stderr.write("assertion failed: %s\n" % node.nice_explanation()) + + +########################################################### +# API / Entry points +# ######################################################### + +def interpret(source, frame, should_fail=False): + module = Interpretable(parse(source, 'exec').node) + #print "got module", module + if isinstance(frame, py.std.types.FrameType): + frame = py.code.Frame(frame) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + return getfailure(e) + except passthroughex: + raise + except: + import traceback + traceback.print_exc() + if should_fail: + return ("(assertion failed, but when it was re-run for " + "printing intermediate values, it did not fail. Suggestions: " + "compute assert expression before the assert or use --nomagic)") + else: + return None + +def getmsg(excinfo): + if isinstance(excinfo, tuple): + excinfo = py.code.ExceptionInfo(excinfo) + #frame, line = gettbline(tb) + #frame = py.code.Frame(frame) + #return interpret(line, frame) + + tb = excinfo.traceback[-1] + source = str(tb.statement).strip() + x = interpret(source, tb.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + return x + +def getfailure(e): + explanation = e.node.nice_explanation() + if str(e.value): + lines = explanation.split('\n') + lines[0] += " << %s" % (e.value,) + explanation = '\n'.join(lines) + text = "%s: %s" % (e.exc.__name__, explanation) + if text.startswith('AssertionError: assert '): + text = text[16:] + return text + +def run(s, frame=None): + if frame is None: + import sys + frame = sys._getframe(1) + frame = py.code.Frame(frame) + module = Interpretable(parse(s, 'exec').node) + try: + module.run(frame) + except Failure: + e = sys.exc_info()[1] + report_failure(e) + + +if __name__ == '__main__': + # example: + def f(): + return 5 + def g(): + return 3 + def h(x): + return 'never' + check("f() * g() == 5") + check("not f()") + check("not (f() and g() or 0)") + check("f() == g()") + i = 4 + check("i == f()") + check("len(f()) == 0") + check("isinstance(2+3+4, float)") + + run("x = i") + check("x == 5") + + run("assert not f(), 'oops'") + run("a, b, c = 1, 2") + run("a, b, c = f()") + + check("max([f(),g()]) == 4") + check("'hello'[g()] == 'h'") + run("'guk%d' % h(f())") Added: pypy/trunk/py/impl/code/assertion.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/assertion.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,75 @@ +import sys +import py + +BuiltinAssertionError = py.builtin.builtins.AssertionError + + +def _format_explanation(explanation): + # uck! See CallFunc for where \n{ and \n} escape sequences are used + raw_lines = (explanation or '').split('\n') + # escape newlines not followed by { and } + lines = [raw_lines[0]] + for l in raw_lines[1:]: + if l.startswith('{') or l.startswith('}'): + lines.append(l) + else: + lines[-1] += '\\n' + l + + result = lines[:1] + stack = [0] + stackcnt = [0] + for line in lines[1:]: + if line.startswith('{'): + if stackcnt[-1]: + s = 'and ' + else: + s = 'where ' + stack.append(len(result)) + stackcnt[-1] += 1 + stackcnt.append(0) + result.append(' +' + ' '*(len(stack)-1) + s + line[1:]) + else: + assert line.startswith('}') + stack.pop() + stackcnt.pop() + result[stack[-1]] += line[1:] + assert len(stack) == 1 + return '\n'.join(result) + + +if sys.version_info >= (2, 6) or (sys.platform.startswith("java")): + from py.impl.code._assertionnew import interpret +else: + from py.impl.code._assertionold import interpret + + +class AssertionError(BuiltinAssertionError): + + def __init__(self, *args): + BuiltinAssertionError.__init__(self, *args) + if args: + try: + self.msg = str(args[0]) + except (KeyboardInterrupt, SystemExit): + raise + except: + self.msg = "<[broken __repr__] %s at %0xd>" %( + args[0].__class__, id(args[0])) + else: + f = py.code.Frame(sys._getframe(1)) + try: + source = f.statement + source = str(source.deindent()).strip() + except py.error.ENOENT: + source = None + # this can also occur during reinterpretation, when the + # co_filename is set to "". + if source: + self.msg = interpret(source, f, should_fail=True) + if not self.args: + self.args = (self.msg,) + else: + self.msg = None + +if sys.version_info > (3, 0): + AssertionError.__module__ = "builtins" Added: pypy/trunk/py/impl/code/code.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/code.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,767 @@ +import py +import sys + +builtin_repr = repr + +repr = py.builtin._tryimport('repr', 'reprlib') + +class Code(object): + """ wrapper around Python code objects """ + def __init__(self, rawcode): + rawcode = py.code.getrawcode(rawcode) + self.raw = rawcode + try: + self.filename = rawcode.co_filename + self.firstlineno = rawcode.co_firstlineno - 1 + self.name = rawcode.co_name + except AttributeError: + raise TypeError("not a code object: %r" %(rawcode,)) + + def __eq__(self, other): + return self.raw == other.raw + + def __ne__(self, other): + return not self == other + + def new(self, rec=False, **kwargs): + """ return new code object with modified attributes. + if rec-cursive is true then dive into code + objects contained in co_consts. + """ + if sys.platform.startswith("java"): + # XXX jython does not support the below co_filename hack + return self.raw + names = [x for x in dir(self.raw) if x[:3] == 'co_'] + for name in kwargs: + if name not in names: + raise TypeError("unknown code attribute: %r" %(name, )) + if rec and hasattr(self.raw, 'co_consts'): # jython + newconstlist = [] + co = self.raw + cotype = type(co) + for c in co.co_consts: + if isinstance(c, cotype): + c = self.__class__(c).new(rec=True, **kwargs) + newconstlist.append(c) + return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) + for name in names: + if name not in kwargs: + kwargs[name] = getattr(self.raw, name) + arglist = [ + kwargs['co_argcount'], + kwargs['co_nlocals'], + kwargs.get('co_stacksize', 0), # jython + kwargs.get('co_flags', 0), # jython + kwargs.get('co_code', ''), # jython + kwargs.get('co_consts', ()), # jython + kwargs.get('co_names', []), # + kwargs['co_varnames'], + kwargs['co_filename'], + kwargs['co_name'], + kwargs['co_firstlineno'], + kwargs.get('co_lnotab', ''), #jython + kwargs.get('co_freevars', None), #jython + kwargs.get('co_cellvars', None), # jython + ] + if sys.version_info >= (3,0): + arglist.insert(1, kwargs['co_kwonlyargcount']) + return self.raw.__class__(*arglist) + else: + return py.std.new.code(*arglist) + + def path(self): + """ return a py.path.local object pointing to the source code """ + fn = self.raw.co_filename + try: + return fn.__path__ + except AttributeError: + p = py.path.local(self.raw.co_filename) + if not p.check(file=1): + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + p = self.raw.co_filename + return p + + path = property(path, None, None, "path of this code object") + + def fullsource(self): + """ return a py.code.Source object for the full source file of the code + """ + from py.impl.code import source + full, _ = source.findsource(self.raw) + return full + fullsource = property(fullsource, None, None, + "full source containing this code object") + + def source(self): + """ return a py.code.Source object for the code object's source only + """ + # return source only for that part of code + return py.code.Source(self.raw) + + def getargs(self): + """ return a tuple with the argument names for the code object + """ + # handfull shortcut for getting args + raw = self.raw + return raw.co_varnames[:raw.co_argcount] + +class Frame(object): + """Wrapper around a Python frame holding f_locals and f_globals + in which expressions can be evaluated.""" + + def __init__(self, frame): + self.code = py.code.Code(frame.f_code) + self.lineno = frame.f_lineno - 1 + self.f_globals = frame.f_globals + self.f_locals = frame.f_locals + self.raw = frame + + def statement(self): + if self.code.fullsource is None: + return py.code.Source("") + return self.code.fullsource.getstatement(self.lineno) + statement = property(statement, None, None, + "statement this frame is at") + + def eval(self, code, **vars): + """ evaluate 'code' in the frame + + 'vars' are optional additional local variables + + returns the result of the evaluation + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + return eval(code, self.f_globals, f_locals) + + def exec_(self, code, **vars): + """ exec 'code' in the frame + + 'vars' are optiona; additional local variables + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + py.builtin.exec_(code, self.f_globals, f_locals ) + + def repr(self, object): + """ return a 'safe' (non-recursive, one-line) string repr for 'object' + """ + return safe_repr(object) + + def is_true(self, object): + return object + + def getargs(self): + """ return a list of tuples (name, value) for all arguments + """ + retval = [] + for arg in self.code.getargs(): + try: + retval.append((arg, self.f_locals[arg])) + except KeyError: + pass # this can occur when using Psyco + return retval + +class TracebackEntry(object): + """ a single entry in a traceback """ + + exprinfo = None + + def __init__(self, rawentry): + self._rawentry = rawentry + self.frame = py.code.Frame(rawentry.tb_frame) + # Ugh. 2.4 and 2.5 differs here when encountering + # multi-line statements. Not sure about the solution, but + # should be portable + self.lineno = rawentry.tb_lineno - 1 + self.relline = self.lineno - self.frame.code.firstlineno + + def __repr__(self): + return "" %(self.frame.code.path, self.lineno+1) + + def statement(self): + """ return a py.code.Source object for the current statement """ + source = self.frame.code.fullsource + return source.getstatement(self.lineno) + statement = property(statement, None, None, + "statement of this traceback entry.") + + def path(self): + return self.frame.code.path + path = property(path, None, None, "path to the full source code") + + def getlocals(self): + return self.frame.f_locals + locals = property(getlocals, None, None, "locals of underlaying frame") + + def reinterpret(self): + """Reinterpret the failing statement and returns a detailed information + about what operations are performed.""" + if self.exprinfo is None: + from py.impl.code import assertion + source = str(self.statement).strip() + x = assertion.interpret(source, self.frame, should_fail=True) + if not isinstance(x, str): + raise TypeError("interpret returned non-string %r" % (x,)) + self.exprinfo = x + return self.exprinfo + + def getfirstlinesource(self): + return self.frame.code.firstlineno + + def getsource(self): + """ return failing source code. """ + source = self.frame.code.fullsource + if source is None: + return None + start = self.getfirstlinesource() + end = self.lineno + try: + _, end = source.getstatementrange(end) + except IndexError: + end = self.lineno + 1 + # heuristic to stop displaying source on e.g. + # if something: # assume this causes a NameError + # # _this_ lines and the one + # below we don't want from entry.getsource() + for i in range(self.lineno, end): + if source[i].rstrip().endswith(':'): + end = i + 1 + break + return source[start:end] + source = property(getsource) + + def ishidden(self): + """ return True if the current frame has a var __tracebackhide__ + resolving to True + + mostly for internal use + """ + try: + return self.frame.eval("__tracebackhide__") + except (SystemExit, KeyboardInterrupt): + raise + except: + return False + + def __str__(self): + try: + fn = str(self.path) + except py.error.Error: + fn = '???' + name = self.frame.code.name + try: + line = str(self.statement).lstrip() + except KeyboardInterrupt: + raise + except: + line = "???" + return " File %r:%d in %s\n %s\n" %(fn, self.lineno+1, name, line) + + def name(self): + return self.frame.code.raw.co_name + name = property(name, None, None, "co_name of underlaying code") + +class Traceback(list): + """ Traceback objects encapsulate and offer higher level + access to Traceback entries. + """ + Entry = TracebackEntry + def __init__(self, tb): + """ initialize from given python traceback object. """ + if hasattr(tb, 'tb_next'): + def f(cur): + while cur is not None: + yield self.Entry(cur) + cur = cur.tb_next + list.__init__(self, f(tb)) + else: + list.__init__(self, tb) + + def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None): + """ return a Traceback instance wrapping part of this Traceback + + by provding any combination of path, lineno and firstlineno, the + first frame to start the to-be-returned traceback is determined + + this allows cutting the first part of a Traceback instance e.g. + for formatting reasons (removing some uninteresting bits that deal + with handling of the exception/traceback) + """ + for x in self: + code = x.frame.code + codepath = code.path + if ((path is None or codepath == path) and + (excludepath is None or (hasattr(codepath, 'relto') and + not codepath.relto(excludepath))) and + (lineno is None or x.lineno == lineno) and + (firstlineno is None or x.frame.code.firstlineno == firstlineno)): + return Traceback(x._rawentry) + return self + + def __getitem__(self, key): + val = super(Traceback, self).__getitem__(key) + if isinstance(key, type(slice(0))): + val = self.__class__(val) + return val + + def filter(self, fn=lambda x: not x.ishidden()): + """ return a Traceback instance with certain items removed + + fn is a function that gets a single argument, a TracebackItem + instance, and should return True when the item should be added + to the Traceback, False when not + + by default this removes all the TracebackItems which are hidden + (see ishidden() above) + """ + return Traceback(filter(fn, self)) + + def getcrashentry(self): + """ return last non-hidden traceback entry that lead + to the exception of a traceback. + """ + tb = self.filter() + if not tb: + tb = self + return tb[-1] + + def recursionindex(self): + """ return the index of the frame/TracebackItem where recursion + originates if appropriate, None if no recursion occurred + """ + cache = {} + for i, entry in enumerate(self): + key = entry.frame.code.path, entry.lineno + #print "checking for recursion at", key + l = cache.setdefault(key, []) + if l: + f = entry.frame + loc = f.f_locals + for otherloc in l: + if f.is_true(f.eval(co_equal, + __recursioncache_locals_1=loc, + __recursioncache_locals_2=otherloc)): + return i + l.append(entry.frame.f_locals) + return None + +co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2', + '?', 'eval') + +class ExceptionInfo(object): + """ wraps sys.exc_info() objects and offers + help for navigating the traceback. + """ + _striptext = '' + def __init__(self, tup=None, exprinfo=None): + # NB. all attributes are private! Subclasses or other + # ExceptionInfo-like classes may have different attributes. + if tup is None: + tup = sys.exc_info() + if exprinfo is None and isinstance(tup[1], py.code._AssertionError): + exprinfo = getattr(tup[1], 'msg', None) + if exprinfo is None: + exprinfo = str(tup[1]) + if exprinfo and exprinfo.startswith('assert '): + self._striptext = 'AssertionError: ' + self._excinfo = tup + self.type, self.value, tb = self._excinfo + self.typename = self.type.__name__ + self.traceback = py.code.Traceback(tb) + + def __repr__(self): + return "" % (self.typename, len(self.traceback)) + + def exconly(self, tryshort=False): + """ return the exception as a string + + when 'tryshort' resolves to True, and the exception is a + py.code._AssertionError, only the actual exception part of + the exception representation is returned (so 'AssertionError: ' is + removed from the beginning) + """ + lines = py.std.traceback.format_exception_only(self.type, self.value) + text = ''.join(lines) + text = text.rstrip() + if tryshort: + if text.startswith(self._striptext): + text = text[len(self._striptext):] + return text + + def errisinstance(self, exc): + """ return True if the exception is an instance of exc """ + return isinstance(self.value, exc) + + def _getreprcrash(self): + exconly = self.exconly(tryshort=True) + entry = self.traceback.getcrashentry() + path, lineno = entry.path, entry.lineno + reprcrash = ReprFileLocation(path, lineno+1, exconly) + return reprcrash + + def getrepr(self, showlocals=False, style="long", + abspath=False, tbfilter=True, funcargs=False): + """ return str()able representation of this exception info. + showlocals: show locals per traceback entry + style: long|short|no traceback style + tbfilter: hide entries (where __tracebackhide__ is true) + """ + fmt = FormattedExcinfo(showlocals=showlocals, style=style, + abspath=abspath, tbfilter=tbfilter, funcargs=funcargs) + return fmt.repr_excinfo(self) + + def __str__(self): + entry = self.traceback[-1] + loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly()) + return str(loc) + +class FormattedExcinfo(object): + """ presenting information about failing Functions and Generators. """ + # for traceback entries + flow_marker = ">" + fail_marker = "E" + + def __init__(self, showlocals=False, style="long", abspath=True, tbfilter=True, funcargs=False): + self.showlocals = showlocals + self.style = style + self.tbfilter = tbfilter + self.funcargs = funcargs + self.abspath = abspath + + def _getindent(self, source): + # figure out indent for given source + try: + s = str(source.getstatement(len(source)-1)) + except KeyboardInterrupt: + raise + except: + try: + s = str(source[-1]) + except KeyboardInterrupt: + raise + except: + return 0 + return 4 + (len(s) - len(s.lstrip())) + + def _getentrysource(self, entry): + source = entry.getsource() + if source is not None: + source = source.deindent() + return source + + def _saferepr(self, obj): + return safe_repr(obj) + + def repr_args(self, entry): + if self.funcargs: + args = [] + for argname, argvalue in entry.frame.getargs(): + args.append((argname, self._saferepr(argvalue))) + return ReprFuncArgs(args) + + def get_source(self, source, line_index=-1, excinfo=None): + """ return formatted and marked up source lines. """ + lines = [] + if source is None: + source = py.code.Source("???") + line_index = 0 + if line_index < 0: + line_index += len(source) + for i in range(len(source)): + if i == line_index: + prefix = self.flow_marker + " " + else: + prefix = " " + line = prefix + source[i] + lines.append(line) + if excinfo is not None: + indent = self._getindent(source) + lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) + return lines + + def get_exconly(self, excinfo, indent=4, markall=False): + lines = [] + indent = " " * indent + # get the real exception information out + exlines = excinfo.exconly(tryshort=True).split('\n') + failindent = self.fail_marker + indent[1:] + for line in exlines: + lines.append(failindent + line) + if not markall: + failindent = indent + return lines + + def repr_locals(self, locals): + if self.showlocals: + lines = [] + keys = list(locals) + keys.sort() + for name in keys: + value = locals[name] + if name == '__builtins__': + lines.append("__builtins__ = ") + else: + # This formatting could all be handled by the + # _repr() function, which is only repr.Repr in + # disguise, so is very configurable. + str_repr = self._saferepr(value) + #if len(str_repr) < 70 or not isinstance(value, + # (list, tuple, dict)): + lines.append("%-10s = %s" %(name, str_repr)) + #else: + # self._line("%-10s =\\" % (name,)) + # # XXX + # py.std.pprint.pprint(value, stream=self.excinfowriter) + return ReprLocals(lines) + + def repr_traceback_entry(self, entry, excinfo=None): + # excinfo is not None if this is the last tb entry + source = self._getentrysource(entry) + if source is None: + source = py.code.Source("???") + line_index = 0 + else: + line_index = entry.lineno - entry.getfirstlinesource() + + lines = [] + if self.style == "long": + reprargs = self.repr_args(entry) + lines.extend(self.get_source(source, line_index, excinfo)) + message = excinfo and excinfo.typename or "" + path = self._makepath(entry.path) + filelocrepr = ReprFileLocation(path, entry.lineno+1, message) + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, filelocrepr) + else: + if self.style == "short": + line = source[line_index].lstrip() + lines.append(' File "%s", line %d, in %s' % ( + entry.path.basename, entry.lineno+1, entry.name)) + lines.append(" " + line) + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None) + + def _makepath(self, path): + if not self.abspath: + np = py.path.local().bestrelpath(path) + if len(np) < len(str(path)): + path = np + return path + + def repr_traceback(self, excinfo): + traceback = excinfo.traceback + if self.tbfilter: + traceback = traceback.filter() + recursionindex = None + if excinfo.errisinstance(RuntimeError): + recursionindex = traceback.recursionindex() + last = traceback[-1] + entries = [] + extraline = None + for index, entry in enumerate(traceback): + einfo = (last == entry) and excinfo or None + reprentry = self.repr_traceback_entry(entry, einfo) + entries.append(reprentry) + if index == recursionindex: + extraline = "!!! Recursion detected (same locals & position)" + break + return ReprTraceback(entries, extraline, style=self.style) + + def repr_excinfo(self, excinfo): + reprtraceback = self.repr_traceback(excinfo) + reprcrash = excinfo._getreprcrash() + return ReprExceptionInfo(reprtraceback, reprcrash) + +class TerminalRepr: + def __str__(self): + tw = py.io.TerminalWriter(stringio=True) + self.toterminal(tw) + return tw.stringio.getvalue().strip() + + def __repr__(self): + return "<%s instance at %0x>" %(self.__class__, id(self)) + +class ReprExceptionInfo(TerminalRepr): + def __init__(self, reprtraceback, reprcrash): + self.reprtraceback = reprtraceback + self.reprcrash = reprcrash + self.sections = [] + + def addsection(self, name, content, sep="-"): + self.sections.append((name, content, sep)) + + def toterminal(self, tw): + self.reprtraceback.toterminal(tw) + for name, content, sep in self.sections: + tw.sep(sep, name) + tw.line(content) + +class ReprTraceback(TerminalRepr): + entrysep = "_ " + + def __init__(self, reprentries, extraline, style): + self.reprentries = reprentries + self.extraline = extraline + self.style = style + + def toterminal(self, tw): + sepok = False + for entry in self.reprentries: + if self.style == "long": + if sepok: + tw.sep(self.entrysep) + tw.line("") + sepok = True + entry.toterminal(tw) + if self.extraline: + tw.line(self.extraline) + +class ReprEntry(TerminalRepr): + localssep = "_ " + + def __init__(self, lines, reprfuncargs, reprlocals, filelocrepr): + self.lines = lines + self.reprfuncargs = reprfuncargs + self.reprlocals = reprlocals + self.reprfileloc = filelocrepr + + def toterminal(self, tw): + if self.reprfuncargs: + self.reprfuncargs.toterminal(tw) + for line in self.lines: + red = line.startswith("E ") + tw.line(line, bold=True, red=red) + if self.reprlocals: + #tw.sep(self.localssep, "Locals") + tw.line("") + self.reprlocals.toterminal(tw) + if self.reprfileloc: + tw.line("") + self.reprfileloc.toterminal(tw) + + def __str__(self): + return "%s\n%s\n%s" % ("\n".join(self.lines), + self.reprlocals, + self.reprfileloc) + +class ReprFileLocation(TerminalRepr): + def __init__(self, path, lineno, message): + self.path = str(path) + self.lineno = lineno + self.message = message + + def toterminal(self, tw): + # filename and lineno output for each entry, + # using an output format that most editors unterstand + msg = self.message + i = msg.find("\n") + if i != -1: + msg = msg[:i] + tw.line("%s:%s: %s" %(self.path, self.lineno, msg)) + +class ReprLocals(TerminalRepr): + def __init__(self, lines): + self.lines = lines + + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + +class ReprFuncArgs(TerminalRepr): + def __init__(self, args): + self.args = args + + def toterminal(self, tw): + if self.args: + linesofar = "" + for name, value in self.args: + ns = "%s = %s" %(name, value) + if len(ns) + len(linesofar) + 2 > tw.fullwidth: + if linesofar: + tw.line(linesofar) + linesofar = ns + else: + if linesofar: + linesofar += ", " + ns + else: + linesofar = ns + if linesofar: + tw.line(linesofar) + tw.line("") + + + +class SafeRepr(repr.Repr): + """ subclass of repr.Repr that limits the resulting size of repr() + and includes information on exceptions raised during the call. + """ + def __init__(self, *args, **kwargs): + repr.Repr.__init__(self, *args, **kwargs) + self.maxstring = 240 # 3 * 80 chars + self.maxother = 160 # 2 * 80 chars + + def repr(self, x): + return self._callhelper(repr.Repr.repr, self, x) + + def repr_instance(self, x, level): + return self._callhelper(builtin_repr, x) + + def _callhelper(self, call, x, *args): + try: + # Try the vanilla repr and make sure that the result is a string + s = call(x, *args) + except (KeyboardInterrupt, MemoryError, SystemExit): + raise + except: + cls, e, tb = sys.exc_info() + try: + exc_name = cls.__name__ + except: + exc_name = 'unknown' + try: + exc_info = str(e) + except: + exc_info = 'unknown' + return '<[%s("%s") raised in repr()] %s object at 0x%x>' % ( + exc_name, exc_info, x.__class__.__name__, id(x)) + else: + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + +safe_repr = SafeRepr().repr + +oldbuiltins = {} + +def patch_builtins(assertion=True, compile=True): + """ put compile and AssertionError builtins to Python's builtins. """ + if assertion: + from py.impl.code import assertion + l = oldbuiltins.setdefault('AssertionError', []) + l.append(py.builtin.builtins.AssertionError) + py.builtin.builtins.AssertionError = assertion.AssertionError + if compile: + l = oldbuiltins.setdefault('compile', []) + l.append(py.builtin.builtins.compile) + py.builtin.builtins.compile = py.code.compile + +def unpatch_builtins(assertion=True, compile=True): + """ remove compile and AssertionError builtins from Python builtins. """ + if assertion: + py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop() + if compile: + py.builtin.builtins.compile = oldbuiltins['compile'].pop() + +def getrawcode(obj): + """ return code object for given function. """ + obj = getattr(obj, 'im_func', obj) + obj = getattr(obj, 'func_code', obj) + obj = getattr(obj, 'f_code', obj) + obj = getattr(obj, '__code__', obj) + return obj + Added: pypy/trunk/py/impl/code/oldmagic.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/oldmagic.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,62 @@ +""" deprecated module for turning on/off some features. """ + +import py + +from py.builtin import builtins as cpy_builtin + +def invoke(assertion=False, compile=False): + """ (deprecated) invoke magic, currently you can specify: + + assertion patches the builtin AssertionError to try to give + more meaningful AssertionErrors, which by means + of deploying a mini-interpreter constructs + a useful error message. + """ + py.log._apiwarn("1.1", + "py.magic.invoke() is deprecated, use py.code.patch_builtins()", + stacklevel=2, + ) + py.code.patch_builtins(assertion=assertion, compile=compile) + +def revoke(assertion=False, compile=False): + """ (deprecated) revoke previously invoked magic (see invoke()).""" + py.log._apiwarn("1.1", + "py.magic.revoke() is deprecated, use py.code.unpatch_builtins()", + stacklevel=2, + ) + py.code.unpatch_builtins(assertion=assertion, compile=compile) + +patched = {} + +def patch(namespace, name, value): + """ (deprecated) rebind the 'name' on the 'namespace' to the 'value', + possibly and remember the original value. Multiple + invocations to the same namespace/name pair will + remember a list of old values. + """ + py.log._apiwarn("1.1", + "py.magic.patch() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + orig = getattr(namespace, name) + patched.setdefault(nref, []).append(orig) + setattr(namespace, name, value) + return orig + +def revert(namespace, name): + """ (deprecated) revert to the orginal value the last patch modified. + Raise ValueError if no such original value exists. + """ + py.log._apiwarn("1.1", + "py.magic.revert() is deprecated, in tests use monkeypatch funcarg.", + stacklevel=2, + ) + nref = (namespace, name) + if nref not in patched or not patched[nref]: + raise ValueError("No original value stored for %s.%s" % nref) + current = getattr(namespace, name) + orig = patched[nref].pop() + setattr(namespace, name, orig) + return current + Added: pypy/trunk/py/impl/code/oldmagic2.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/oldmagic2.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,6 @@ + +import py + +py.log._apiwarn("1.1", "py.magic.AssertionError is deprecated, use py.code._AssertionError", stacklevel=2) + +from py.code import _AssertionError as AssertionError Added: pypy/trunk/py/impl/code/source.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/code/source.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,347 @@ +from __future__ import generators +import sys +import inspect, tokenize +import py +cpy_compile = compile + +try: + import _ast + from _ast import PyCF_ONLY_AST as _AST_FLAG +except ImportError: + _AST_FLAG = 0 + _ast = None + + +class Source(object): + """ a immutable object holding a source code fragment, + possibly deindenting it. + """ + def __init__(self, *parts, **kwargs): + self.lines = lines = [] + de = kwargs.get('deindent', True) + rstrip = kwargs.get('rstrip', True) + for part in parts: + if not part: + partlines = [] + if isinstance(part, Source): + partlines = part.lines + elif isinstance(part, py.builtin._basestring): + partlines = part.split('\n') + if rstrip: + while partlines: + if partlines[-1].strip(): + break + partlines.pop() + else: + partlines = getsource(part, deindent=de).lines + if de: + partlines = deindent(partlines) + lines.extend(partlines) + + def __eq__(self, other): + try: + return self.lines == other.lines + except AttributeError: + if isinstance(other, str): + return str(self) == other + return False + + def __getitem__(self, key): + if isinstance(key, int): + return self.lines[key] + else: + if key.step not in (None, 1): + raise IndexError("cannot slice a Source with a step") + return self.__getslice__(key.start, key.stop) + + def __len__(self): + return len(self.lines) + + def __getslice__(self, start, end): + newsource = Source() + newsource.lines = self.lines[start:end] + return newsource + + def strip(self): + """ return new source object with trailing + and leading blank lines removed. + """ + start, end = 0, len(self) + while start < end and not self.lines[start].strip(): + start += 1 + while end > start and not self.lines[end-1].strip(): + end -= 1 + source = Source() + source.lines[:] = self.lines[start:end] + return source + + def putaround(self, before='', after='', indent=' ' * 4): + """ return a copy of the source object with + 'before' and 'after' wrapped around it. + """ + before = Source(before) + after = Source(after) + newsource = Source() + lines = [ (indent + line) for line in self.lines] + newsource.lines = before.lines + lines + after.lines + return newsource + + def indent(self, indent=' ' * 4): + """ return a copy of the source object with + all lines indented by the given indent-string. + """ + newsource = Source() + newsource.lines = [(indent+line) for line in self.lines] + return newsource + + def getstatement(self, lineno): + """ return Source statement which contains the + given linenumber (counted from 0). + """ + start, end = self.getstatementrange(lineno) + return self[start:end] + + def getstatementrange(self, lineno): + """ return (start, end) tuple which spans the minimal + statement region which containing the given lineno. + """ + # XXX there must be a better than these heuristic ways ... + # XXX there may even be better heuristics :-) + if not (0 <= lineno < len(self)): + raise IndexError("lineno out of range") + + # 1. find the start of the statement + from codeop import compile_command + for start in range(lineno, -1, -1): + trylines = self.lines[start:lineno+1] + # quick hack to indent the source and get it as a string in one go + trylines.insert(0, 'def xxx():') + trysource = '\n '.join(trylines) + # ^ space here + try: + compile_command(trysource) + except (SyntaxError, OverflowError, ValueError): + pass + else: + break # got a valid or incomplete statement + + # 2. find the end of the statement + for end in range(lineno+1, len(self)+1): + trysource = self[start:end] + if trysource.isparseable(): + break + + return start, end + + def getblockend(self, lineno): + # XXX + lines = [x + '\n' for x in self.lines[lineno:]] + blocklines = inspect.getblock(lines) + #print blocklines + return lineno + len(blocklines) - 1 + + def deindent(self, offset=None): + """ return a new source object deindented by offset. + If offset is None then guess an indentation offset from + the first non-blank line. Subsequent lines which have a + lower indentation offset will be copied verbatim as + they are assumed to be part of multilines. + """ + # XXX maybe use the tokenizer to properly handle multiline + # strings etc.pp? + newsource = Source() + newsource.lines[:] = deindent(self.lines, offset) + return newsource + + def isparseable(self, deindent=True): + """ return True if source is parseable, heuristically + deindenting it by default. + """ + try: + import parser + except ImportError: + syntax_checker = lambda x: compile(x, 'asd', 'exec') + else: + syntax_checker = parser.suite + + if deindent: + source = str(self.deindent()) + else: + source = str(self) + try: + #compile(source+'\n', "x", "exec") + syntax_checker(source+'\n') + except SyntaxError: + return False + else: + return True + + def __str__(self): + return "\n".join(self.lines) + + def compile(self, filename=None, mode='exec', + flag=generators.compiler_flag, + dont_inherit=0, _genframe=None): + """ return compiled code object. if filename is None + invent an artificial filename which displays + the source/line position of the caller frame. + """ + if not filename or py.path.local(filename).check(file=0): + if _genframe is None: + _genframe = sys._getframe(1) # the caller + fn,lineno = _genframe.f_code.co_filename, _genframe.f_lineno + if not filename: + filename = '' % (fn, lineno) + else: + filename = '' % (filename, fn, lineno) + source = "\n".join(self.lines) + '\n' + try: + co = cpy_compile(source, filename, mode, flag) + except SyntaxError: + ex = sys.exc_info()[1] + # re-represent syntax errors from parsing python strings + msglines = self.lines[:ex.lineno] + if ex.offset: + msglines.append(" "*ex.offset + '^') + msglines.append("syntax error probably generated here: %s" % filename) + newex = SyntaxError('\n'.join(msglines)) + newex.offset = ex.offset + newex.lineno = ex.lineno + newex.text = ex.text + raise newex + else: + if flag & _AST_FLAG: + return co + co_filename = MyStr(filename) + co_filename.__source__ = self + return py.code.Code(co).new(rec=1, co_filename=co_filename) + #return newcode_withfilename(co, co_filename) + +# +# public API shortcut functions +# + +def compile_(source, filename=None, mode='exec', flags= + generators.compiler_flag, dont_inherit=0): + """ compile the given source to a raw code object, + which points back to the source code through + "co_filename.__source__". All code objects + contained in the code object will recursively + also have this special subclass-of-string + filename. + """ + if _ast is not None and isinstance(source, _ast.AST): + # XXX should Source support having AST? + return cpy_compile(source, filename, mode, flags, dont_inherit) + _genframe = sys._getframe(1) # the caller + s = Source(source) + co = s.compile(filename, mode, flags, _genframe=_genframe) + return co + + +def getfslineno(obj): + try: + code = py.code.Code(obj) + except TypeError: + # fallback to + fn = (py.std.inspect.getsourcefile(obj) or + py.std.inspect.getfile(obj)) + fspath = fn and py.path.local(fn) or None + if fspath: + try: + _, lineno = findsource(obj) + except IOError: + lineno = None + else: + lineno = None + else: + fspath = code.path + lineno = code.firstlineno + return fspath, lineno + +# +# helper functions +# +class MyStr(str): + """ custom string which allows to add attributes. """ + +def findsource(obj): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + sourcelines, lineno = py.std.inspect.findsource(obj) + except (KeyboardInterrupt, SystemExit): + raise + except: + return None, None + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno + else: + lineno = obj.co_firstlineno - 1 + return fullsource, lineno + + +def getsource(obj, **kwargs): + obj = py.code.getrawcode(obj) + try: + fullsource = obj.co_filename.__source__ + except AttributeError: + try: + strsrc = inspect.getsource(obj) + except IndentationError: + strsrc = "\"Buggy python version consider upgrading, cannot get source\"" + assert isinstance(strsrc, str) + return Source(strsrc, **kwargs) + else: + lineno = obj.co_firstlineno - 1 + end = fullsource.getblockend(lineno) + return Source(fullsource[lineno:end+1], deident=True) + + +def deindent(lines, offset=None): + if offset is None: + for line in lines: + line = line.expandtabs() + s = line.lstrip() + if s: + offset = len(line)-len(s) + break + else: + offset = 0 + if offset == 0: + return list(lines) + newlines = [] + def readline_generator(lines): + for line in lines: + yield line + '\n' + while True: + yield '' + + r = readline_generator(lines) + try: + readline = r.next + except AttributeError: + readline = r.__next__ + + try: + for _, _, (sline, _), (eline, _), _ in tokenize.generate_tokens(readline): + if sline > len(lines): + break # End of input reached + if sline > len(newlines): + line = lines[sline - 1].expandtabs() + if line.lstrip() and line[:offset].isspace(): + line = line[offset:] # Deindent + newlines.append(line) + + for i in range(sline, eline): + # Don't deindent continuing lines of + # multiline tokens (i.e. multiline strings) + newlines.append(lines[i]) + except (IndentationError, tokenize.TokenError): + pass + # Add any lines we didn't see. E.g. if an exception was raised. + newlines.extend(lines[len(newlines):]) + return newlines Added: pypy/trunk/py/impl/compat/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/compat/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ +""" compatibility modules (taken from 2.4.4) """ + Added: pypy/trunk/py/impl/compat/dep_doctest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/compat/dep_doctest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,4 @@ +import py + +py.log._apiwarn("1.1", "py.compat.doctest deprecated, use standard library version.", stacklevel="initpkg") +doctest = py.std.doctest Added: pypy/trunk/py/impl/compat/dep_optparse.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/compat/dep_optparse.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,4 @@ +import py +py.log._apiwarn("1.1", "py.compat.optparse deprecated, use standard library version.", stacklevel="initpkg") + +optparse = py.std.optparse Added: pypy/trunk/py/impl/compat/dep_subprocess.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/compat/dep_subprocess.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,4 @@ + +import py +py.log._apiwarn("1.1", "py.compat.subprocess deprecated, use standard library version.", stacklevel="initpkg") +subprocess = py.std.subprocess Added: pypy/trunk/py/impl/compat/dep_textwrap.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/compat/dep_textwrap.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,4 @@ +import py + +py.log._apiwarn("1.1", "py.compat.textwrap deprecated, use standard library version.", stacklevel="initpkg") +textwrap = py.std.textwrap Added: pypy/trunk/py/impl/error.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/error.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,83 @@ +""" +create errno-specific classes for IO or os calls. + +""" +import sys, os, errno + +class Error(EnvironmentError): + def __repr__(self): + return "%s.%s %r: %s " %(self.__class__.__module__, + self.__class__.__name__, + self.__class__.__doc__, + " ".join(map(str, self.args)), + #repr(self.args) + ) + + def __str__(self): + s = "[%s]: %s" %(self.__class__.__doc__, + " ".join(map(str, self.args)), + ) + return s + +_winerrnomap = { + 2: errno.ENOENT, + 3: errno.ENOENT, + 17: errno.EEXIST, + 22: errno.ENOTDIR, + 267: errno.ENOTDIR, + 5: errno.EACCES, # anything better? +} + +class ErrorMaker(object): + """ lazily provides Exception classes for each possible POSIX errno + (as defined per the 'errno' module). All such instances + subclass EnvironmentError. + """ + Error = Error + _errno2class = {} + + def __getattr__(self, name): + eno = getattr(errno, name) + cls = self._geterrnoclass(eno) + setattr(self, name, cls) + return cls + + def _geterrnoclass(self, eno): + try: + return self._errno2class[eno] + except KeyError: + clsname = errno.errorcode.get(eno, "UnknownErrno%d" %(eno,)) + errorcls = type(Error)(clsname, (Error,), + {'__module__':'py.error', + '__doc__': os.strerror(eno)}) + self._errno2class[eno] = errorcls + return errorcls + + def checked_call(self, func, *args): + """ call a function and raise an errno-exception if applicable. """ + __tracebackhide__ = True + try: + return func(*args) + except self.Error: + raise + except EnvironmentError: + cls, value, tb = sys.exc_info() + if not hasattr(value, 'errno'): + raise + __tracebackhide__ = False + errno = value.errno + try: + if not isinstance(value, WindowsError): + raise NameError + except NameError: + # we are not on Windows, or we got a proper OSError + cls = self._geterrnoclass(errno) + else: + try: + cls = self._geterrnoclass(_winerrnomap[errno]) + except KeyError: + raise value + raise cls("%s%r" % (func.__name__, args)) + __tracebackhide__ = True + +error = ErrorMaker() Added: pypy/trunk/py/impl/io/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/io/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +""" input/output helping """ Added: pypy/trunk/py/impl/io/capture.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/io/capture.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,344 @@ +import os +import sys +import py +import tempfile + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +if sys.version_info < (3,0): + class TextIO(StringIO): + def write(self, data): + if not isinstance(data, unicode): + data = unicode(data, getattr(self, '_encoding', 'UTF-8')) + StringIO.write(self, data) +else: + TextIO = StringIO + +try: + from io import BytesIO +except ImportError: + class BytesIO(StringIO): + def write(self, data): + if isinstance(data, unicode): + raise TypeError("not a byte value: %r" %(data,)) + StringIO.write(self, data) + +class FDCapture: + """ Capture IO to/from a given os-level filedescriptor. """ + + def __init__(self, targetfd, tmpfile=None): + """ save targetfd descriptor, and open a new + temporary file there. If no tmpfile is + specified a tempfile.Tempfile() will be opened + in text mode. + """ + self.targetfd = targetfd + if tmpfile is None: + f = tempfile.TemporaryFile('wb+') + tmpfile = dupfile(f, encoding="UTF-8") + f.close() + self.tmpfile = tmpfile + self._savefd = os.dup(targetfd) + os.dup2(self.tmpfile.fileno(), targetfd) + self._patched = [] + + def setasfile(self, name, module=sys): + """ patch . to self.tmpfile + """ + key = (module, name) + self._patched.append((key, getattr(module, name))) + setattr(module, name, self.tmpfile) + + def unsetfiles(self): + """ unpatch all patched items + """ + while self._patched: + (module, name), value = self._patched.pop() + setattr(module, name, value) + + def done(self): + """ unpatch and clean up, returns the self.tmpfile (file object) + """ + os.dup2(self._savefd, self.targetfd) + self.unsetfiles() + os.close(self._savefd) + self.tmpfile.seek(0) + return self.tmpfile + + def writeorg(self, data): + """ write a string to the original file descriptor + """ + tempfp = tempfile.TemporaryFile() + try: + os.dup2(self._savefd, tempfp.fileno()) + tempfp.write(data) + finally: + tempfp.close() + + +def dupfile(f, mode=None, buffering=0, raising=False, encoding=None): + """ return a new open file object that's a duplicate of f + + mode is duplicated if not given, 'buffering' controls + buffer size (defaulting to no buffering) and 'raising' + defines whether an exception is raised when an incompatible + file object is passed in (if raising is False, the file + object itself will be returned) + """ + try: + fd = f.fileno() + except AttributeError: + if raising: + raise + return f + newfd = os.dup(fd) + mode = mode and mode or f.mode + if sys.version_info >= (3,0): + if encoding is not None: + mode = mode.replace("b", "") + buffering = True + return os.fdopen(newfd, mode, buffering, encoding, closefd=False) + else: + f = os.fdopen(newfd, mode, buffering) + if encoding is not None: + return EncodedFile(f, encoding) + return f + +class EncodedFile(object): + def __init__(self, _stream, encoding): + self._stream = _stream + self.encoding = encoding + + def write(self, obj): + if isinstance(obj, unicode): + obj = obj.encode(self.encoding) + elif isinstance(obj, str): + pass + else: + obj = str(obj) + self._stream.write(obj) + + def writelines(self, linelist): + data = ''.join(linelist) + self.write(data) + + def __getattr__(self, name): + return getattr(self._stream, name) + +class Capture(object): + def call(cls, func, *args, **kwargs): + """ return a (res, out, err) tuple where + out and err represent the output/error output + during function execution. + call the given function with args/kwargs + and capture output/error during its execution. + """ + so = cls() + try: + res = func(*args, **kwargs) + finally: + out, err = so.reset() + return res, out, err + call = classmethod(call) + + def reset(self): + """ reset sys.stdout/stderr and return captured output as strings. """ + if hasattr(self, '_suspended'): + outfile = self._kwargs['out'] + errfile = self._kwargs['err'] + del self._kwargs + else: + outfile, errfile = self.done() + out, err = "", "" + if outfile: + out = outfile.read() + outfile.close() + if errfile and errfile != outfile: + err = errfile.read() + errfile.close() + return out, err + + def suspend(self): + """ return current snapshot captures, memorize tempfiles. """ + assert not hasattr(self, '_suspended') + self._suspended = True + outerr = self.readouterr() + outfile, errfile = self.done() + self._kwargs['out'] = outfile + self._kwargs['err'] = errfile + return outerr + + def resume(self): + """ resume capturing with original temp files. """ + assert self._suspended + self._initialize(**self._kwargs) + del self._suspended + + +class StdCaptureFD(Capture): + """ This class allows to capture writes to FD1 and FD2 + and may connect a NULL file to FD0 (and prevent + reads from sys.stdin) + """ + def __init__(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out=True, err=True, + mixed=False, in_=True, patchsys=True): + if in_: + self._oldin = (sys.stdin, os.dup(0)) + sys.stdin = DontReadFromInput() + fd = os.open(devnullpath, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if out: + tmpfile = None + if hasattr(out, 'write'): + tmpfile = out + self.out = py.io.FDCapture(1, tmpfile=tmpfile) + if patchsys: + self.out.setasfile('stdout') + if err: + if mixed and out: + tmpfile = self.out.tmpfile + elif hasattr(err, 'write'): + tmpfile = err + else: + tmpfile = None + self.err = py.io.FDCapture(2, tmpfile=tmpfile) + if patchsys: + self.err.setasfile('stderr') + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + if hasattr(self, 'out'): + outfile = self.out.done() + else: + outfile = None + if hasattr(self, 'err'): + errfile = self.err.done() + else: + errfile = None + if hasattr(self, '_oldin'): + oldsys, oldfd = self._oldin + os.dup2(oldfd, 0) + os.close(oldfd) + sys.stdin = oldsys + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + l = [] + for name in ('out', 'err'): + res = "" + if hasattr(self, name): + f = getattr(self, name).tmpfile + f.seek(0) + res = f.read() + f.truncate(0) + f.seek(0) + l.append(res) + return l + +class StdCapture(Capture): + """ This class allows to capture writes to sys.stdout|stderr "in-memory" + and will raise errors on tries to read from sys.stdin. It only + modifies sys.stdout|stderr|stdin attributes and does not + touch underlying File Descriptors (use StdCaptureFD for that). + """ + def __init__(self, out=True, err=True, in_=True, mixed=False): + self._kwargs = locals().copy() + del self._kwargs['self'] + self._initialize(**self._kwargs) + + def _initialize(self, out, err, in_, mixed): + self._out = out + self._err = err + self._in = in_ + if out: + self._oldout = sys.stdout + if not hasattr(out, 'write'): + out = TextIO() + sys.stdout = self.out = out + if err: + self._olderr = sys.stderr + if out and mixed: + err = self.out + elif not hasattr(err, 'write'): + err = TextIO() + sys.stderr = self.err = err + if in_: + self._oldin = sys.stdin + sys.stdin = self.newin = DontReadFromInput() + + def done(self): + """ return (outfile, errfile) and stop capturing. """ + o,e = sys.stdout, sys.stderr + if self._out: + try: + sys.stdout = self._oldout + except AttributeError: + raise IOError("stdout capturing already reset") + del self._oldout + outfile = self.out + outfile.seek(0) + else: + outfile = None + if self._err: + try: + sys.stderr = self._olderr + except AttributeError: + raise IOError("stderr capturing already reset") + del self._olderr + errfile = self.err + errfile.seek(0) + else: + errfile = None + if self._in: + sys.stdin = self._oldin + return outfile, errfile + + def readouterr(self): + """ return snapshot value of stdout/stderr capturings. """ + out = err = "" + if self._out: + out = sys.stdout.getvalue() + sys.stdout.truncate(0) + if self._err: + err = sys.stderr.getvalue() + sys.stderr.truncate(0) + return out, err + +class DontReadFromInput: + """Temporary stub class. Ideally when stdin is accessed, the + capturing should be turned off, with possibly all data captured + so far sent to the screen. This should be configurable, though, + because in automated test runs it is better to crash than + hang indefinitely. + """ + def read(self, *args): + raise IOError("reading from stdin while output is captured") + readline = read + readlines = read + __iter__ = read + + def fileno(self): + raise ValueError("redirected Stdin is pseudofile, has no fileno()") + def isatty(self): + return False + +try: + devnullpath = os.devnull +except AttributeError: + if os.name == 'nt': + devnullpath = 'NUL' + else: + devnullpath = '/dev/null' + + Added: pypy/trunk/py/impl/io/terminalwriter.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/io/terminalwriter.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,264 @@ +""" + +Helper functions for writing to terminals and files. + +""" + + +import sys, os +import py + +def _getdimensions(): + import termios,fcntl,struct + call = fcntl.ioctl(0,termios.TIOCGWINSZ,"\000"*8) + height,width = struct.unpack( "hhhh", call ) [:2] + return height, width + +if sys.platform == 'win32': + # ctypes access to the Windows console + + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + FOREGROUND_BLUE = 0x0001 # text color contains blue. + FOREGROUND_GREEN = 0x0002 # text color contains green. + FOREGROUND_RED = 0x0004 # text color contains red. + FOREGROUND_WHITE = 0x0007 + FOREGROUND_INTENSITY = 0x0008 # text color is intensified. + BACKGROUND_BLUE = 0x0010 # background color contains blue. + BACKGROUND_GREEN = 0x0020 # background color contains green. + BACKGROUND_RED = 0x0040 # background color contains red. + BACKGROUND_WHITE = 0x0070 + BACKGROUND_INTENSITY = 0x0080 # background color is intensified. + + def GetStdHandle(kind): + import ctypes + return ctypes.windll.kernel32.GetStdHandle(kind) + + def SetConsoleTextAttribute(handle, attr): + import ctypes + ctypes.windll.kernel32.SetConsoleTextAttribute( + handle, attr) + + def _getdimensions(): + import ctypes + from ctypes import wintypes + + SHORT = ctypes.c_short + class COORD(ctypes.Structure): + _fields_ = [('X', SHORT), + ('Y', SHORT)] + class SMALL_RECT(ctypes.Structure): + _fields_ = [('Left', SHORT), + ('Top', SHORT), + ('Right', SHORT), + ('Bottom', SHORT)] + class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): + _fields_ = [('dwSize', COORD), + ('dwCursorPosition', COORD), + ('wAttributes', wintypes.WORD), + ('srWindow', SMALL_RECT), + ('dwMaximumWindowSize', COORD)] + STD_OUTPUT_HANDLE = -11 + handle = GetStdHandle(STD_OUTPUT_HANDLE) + info = CONSOLE_SCREEN_BUFFER_INFO() + ctypes.windll.kernel32.GetConsoleScreenBufferInfo( + handle, ctypes.byref(info)) + # Substract one from the width, otherwise the cursor wraps + # and the ending \n causes an empty line to display. + return info.dwSize.Y, info.dwSize.X - 1 + +def get_terminal_width(): + try: + height, width = _getdimensions() + except (SystemExit, KeyboardInterrupt): + raise + except: + # FALLBACK + width = int(os.environ.get('COLUMNS', 80))-1 + # XXX the windows getdimensions may be bogus, let's sanify a bit + width = max(width, 40) # we alaways need 40 chars + return width + +terminal_width = get_terminal_width() + +# XXX unify with _escaped func below +def ansi_print(text, esc, file=None, newline=True, flush=False): + if file is None: + file = sys.stderr + text = text.rstrip() + if esc and not isinstance(esc, tuple): + esc = (esc,) + if esc and sys.platform != "win32" and file.isatty(): + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text + + '\x1b[0m') # ANSI color code "reset" + if newline: + text += '\n' + + if esc and sys.platform == "win32" and file.isatty(): + if 1 in esc: + bold = True + esc = tuple([x for x in esc if x != 1]) + else: + bold = False + esctable = {() : FOREGROUND_WHITE, # normal + (31,): FOREGROUND_RED, # red + (32,): FOREGROUND_GREEN, # green + (33,): FOREGROUND_GREEN|FOREGROUND_RED, # yellow + (34,): FOREGROUND_BLUE, # blue + (35,): FOREGROUND_BLUE|FOREGROUND_RED, # purple + (36,): FOREGROUND_BLUE|FOREGROUND_GREEN, # cyan + (37,): FOREGROUND_WHITE, # white + (39,): FOREGROUND_WHITE, # reset + } + attr = esctable.get(esc, FOREGROUND_WHITE) + if bold: + attr |= FOREGROUND_INTENSITY + STD_OUTPUT_HANDLE = -11 + STD_ERROR_HANDLE = -12 + if file is sys.stderr: + handle = GetStdHandle(STD_ERROR_HANDLE) + else: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + SetConsoleTextAttribute(handle, attr) + file.write(text) + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + else: + file.write(text) + + if flush: + file.flush() + +def should_do_markup(file): + return hasattr(file, 'isatty') and file.isatty() \ + and os.environ.get('TERM') != 'dumb' + +class TerminalWriter(object): + _esctable = dict(black=30, red=31, green=32, yellow=33, + blue=34, purple=35, cyan=36, white=37, + Black=40, Red=41, Green=42, Yellow=43, + Blue=44, Purple=45, Cyan=46, White=47, + bold=1, light=2, blink=5, invert=7) + + # XXX deprecate stringio argument + def __init__(self, file=None, stringio=False, encoding=None): + self.encoding = encoding + + if file is None: + if stringio: + self.stringio = file = py.io.TextIO() + else: + file = py.std.sys.stdout + elif hasattr(file, '__call__'): + file = WriteFile(file, encoding=encoding) + self._file = file + self.fullwidth = get_terminal_width() + self.hasmarkup = should_do_markup(file) + + def _escaped(self, text, esc): + if esc and self.hasmarkup: + text = (''.join(['\x1b[%sm' % cod for cod in esc]) + + text +'\x1b[0m') + return text + + def markup(self, text, **kw): + esc = [] + for name in kw: + if name not in self._esctable: + raise ValueError("unknown markup: %r" %(name,)) + if kw[name]: + esc.append(self._esctable[name]) + return self._escaped(text, tuple(esc)) + + def sep(self, sepchar, title=None, fullwidth=None, **kw): + if fullwidth is None: + fullwidth = self.fullwidth + # the goal is to have the line be as long as possible + # under the condition that len(line) <= fullwidth + if title is not None: + # we want 2 + 2*len(fill) + len(title) <= fullwidth + # i.e. 2 + 2*len(sepchar)*N + len(title) <= fullwidth + # 2*len(sepchar)*N <= fullwidth - len(title) - 2 + # N <= (fullwidth - len(title) - 2) // (2*len(sepchar)) + N = (fullwidth - len(title) - 2) // (2*len(sepchar)) + fill = sepchar * N + line = "%s %s %s" % (fill, title, fill) + else: + # we want len(sepchar)*N <= fullwidth + # i.e. N <= fullwidth // len(sepchar) + line = sepchar * (fullwidth // len(sepchar)) + # in some situations there is room for an extra sepchar at the right, + # in particular if we consider that with a sepchar like "_ " the + # trailing space is not important at the end of the line + if len(line) + len(sepchar.rstrip()) <= fullwidth: + line += sepchar.rstrip() + + self.line(line, **kw) + + def write(self, s, **kw): + if s: + s = self._getbytestring(s) + if self.hasmarkup and kw: + s = self.markup(s, **kw) + self._file.write(s) + self._file.flush() + + def _getbytestring(self, s): + # XXX review this and the whole logic + if self.encoding and sys.version_info < (3,0) and isinstance(s, unicode): + return s.encode(self.encoding) + elif not isinstance(s, str): + return str(s) + return s + + def line(self, s='', **kw): + self.write(s, **kw) + self.write('\n') + +class Win32ConsoleWriter(TerminalWriter): + def write(self, s, **kw): + if s: + s = self._getbytestring(s) + if self.hasmarkup: + handle = GetStdHandle(STD_OUTPUT_HANDLE) + + if self.hasmarkup and kw: + attr = 0 + if kw.pop('bold', False): + attr |= FOREGROUND_INTENSITY + + if kw.pop('red', False): + attr |= FOREGROUND_RED + elif kw.pop('blue', False): + attr |= FOREGROUND_BLUE + elif kw.pop('green', False): + attr |= FOREGROUND_GREEN + else: + attr |= FOREGROUND_WHITE + + SetConsoleTextAttribute(handle, attr) + self._file.write(s) + self._file.flush() + if self.hasmarkup: + SetConsoleTextAttribute(handle, FOREGROUND_WHITE) + + def line(self, s="", **kw): + self.write(s+"\n", **kw) + +if sys.platform == 'win32': + TerminalWriter = Win32ConsoleWriter + +class WriteFile(object): + def __init__(self, writemethod, encoding=None): + self.encoding = encoding + self._writemethod = writemethod + + def write(self, data): + if self.encoding: + data = data.encode(self.encoding) + self._writemethod(data) + + def flush(self): + return + + Added: pypy/trunk/py/impl/log/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/log/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,2 @@ +""" logging API ('producers' and 'consumers' connected via keywords) """ + Added: pypy/trunk/py/impl/log/log.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/log/log.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,184 @@ +""" +basic logging functionality based on a producer/consumer scheme. + +XXX implement this API: (maybe put it into slogger.py?) + + log = Logger( + info=py.log.STDOUT, + debug=py.log.STDOUT, + command=None) + log.info("hello", "world") + log.command("hello", "world") + + log = Logger(info=Logger(something=...), + debug=py.log.STDOUT, + command=None) +""" +import py, sys + +class Message(object): + def __init__(self, keywords, args): + self.keywords = keywords + self.args = args + + def content(self): + return " ".join(map(str, self.args)) + + def prefix(self): + return "[%s] " % (":".join(self.keywords)) + + def __str__(self): + return self.prefix() + self.content() + + +class Producer(object): + """ (deprecated) Log producer API which sends messages to be logged + to a 'consumer' object, which then prints them to stdout, + stderr, files, etc. Used extensively by PyPy-1.1. + """ + + Message = Message # to allow later customization + keywords2consumer = {} + + def __init__(self, keywords, keywordmapper=None, **kw): + if hasattr(keywords, 'split'): + keywords = tuple(keywords.split()) + self._keywords = keywords + if keywordmapper is None: + keywordmapper = default_keywordmapper + self._keywordmapper = keywordmapper + + def __repr__(self): + return "" % ":".join(self._keywords) + + def __getattr__(self, name): + if '_' in name: + raise AttributeError(name) + producer = self.__class__(self._keywords + (name,)) + setattr(self, name, producer) + return producer + + def __call__(self, *args): + """ write a message to the appropriate consumer(s) """ + func = self._keywordmapper.getconsumer(self._keywords) + if func is not None: + func(self.Message(self._keywords, args)) + +class KeywordMapper: + def __init__(self): + self.keywords2consumer = {} + + def getstate(self): + return self.keywords2consumer.copy() + def setstate(self, state): + self.keywords2consumer.clear() + self.keywords2consumer.update(state) + + def getconsumer(self, keywords): + """ return a consumer matching the given keywords. + + tries to find the most suitable consumer by walking, starting from + the back, the list of keywords, the first consumer matching a + keyword is returned (falling back to py.log.default) + """ + for i in range(len(keywords), 0, -1): + try: + return self.keywords2consumer[keywords[:i]] + except KeyError: + continue + return self.keywords2consumer.get('default', default_consumer) + + def setconsumer(self, keywords, consumer): + """ set a consumer for a set of keywords. """ + # normalize to tuples + if isinstance(keywords, str): + keywords = tuple(filter(None, keywords.split())) + elif hasattr(keywords, '_keywords'): + keywords = keywords._keywords + elif not isinstance(keywords, tuple): + raise TypeError("key %r is not a string or tuple" % (keywords,)) + if consumer is not None and not py.builtin.callable(consumer): + if not hasattr(consumer, 'write'): + raise TypeError( + "%r should be None, callable or file-like" % (consumer,)) + consumer = File(consumer) + self.keywords2consumer[keywords] = consumer + +def default_consumer(msg): + """ the default consumer, prints the message to stdout (using 'print') """ + sys.stderr.write(str(msg)+"\n") + +default_keywordmapper = KeywordMapper() + +def setconsumer(keywords, consumer): + default_keywordmapper.setconsumer(keywords, consumer) + +def setstate(state): + default_keywordmapper.setstate(state) +def getstate(): + return default_keywordmapper.getstate() + +# +# Consumers +# + +class File(object): + """ log consumer wrapping a file(-like) object """ + def __init__(self, f): + assert hasattr(f, 'write') + #assert isinstance(f, file) or not hasattr(f, 'open') + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + self._file.write(str(msg) + "\n") + +class Path(object): + """ log consumer that opens and writes to a Path """ + def __init__(self, filename, append=False, + delayed_create=False, buffering=False): + self._append = append + self._filename = str(filename) + self._buffering = buffering + if not delayed_create: + self._openfile() + + def _openfile(self): + mode = self._append and 'a' or 'w' + f = open(self._filename, mode) + self._file = f + + def __call__(self, msg): + """ write a message to the log """ + if not hasattr(self, "_file"): + self._openfile() + self._file.write(str(msg) + "\n") + if not self._buffering: + self._file.flush() + +def STDOUT(msg): + """ consumer that writes to sys.stdout """ + sys.stdout.write(str(msg)+"\n") + +def STDERR(msg): + """ consumer that writes to sys.stderr """ + sys.stderr.write(str(msg)+"\n") + +class Syslog: + """ consumer that writes to the syslog daemon """ + + def __init__(self, priority = None): + if priority is None: + priority = self.LOG_INFO + self.priority = priority + + def __call__(self, msg): + """ write a message to the log """ + py.std.syslog.syslog(self.priority, str(msg)) + +for _prio in "EMERG ALERT CRIT ERR WARNING NOTICE INFO DEBUG".split(): + _prio = "LOG_" + _prio + try: + setattr(Syslog, _prio, getattr(py.std.syslog, _prio)) + except AttributeError: + pass Added: pypy/trunk/py/impl/log/warning.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/log/warning.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,72 @@ +import py, sys + +class Warning(DeprecationWarning): + def __init__(self, msg, path, lineno): + self.msg = msg + self.path = path + self.lineno = lineno + def __repr__(self): + return "%s:%d: %s" %(self.path, self.lineno+1, self.msg) + def __str__(self): + return self.msg + +def _apiwarn(startversion, msg, stacklevel=2, function=None): + # below is mostly COPIED from python2.4/warnings.py's def warn() + # Get context information + if stacklevel == "initpkg": + frame = sys._getframe(stacklevel == "initpkg" and 1 or stacklevel) + level = 2 + while frame: + co = frame.f_code + if co.co_name == "__getattr__" and co.co_filename.find("initpkg") !=-1: + stacklevel = level + break + level += 1 + frame = frame.f_back + else: + stacklevel = 1 + msg = "%s (since version %s)" %(msg, startversion) + warn(msg, stacklevel=stacklevel+1, function=function) + +def warn(msg, stacklevel=1, function=None): + if function is not None: + filename = py.std.inspect.getfile(function) + lineno = py.code.getrawcode(function).co_firstlineno + else: + try: + caller = sys._getframe(stacklevel) + except ValueError: + globals = sys.__dict__ + lineno = 1 + else: + globals = caller.f_globals + lineno = caller.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + filename = globals.get('__file__') + if filename: + fnl = filename.lower() + if fnl.endswith(".pyc") or fnl.endswith(".pyo"): + filename = filename[:-1] + elif fnl.endswith("$py.class"): + filename = filename.replace('$py.class', '.py') + else: + if module == "__main__": + try: + filename = sys.argv[0] + except AttributeError: + # embedded interpreters don't have sys.argv, see bug #839151 + filename = '__main__' + if not filename: + filename = module + path = py.path.local(filename) + warning = Warning(msg, path, lineno) + py.std.warnings.warn_explicit(warning, category=Warning, + filename=str(warning.path), + lineno=warning.lineno, + registry=py.std.warnings.__dict__.setdefault( + "__warningsregistry__", {}) + ) + Added: pypy/trunk/py/impl/path/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +""" unified file system api """ Added: pypy/trunk/py/impl/path/cacheutil.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/cacheutil.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,111 @@ +""" +This module contains multithread-safe cache implementations. + +All Caches have + + getorbuild(key, builder) + delentry(key) + +methods and allow configuration when instantiating the cache class. +""" +from time import time as gettime + +class BasicCache(object): + def __init__(self, maxentries=128): + self.maxentries = maxentries + self.prunenum = int(maxentries - maxentries/8) + self._dict = {} + + def _getentry(self, key): + return self._dict[key] + + def _putentry(self, key, entry): + self._prunelowestweight() + self._dict[key] = entry + + def delentry(self, key, raising=False): + try: + del self._dict[key] + except KeyError: + if raising: + raise + + def getorbuild(self, key, builder): + try: + entry = self._getentry(key) + except KeyError: + entry = self._build(key, builder) + self._putentry(key, entry) + return entry.value + + def _prunelowestweight(self): + """ prune out entries with lowest weight. """ + numentries = len(self._dict) + if numentries >= self.maxentries: + # evict according to entry's weight + items = [(entry.weight, key) + for key, entry in self._dict.items()] + items.sort() + index = numentries - self.prunenum + if index > 0: + for weight, key in items[:index]: + # in MT situations the element might be gone + self.delentry(key, raising=False) + +class BuildcostAccessCache(BasicCache): + """ A BuildTime/Access-counting cache implementation. + the weight of a value is computed as the product of + + num-accesses-of-a-value * time-to-build-the-value + + The values with the least such weights are evicted + if the cache maxentries threshold is superceded. + For implementation flexibility more than one object + might be evicted at a time. + """ + # time function to use for measuring build-times + + def _build(self, key, builder): + start = gettime() + val = builder() + end = gettime() + return WeightedCountingEntry(val, end-start) + + +class WeightedCountingEntry(object): + def __init__(self, value, oneweight): + self._value = value + self.weight = self._oneweight = oneweight + + def value(self): + self.weight += self._oneweight + return self._value + value = property(value) + +class AgingCache(BasicCache): + """ This cache prunes out cache entries that are too old. + """ + def __init__(self, maxentries=128, maxseconds=10.0): + super(AgingCache, self).__init__(maxentries) + self.maxseconds = maxseconds + + def _getentry(self, key): + entry = self._dict[key] + if entry.isexpired(): + self.delentry(key) + raise KeyError(key) + return entry + + def _build(self, key, builder): + val = builder() + entry = AgingEntry(val, gettime() + self.maxseconds) + return entry + +class AgingEntry(object): + def __init__(self, value, expirationtime): + self.value = value + self.weight = expirationtime + + def isexpired(self): + t = gettime() + return t >= self.weight Added: pypy/trunk/py/impl/path/common.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/common.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,333 @@ +""" +""" +import os, sys +import py + +class Checkers: + _depend_on_existence = 'exists', 'link', 'dir', 'file' + + def __init__(self, path): + self.path = path + + def dir(self): + raise NotImplementedError + + def file(self): + raise NotImplementedError + + def dotfile(self): + return self.path.basename.startswith('.') + + def ext(self, arg): + if not arg.startswith('.'): + arg = '.' + arg + return self.path.ext == arg + + def exists(self): + raise NotImplementedError + + def basename(self, arg): + return self.path.basename == arg + + def basestarts(self, arg): + return self.path.basename.startswith(arg) + + def relto(self, arg): + return self.path.relto(arg) + + def fnmatch(self, arg): + return FNMatcher(arg)(self.path) + + def endswith(self, arg): + return str(self.path).endswith(arg) + + def _evaluate(self, kw): + for name, value in kw.items(): + invert = False + meth = None + try: + meth = getattr(self, name) + except AttributeError: + if name[:3] == 'not': + invert = True + try: + meth = getattr(self, name[3:]) + except AttributeError: + pass + if meth is None: + raise TypeError( + "no %r checker available for %r" % (name, self.path)) + try: + if py.code.getrawcode(meth).co_argcount > 1: + if (not meth(value)) ^ invert: + return False + else: + if bool(value) ^ bool(meth()) ^ invert: + return False + except (py.error.ENOENT, py.error.ENOTDIR): + for name in self._depend_on_existence: + if name in kw: + if kw.get(name): + return False + name = 'not' + name + if name in kw: + if not kw.get(name): + return False + return True + +class NeverRaised(Exception): + pass + +class PathBase(object): + """ shared implementation for filesystem path objects.""" + Checkers = Checkers + + def __div__(self, other): + return self.join(str(other)) + __truediv__ = __div__ # py3k + + def basename(self): + """ basename part of path. """ + return self._getbyspec('basename')[0] + basename = property(basename, None, None, basename.__doc__) + + def purebasename(self): + """ pure base name of the path.""" + return self._getbyspec('purebasename')[0] + purebasename = property(purebasename, None, None, purebasename.__doc__) + + def ext(self): + """ extension of the path (including the '.').""" + return self._getbyspec('ext')[0] + ext = property(ext, None, None, ext.__doc__) + + def dirpath(self, *args, **kwargs): + """ return the directory Path of the current Path joined + with any given path arguments. + """ + return self.new(basename='').join(*args, **kwargs) + + def read(self, mode='r'): + """ read and return a bytestring from reading the path. """ + if sys.version_info < (2,3): + for x in 'u', 'U': + if x in mode: + mode = mode.replace(x, '') + f = self.open(mode) + try: + return f.read() + finally: + f.close() + + def readlines(self, cr=1): + """ read and return a list of lines from the path. if cr is False, the +newline will be removed from the end of each line. """ + if not cr: + content = self.read('rU') + return content.split('\n') + else: + f = self.open('rU') + try: + return f.readlines() + finally: + f.close() + + def load(self): + """ (deprecated) return object unpickled from self.read() """ + f = self.open('rb') + try: + return py.error.checked_call(py.std.pickle.load, f) + finally: + f.close() + + def move(self, target): + """ move this path to target. """ + if target.relto(self): + raise py.error.EINVAL(target, + "cannot move path into a subdirectory of itself") + try: + self.rename(target) + except py.error.EXDEV: # invalid cross-device link + self.copy(target) + self.remove() + + def __repr__(self): + """ return a string representation of this path. """ + return repr(str(self)) + + def check(self, **kw): + """ check a path for existence, or query its properties + + without arguments, this returns True if the path exists (on the + filesystem), False if not + + with (keyword only) arguments, the object compares the value + of the argument with the value of a property with the same name + (if it has one, else it raises a TypeError) + + when for example the keyword argument 'ext' is '.py', this will + return True if self.ext == '.py', False otherwise + """ + if not kw: + kw = {'exists' : 1} + return self.Checkers(self)._evaluate(kw) + + def relto(self, relpath): + """ return a string which is the relative part of the path + to the given 'relpath'. + """ + if not isinstance(relpath, (str, PathBase)): + raise TypeError("%r: not a string or path object" %(relpath,)) + strrelpath = str(relpath) + if strrelpath and strrelpath[-1] != self.sep: + strrelpath += self.sep + #assert strrelpath[-1] == self.sep + #assert strrelpath[-2] != self.sep + strself = str(self) + if sys.platform == "win32": + if os.path.normcase(strself).startswith( + os.path.normcase(strrelpath)): + return strself[len(strrelpath):] + elif strself.startswith(strrelpath): + return strself[len(strrelpath):] + return "" + + def bestrelpath(self, dest): + """ return a string which is a relative path from self + to dest such that self.join(bestrelpath) == dest and + if not such path can be determined return dest. + """ + try: + base = self.common(dest) + if not base: # can be the case on windows + return str(dest) + self2base = self.relto(base) + reldest = dest.relto(base) + if self2base: + n = self2base.count(self.sep) + 1 + else: + n = 0 + l = ['..'] * n + if reldest: + l.append(reldest) + target = dest.sep.join(l) + return target + except AttributeError: + return str(dest) + + + def parts(self, reverse=False): + """ return a root-first list of all ancestor directories + plus the path itself. + """ + current = self + l = [self] + while 1: + last = current + current = current.dirpath() + if last == current: + break + l.insert(0, current) + if reverse: + l.reverse() + return l + + def common(self, other): + """ return the common part shared with the other path + or None if there is no common part. + """ + last = None + for x, y in zip(self.parts(), other.parts()): + if x != y: + return last + last = x + return last + + def __add__(self, other): + """ return new path object with 'other' added to the basename""" + return self.new(basename=self.basename+str(other)) + + def __cmp__(self, other): + """ return sort value (-1, 0, +1). """ + try: + return cmp(self.strpath, other.strpath) + except AttributeError: + return cmp(str(self), str(other)) # self.path, other.path) + + def __lt__(self, other): + try: + return self.strpath < other.strpath + except AttributeError: + return str(self) < str(other) + + def visit(self, fil=None, rec=None, ignore=NeverRaised): + """ yields all paths below the current one + + fil is a filter (glob pattern or callable), if not matching the + path will not be yielded, defaulting to None (everything is + returned) + + rec is a filter (glob pattern or callable) that controls whether + a node is descended, defaulting to None + + ignore is an Exception class that is ignoredwhen calling dirlist() + on any of the paths (by default, all exceptions are reported) + """ + if isinstance(fil, str): + fil = FNMatcher(fil) + if rec: + if isinstance(rec, str): + rec = fnmatch(fil) + elif not hasattr(rec, '__call__'): + rec = None + try: + entries = self.listdir() + except ignore: + return + dirs = [p for p in entries + if p.check(dir=1) and (rec is None or rec(p))] + for subdir in dirs: + for p in subdir.visit(fil=fil, rec=rec, ignore=ignore): + yield p + for p in entries: + if fil is None or fil(p): + yield p + + def _sortlist(self, res, sort): + if sort: + if hasattr(sort, '__call__'): + res.sort(sort) + else: + res.sort() + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return self.strpath == str(other) + +class FNMatcher: + def __init__(self, pattern): + self.pattern = pattern + def __call__(self, path): + """return true if the basename/fullname matches the glob-'pattern'. + + * matches everything + ? matches any single character + [seq] matches any character in seq + [!seq] matches any char not in seq + + if the pattern contains a path-separator then the full path + is used for pattern matching and a '*' is prepended to the + pattern. + + if the pattern doesn't contain a path-separator the pattern + is only matched against the basename. + """ + pattern = self.pattern + if pattern.find(path.sep) == -1: + name = path.basename + else: + name = str(path) # path.strpath # XXX svn? + pattern = '*' + path.sep + pattern + from fnmatch import fnmatch + return fnmatch(name, pattern) + Added: pypy/trunk/py/impl/path/gateway/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/gateway/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +# Added: pypy/trunk/py/impl/path/gateway/channeltest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/gateway/channeltest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,65 @@ +import threading + + +class PathServer: + + def __init__(self, channel): + self.channel = channel + self.C2P = {} + self.next_id = 0 + threading.Thread(target=self.serve).start() + + def p2c(self, path): + id = self.next_id + self.next_id += 1 + self.C2P[id] = path + return id + + def command_LIST(self, id, *args): + path = self.C2P[id] + answer = [(self.p2c(p), p.basename) for p in path.listdir(*args)] + self.channel.send(answer) + + def command_DEL(self, id): + del self.C2P[id] + + def command_GET(self, id, spec): + path = self.C2P[id] + self.channel.send(path._getbyspec(spec)) + + def command_READ(self, id): + path = self.C2P[id] + self.channel.send(path.read()) + + def command_JOIN(self, id, resultid, *args): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.join(*args) + + def command_DIRPATH(self, id, resultid): + path = self.C2P[id] + assert resultid not in self.C2P + self.C2P[resultid] = path.dirpath() + + def serve(self): + try: + while 1: + msg = self.channel.receive() + meth = getattr(self, 'command_' + msg[0]) + meth(*msg[1:]) + except EOFError: + pass + +if __name__ == '__main__': + import py + gw = execnet.PopenGateway() + channel = gw._channelfactory.new() + srv = PathServer(channel) + c = gw.remote_exec(""" + import remotepath + p = remotepath.RemotePath(channel.receive(), channel.receive()) + channel.send(len(p.listdir())) + """) + c.send(channel) + c.send(srv.p2c(py.path.local('/tmp'))) + print(c.receive()) Added: pypy/trunk/py/impl/path/gateway/channeltest2.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/gateway/channeltest2.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,21 @@ +import py +from remotepath import RemotePath + + +SRC = open('channeltest.py', 'r').read() + +SRC += ''' +import py +srv = PathServer(channel.receive()) +channel.send(srv.p2c(py.path.local("/tmp"))) +''' + + +#gw = execnet.SshGateway('codespeak.net') +gw = execnet.PopenGateway() +gw.remote_init_threads(5) +c = gw.remote_exec(SRC, stdout=py.std.sys.stdout, stderr=py.std.sys.stderr) +subchannel = gw._channelfactory.new() +c.send(subchannel) + +p = RemotePath(subchannel, c.receive()) Added: pypy/trunk/py/impl/path/gateway/remotepath.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/gateway/remotepath.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,47 @@ +import py, itertools +from py.impl.path import common + +COUNTER = itertools.count() + +class RemotePath(common.PathBase): + sep = '/' + + def __init__(self, channel, id, basename=None): + self._channel = channel + self._id = id + self._basename = basename + self._specs = {} + + def __del__(self): + self._channel.send(('DEL', self._id)) + + def __repr__(self): + return 'RemotePath(%s)' % self.basename + + def listdir(self, *args): + self._channel.send(('LIST', self._id) + args) + return [RemotePath(self._channel, id, basename) + for (id, basename) in self._channel.receive()] + + def dirpath(self): + id = ~COUNTER.next() + self._channel.send(('DIRPATH', self._id, id)) + return RemotePath(self._channel, id) + + def join(self, *args): + id = ~COUNTER.next() + self._channel.send(('JOIN', self._id, id) + args) + return RemotePath(self._channel, id) + + def _getbyspec(self, spec): + parts = spec.split(',') + ask = [x for x in parts if x not in self._specs] + if ask: + self._channel.send(('GET', self._id, ",".join(ask))) + for part, value in zip(ask, self._channel.receive()): + self._specs[part] = value + return [self._specs[x] for x in parts] + + def read(self): + self._channel.send(('READ', self._id)) + return self._channel.receive() Added: pypy/trunk/py/impl/path/local.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/local.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,799 @@ +""" +local path implementation. +""" +import sys, os, stat, re, atexit +import py +from py.impl.path import common + +iswin32 = sys.platform == "win32" + +class Stat(object): + def __getattr__(self, name): + return getattr(self._osstatresult, "st_" + name) + + def __init__(self, path, osstatresult): + self.path = path + self._osstatresult = osstatresult + + def owner(self): + if iswin32: + raise NotImplementedError("XXX win32") + import pwd + entry = py.error.checked_call(pwd.getpwuid, self.uid) + return entry[0] + owner = property(owner, None, None, "owner of path") + + def group(self): + """ return group name of file. """ + if iswin32: + raise NotImplementedError("XXX win32") + import grp + entry = py.error.checked_call(grp.getgrgid, self.gid) + return entry[0] + group = property(group) + +class PosixPath(common.PathBase): + def chown(self, user, group, rec=0): + """ change ownership to the given user and group. + user and group may be specified by a number or + by a name. if rec is True change ownership + recursively. + """ + uid = getuserid(user) + gid = getgroupid(group) + if rec: + for x in self.visit(rec=lambda x: x.check(link=0)): + if x.check(link=0): + py.error.checked_call(os.chown, str(x), uid, gid) + py.error.checked_call(os.chown, str(self), uid, gid) + + def readlink(self): + """ return value of a symbolic link. """ + return py.error.checked_call(os.readlink, self.strpath) + + def mklinkto(self, oldname): + """ posix style hard link to another name. """ + py.error.checked_call(os.link, str(oldname), str(self)) + + def mksymlinkto(self, value, absolute=1): + """ create a symbolic link with the given value (pointing to another name). """ + if absolute: + py.error.checked_call(os.symlink, str(value), self.strpath) + else: + base = self.common(value) + # with posix local paths '/' is always a common base + relsource = self.__class__(value).relto(base) + reldest = self.relto(base) + n = reldest.count(self.sep) + target = self.sep.join(('..', )*n + (relsource, )) + py.error.checked_call(os.symlink, target, self.strpath) + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return py.error.checked_call(os.path.samefile, str(self), str(other)) + +def getuserid(user): + import pwd + if not isinstance(user, int): + user = pwd.getpwnam(user)[2] + return user + +def getgroupid(group): + import grp + if not isinstance(group, int): + group = grp.getgrnam(group)[2] + return group + +FSBase = not iswin32 and PosixPath or common.PathBase + +class LocalPath(FSBase): + """ object oriented interface to os.path and other local filesystem + related information. + """ + sep = os.sep + class Checkers(common.Checkers): + def _stat(self): + try: + return self._statcache + except AttributeError: + try: + self._statcache = self.path.stat() + except py.error.ELOOP: + self._statcache = self.path.lstat() + return self._statcache + + def dir(self): + return stat.S_ISDIR(self._stat().mode) + + def file(self): + return stat.S_ISREG(self._stat().mode) + + def exists(self): + return self._stat() + + def link(self): + st = self.path.lstat() + return stat.S_ISLNK(st.mode) + + def __new__(cls, path=None): + """ Initialize and return a local Path instance. + + Path can be relative to the current directory. + If it is None then the current working directory is taken. + Note that Path instances always carry an absolute path. + Note also that passing in a local path object will simply return + the exact same path object. Use new() to get a new copy. + """ + if isinstance(path, common.PathBase): + if path.__class__ == cls: + return path + path = path.strpath + # initialize the path + self = object.__new__(cls) + if not path: + self.strpath = os.getcwd() + elif isinstance(path, py.builtin._basestring): + self.strpath = os.path.abspath(os.path.normpath(str(path))) + else: + raise ValueError("can only pass None, Path instances " + "or non-empty strings to LocalPath") + assert isinstance(self.strpath, str) + return self + + def __hash__(self): + return hash(self.strpath) + + def __eq__(self, other): + s1 = str(self) + s2 = str(other) + if iswin32: + s1 = s1.lower() + s2 = s2.lower() + return s1 == s2 + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return str(self) < str(other) + + def remove(self, rec=1): + """ remove a file or directory (or a directory tree if rec=1). """ + if self.check(dir=1, link=0): + if rec: + # force remove of readonly files on windows + if iswin32: + self.chmod(448, rec=1) # octcal 0700 + py.error.checked_call(py.std.shutil.rmtree, self.strpath) + else: + py.error.checked_call(os.rmdir, self.strpath) + else: + if iswin32: + self.chmod(448) # octcal 0700 + py.error.checked_call(os.remove, self.strpath) + + def computehash(self, hashtype="md5", chunksize=524288): + """ return hexdigest of hashvalue for this file. """ + try: + try: + import hashlib as mod + except ImportError: + if hashtype == "sha1": + hashtype = "sha" + mod = __import__(hashtype) + hash = getattr(mod, hashtype)() + except (AttributeError, ImportError): + raise ValueError("Don't know how to compute %r hash" %(hashtype,)) + f = self.open('rb') + try: + while 1: + buf = f.read(chunksize) + if not buf: + return hash.hexdigest() + hash.update(buf) + finally: + f.close() + + def new(self, **kw): + """ create a modified version of this path. + the following keyword arguments modify various path parts: + + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + drive, dirname, basename, purebasename,ext = self._getbyspec( + "drive,dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + try: + ext = kw['ext'] + except KeyError: + pass + else: + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('drive', drive) + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + obj.strpath = os.path.normpath( + "%(drive)s%(dirname)s%(sep)s%(basename)s" % kw) + return obj + + def _getbyspec(self, spec): + """ return a sequence of specified path parts. 'spec' is + a comma separated string containing path part names. + according to the following convention: + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + + args = filter(None, spec.split(',') ) + append = res.append + for name in args: + if name == 'drive': + append(parts[0]) + elif name == 'dirname': + append(self.sep.join(['']+parts[1:-1])) + else: + basename = parts[-1] + if name == 'basename': + append(basename) + else: + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + append(purebasename) + elif name == 'ext': + append(ext) + else: + raise ValueError("invalid part specification %r" % name) + return res + + def join(self, *args, **kwargs): + """ return a new path by appending all 'args' as path + components. if abs=1 is used restart from root if any + of the args is an absolute path. + """ + if not args: + return self + strpath = self.strpath + sep = self.sep + strargs = [str(x) for x in args] + if kwargs.get('abs', 0): + for i in range(len(strargs)-1, -1, -1): + if os.path.isabs(strargs[i]): + strpath = strargs[i] + strargs = strargs[i+1:] + break + for arg in strargs: + arg = arg.strip(sep) + if iswin32: + # allow unix style paths even on windows. + arg = arg.strip('/') + arg = arg.replace('/', sep) + if arg: + if not strpath.endswith(sep): + strpath += sep + strpath += arg + obj = self.new() + obj.strpath = os.path.normpath(strpath) + return obj + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return py.error.checked_call(open, self.strpath, mode) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + res = [] + for name in py.error.checked_call(os.listdir, self.strpath): + childurl = self.join(name) + if fil is None or fil(childurl): + res.append(childurl) + self._sortlist(res, sort) + return res + + def size(self): + """ return size of the underlying file object """ + return self.stat().size + + def mtime(self): + """ return last modification time of the path. """ + return self.stat().mtime + + def copy(self, target, archive=False): + """ copy path to target.""" + assert not archive, "XXX archive-mode not supported" + if self.check(file=1): + if target.check(dir=1): + target = target.join(self.basename) + assert self!=target + copychunked(self, target) + else: + def rec(p): + return p.check(link=0) + for x in self.visit(rec=rec): + relpath = x.relto(self) + newx = target.join(relpath) + newx.dirpath().ensure(dir=1) + if x.check(link=1): + newx.mksymlinkto(x.readlink()) + elif x.check(file=1): + copychunked(x, newx) + elif x.check(dir=1): + newx.ensure(dir=1) + + def rename(self, target): + """ rename this path to target. """ + return py.error.checked_call(os.rename, str(self), str(target)) + + def dump(self, obj, bin=1): + """ pickle object into path location""" + f = self.open('wb') + try: + py.error.checked_call(py.std.pickle.dump, obj, f, bin) + finally: + f.close() + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + p = self.join(*args) + py.error.checked_call(os.mkdir, str(p)) + return p + + def write(self, data, mode='w'): + """ write data into path. """ + if 'b' in mode: + if not py.builtin._isbytes(data): + raise ValueError("can only process bytes") + else: + if not py.builtin._istext(data): + if not py.builtin._isbytes(data): + data = str(data) + else: + data = py.builtin._totext(data, sys.getdefaultencoding()) + f = self.open(mode) + try: + f.write(data) + finally: + f.close() + + def _ensuredirs(self): + parent = self.dirpath() + if parent == self: + return self + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + try: + self.mkdir() + except py.error.EEXIST: + # race condition: file/dir created by another thread/process. + # complain if it is not a dir + if self.check(dir=0): + raise + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if kwargs.get('dir', 0): + return p._ensuredirs() + else: + p.dirpath()._ensuredirs() + if not p.check(file=1): + p.open('w').close() + return p + + def stat(self): + """ Return an os.stat() tuple. """ + return Stat(self, py.error.checked_call(os.stat, self.strpath)) + + def lstat(self): + """ Return an os.lstat() tuple. """ + return Stat(self, py.error.checked_call(os.lstat, self.strpath)) + + def setmtime(self, mtime=None): + """ set modification time for the given path. if 'mtime' is None + (the default) then the file's mtime is set to current time. + + Note that the resolution for 'mtime' is platform dependent. + """ + if mtime is None: + return py.error.checked_call(os.utime, self.strpath, mtime) + try: + return py.error.checked_call(os.utime, self.strpath, (-1, mtime)) + except py.error.EINVAL: + return py.error.checked_call(os.utime, self.strpath, (self.atime(), mtime)) + + def chdir(self): + """ change directory to self and return old current directory """ + old = self.__class__() + py.error.checked_call(os.chdir, self.strpath) + return old + + def realpath(self): + """ return a new path which contains no symbolic links.""" + return self.__class__(os.path.realpath(self.strpath)) + + def atime(self): + """ return last access time of the path. """ + return self.stat().atime + + def __repr__(self): + return 'local(%r)' % self.strpath + + def __str__(self): + """ return string representation of the Path. """ + return self.strpath + + def pypkgpath(self, pkgname=None): + """ return the path's package path by looking for the given + pkgname. If pkgname is None then look for the last + directory upwards which still contains an __init__.py. + Return None if a pkgpath can not be determined. + """ + pkgpath = None + for parent in self.parts(reverse=True): + if pkgname is None: + if parent.check(file=1): + continue + if parent.join('__init__.py').check(): + pkgpath = parent + continue + return pkgpath + else: + if parent.basename == pkgname: + return parent + return pkgpath + + def _prependsyspath(self, path): + s = str(path) + if s != sys.path[0]: + #print "prepending to sys.path", s + sys.path.insert(0, s) + + def chmod(self, mode, rec=0): + """ change permissions to the given mode. If mode is an + integer it directly encodes the os-specific modes. + if rec is True perform recursively. + """ + if not isinstance(mode, int): + raise TypeError("mode %r must be an integer" % (mode,)) + if rec: + for x in self.visit(rec=rec): + py.error.checked_call(os.chmod, str(x), mode) + py.error.checked_call(os.chmod, str(self), mode) + + def pyimport(self, modname=None, ensuresyspath=True): + """ return path as an imported python module. + if modname is None, look for the containing package + and construct an according module name. + The module will be put/looked up in sys.modules. + """ + if not self.check(): + raise py.error.ENOENT(self) + #print "trying to import", self + pkgpath = None + if modname is None: + pkgpath = self.pypkgpath() + if pkgpath is not None: + if ensuresyspath: + self._prependsyspath(pkgpath.dirpath()) + pkg = __import__(pkgpath.basename, None, None, []) + names = self.new(ext='').relto(pkgpath.dirpath()) + names = names.split(self.sep) + modname = ".".join(names) + else: + # no package scope, still make it possible + if ensuresyspath: + self._prependsyspath(self.dirpath()) + modname = self.purebasename + mod = __import__(modname, None, None, ['__doc__']) + modfile = mod.__file__ + if modfile[-4:] in ('.pyc', '.pyo'): + modfile = modfile[:-1] + elif modfile.endswith('$py.class'): + modfile = modfile[:-9] + '.py' + if not self.samefile(modfile): + raise EnvironmentError("mismatch:\n" + "imported module %r\n" + "does not stem from %r\n" + "maybe __init__.py files are missing?" % (mod, str(self))) + return mod + else: + try: + return sys.modules[modname] + except KeyError: + # we have a custom modname, do a pseudo-import + mod = py.std.types.ModuleType(modname) + mod.__file__ = str(self) + sys.modules[modname] = mod + try: + py.builtin.execfile(str(self), mod.__dict__) + except: + del sys.modules[modname] + raise + return mod + + def sysexec(self, *argv, **popen_opts): + """ return stdout text from executing a system child process, + where the 'self' path points to executable. + The process is directly invoked and not through a system shell. + """ + from subprocess import Popen, PIPE + argv = map(str, argv) + popen_opts['stdout'] = popen_opts['stderr'] = PIPE + proc = Popen([str(self)] + list(argv), **popen_opts) + stdout, stderr = proc.communicate() + ret = proc.wait() + if py.builtin._isbytes(stdout): + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + if ret != 0: + if py.builtin._isbytes(stderr): + stderr = py.builtin._totext(stderr, sys.getdefaultencoding()) + raise py.process.cmdexec.Error(ret, ret, str(self), + stdout, stderr,) + return stdout + + def sysfind(cls, name, checker=None): + """ return a path object found by looking at the systems + underlying PATH specification. If the checker is not None + it will be invoked to filter matching paths. If a binary + cannot be found, None is returned + Note: This is probably not working on plain win32 systems + but may work on cygwin. + """ + if os.path.isabs(name): + p = py.path.local(name) + if p.check(file=1): + return p + else: + if iswin32: + paths = py.std.os.environ['Path'].split(';') + if '' not in paths and '.' not in paths: + paths.append('.') + try: + systemroot = os.environ['SYSTEMROOT'] + except KeyError: + pass + else: + paths = [re.sub('%SystemRoot%', systemroot, path) + for path in paths] + tryadd = '', '.exe', '.com', '.bat' # XXX add more? + else: + paths = py.std.os.environ['PATH'].split(':') + tryadd = ('',) + + for x in paths: + for addext in tryadd: + p = py.path.local(x).join(name, abs=True) + addext + try: + if p.check(file=1): + if checker: + if not checker(p): + continue + return p + except py.error.EACCES: + pass + return None + sysfind = classmethod(sysfind) + + def _gethomedir(cls): + try: + x = os.environ['HOME'] + except KeyError: + x = os.environ['HOMEPATH'] + return cls(x) + _gethomedir = classmethod(_gethomedir) + + #""" + #special class constructors for local filesystem paths + #""" + def get_temproot(cls): + """ return the system's temporary directory + (where tempfiles are usually created in) + """ + return py.path.local(py.std.tempfile.gettempdir()) + get_temproot = classmethod(get_temproot) + + def mkdtemp(cls): + """ return a Path object pointing to a fresh new temporary directory + (which we created ourself). + """ + import tempfile + tries = 10 + for i in range(tries): + dname = tempfile.mktemp() + dpath = cls(tempfile.mktemp()) + try: + dpath.mkdir() + except (py.error.EEXIST, py.error.EPERM, py.error.EACCES): + continue + return dpath + raise py.error.ENOENT(dpath, "could not create tempdir, %d tries" % tries) + mkdtemp = classmethod(mkdtemp) + + def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800): # two days + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = cls.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError): # AttributeError on win32 + pass + + return udir + make_numbered_dir = classmethod(make_numbered_dir) + +def copychunked(src, dest): + chunksize = 524288 # half a meg of bytes + fsrc = src.open('rb') + try: + fdest = dest.open('wb') + try: + while 1: + buf = fsrc.read(chunksize) + if not buf: + break + fdest.write(buf) + finally: + fdest.close() + finally: + fsrc.close() + +def autopath(globs=None): + """ (deprecated) return the (local) path of the "current" file pointed to by globals or - if it is none - alternatively the callers frame globals. + + the path will always point to a .py file or to None. + the path will have the following payload: + pkgdir is the last parent directory path containing __init__.py + """ + py.log._apiwarn("1.1", "py.magic.autopath deprecated, " + "use py.path.local(__file__) and maybe pypkgpath/pyimport().") + if globs is None: + globs = sys._getframe(1).f_globals + try: + __file__ = globs['__file__'] + except KeyError: + if not sys.argv[0]: + raise ValueError("cannot compute autopath in interactive mode") + __file__ = os.path.abspath(sys.argv[0]) + + ret = py.path.local(__file__) + if ret.ext in ('.pyc', '.pyo'): + ret = ret.new(ext='.py') + current = pkgdir = ret.dirpath() + while 1: + if current.join('__init__.py').check(): + pkgdir = current + current = current.dirpath() + if pkgdir != current: + continue + elif str(current) not in sys.path: + sys.path.insert(0, str(current)) + break + ret.pkgdir = pkgdir + return ret + Added: pypy/trunk/py/impl/path/local.py.orig ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/local.py.orig Wed Nov 11 18:54:49 2009 @@ -0,0 +1,802 @@ +""" +local path implementation. +""" +import sys, os, stat, re, atexit +import py +from py.impl.path import common + +iswin32 = sys.platform == "win32" + +class Stat(object): + def __getattr__(self, name): + return getattr(self._osstatresult, "st_" + name) + + def __init__(self, path, osstatresult): + self.path = path + self._osstatresult = osstatresult + + def owner(self): + if iswin32: + raise NotImplementedError("XXX win32") + import pwd + entry = py.error.checked_call(pwd.getpwuid, self.uid) + return entry[0] + owner = property(owner, None, None, "owner of path") + + def group(self): + """ return group name of file. """ + if iswin32: + raise NotImplementedError("XXX win32") + import grp + entry = py.error.checked_call(grp.getgrgid, self.gid) + return entry[0] + group = property(group) + +class PosixPath(common.PathBase): + def chown(self, user, group, rec=0): + """ change ownership to the given user and group. + user and group may be specified by a number or + by a name. if rec is True change ownership + recursively. + """ + uid = getuserid(user) + gid = getgroupid(group) + if rec: + for x in self.visit(rec=lambda x: x.check(link=0)): + if x.check(link=0): + py.error.checked_call(os.chown, str(x), uid, gid) + py.error.checked_call(os.chown, str(self), uid, gid) + + def readlink(self): + """ return value of a symbolic link. """ + return py.error.checked_call(os.readlink, self.strpath) + + def mklinkto(self, oldname): + """ posix style hard link to another name. """ + py.error.checked_call(os.link, str(oldname), str(self)) + + def mksymlinkto(self, value, absolute=1): + """ create a symbolic link with the given value (pointing to another name). """ + if absolute: + py.error.checked_call(os.symlink, str(value), self.strpath) + else: + base = self.common(value) + # with posix local paths '/' is always a common base + relsource = self.__class__(value).relto(base) + reldest = self.relto(base) + n = reldest.count(self.sep) + target = self.sep.join(('..', )*n + (relsource, )) + py.error.checked_call(os.symlink, target, self.strpath) + + def samefile(self, other): + """ return True if other refers to the same stat object as self. """ + return py.error.checked_call(os.path.samefile, str(self), str(other)) + +def getuserid(user): + import pwd + if not isinstance(user, int): + user = pwd.getpwnam(user)[2] + return user + +def getgroupid(group): + import grp + if not isinstance(group, int): + group = grp.getgrnam(group)[2] + return group + +FSBase = not iswin32 and PosixPath or common.PathBase + +class LocalPath(FSBase): + """ object oriented interface to os.path and other local filesystem + related information. + """ + sep = os.sep + class Checkers(common.Checkers): + def _stat(self): + try: + return self._statcache + except AttributeError: + try: + self._statcache = self.path.stat() + except py.error.ELOOP: + self._statcache = self.path.lstat() + return self._statcache + + def dir(self): + return stat.S_ISDIR(self._stat().mode) + + def file(self): + return stat.S_ISREG(self._stat().mode) + + def exists(self): + return self._stat() + + def link(self): + st = self.path.lstat() + return stat.S_ISLNK(st.mode) + + def __new__(cls, path=None): + """ Initialize and return a local Path instance. + + Path can be relative to the current directory. + If it is None then the current working directory is taken. + Note that Path instances always carry an absolute path. + Note also that passing in a local path object will simply return + the exact same path object. Use new() to get a new copy. + """ + if isinstance(path, common.PathBase): + if path.__class__ == cls: + return path + path = path.strpath + # initialize the path + self = object.__new__(cls) + if not path: + self.strpath = os.getcwd() + elif isinstance(path, py.builtin._basestring): + self.strpath = os.path.abspath(os.path.normpath(str(path))) + else: + raise ValueError("can only pass None, Path instances " + "or non-empty strings to LocalPath") + assert isinstance(self.strpath, str) + return self + + def __hash__(self): + return hash(self.strpath) + + def __eq__(self, other): + s1 = str(self) + s2 = str(other) + if iswin32: + s1 = s1.lower() + s2 = s2.lower() + return s1 == s2 + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return str(self) < str(other) + + def remove(self, rec=1): + """ remove a file or directory (or a directory tree if rec=1). """ + if self.check(dir=1, link=0): + if rec: + # force remove of readonly files on windows + if iswin32: + self.chmod(448, rec=1) # octcal 0700 + py.error.checked_call(py.std.shutil.rmtree, self.strpath) + else: + py.error.checked_call(os.rmdir, self.strpath) + else: + if iswin32: + self.chmod(448) # octcal 0700 + py.error.checked_call(os.remove, self.strpath) + + def computehash(self, hashtype="md5", chunksize=524288): + """ return hexdigest of hashvalue for this file. """ + try: + try: + import hashlib as mod + except ImportError: + if hashtype == "sha1": + hashtype = "sha" + mod = __import__(hashtype) + hash = getattr(mod, hashtype)() + except (AttributeError, ImportError): + raise ValueError("Don't know how to compute %r hash" %(hashtype,)) + f = self.open('rb') + try: + while 1: + buf = f.read(chunksize) + if not buf: + return hash.hexdigest() + hash.update(buf) + finally: + f.close() + + def new(self, **kw): + """ create a modified version of this path. + the following keyword arguments modify various path parts: + + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + drive, dirname, basename, purebasename,ext = self._getbyspec( + "drive,dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + try: + ext = kw['ext'] + except KeyError: + pass + else: + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('drive', drive) + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + obj.strpath = os.path.normpath( + "%(drive)s%(dirname)s%(sep)s%(basename)s" % kw) + return obj + + def _getbyspec(self, spec): + """ return a sequence of specified path parts. 'spec' is + a comma separated string containing path part names. + according to the following convention: + a:/some/path/to/a/file.ext + || drive + |-------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + + args = filter(None, spec.split(',') ) + append = res.append + for name in args: + if name == 'drive': + append(parts[0]) + elif name == 'dirname': + append(self.sep.join(['']+parts[1:-1])) + else: + basename = parts[-1] + if name == 'basename': + append(basename) + else: + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + append(purebasename) + elif name == 'ext': + append(ext) + else: + raise ValueError("invalid part specification %r" % name) + return res + + def join(self, *args, **kwargs): + """ return a new path by appending all 'args' as path + components. if abs=1 is used restart from root if any + of the args is an absolute path. + """ + if not args: + return self + strpath = self.strpath + sep = self.sep + strargs = [str(x) for x in args] + if kwargs.get('abs', 0): + for i in range(len(strargs)-1, -1, -1): + if os.path.isabs(strargs[i]): + strpath = strargs[i] + strargs = strargs[i+1:] + break + for arg in strargs: + arg = arg.strip(sep) + if iswin32: + # allow unix style paths even on windows. + arg = arg.strip('/') + arg = arg.replace('/', sep) + if arg: + if not strpath.endswith(sep): + strpath += sep + strpath += arg + obj = self.new() + obj.strpath = os.path.normpath(strpath) + return obj + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return py.error.checked_call(open, self.strpath, mode) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + res = [] + for name in py.error.checked_call(os.listdir, self.strpath): + childurl = self.join(name) + if fil is None or fil(childurl): + res.append(childurl) + self._sortlist(res, sort) + return res + + def size(self): + """ return size of the underlying file object """ + return self.stat().size + + def mtime(self): + """ return last modification time of the path. """ + return self.stat().mtime + + def copy(self, target, archive=False): + """ copy path to target.""" + assert not archive, "XXX archive-mode not supported" + if self.check(file=1): + if target.check(dir=1): + target = target.join(self.basename) + assert self!=target + copychunked(self, target) + else: + def rec(p): + return p.check(link=0) + for x in self.visit(rec=rec): + relpath = x.relto(self) + newx = target.join(relpath) + newx.dirpath().ensure(dir=1) + if x.check(link=1): + newx.mksymlinkto(x.readlink()) + elif x.check(file=1): + copychunked(x, newx) + elif x.check(dir=1): + newx.ensure(dir=1) + + def rename(self, target): + """ rename this path to target. """ + return py.error.checked_call(os.rename, str(self), str(target)) + + def dump(self, obj, bin=1): + """ pickle object into path location""" + f = self.open('wb') + try: + py.error.checked_call(py.std.pickle.dump, obj, f, bin) + finally: + f.close() + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + p = self.join(*args) + py.error.checked_call(os.mkdir, str(p)) + return p + + def write(self, data, mode='w'): + """ write data into path. """ + if 'b' in mode: + if not py.builtin._isbytes(data): + raise ValueError("can only process bytes") + else: + if not py.builtin._istext(data): + if not py.builtin._isbytes(data): + data = str(data) + else: + try: + data = py.builtin._totext(data, sys.getdefaultencoding()) + except UnicodeDecodeError: + pass + f = self.open(mode) + try: + f.write(data) + finally: + f.close() + + def _ensuredirs(self): + parent = self.dirpath() + if parent == self: + return self + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + try: + self.mkdir() + except py.error.EEXIST: + # race condition: file/dir created by another thread/process. + # complain if it is not a dir + if self.check(dir=0): + raise + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if kwargs.get('dir', 0): + return p._ensuredirs() + else: + p.dirpath()._ensuredirs() + if not p.check(file=1): + p.open('w').close() + return p + + def stat(self): + """ Return an os.stat() tuple. """ + return Stat(self, py.error.checked_call(os.stat, self.strpath)) + + def lstat(self): + """ Return an os.lstat() tuple. """ + return Stat(self, py.error.checked_call(os.lstat, self.strpath)) + + def setmtime(self, mtime=None): + """ set modification time for the given path. if 'mtime' is None + (the default) then the file's mtime is set to current time. + + Note that the resolution for 'mtime' is platform dependent. + """ + if mtime is None: + return py.error.checked_call(os.utime, self.strpath, mtime) + try: + return py.error.checked_call(os.utime, self.strpath, (-1, mtime)) + except py.error.EINVAL: + return py.error.checked_call(os.utime, self.strpath, (self.atime(), mtime)) + + def chdir(self): + """ change directory to self and return old current directory """ + old = self.__class__() + py.error.checked_call(os.chdir, self.strpath) + return old + + def realpath(self): + """ return a new path which contains no symbolic links.""" + return self.__class__(os.path.realpath(self.strpath)) + + def atime(self): + """ return last access time of the path. """ + return self.stat().atime + + def __repr__(self): + return 'local(%r)' % self.strpath + + def __str__(self): + """ return string representation of the Path. """ + return self.strpath + + def pypkgpath(self, pkgname=None): + """ return the path's package path by looking for the given + pkgname. If pkgname is None then look for the last + directory upwards which still contains an __init__.py. + Return None if a pkgpath can not be determined. + """ + pkgpath = None + for parent in self.parts(reverse=True): + if pkgname is None: + if parent.check(file=1): + continue + if parent.join('__init__.py').check(): + pkgpath = parent + continue + return pkgpath + else: + if parent.basename == pkgname: + return parent + return pkgpath + + def _prependsyspath(self, path): + s = str(path) + if s != sys.path[0]: + #print "prepending to sys.path", s + sys.path.insert(0, s) + + def chmod(self, mode, rec=0): + """ change permissions to the given mode. If mode is an + integer it directly encodes the os-specific modes. + if rec is True perform recursively. + """ + if not isinstance(mode, int): + raise TypeError("mode %r must be an integer" % (mode,)) + if rec: + for x in self.visit(rec=rec): + py.error.checked_call(os.chmod, str(x), mode) + py.error.checked_call(os.chmod, str(self), mode) + + def pyimport(self, modname=None, ensuresyspath=True): + """ return path as an imported python module. + if modname is None, look for the containing package + and construct an according module name. + The module will be put/looked up in sys.modules. + """ + if not self.check(): + raise py.error.ENOENT(self) + #print "trying to import", self + pkgpath = None + if modname is None: + pkgpath = self.pypkgpath() + if pkgpath is not None: + if ensuresyspath: + self._prependsyspath(pkgpath.dirpath()) + pkg = __import__(pkgpath.basename, None, None, []) + names = self.new(ext='').relto(pkgpath.dirpath()) + names = names.split(self.sep) + modname = ".".join(names) + else: + # no package scope, still make it possible + if ensuresyspath: + self._prependsyspath(self.dirpath()) + modname = self.purebasename + mod = __import__(modname, None, None, ['__doc__']) + modfile = mod.__file__ + if modfile[-4:] in ('.pyc', '.pyo'): + modfile = modfile[:-1] + elif modfile.endswith('$py.class'): + modfile = modfile[:-9] + '.py' + if not self.samefile(modfile): + raise EnvironmentError("mismatch:\n" + "imported module %r\n" + "does not stem from %r\n" + "maybe __init__.py files are missing?" % (mod, str(self))) + return mod + else: + try: + return sys.modules[modname] + except KeyError: + # we have a custom modname, do a pseudo-import + mod = py.std.types.ModuleType(modname) + mod.__file__ = str(self) + sys.modules[modname] = mod + try: + py.builtin.execfile(str(self), mod.__dict__) + except: + del sys.modules[modname] + raise + return mod + + def sysexec(self, *argv, **popen_opts): + """ return stdout text from executing a system child process, + where the 'self' path points to executable. + The process is directly invoked and not through a system shell. + """ + from subprocess import Popen, PIPE + argv = map(str, argv) + popen_opts['stdout'] = popen_opts['stderr'] = PIPE + proc = Popen([str(self)] + list(argv), **popen_opts) + stdout, stderr = proc.communicate() + ret = proc.wait() + if py.builtin._isbytes(stdout): + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + if ret != 0: + if py.builtin._isbytes(stderr): + stderr = py.builtin._totext(stderr, sys.getdefaultencoding()) + raise py.process.cmdexec.Error(ret, ret, str(self), + stdout, stderr,) + return stdout + + def sysfind(cls, name, checker=None): + """ return a path object found by looking at the systems + underlying PATH specification. If the checker is not None + it will be invoked to filter matching paths. If a binary + cannot be found, None is returned + Note: This is probably not working on plain win32 systems + but may work on cygwin. + """ + if os.path.isabs(name): + p = py.path.local(name) + if p.check(file=1): + return p + else: + if iswin32: + paths = py.std.os.environ['Path'].split(';') + if '' not in paths and '.' not in paths: + paths.append('.') + try: + systemroot = os.environ['SYSTEMROOT'] + except KeyError: + pass + else: + paths = [re.sub('%SystemRoot%', systemroot, path) + for path in paths] + tryadd = '', '.exe', '.com', '.bat' # XXX add more? + else: + paths = py.std.os.environ['PATH'].split(':') + tryadd = ('',) + + for x in paths: + for addext in tryadd: + p = py.path.local(x).join(name, abs=True) + addext + try: + if p.check(file=1): + if checker: + if not checker(p): + continue + return p + except py.error.EACCES: + pass + return None + sysfind = classmethod(sysfind) + + def _gethomedir(cls): + try: + x = os.environ['HOME'] + except KeyError: + x = os.environ['HOMEPATH'] + return cls(x) + _gethomedir = classmethod(_gethomedir) + + #""" + #special class constructors for local filesystem paths + #""" + def get_temproot(cls): + """ return the system's temporary directory + (where tempfiles are usually created in) + """ + return py.path.local(py.std.tempfile.gettempdir()) + get_temproot = classmethod(get_temproot) + + def mkdtemp(cls): + """ return a Path object pointing to a fresh new temporary directory + (which we created ourself). + """ + import tempfile + tries = 10 + for i in range(tries): + dname = tempfile.mktemp() + dpath = cls(tempfile.mktemp()) + try: + dpath.mkdir() + except (py.error.EEXIST, py.error.EPERM, py.error.EACCES): + continue + return dpath + raise py.error.ENOENT(dpath, "could not create tempdir, %d tries" % tries) + mkdtemp = classmethod(mkdtemp) + + def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3, + lock_timeout = 172800): # two days + """ return unique directory with a number greater than the current + maximum one. The number is assumed to start directly after prefix. + if keep is true directories with a number less than (maxnum-keep) + will be removed. + """ + if rootdir is None: + rootdir = cls.get_temproot() + + def parse_num(path): + """ parse the number out of a path (if it matches the prefix) """ + bn = path.basename + if bn.startswith(prefix): + try: + return int(bn[len(prefix):]) + except ValueError: + pass + + # compute the maximum number currently in use with the + # prefix + lastmax = None + while True: + maxnum = -1 + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None: + maxnum = max(maxnum, num) + + # make the new directory + try: + udir = rootdir.mkdir(prefix + str(maxnum+1)) + except py.error.EEXIST: + # race condition: another thread/process created the dir + # in the meantime. Try counting again + if lastmax == maxnum: + raise + lastmax = maxnum + continue + break + + # put a .lock file in the new directory that will be removed at + # process exit + if lock_timeout: + lockfile = udir.join('.lock') + mypid = os.getpid() + if hasattr(lockfile, 'mksymlinkto'): + lockfile.mksymlinkto(str(mypid)) + else: + lockfile.write(str(mypid)) + def try_remove_lockfile(): + # in a fork() situation, only the last process should + # remove the .lock, otherwise the other processes run the + # risk of seeing their temporary dir disappear. For now + # we remove the .lock in the parent only (i.e. we assume + # that the children finish before the parent). + if os.getpid() != mypid: + return + try: + lockfile.remove() + except py.error.Error: + pass + atexit.register(try_remove_lockfile) + + # prune old directories + if keep: + for path in rootdir.listdir(): + num = parse_num(path) + if num is not None and num <= (maxnum - keep): + lf = path.join('.lock') + try: + t1 = lf.lstat().mtime + t2 = lockfile.lstat().mtime + if not lock_timeout or abs(t2-t1) < lock_timeout: + continue # skip directories still locked + except py.error.Error: + pass # assume that it means that there is no 'lf' + try: + path.remove(rec=1) + except KeyboardInterrupt: + raise + except: # this might be py.error.Error, WindowsError ... + pass + + # make link... + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = 'current' + + src = str(udir) + dest = src[:src.rfind('-')] + '-' + username + try: + os.unlink(dest) + except OSError: + pass + try: + os.symlink(src, dest) + except (OSError, AttributeError): # AttributeError on win32 + pass + + return udir + make_numbered_dir = classmethod(make_numbered_dir) + +def copychunked(src, dest): + chunksize = 524288 # half a meg of bytes + fsrc = src.open('rb') + try: + fdest = dest.open('wb') + try: + while 1: + buf = fsrc.read(chunksize) + if not buf: + break + fdest.write(buf) + finally: + fdest.close() + finally: + fsrc.close() + +def autopath(globs=None): + """ (deprecated) return the (local) path of the "current" file pointed to by globals or - if it is none - alternatively the callers frame globals. + + the path will always point to a .py file or to None. + the path will have the following payload: + pkgdir is the last parent directory path containing __init__.py + """ + py.log._apiwarn("1.1", "py.magic.autopath deprecated, " + "use py.path.local(__file__) and maybe pypkgpath/pyimport().") + if globs is None: + globs = sys._getframe(1).f_globals + try: + __file__ = globs['__file__'] + except KeyError: + if not sys.argv[0]: + raise ValueError("cannot compute autopath in interactive mode") + __file__ = os.path.abspath(sys.argv[0]) + + ret = py.path.local(__file__) + if ret.ext in ('.pyc', '.pyo'): + ret = ret.new(ext='.py') + current = pkgdir = ret.dirpath() + while 1: + if current.join('__init__.py').check(): + pkgdir = current + current = current.dirpath() + if pkgdir != current: + continue + elif str(current) not in sys.path: + sys.path.insert(0, str(current)) + break + ret.pkgdir = pkgdir + return ret + Added: pypy/trunk/py/impl/path/svnurl.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/svnurl.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,365 @@ +""" +module defining a subversion path object based on the external +command 'svn'. This modules aims to work with svn 1.3 and higher +but might also interact well with earlier versions. +""" + +import os, sys, time, re +import py +from py import path, process +from py.impl.path import common +from py.impl.path import svnwc as svncommon +from py.impl.path.cacheutil import BuildcostAccessCache, AgingCache + +DEBUG=False + +class SvnCommandPath(svncommon.SvnPathBase): + """ path implementation that offers access to (possibly remote) subversion + repositories. """ + + _lsrevcache = BuildcostAccessCache(maxentries=128) + _lsnorevcache = AgingCache(maxentries=1000, maxseconds=60.0) + + def __new__(cls, path, rev=None, auth=None): + self = object.__new__(cls) + if isinstance(path, cls): + rev = path.rev + auth = path.auth + path = path.strpath + svncommon.checkbadchars(path) + path = path.rstrip('/') + self.strpath = path + self.rev = rev + self.auth = auth + return self + + def __repr__(self): + if self.rev == -1: + return 'svnurl(%r)' % self.strpath + else: + return 'svnurl(%r, %r)' % (self.strpath, self.rev) + + def _svnwithrev(self, cmd, *args): + """ execute an svn command, append our own url and revision """ + if self.rev is None: + return self._svnwrite(cmd, *args) + else: + args = ['-r', self.rev] + list(args) + return self._svnwrite(cmd, *args) + + def _svnwrite(self, cmd, *args): + """ execute an svn command, append our own url """ + l = ['svn %s' % cmd] + args = ['"%s"' % self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._encodedurl()) + # fixing the locale because we can't otherwise parse + string = " ".join(l) + if DEBUG: + print("execing %s" % string) + out = self._svncmdexecauth(string) + return out + + def _svncmdexecauth(self, cmd): + """ execute an svn command 'as is' """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._cmdexec(cmd) + + def _cmdexec(self, cmd): + try: + out = process.cmdexec(cmd) + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if (e.err.find('File Exists') != -1 or + e.err.find('File already exists') != -1): + raise py.error.EEXIST(self) + raise + return out + + def _svnpopenauth(self, cmd): + """ execute an svn command, return a pipe for reading stdin """ + cmd = svncommon.fixlocale() + cmd + if self.auth is not None: + cmd += ' ' + self.auth.makecmdoptions() + return self._popen(cmd) + + def _popen(self, cmd): + return os.popen(cmd) + + def _encodedurl(self): + return self._escape(self.strpath) + + def _norev_delentry(self, path): + auth = self.auth and self.auth.makecmdoptions() or None + self._lsnorevcache.delentry((str(path), auth)) + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + if mode not in ("r", "rU",): + raise ValueError("mode %r not supported" % (mode,)) + assert self.check(file=1) # svn cat returns an empty file otherwise + if self.rev is None: + return self._svnpopenauth('svn cat "%s"' % ( + self._escape(self.strpath), )) + else: + return self._svnpopenauth('svn cat -r %s "%s"' % ( + self.rev, self._escape(self.strpath))) + + def dirpath(self, *args, **kwargs): + """ return the directory path of the current path joined + with any given path arguments. + """ + l = self.strpath.split(self.sep) + if len(l) < 4: + raise py.error.EINVAL(self, "base is not valid") + elif len(l) == 4: + return self.join(*args, **kwargs) + else: + return self.new(basename='').join(*args, **kwargs) + + # modifying methods (cache must be invalidated) + def mkdir(self, *args, **kwargs): + """ create & return the directory joined with args. + pass a 'msg' keyword argument to set the commit message. + """ + commit_msg = kwargs.get('msg', "mkdir by py lib invocation") + createpath = self.join(*args) + createpath._svnwrite('mkdir', '-m', commit_msg) + self._norev_delentry(createpath.dirpath()) + return createpath + + def copy(self, target, msg='copied by py lib invocation'): + """ copy path to target with checkin message msg.""" + if getattr(target, 'rev', None) is not None: + raise py.error.EINVAL(target, "revisions are immutable") + self._svncmdexecauth('svn copy -m "%s" "%s" "%s"' %(msg, + self._escape(self), self._escape(target))) + self._norev_delentry(target.dirpath()) + + def rename(self, target, msg="renamed by py lib invocation"): + """ rename this path to target with checkin message msg. """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn move -m "%s" --force "%s" "%s"' %( + msg, self._escape(self), self._escape(target))) + self._norev_delentry(self.dirpath()) + self._norev_delentry(self) + + def remove(self, rec=1, msg='removed by py lib invocation'): + """ remove a file or directory (or a directory tree if rec=1) with +checkin message msg.""" + if self.rev is not None: + raise py.error.EINVAL(self, "revisions are immutable") + self._svncmdexecauth('svn rm -m "%s" "%s"' %(msg, self._escape(self))) + self._norev_delentry(self.dirpath()) + + def export(self, topath): + """ export to a local path + + topath should not exist prior to calling this, returns a + py.path.local instance + """ + topath = py.path.local(topath) + args = ['"%s"' % (self._escape(self),), + '"%s"' % (self._escape(topath),)] + if self.rev is not None: + args = ['-r', str(self.rev)] + args + self._svncmdexecauth('svn export %s' % (' '.join(args),)) + return topath + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). If you specify a keyword argument 'dir=True' + then the path is forced to be a directory path. + """ + if getattr(self, 'rev', None) is not None: + raise py.error.EINVAL(self, "revisions are immutable") + target = self.join(*args) + dir = kwargs.get('dir', 0) + for x in target.parts(reverse=True): + if x.check(): + break + else: + raise py.error.ENOENT(target, "has not any valid base!") + if x == target: + if not x.check(dir=dir): + raise dir and py.error.ENOTDIR(x) or py.error.EISDIR(x) + return x + tocreate = target.relto(x) + basename = tocreate.split(self.sep, 1)[0] + tempdir = py.path.local.mkdtemp() + try: + tempdir.ensure(tocreate, dir=dir) + cmd = 'svn import -m "%s" "%s" "%s"' % ( + "ensure %s" % self._escape(tocreate), + self._escape(tempdir.join(basename)), + x.join(basename)._encodedurl()) + self._svncmdexecauth(cmd) + self._norev_delentry(x) + finally: + tempdir.remove() + return target + + # end of modifying methods + def _propget(self, name): + res = self._svnwithrev('propget', name) + return res[:-1] # strip trailing newline + + def _proplist(self): + res = self._svnwithrev('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return svncommon.PropListDict(self, lines) + + def _listdir_nameinfo(self): + """ return sequence of name-info directory entries of self """ + def builder(): + try: + res = self._svnwithrev('ls', '-v') + except process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('non-existent in that revision') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('File not found') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('not part of a repository')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.find('Unable to open')!=-1: + raise py.error.ENOENT(self, e.err) + elif e.err.lower().find('method not allowed')!=-1: + raise py.error.EACCES(self, e.err) + raise py.error.Error(e.err) + lines = res.split('\n') + nameinfo_seq = [] + for lsline in lines: + if lsline: + info = InfoSvnCommand(lsline) + if info._name != '.': # svn 1.5 produces '.' dirs, + nameinfo_seq.append((info._name, info)) + nameinfo_seq.sort() + return nameinfo_seq + auth = self.auth and self.auth.makecmdoptions() or None + if self.rev is not None: + return self._lsrevcache.getorbuild((self.strpath, self.rev, auth), + builder) + else: + return self._lsnorevcache.getorbuild((self.strpath, auth), + builder) + + def listdir(self, fil=None, sort=None): + """ list directory contents, possibly filter by the given fil func + and possibly sorted. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + nameinfo_seq = self._listdir_nameinfo() + if len(nameinfo_seq) == 1: + name, info = nameinfo_seq[0] + if name == self.basename and info.kind == 'file': + #if not self.check(dir=1): + raise py.error.ENOTDIR(self) + paths = [self.join(name) for (name, info) in nameinfo_seq] + if fil: + paths = [x for x in paths if fil(x)] + self._sortlist(paths, sort) + return paths + + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() #make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + xmlpipe = self._svnpopenauth('svn log --xml %s %s "%s"' % + (rev_opt, verbose_opt, self.strpath)) + from xml.dom import minidom + tree = minidom.parse(xmlpipe) + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(svncommon.LogEntry(logentry)) + return result + +#01234567890123456789012345678901234567890123467 +# 2256 hpk 165 Nov 24 17:55 __init__.py +# XXX spotted by Guido, SVN 1.3.0 has different aligning, breaks the code!!! +# 1312 johnny 1627 May 05 14:32 test_decorators.py +# +class InfoSvnCommand: + # the '0?' part in the middle is an indication of whether the resource is + # locked, see 'svn help ls' + lspattern = re.compile( + r'^ *(?P\d+) +(?P.+?) +(0? *(?P\d+))? ' + '*(?P\w+ +\d{2} +[\d:]+) +(?P.*)$') + def __init__(self, line): + # this is a typical line from 'svn ls http://...' + #_ 1127 jum 0 Jul 13 15:28 branch/ + match = self.lspattern.match(line) + data = match.groupdict() + self._name = data['file'] + if self._name[-1] == '/': + self._name = self._name[:-1] + self.kind = 'dir' + else: + self.kind = 'file' + #self.has_props = l.pop(0) == 'P' + self.created_rev = int(data['rev']) + self.last_author = data['author'] + self.size = data['size'] and int(data['size']) or 0 + self.mtime = parse_time_with_missing_year(data['date']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + +#____________________________________________________ +# +# helper functions +#____________________________________________________ +def parse_time_with_missing_year(timestr): + """ analyze the time part from a single line of "svn ls -v" + the svn output doesn't show the year makes the 'timestr' + ambigous. + """ + import calendar + t_now = time.gmtime() + + tparts = timestr.split() + month = time.strptime(tparts.pop(0), '%b')[1] + day = time.strptime(tparts.pop(0), '%d')[2] + last = tparts.pop(0) # year or hour:minute + try: + year = time.strptime(last, '%Y')[0] + hour = minute = 0 + except ValueError: + hour, minute = time.strptime(last, '%H:%M')[3:5] + year = t_now[0] + + t_result = (year, month, day, hour, minute, 0,0,0,0) + if t_result > t_now: + year -= 1 + t_result = (year, month, day, hour, minute, 0,0,0,0) + return calendar.timegm(t_result) + +class PathEntry: + def __init__(self, ppart): + self.strpath = ppart.firstChild.nodeValue.encode('UTF-8') + self.action = ppart.getAttribute('action').encode('UTF-8') + if self.action == 'A': + self.copyfrom_path = ppart.getAttribute('copyfrom-path').encode('UTF-8') + if self.copyfrom_path: + self.copyfrom_rev = int(ppart.getAttribute('copyfrom-rev')) + Added: pypy/trunk/py/impl/path/svnwc.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/path/svnwc.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,1236 @@ +""" +svn-Command based Implementation of a Subversion WorkingCopy Path. + + SvnWCCommandPath is the main class. + +""" + +import os, sys, time, re, calendar +import py +import subprocess +from py.impl.path import common + +#----------------------------------------------------------- +# Caching latest repository revision and repo-paths +# (getting them is slow with the current implementations) +# +# XXX make mt-safe +#----------------------------------------------------------- + +class cache: + proplist = {} + info = {} + entries = {} + prop = {} + +class RepoEntry: + def __init__(self, url, rev, timestamp): + self.url = url + self.rev = rev + self.timestamp = timestamp + + def __str__(self): + return "repo: %s;%s %s" %(self.url, self.rev, self.timestamp) + +class RepoCache: + """ The Repocache manages discovered repository paths + and their revisions. If inside a timeout the cache + will even return the revision of the root. + """ + timeout = 20 # seconds after which we forget that we know the last revision + + def __init__(self): + self.repos = [] + + def clear(self): + self.repos = [] + + def put(self, url, rev, timestamp=None): + if rev is None: + return + if timestamp is None: + timestamp = time.time() + + for entry in self.repos: + if url == entry.url: + entry.timestamp = timestamp + entry.rev = rev + #print "set repo", entry + break + else: + entry = RepoEntry(url, rev, timestamp) + self.repos.append(entry) + #print "appended repo", entry + + def get(self, url): + now = time.time() + for entry in self.repos: + if url.startswith(entry.url): + if now < entry.timestamp + self.timeout: + #print "returning immediate Etrny", entry + return entry.url, entry.rev + return entry.url, -1 + return url, -1 + +repositories = RepoCache() + + +# svn support code + +ALLOWED_CHARS = "_ -/\\=$.~+" #add characters as necessary when tested +if sys.platform == "win32": + ALLOWED_CHARS += ":" +ALLOWED_CHARS_HOST = ALLOWED_CHARS + '@:' + +def _getsvnversion(ver=[]): + try: + return ver[0] + except IndexError: + v = py.process.cmdexec("svn -q --version") + v.strip() + v = '.'.join(v.split('.')[:2]) + ver.append(v) + return v + +def _escape_helper(text): + text = str(text) + if py.std.sys.platform != 'win32': + text = str(text).replace('$', '\\$') + return text + +def _check_for_bad_chars(text, allowed_chars=ALLOWED_CHARS): + for c in str(text): + if c.isalnum(): + continue + if c in allowed_chars: + continue + return True + return False + +def checkbadchars(url): + # (hpk) not quite sure about the exact purpose, guido w.? + proto, uri = url.split("://", 1) + if proto != "file": + host, uripath = uri.split('/', 1) + # only check for bad chars in the non-protocol parts + if (_check_for_bad_chars(host, ALLOWED_CHARS_HOST) \ + or _check_for_bad_chars(uripath, ALLOWED_CHARS)): + raise ValueError("bad char in %r" % (url, )) + + +#_______________________________________________________________ + +class SvnPathBase(common.PathBase): + """ Base implementation for SvnPath implementations. """ + sep = '/' + + def _geturl(self): + return self.strpath + url = property(_geturl, None, None, "url of this svn-path.") + + def __str__(self): + """ return a string representation (including rev-number) """ + return self.strpath + + def __hash__(self): + return hash(self.strpath) + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + obj = object.__new__(self.__class__) + obj.rev = kw.get('rev', self.rev) + obj.auth = kw.get('auth', self.auth) + dirname, basename, purebasename, ext = self._getbyspec( + "dirname,basename,purebasename,ext") + if 'basename' in kw: + if 'purebasename' in kw or 'ext' in kw: + raise ValueError("invalid specification %r" % kw) + else: + pb = kw.setdefault('purebasename', purebasename) + ext = kw.setdefault('ext', ext) + if ext and not ext.startswith('.'): + ext = '.' + ext + kw['basename'] = pb + ext + + kw.setdefault('dirname', dirname) + kw.setdefault('sep', self.sep) + if kw['basename']: + obj.strpath = "%(dirname)s%(sep)s%(basename)s" % kw + else: + obj.strpath = "%(dirname)s" % kw + return obj + + def _getbyspec(self, spec): + """ get specified parts of the path. 'arg' is a string + with comma separated path parts. The parts are returned + in exactly the order of the specification. + + you may specify the following parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + res = [] + parts = self.strpath.split(self.sep) + for name in spec.split(','): + name = name.strip() + if name == 'dirname': + res.append(self.sep.join(parts[:-1])) + elif name == 'basename': + res.append(parts[-1]) + else: + basename = parts[-1] + i = basename.rfind('.') + if i == -1: + purebasename, ext = basename, '' + else: + purebasename, ext = basename[:i], basename[i:] + if name == 'purebasename': + res.append(purebasename) + elif name == 'ext': + res.append(ext) + else: + raise NameError("Don't know part %r" % name) + return res + + def __eq__(self, other): + """ return true if path and rev attributes each match """ + return (str(self) == str(other) and + (self.rev == other.rev or self.rev == other.rev)) + + def __ne__(self, other): + return not self == other + + def join(self, *args): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + + args = tuple([arg.strip(self.sep) for arg in args]) + parts = (self.strpath, ) + args + newpath = self.__class__(self.sep.join(parts), self.rev, self.auth) + return newpath + + def propget(self, name): + """ return the content of the given property. """ + value = self._propget(name) + return value + + def proplist(self): + """ list all property names. """ + content = self._proplist() + return content + + def info(self): + """ return an Info structure with svn-provided information. """ + parent = self.dirpath() + nameinfo_seq = parent._listdir_nameinfo() + bn = self.basename + for name, info in nameinfo_seq: + if name == bn: + return info + raise py.error.ENOENT(self) + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + # shared help methods + + def _escape(self, cmd): + return _escape_helper(cmd) + + + #def _childmaxrev(self): + # """ return maximum revision number of childs (or self.rev if no childs) """ + # rev = self.rev + # for name, info in self._listdir_nameinfo(): + # rev = max(rev, info.created_rev) + # return rev + + #def _getlatestrevision(self): + # """ return latest repo-revision for this path. """ + # url = self.strpath + # path = self.__class__(url, None) + # + # # we need a long walk to find the root-repo and revision + # while 1: + # try: + # rev = max(rev, path._childmaxrev()) + # previous = path + # path = path.dirpath() + # except (IOError, process.cmdexec.Error): + # break + # if rev is None: + # raise IOError, "could not determine newest repo revision for %s" % self + # return rev + + class Checkers(common.Checkers): + def dir(self): + try: + return self.path.info().kind == 'dir' + except py.error.Error: + return self._listdirworks() + + def _listdirworks(self): + try: + self.path.listdir() + except py.error.ENOENT: + return False + else: + return True + + def file(self): + try: + return self.path.info().kind == 'file' + except py.error.ENOENT: + return False + + def exists(self): + try: + return self.path.info() + except py.error.ENOENT: + return self._listdirworks() + +def parse_apr_time(timestr): + i = timestr.rfind('.') + if i == -1: + raise ValueError("could not parse %s" % timestr) + timestr = timestr[:i] + parsedtime = time.strptime(timestr, "%Y-%m-%dT%H:%M:%S") + return time.mktime(parsedtime) + +class PropListDict(dict): + """ a Dictionary which fetches values (InfoSvnCommand instances) lazily""" + def __init__(self, path, keynames): + dict.__init__(self, [(x, None) for x in keynames]) + self.path = path + + def __getitem__(self, key): + value = dict.__getitem__(self, key) + if value is None: + value = self.path.propget(key) + dict.__setitem__(self, key, value) + return value + +def fixlocale(): + if sys.platform != 'win32': + return 'LC_ALL=C ' + return '' + +# some nasty chunk of code to solve path and url conversion and quoting issues +ILLEGAL_CHARS = '* | \ / : < > ? \t \n \x0b \x0c \r'.split(' ') +if os.sep in ILLEGAL_CHARS: + ILLEGAL_CHARS.remove(os.sep) +ISWINDOWS = sys.platform == 'win32' +_reg_allow_disk = re.compile(r'^([a-z]\:\\)?[^:]+$', re.I) +def _check_path(path): + illegal = ILLEGAL_CHARS[:] + sp = path.strpath + if ISWINDOWS: + illegal.remove(':') + if not _reg_allow_disk.match(sp): + raise ValueError('path may not contain a colon (:)') + for char in sp: + if char not in string.printable or char in illegal: + raise ValueError('illegal character %r in path' % (char,)) + +def path_to_fspath(path, addat=True): + _check_path(path) + sp = path.strpath + if addat and path.rev != -1: + sp = '%s@%s' % (sp, path.rev) + elif addat: + sp = '%s at HEAD' % (sp,) + return sp + +def url_from_path(path): + fspath = path_to_fspath(path, False) + quote = py.std.urllib.quote + if ISWINDOWS: + match = _reg_allow_disk.match(fspath) + fspath = fspath.replace('\\', '/') + if match.group(1): + fspath = '/%s%s' % (match.group(1).replace('\\', '/'), + quote(fspath[len(match.group(1)):])) + else: + fspath = quote(fspath) + else: + fspath = quote(fspath) + if path.rev != -1: + fspath = '%s@%s' % (fspath, path.rev) + else: + fspath = '%s at HEAD' % (fspath,) + return 'file://%s' % (fspath,) + +class SvnAuth(object): + """ container for auth information for Subversion """ + def __init__(self, username, password, cache_auth=True, interactive=True): + self.username = username + self.password = password + self.cache_auth = cache_auth + self.interactive = interactive + + def makecmdoptions(self): + uname = self.username.replace('"', '\\"') + passwd = self.password.replace('"', '\\"') + ret = [] + if uname: + ret.append('--username="%s"' % (uname,)) + if passwd: + ret.append('--password="%s"' % (passwd,)) + if not self.cache_auth: + ret.append('--no-auth-cache') + if not self.interactive: + ret.append('--non-interactive') + return ' '.join(ret) + + def __str__(self): + return "" %(self.username,) + +rex_blame = re.compile(r'\s*(\d+)\s*(\S+) (.*)') + +class SvnWCCommandPath(common.PathBase): + """ path implementation offering access/modification to svn working copies. + It has methods similar to the functions in os.path and similar to the + commands of the svn client. + """ + sep = os.sep + + def __new__(cls, wcpath=None, auth=None): + self = object.__new__(cls) + if isinstance(wcpath, cls): + if wcpath.__class__ == cls: + return wcpath + wcpath = wcpath.localpath + if _check_for_bad_chars(str(wcpath), + ALLOWED_CHARS): + raise ValueError("bad char in wcpath %s" % (wcpath, )) + self.localpath = py.path.local(wcpath) + self.auth = auth + return self + + strpath = property(lambda x: str(x.localpath), None, None, "string path") + + def __eq__(self, other): + return self.localpath == getattr(other, 'localpath', None) + + def _geturl(self): + if getattr(self, '_url', None) is None: + info = self.info() + self._url = info.url #SvnPath(info.url, info.rev) + assert isinstance(self._url, py.builtin._basestring) + return self._url + + url = property(_geturl, None, None, "url of this WC item") + + def _escape(self, cmd): + return _escape_helper(cmd) + + def dump(self, obj): + """ pickle object into path location""" + return self.localpath.dump(obj) + + def svnurl(self): + """ return current SvnPath for this WC-item. """ + info = self.info() + return py.path.svnurl(info.url) + + def __repr__(self): + return "svnwc(%r)" % (self.strpath) # , self._url) + + def __str__(self): + return str(self.localpath) + + def _makeauthoptions(self): + if self.auth is None: + return '' + return self.auth.makecmdoptions() + + def _authsvn(self, cmd, args=None): + args = args and list(args) or [] + args.append(self._makeauthoptions()) + return self._svn(cmd, *args) + + def _svn(self, cmd, *args): + l = ['svn %s' % cmd] + args = [self._escape(item) for item in args] + l.extend(args) + l.append('"%s"' % self._escape(self.strpath)) + # try fixing the locale because we can't otherwise parse + string = fixlocale() + " ".join(l) + try: + try: + key = 'LC_MESSAGES' + hold = os.environ.get(key) + os.environ[key] = 'C' + out = py.process.cmdexec(string) + finally: + if hold: + os.environ[key] = hold + else: + del os.environ[key] + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + strerr = e.err.lower() + if strerr.find('file not found') != -1: + raise py.error.ENOENT(self) + if (strerr.find('file exists') != -1 or + strerr.find('file already exists') != -1 or + strerr.find("can't create directory") != -1): + raise py.error.EEXIST(self) + raise + return out + + def switch(self, url): + """ switch to given URL. """ + self._authsvn('switch', [url]) + + def checkout(self, url=None, rev=None): + """ checkout from url to local wcpath. """ + args = [] + if url is None: + url = self.url + if rev is None or rev == -1: + if (py.std.sys.platform != 'win32' and + _getsvnversion() == '1.3'): + url += "@HEAD" + else: + if _getsvnversion() == '1.3': + url += "@%d" % rev + else: + args.append('-r' + str(rev)) + args.append(url) + self._authsvn('co', args) + + def update(self, rev='HEAD'): + """ update working copy item to given revision. (None -> HEAD). """ + self._authsvn('up', ['-r', rev, "--non-interactive"],) + + def write(self, content, mode='w'): + """ write content into local filesystem wc. """ + self.localpath.write(content, mode) + + def dirpath(self, *args): + """ return the directory Path of the current Path. """ + return self.__class__(self.localpath.dirpath(*args), auth=self.auth) + + def _ensuredirs(self): + parent = self.dirpath() + if parent.check(dir=0): + parent._ensuredirs() + if self.check(dir=0): + self.mkdir() + return self + + def ensure(self, *args, **kwargs): + """ ensure that an args-joined path exists (by default as + a file). if you specify a keyword argument 'directory=True' + then the path is forced to be a directory path. + """ + p = self.join(*args) + if p.check(): + if p.check(versioned=False): + p.add() + return p + if kwargs.get('dir', 0): + return p._ensuredirs() + parent = p.dirpath() + parent._ensuredirs() + p.write("") + p.add() + return p + + def mkdir(self, *args): + """ create & return the directory joined with args. """ + if args: + return self.join(*args).mkdir() + else: + self._svn('mkdir') + return self + + def add(self): + """ add ourself to svn """ + self._svn('add') + + def remove(self, rec=1, force=1): + """ remove a file or a directory tree. 'rec'ursive is + ignored and considered always true (because of + underlying svn semantics. + """ + assert rec, "svn cannot remove non-recursively" + if not self.check(versioned=True): + # not added to svn (anymore?), just remove + py.path.local(self).remove() + return + flags = [] + if force: + flags.append('--force') + self._svn('remove', *flags) + + def copy(self, target): + """ copy path to target.""" + py.process.cmdexec("svn copy %s %s" %(str(self), str(target))) + + def rename(self, target): + """ rename this path to target. """ + py.process.cmdexec("svn move --force %s %s" %(str(self), str(target))) + + def lock(self): + """ set a lock (exclusive) on the resource """ + out = self._authsvn('lock').strip() + if not out: + # warning or error, raise exception + raise Exception(out[4:]) + + def unlock(self): + """ unset a previously set lock """ + out = self._authsvn('unlock').strip() + if out.startswith('svn:'): + # warning or error, raise exception + raise Exception(out[4:]) + + def cleanup(self): + """ remove any locks from the resource """ + # XXX should be fixed properly!!! + try: + self.unlock() + except: + pass + + def status(self, updates=0, rec=0, externals=0): + """ return (collective) Status object for this file. """ + # http://svnbook.red-bean.com/book.html#svn-ch-3-sect-4.3.1 + # 2201 2192 jum test + # XXX + if externals: + raise ValueError("XXX cannot perform status() " + "on external items yet") + else: + #1.2 supports: externals = '--ignore-externals' + externals = '' + if rec: + rec= '' + else: + rec = '--non-recursive' + + # XXX does not work on all subversion versions + #if not externals: + # externals = '--ignore-externals' + + if updates: + updates = '-u' + else: + updates = '' + + try: + cmd = 'status -v --xml --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + except py.process.cmdexec.Error: + cmd = 'status -v --no-ignore %s %s %s' % ( + updates, rec, externals) + out = self._authsvn(cmd) + rootstatus = WCStatus(self).fromstring(out, self) + else: + rootstatus = XMLWCStatus(self).fromstring(out, self) + return rootstatus + + def diff(self, rev=None): + """ return a diff of the current path against revision rev (defaulting + to the last one). + """ + args = [] + if rev is not None: + args.append("-r %d" % rev) + out = self._authsvn('diff', args) + return out + + def blame(self): + """ return a list of tuples of three elements: + (revision, commiter, line) + """ + out = self._svn('blame') + result = [] + blamelines = out.splitlines() + reallines = py.path.svnurl(self.url).readlines() + for i, (blameline, line) in enumerate( + zip(blamelines, reallines)): + m = rex_blame.match(blameline) + if not m: + raise ValueError("output line %r of svn blame does not match " + "expected format" % (line, )) + rev, name, _ = m.groups() + result.append((int(rev), name, line)) + return result + + _rex_commit = re.compile(r'.*Committed revision (\d+)\.$', re.DOTALL) + def commit(self, msg='', rec=1): + """ commit with support for non-recursive commits """ + # XXX i guess escaping should be done better here?!? + cmd = 'commit -m "%s" --force-log' % (msg.replace('"', '\\"'),) + if not rec: + cmd += ' -N' + out = self._authsvn(cmd) + try: + del cache.info[self] + except KeyError: + pass + if out: + m = self._rex_commit.match(out) + return int(m.group(1)) + + def propset(self, name, value, *args): + """ set property name to value on this path. """ + d = py.path.local.mkdtemp() + try: + p = d.join('value') + p.write(value) + self._svn('propset', name, '--file', str(p), *args) + finally: + d.remove() + + def propget(self, name): + """ get property name on this path. """ + res = self._svn('propget', name) + return res[:-1] # strip trailing newline + + def propdel(self, name): + """ delete property name on this path. """ + res = self._svn('propdel', name) + return res[:-1] # strip trailing newline + + def proplist(self, rec=0): + """ return a mapping of property names to property values. +If rec is True, then return a dictionary mapping sub-paths to such mappings. +""" + if rec: + res = self._svn('proplist -R') + return make_recursive_propdict(self, res) + else: + res = self._svn('proplist') + lines = res.split('\n') + lines = [x.strip() for x in lines[1:]] + return PropListDict(self, lines) + + def revert(self, rec=0): + """ revert the local changes of this path. if rec is True, do so +recursively. """ + if rec: + result = self._svn('revert -R') + else: + result = self._svn('revert') + return result + + def new(self, **kw): + """ create a modified version of this path. A 'rev' argument + indicates a new revision. + the following keyword arguments modify various path parts: + + http://host.com/repo/path/file.ext + |-----------------------| dirname + |------| basename + |--| purebasename + |--| ext + """ + if kw: + localpath = self.localpath.new(**kw) + else: + localpath = self.localpath + return self.__class__(localpath, auth=self.auth) + + def join(self, *args, **kwargs): + """ return a new Path (with the same revision) which is composed + of the self Path followed by 'args' path components. + """ + if not args: + return self + localpath = self.localpath.join(*args, **kwargs) + return self.__class__(localpath, auth=self.auth) + + def info(self, usecache=1): + """ return an Info structure with svn-provided information. """ + info = usecache and cache.info.get(self) + if not info: + try: + output = self._svn('info') + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('Path is not a working copy directory') != -1: + raise py.error.ENOENT(self, e.err) + elif e.err.find("is not under version control") != -1: + raise py.error.ENOENT(self, e.err) + raise + # XXX SVN 1.3 has output on stderr instead of stdout (while it does + # return 0!), so a bit nasty, but we assume no output is output + # to stderr... + if (output.strip() == '' or + output.lower().find('not a versioned resource') != -1): + raise py.error.ENOENT(self, output) + info = InfoSvnWCCommand(output) + + # Can't reliably compare on Windows without access to win32api + if py.std.sys.platform != 'win32': + if info.path != self.localpath: + raise py.error.ENOENT(self, "not a versioned resource:" + + " %s != %s" % (info.path, self.localpath)) + cache.info[self] = info + self.rev = info.rev + return info + + def listdir(self, fil=None, sort=None): + """ return a sequence of Paths. + + listdir will return either a tuple or a list of paths + depending on implementation choices. + """ + if isinstance(fil, str): + fil = common.FNMatcher(fil) + # XXX unify argument naming with LocalPath.listdir + def notsvn(path): + return path.basename != '.svn' + + paths = [self.__class__(p, auth=self.auth) + for p in self.localpath.listdir() + if notsvn(p) and (not fil or fil(p))] + self._sortlist(paths, sort) + return paths + + def open(self, mode='r'): + """ return an opened file with the given mode. """ + return open(self.strpath, mode) + + def _getbyspec(self, spec): + return self.localpath._getbyspec(spec) + + class Checkers(py.path.local.Checkers): + def __init__(self, path): + self.svnwcpath = path + self.path = path.localpath + def versioned(self): + try: + s = self.svnwcpath.info() + except (py.error.ENOENT, py.error.EEXIST): + return False + except py.process.cmdexec.Error: + e = sys.exc_info()[1] + if e.err.find('is not a working copy')!=-1: + return False + if e.err.lower().find('not a versioned resource') != -1: + return False + raise + else: + return True + + def log(self, rev_start=None, rev_end=1, verbose=False): + """ return a list of LogEntry instances for this path. +rev_start is the starting revision (defaulting to the first one). +rev_end is the last revision (defaulting to HEAD). +if verbose is True, then the LogEntry instances also know which files changed. +""" + assert self.check() # make it simpler for the pipe + rev_start = rev_start is None and "HEAD" or rev_start + rev_end = rev_end is None and "HEAD" or rev_end + if rev_start == "HEAD" and rev_end == 1: + rev_opt = "" + else: + rev_opt = "-r %s:%s" % (rev_start, rev_end) + verbose_opt = verbose and "-v" or "" + locale_env = fixlocale() + # some blather on stderr + auth_opt = self._makeauthoptions() + #stdin, stdout, stderr = os.popen3(locale_env + + # 'svn log --xml %s %s %s "%s"' % ( + # rev_opt, verbose_opt, auth_opt, + # self.strpath)) + cmd = locale_env + 'svn log --xml %s %s %s "%s"' % ( + rev_opt, verbose_opt, auth_opt, self.strpath) + + popen = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + stdout, stderr = popen.communicate() + stdout = py.builtin._totext(stdout, sys.getdefaultencoding()) + minidom,ExpatError = importxml() + try: + tree = minidom.parseString(stdout) + except ExpatError: + raise ValueError('no such revision') + result = [] + for logentry in filter(None, tree.firstChild.childNodes): + if logentry.nodeType == logentry.ELEMENT_NODE: + result.append(LogEntry(logentry)) + return result + + def size(self): + """ Return the size of the file content of the Path. """ + return self.info().size + + def mtime(self): + """ Return the last modification time of the file. """ + return self.info().mtime + + def __hash__(self): + return hash((self.strpath, self.__class__, self.auth)) + + +class WCStatus: + attrnames = ('modified','added', 'conflict', 'unchanged', 'external', + 'deleted', 'prop_modified', 'unknown', 'update_available', + 'incomplete', 'kindmismatch', 'ignored', 'locked', 'replaced' + ) + + def __init__(self, wcpath, rev=None, modrev=None, author=None): + self.wcpath = wcpath + self.rev = rev + self.modrev = modrev + self.author = author + + for name in self.attrnames: + setattr(self, name, []) + + def allpath(self, sort=True, **kw): + d = {} + for name in self.attrnames: + if name not in kw or kw[name]: + for path in getattr(self, name): + d[path] = 1 + l = d.keys() + if sort: + l.sort() + return l + + # XXX a bit scary to assume there's always 2 spaces between username and + # path, however with win32 allowing spaces in user names there doesn't + # seem to be a more solid approach :( + _rex_status = re.compile(r'\s+(\d+|-)\s+(\S+)\s+(.+?)\s{2,}(.*)') + + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ return a new WCStatus object from data 's' + """ + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + for line in data.split('\n'): + if not line.strip(): + continue + #print "processing %r" % line + flags, rest = line[:8], line[8:] + # first column + c0,c1,c2,c3,c4,c5,x6,c7 = flags + #if '*' in line: + # print "flags", repr(flags), "rest", repr(rest) + + if c0 in '?XI': + fn = line.split(None, 1)[1] + if c0 == '?': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.unknown.append(wcpath) + elif c0 == 'X': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(fn, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + elif c0 == 'I': + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.ignored.append(wcpath) + + continue + + #elif c0 in '~!' or c4 == 'S': + # raise NotImplementedError("received flag %r" % c0) + + m = WCStatus._rex_status.match(rest) + if not m: + if c7 == '*': + fn = rest.strip() + wcpath = rootwcpath.join(fn, abs=1) + rootstatus.update_available.append(wcpath) + continue + if line.lower().find('against revision:')!=-1: + update_rev = int(rest.split(':')[1].strip()) + continue + if line.lower().find('status on external') > -1: + # XXX not sure what to do here... perhaps we want to + # store some state instead of just continuing, as right + # now it makes the top-level external get added twice + # (once as external, once as 'normal' unchanged item) + # because of the way SVN presents external items + continue + # keep trying + raise ValueError("could not parse line %r" % line) + else: + rev, modrev, author, fn = m.groups() + wcpath = rootwcpath.join(fn, abs=1) + #assert wcpath.check() + if c0 == 'M': + assert wcpath.check(file=1), "didn't expect a directory with changed content here" + rootstatus.modified.append(wcpath) + elif c0 == 'A' or c3 == '+' : + rootstatus.added.append(wcpath) + elif c0 == 'D': + rootstatus.deleted.append(wcpath) + elif c0 == 'C': + rootstatus.conflict.append(wcpath) + elif c0 == '~': + rootstatus.kindmismatch.append(wcpath) + elif c0 == '!': + rootstatus.incomplete.append(wcpath) + elif c0 == 'R': + rootstatus.replaced.append(wcpath) + elif not c0.strip(): + rootstatus.unchanged.append(wcpath) + else: + raise NotImplementedError("received flag %r" % c0) + + if c1 == 'M': + rootstatus.prop_modified.append(wcpath) + # XXX do we cover all client versions here? + if c2 == 'L' or c5 == 'K': + rootstatus.locked.append(wcpath) + if c7 == '*': + rootstatus.update_available.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + if update_rev: + rootstatus.update_rev = update_rev + continue + return rootstatus + fromstring = staticmethod(fromstring) + +class XMLWCStatus(WCStatus): + def fromstring(data, rootwcpath, rev=None, modrev=None, author=None): + """ parse 'data' (XML string as outputted by svn st) into a status obj + """ + # XXX for externals, the path is shown twice: once + # with external information, and once with full info as if + # the item was a normal non-external... the current way of + # dealing with this issue is by ignoring it - this does make + # externals appear as external items as well as 'normal', + # unchanged ones in the status object so this is far from ideal + rootstatus = WCStatus(rootwcpath, rev, modrev, author) + update_rev = None + minidom, ExpatError = importxml() + try: + doc = minidom.parseString(data) + except ExpatError: + e = sys.exc_info()[1] + raise ValueError(str(e)) + urevels = doc.getElementsByTagName('against') + if urevels: + rootstatus.update_rev = urevels[-1].getAttribute('revision') + for entryel in doc.getElementsByTagName('entry'): + path = entryel.getAttribute('path') + statusel = entryel.getElementsByTagName('wc-status')[0] + itemstatus = statusel.getAttribute('item') + + if itemstatus == 'unversioned': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.unknown.append(wcpath) + continue + elif itemstatus == 'external': + wcpath = rootwcpath.__class__( + rootwcpath.localpath.join(path, abs=1), + auth=rootwcpath.auth) + rootstatus.external.append(wcpath) + continue + elif itemstatus == 'ignored': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.ignored.append(wcpath) + continue + elif itemstatus == 'incomplete': + wcpath = rootwcpath.join(path, abs=1) + rootstatus.incomplete.append(wcpath) + continue + + rev = statusel.getAttribute('revision') + if itemstatus == 'added' or itemstatus == 'none': + rev = '0' + modrev = '?' + author = '?' + date = '' + else: + #print entryel.toxml() + commitel = entryel.getElementsByTagName('commit')[0] + if commitel: + modrev = commitel.getAttribute('revision') + author = '' + author_els = commitel.getElementsByTagName('author') + if author_els: + for c in author_els[0].childNodes: + author += c.nodeValue + date = '' + for c in commitel.getElementsByTagName('date')[0]\ + .childNodes: + date += c.nodeValue + + wcpath = rootwcpath.join(path, abs=1) + + assert itemstatus != 'modified' or wcpath.check(file=1), ( + 'did\'t expect a directory with changed content here') + + itemattrname = { + 'normal': 'unchanged', + 'unversioned': 'unknown', + 'conflicted': 'conflict', + 'none': 'added', + }.get(itemstatus, itemstatus) + + attr = getattr(rootstatus, itemattrname) + attr.append(wcpath) + + propsstatus = statusel.getAttribute('props') + if propsstatus not in ('none', 'normal'): + rootstatus.prop_modified.append(wcpath) + + if wcpath == rootwcpath: + rootstatus.rev = rev + rootstatus.modrev = modrev + rootstatus.author = author + rootstatus.date = date + + # handle repos-status element (remote info) + rstatusels = entryel.getElementsByTagName('repos-status') + if rstatusels: + rstatusel = rstatusels[0] + ritemstatus = rstatusel.getAttribute('item') + if ritemstatus in ('added', 'modified'): + rootstatus.update_available.append(wcpath) + + lockels = entryel.getElementsByTagName('lock') + if len(lockels): + rootstatus.locked.append(wcpath) + + return rootstatus + fromstring = staticmethod(fromstring) + +class InfoSvnWCCommand: + def __init__(self, output): + # Path: test + # URL: http://codespeak.net/svn/std.path/trunk/dist/std.path/test + # Repository UUID: fd0d7bf2-dfb6-0310-8d31-b7ecfe96aada + # Revision: 2151 + # Node Kind: directory + # Schedule: normal + # Last Changed Author: hpk + # Last Changed Rev: 2100 + # Last Changed Date: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + # Properties Last Updated: 2003-11-03 14:47:48 +0100 (Mon, 03 Nov 2003) + + d = {} + for line in output.split('\n'): + if not line.strip(): + continue + key, value = line.split(':', 1) + key = key.lower().replace(' ', '') + value = value.strip() + d[key] = value + try: + self.url = d['url'] + except KeyError: + raise ValueError("Not a versioned resource") + #raise ValueError, "Not a versioned resource %r" % path + self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind'] + self.rev = int(d['revision']) + self.path = py.path.local(d['path']) + self.size = self.path.size() + if 'lastchangedrev' in d: + self.created_rev = int(d['lastchangedrev']) + if 'lastchangedauthor' in d: + self.last_author = d['lastchangedauthor'] + if 'lastchangeddate' in d: + self.mtime = parse_wcinfotime(d['lastchangeddate']) + self.time = self.mtime * 1000000 + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + +def parse_wcinfotime(timestr): + """ Returns seconds since epoch, UTC. """ + # example: 2003-10-27 20:43:14 +0100 (Mon, 27 Oct 2003) + m = re.match(r'(\d+-\d+-\d+ \d+:\d+:\d+) ([+-]\d+) .*', timestr) + if not m: + raise ValueError("timestring %r does not match" % timestr) + timestr, timezone = m.groups() + # do not handle timezone specially, return value should be UTC + parsedtime = time.strptime(timestr, "%Y-%m-%d %H:%M:%S") + return calendar.timegm(parsedtime) + +def make_recursive_propdict(wcroot, + output, + rex = re.compile("Properties on '(.*)':")): + """ Return a dictionary of path->PropListDict mappings. """ + lines = [x for x in output.split('\n') if x] + pdict = {} + while lines: + line = lines.pop(0) + m = rex.match(line) + if not m: + raise ValueError("could not parse propget-line: %r" % line) + path = m.groups()[0] + wcpath = wcroot.join(path, abs=1) + propnames = [] + while lines and lines[0].startswith(' '): + propname = lines.pop(0).strip() + propnames.append(propname) + assert propnames, "must have found properties!" + pdict[wcpath] = PropListDict(wcpath, propnames) + return pdict + + +def importxml(cache=[]): + if cache: + return cache + from xml.dom import minidom + from xml.parsers.expat import ExpatError + cache.extend([minidom, ExpatError]) + return cache + +class LogEntry: + def __init__(self, logentry): + self.rev = int(logentry.getAttribute('revision')) + for lpart in filter(None, logentry.childNodes): + if lpart.nodeType == lpart.ELEMENT_NODE: + if lpart.nodeName == 'author': + self.author = lpart.firstChild.nodeValue + elif lpart.nodeName == 'msg': + if lpart.firstChild: + self.msg = lpart.firstChild.nodeValue + else: + self.msg = '' + elif lpart.nodeName == 'date': + #2003-07-29T20:05:11.598637Z + timestr = lpart.firstChild.nodeValue + self.date = parse_apr_time(timestr) + elif lpart.nodeName == 'paths': + self.strpaths = [] + for ppart in filter(None, lpart.childNodes): + if ppart.nodeType == ppart.ELEMENT_NODE: + self.strpaths.append(PathEntry(ppart)) + def __repr__(self): + return '' % ( + self.rev, self.author, self.date) + + Added: pypy/trunk/py/impl/process/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/process/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +""" high-level sub-process handling """ Added: pypy/trunk/py/impl/process/cmdexec.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/process/cmdexec.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,44 @@ +""" + +""" + +import os, sys +import subprocess +import py +from subprocess import Popen, PIPE + +def cmdexec(cmd): + """ return output of executing 'cmd' in a separate process. + + raise cmdexec.ExecutionFailed exeception if the command failed. + the exception will provide an 'err' attribute containing + the error-output from the command. + """ + process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = process.communicate() + out = py.builtin._totext(out, sys.getdefaultencoding()) + err = py.builtin._totext(err, sys.getdefaultencoding()) + status = process.poll() + if status: + raise ExecutionFailed(status, status, cmd, out, err) + return out + +class ExecutionFailed(py.error.Error): + def __init__(self, status, systemstatus, cmd, out, err): + Exception.__init__(self) + self.status = status + self.systemstatus = systemstatus + self.cmd = cmd + self.err = err + self.out = out + + def __str__(self): + return "ExecutionFailed: %d %s\n%s" %(self.status, self.cmd, self.err) + +# export the exception under the name 'py.process.cmdexec.Error' +cmdexec.Error = ExecutionFailed +try: + ExecutionFailed.__module__ = 'py.process.cmdexec' + ExecutionFailed.__name__ = 'Error' +except (AttributeError, TypeError): + pass Added: pypy/trunk/py/impl/process/forkedfunc.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/process/forkedfunc.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,108 @@ + +""" + ForkedFunc provides a way to run a function in a forked process + and get at its return value, stdout and stderr output as well + as signals and exitstatusus. + + XXX see if tempdir handling is sane +""" + +import py +import os +import sys +import marshal + +class ForkedFunc(object): + EXITSTATUS_EXCEPTION = 3 + def __init__(self, fun, args=None, kwargs=None, nice_level=0): + if args is None: + args = [] + if kwargs is None: + kwargs = {} + self.fun = fun + self.args = args + self.kwargs = kwargs + self.tempdir = tempdir = py.path.local.mkdtemp() + self.RETVAL = tempdir.ensure('retval') + self.STDOUT = tempdir.ensure('stdout') + self.STDERR = tempdir.ensure('stderr') + + pid = os.fork() + if pid: # in parent process + self.pid = pid + else: # in child process + self._child(nice_level) + + def _child(self, nice_level): + # right now we need to call a function, but first we need to + # map all IO that might happen + # make sure sys.stdout points to file descriptor one + sys.stdout = stdout = self.STDOUT.open('w') + sys.stdout.flush() + fdstdout = stdout.fileno() + if fdstdout != 1: + os.dup2(fdstdout, 1) + sys.stderr = stderr = self.STDERR.open('w') + fdstderr = stderr.fileno() + if fdstderr != 2: + os.dup2(fdstderr, 2) + retvalf = self.RETVAL.open("wb") + EXITSTATUS = 0 + try: + if nice_level: + os.nice(nice_level) + try: + retval = self.fun(*self.args, **self.kwargs) + retvalf.write(marshal.dumps(retval)) + except: + excinfo = py.code.ExceptionInfo() + stderr.write(excinfo.exconly()) + EXITSTATUS = self.EXITSTATUS_EXCEPTION + finally: + stdout.close() + stderr.close() + retvalf.close() + os.close(1) + os.close(2) + os._exit(EXITSTATUS) + + def waitfinish(self, waiter=os.waitpid): + pid, systemstatus = waiter(self.pid, 0) + if systemstatus: + if os.WIFSIGNALED(systemstatus): + exitstatus = os.WTERMSIG(systemstatus) + 128 + else: + exitstatus = os.WEXITSTATUS(systemstatus) + #raise ExecutionFailed(status, systemstatus, cmd, + # ''.join(out), ''.join(err)) + else: + exitstatus = 0 + signal = systemstatus & 0x7f + if not exitstatus and not signal: + retval = self.RETVAL.open('rb') + try: + retval_data = retval.read() + finally: + retval.close() + retval = marshal.loads(retval_data) + else: + retval = None + stdout = self.STDOUT.read() + stderr = self.STDERR.read() + self._removetemp() + return Result(exitstatus, signal, retval, stdout, stderr) + + def _removetemp(self): + if self.tempdir.check(): + self.tempdir.remove() + + def __del__(self): + self._removetemp() + +class Result(object): + def __init__(self, exitstatus, signal, retval, stdout, stderr): + self.exitstatus = exitstatus + self.signal = signal + self.retval = retval + self.out = stdout + self.err = stderr Added: pypy/trunk/py/impl/process/killproc.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/process/killproc.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,23 @@ +import py +import os, sys + +if sys.platform == "win32": + try: + import ctypes + except ImportError: + def dokill(pid): + py.process.cmdexec("taskkill /F /PID %d" %(pid,)) + else: + def dokill(pid): + PROCESS_TERMINATE = 1 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_TERMINATE, False, pid) + ctypes.windll.kernel32.TerminateProcess(handle, -1) + ctypes.windll.kernel32.CloseHandle(handle) +else: + def dokill(pid): + os.kill(pid, 15) + +def kill(pid): + """ kill process by id. """ + dokill(pid) Added: pypy/trunk/py/impl/std.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/std.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,18 @@ +import sys + +class Std(object): + """ makes top-level python modules available as an attribute, + importing them on first access. + """ + + def __init__(self): + self.__dict__ = sys.modules + + def __getattr__(self, name): + try: + m = __import__(name) + except ImportError: + raise AttributeError("py.std: could not import %s" % name) + return m + +std = Std() Added: pypy/trunk/py/impl/test/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +""" versatile unit-testing tool + libraries """ Added: pypy/trunk/py/impl/test/cmdline.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/cmdline.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,23 @@ +import py +import sys + +# +# main entry point +# + +def main(args=None): + if args is None: + args = sys.argv[1:] + config = py.test.config + try: + config.parse(args) + config.pluginmanager.do_configure(config) + session = config.initsession() + exitstatus = session.main() + config.pluginmanager.do_unconfigure(config) + raise SystemExit(exitstatus) + except config.Error: + e = sys.exc_info()[1] + sys.stderr.write("ERROR: %s\n" %(e.args[0],)) + raise SystemExit(3) + Added: pypy/trunk/py/impl/test/collect.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/collect.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,466 @@ +""" +base test collection objects. +Collectors and test Items form a tree +that is usually built iteratively. +""" +import py + +def configproperty(name): + def fget(self): + #print "retrieving %r property from %s" %(name, self.fspath) + return self.config.getvalue(name, self.fspath) + return property(fget) + +class Node(object): + """ base class for Nodes in the collection tree. + Collector nodes have children and + Item nodes are terminal. + + All nodes of the collection tree carry a _config + attribute for these reasons: + - to access custom Collection Nodes from a project + (defined in conftest's) + - to pickle themselves relatively to the "topdir" + - configuration/options for setup/teardown + stdout/stderr capturing and execution of test items + """ + def __init__(self, name, parent=None): + self.name = name + self.parent = parent + self.config = getattr(parent, 'config', None) + self.fspath = getattr(parent, 'fspath', None) + + def _checkcollectable(self): + if not hasattr(self, 'fspath'): + self.parent._memocollect() # to reraise exception + + # + # note to myself: Pickling is uh. + # + def __getstate__(self): + return (self.name, self.parent) + def __setstate__(self, nameparent): + name, parent = nameparent + try: + colitems = parent._memocollect() + except KeyboardInterrupt: + raise + except Exception: + # seems our parent can't collect us + # so let's be somewhat operable + # _checkcollectable() is to tell outsiders about the fact + self.name = name + self.parent = parent + self.config = parent.config + #self._obj = "could not unpickle" + else: + for colitem in colitems: + if colitem.name == name: + # we are a copy that will not be returned + # by our parent + self.__dict__ = colitem.__dict__ + break + + def __repr__(self): + if getattr(self.config.option, 'debug', False): + return "<%s %r %0x>" %(self.__class__.__name__, + getattr(self, 'name', None), id(self)) + else: + return "<%s %r>" %(self.__class__.__name__, + getattr(self, 'name', None)) + + # methods for ordering nodes + + def __eq__(self, other): + if not isinstance(other, Node): + return False + return self.name == other.name and self.parent == other.parent + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.name, self.parent)) + + def setup(self): + pass + + def teardown(self): + pass + + def _memoizedcall(self, attrname, function): + exattrname = "_ex_" + attrname + failure = getattr(self, exattrname, None) + if failure is not None: + py.builtin._reraise(failure[0], failure[1], failure[2]) + if hasattr(self, attrname): + return getattr(self, attrname) + try: + res = function() + except (KeyboardInterrupt, SystemExit): + raise + except: + failure = py.std.sys.exc_info() + setattr(self, exattrname, failure) + raise + setattr(self, attrname, res) + return res + + def listchain(self, rootfirst=False): + """ return list of all parent collectors up to self, + starting form root of collection tree. """ + l = [self] + while 1: + x = l[-1] + if x.parent is not None: + l.append(x.parent) + else: + if not rootfirst: + l.reverse() + return l + + def listnames(self): + return [x.name for x in self.listchain()] + + def getparent(self, cls): + current = self + while current and not isinstance(current, cls): + current = current.parent + return current + + def _getitembynames(self, namelist): + cur = self + for name in namelist: + if name: + next = cur.collect_by_name(name) + if next is None: + existingnames = [x.name for x in self._memocollect()] + msg = ("Collector %r does not have name %r " + "existing names are: %s" % + (cur, name, existingnames)) + raise AssertionError(msg) + cur = next + return cur + + + def _getfsnode(self, path): + # this method is usually called from + # config.getfsnode() which returns a colitem + # from filename arguments + # + # pytest's collector tree does not neccessarily + # follow the filesystem and we thus need to do + # some special matching code here because + # _getitembynames() works by colitem names, not + # basenames. + if path == self.fspath: + return self + basenames = path.relto(self.fspath).split(path.sep) + cur = self + while basenames: + basename = basenames.pop(0) + assert basename + fspath = cur.fspath.join(basename) + colitems = cur._memocollect() + l = [] + for colitem in colitems: + if colitem.fspath == fspath or colitem.name == basename: + l.append(colitem) + if not l: + raise self.config.Error("can't collect: %s" %(fspath,)) + if basenames: + if len(l) > 1: + msg = ("Collector %r has more than one %r colitem " + "existing colitems are: %s" % + (cur, fspath, colitems)) + raise self.config.Error("xxx-too many test types for: %s" % (fspath, )) + cur = l[0] + else: + if len(l) > 1: + cur = l + else: + cur = l[0] + break + return cur + + def readkeywords(self): + return dict([(x, True) for x in self._keywords()]) + + def _keywords(self): + return [self.name] + + def _skipbykeyword(self, keywordexpr): + """ return True if they given keyword expression means to + skip this collector/item. + """ + if not keywordexpr: + return + chain = self.listchain() + for key in filter(None, keywordexpr.split()): + eor = key[:1] == '-' + if eor: + key = key[1:] + if not (eor ^ self._matchonekeyword(key, chain)): + return True + + def _matchonekeyword(self, key, chain): + elems = key.split(".") + # XXX O(n^2), anyone cares? + chain = [item.readkeywords() for item in chain if item._keywords()] + for start, _ in enumerate(chain): + if start + len(elems) > len(chain): + return False + for num, elem in enumerate(elems): + for keyword in chain[num + start]: + ok = False + if elem in keyword: + ok = True + break + if not ok: + break + if num == len(elems) - 1 and ok: + return True + return False + + def _prunetraceback(self, traceback): + return traceback + + def _totrail(self): + """ provide a trail relative to the topdir, + which can be used to reconstruct the + collector (possibly on a different host + starting from a different topdir). + """ + chain = self.listchain() + topdir = self.config.topdir + relpath = chain[0].fspath.relto(topdir) + if not relpath: + if chain[0].fspath == topdir: + relpath = "." + else: + raise ValueError("%r not relative to topdir %s" + %(chain[0].fspath, topdir)) + return relpath, tuple([x.name for x in chain[1:]]) + + def _fromtrail(trail, config): + relpath, names = trail + fspath = config.topdir.join(relpath) + col = config.getfsnode(fspath) + return col._getitembynames(names) + _fromtrail = staticmethod(_fromtrail) + + def _repr_failure_py(self, excinfo, outerr=None): + assert outerr is None, "XXX deprecated" + excinfo.traceback = self._prunetraceback(excinfo.traceback) + # XXX temporary hack: getrepr() should not take a 'style' argument + # at all; it should record all data in all cases, and the style + # should be parametrized in toterminal(). + if self.config.option.tbstyle == "short": + style = "short" + else: + style = "long" + return excinfo.getrepr(funcargs=True, + showlocals=self.config.option.showlocals, + style=style) + + repr_failure = _repr_failure_py + shortfailurerepr = "F" + +class Collector(Node): + """ + Collector instances create children through collect() + and thus iteratively build a tree. attributes:: + + parent: attribute pointing to the parent collector + (or None if this is the root collector) + name: basename of this collector object + """ + Directory = configproperty('Directory') + Module = configproperty('Module') + + def collect(self): + """ returns a list of children (items and collectors) + for this collection node. + """ + raise NotImplementedError("abstract") + + def collect_by_name(self, name): + """ return a child matching the given name, else None. """ + for colitem in self._memocollect(): + if colitem.name == name: + return colitem + + def repr_failure(self, excinfo, outerr=None): + """ represent a failure. """ + assert outerr is None, "XXX deprecated" + return self._repr_failure_py(excinfo) + + def _memocollect(self): + """ internal helper method to cache results of calling collect(). """ + return self._memoizedcall('_collected', self.collect) + + # ********************************************************************** + # DEPRECATED METHODS + # ********************************************************************** + + def _deprecated_collect(self): + # avoid recursion: + # collect -> _deprecated_collect -> custom run() -> + # super().run() -> collect + attrname = '_depcollectentered' + if hasattr(self, attrname): + return + setattr(self, attrname, True) + method = getattr(self.__class__, 'run', None) + if method is not None and method != Collector.run: + warnoldcollect(function=method) + names = self.run() + return [x for x in [self.join(name) for name in names] if x] + + def run(self): + """ DEPRECATED: returns a list of names available from this collector. + You can return an empty list. Callers of this method + must take care to catch exceptions properly. + """ + return [colitem.name for colitem in self._memocollect()] + + def join(self, name): + """ DEPRECATED: return a child collector or item for the given name. + If the return value is None there is no such child. + """ + return self.collect_by_name(name) + + def _prunetraceback(self, traceback): + if hasattr(self, 'fspath'): + path = self.fspath + ntraceback = traceback.cut(path=self.fspath) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._dir) + traceback = ntraceback.filter() + return traceback + +class FSCollector(Collector): + def __init__(self, fspath, parent=None): + fspath = py.path.local(fspath) + super(FSCollector, self).__init__(fspath.basename, parent) + self.fspath = fspath + + def __getstate__(self): + if self.parent is None: + # the root node needs to pickle more context info + topdir = self.config.topdir + relpath = self.fspath.relto(topdir) + if not relpath: + if self.fspath == topdir: + relpath = "." + else: + raise ValueError("%r not relative to topdir %s" + %(self.fspath, topdir)) + return (self.name, self.config, relpath) + else: + return (self.name, self.parent) + + def __setstate__(self, picklestate): + if len(picklestate) == 3: + # root node + name, config, relpath = picklestate + fspath = config.topdir.join(relpath) + fsnode = config.getfsnode(fspath) + self.__dict__.update(fsnode.__dict__) + else: + name, parent = picklestate + self.__init__(parent.fspath.join(name), parent=parent) + +class File(FSCollector): + """ base class for collecting tests from a file. """ + +class Directory(FSCollector): + def recfilter(self, path): + if path.check(dir=1, dotfile=0): + return path.basename not in ('CVS', '_darcs', '{arch}') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + l = [] + for path in self.fspath.listdir(sort=True): + res = self.consider(path) + if res is not None: + if isinstance(res, (list, tuple)): + l.extend(res) + else: + l.append(res) + return l + + def _ignore(self, path): + ignore_paths = self.config.getconftest_pathlist("collect_ignore", path=path) + return ignore_paths and path in ignore_paths + # XXX more refined would be: + if ignore_paths: + for p in ignore_paths: + if path == p or path.relto(p): + return True + + def consider(self, path): + if self._ignore(path): + return + if path.check(file=1): + res = self.consider_file(path) + elif path.check(dir=1): + res = self.consider_dir(path) + else: + res = None + if isinstance(res, list): + # throw out identical results + l = [] + for x in res: + if x not in l: + assert x.parent == self, "wrong collection tree construction" + l.append(x) + res = l + return res + + def consider_file(self, path): + return self.config.hook.pytest_collect_file(path=path, parent=self) + + def consider_dir(self, path, usefilters=None): + if usefilters is not None: + py.log._apiwarn("0.99", "usefilters argument not needed") + return self.config.hook.pytest_collect_directory( + path=path, parent=self) + +class Item(Node): + """ a basic test item. """ + def _deprecated_testexecution(self): + if self.__class__.run != Item.run: + warnoldtestrun(function=self.run) + elif self.__class__.execute != Item.execute: + warnoldtestrun(function=self.execute) + else: + return False + self.run() + return True + + def run(self): + """ deprecated, here because subclasses might call it. """ + return self.execute(self.obj) + + def execute(self, obj): + """ deprecated, here because subclasses might call it. """ + return obj() + + def reportinfo(self): + return self.fspath, None, "" + +def warnoldcollect(function=None): + py.log._apiwarn("1.0", + "implement collector.collect() instead of " + "collector.run() and collector.join()", + stacklevel=2, function=function) + +def warnoldtestrun(function=None): + py.log._apiwarn("1.0", + "implement item.runtest() instead of " + "item.run() and item.execute()", + stacklevel=2, function=function) Added: pypy/trunk/py/impl/test/compat.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/compat.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,58 @@ +import py + +from py.test.collect import Function + +class TestCaseUnit(Function): + """ compatibility Unit executor for TestCase methods + honouring setUp and tearDown semantics. + """ + def runtest(self, _deprecated=None): + boundmethod = self.obj + instance = py.builtin._getimself(boundmethod) + instance.setUp() + try: + boundmethod() + finally: + instance.tearDown() + +class TestCase(object): + """compatibility class of unittest's TestCase. """ + Function = TestCaseUnit + + def setUp(self): + pass + + def tearDown(self): + pass + + def fail(self, msg=None): + """ fail immediate with given message. """ + py.test.fail(msg) + + def assertRaises(self, excclass, func, *args, **kwargs): + py.test.raises(excclass, func, *args, **kwargs) + failUnlessRaises = assertRaises + + # dynamically construct (redundant) methods + aliasmap = [ + ('x', 'not x', 'assert_, failUnless'), + ('x', 'x', 'failIf'), + ('x,y', 'x!=y', 'failUnlessEqual,assertEqual, assertEquals'), + ('x,y', 'x==y', 'failIfEqual,assertNotEqual, assertNotEquals'), + ] + items = [] + for sig, expr, names in aliasmap: + names = map(str.strip, names.split(',')) + sigsubst = expr.replace('y', '%s').replace('x', '%s') + for name in names: + items.append(""" + def %(name)s(self, %(sig)s, msg=""): + __tracebackhide__ = True + if %(expr)s: + py.test.fail(msg=msg + (%(sigsubst)r %% (%(sig)s))) + """ % locals() ) + + source = "".join(items) + exec(py.code.Source(source).compile()) + +__all__ = ['TestCase'] Added: pypy/trunk/py/impl/test/config.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/config.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,307 @@ +import py, os +from py.impl.test.conftesthandle import Conftest + +from py.impl.test import parseopt + +def ensuretemp(string, dir=1): + """ return temporary directory path with + the given string as the trailing part. + """ + return py.test.config.ensuretemp(string, dir=dir) + +class CmdOptions(object): + """ pure container instance for holding cmdline options + as attributes. + """ + def __repr__(self): + return "" %(self.__dict__,) + +class Error(Exception): + """ Test Configuration Error. """ + +class Config(object): + """ test configuration object, provides access to config valueso, + the pluginmanager and plugin api. + """ + Option = py.std.optparse.Option + Error = Error + basetemp = None + _sessionclass = None + + def __init__(self, pluginmanager=None, topdir=None): + self.option = CmdOptions() + self.topdir = topdir + self._parser = parseopt.Parser( + usage="usage: %prog [options] [file_or_dir] [file_or_dir] [...]", + processopt=self._processopt, + ) + if pluginmanager is None: + pluginmanager = py.test._PluginManager() + assert isinstance(pluginmanager, py.test._PluginManager) + self.pluginmanager = pluginmanager + self._conftest = Conftest(onimport=self._onimportconftest) + self.hook = pluginmanager.hook + + def _onimportconftest(self, conftestmodule): + self.trace("loaded conftestmodule %r" %(conftestmodule,)) + self.pluginmanager.consider_conftest(conftestmodule) + + def trace(self, msg): + if getattr(self.option, 'traceconfig', None): + self.hook.pytest_trace(category="config", msg=msg) + + def _processopt(self, opt): + if hasattr(opt, 'default') and opt.dest: + val = os.environ.get("PYTEST_OPTION_" + opt.dest.upper(), None) + if val is not None: + if opt.type == "int": + val = int(val) + elif opt.type == "long": + val = long(val) + elif opt.type == "float": + val = float(val) + elif not opt.type and opt.action in ("store_true", "store_false"): + val = eval(val) + opt.default = val + else: + name = "option_" + opt.dest + try: + opt.default = self._conftest.rget(name) + except (ValueError, KeyError): + pass + if not hasattr(self.option, opt.dest): + setattr(self.option, opt.dest, opt.default) + + def _preparse(self, args): + self._conftest.setinitial(args) + self.pluginmanager.consider_preparse(args) + self.pluginmanager.consider_env() + self.pluginmanager.do_addoption(self._parser) + + def parse(self, args): + """ parse cmdline arguments into this config object. + Note that this can only be called once per testing process. + """ + assert not hasattr(self, 'args'), ( + "can only parse cmdline args at most once per Config object") + self._preparse(args) + args = self._parser.parse_setoption(args, self.option) + if not args: + args.append(py.std.os.getcwd()) + self.topdir = gettopdir(args) + self.args = [py.path.local(x) for x in args] + + # config objects are usually pickled across system + # barriers but they contain filesystem paths. + # upon getstate/setstate we take care to do everything + # relative to "topdir". + def __getstate__(self): + l = [] + for path in self.args: + path = py.path.local(path) + l.append(path.relto(self.topdir)) + return l, self.option + + def __setstate__(self, repr): + # warning global side effects: + # * registering to py lib plugins + # * setting py.test.config + self.__init__( + pluginmanager=py.test._PluginManager(py._com.comregistry), + topdir=py.path.local(), + ) + # we have to set py.test.config because preparse() + # might load conftest files which have + # py.test.config.addoptions() lines in them + py.test.config = self + args, cmdlineopts = repr + args = [self.topdir.join(x) for x in args] + self.option = cmdlineopts + self._preparse(args) + self.args = args + + def ensuretemp(self, string, dir=True): + return self.getbasetemp().ensure(string, dir=dir) + + def getbasetemp(self): + if self.basetemp is None: + basetemp = self.option.basetemp + if basetemp: + basetemp = py.path.local(basetemp) + if not basetemp.check(dir=1): + basetemp.mkdir() + else: + basetemp = py.path.local.make_numbered_dir(prefix='pytest-') + self.basetemp = basetemp + return self.basetemp + + def mktemp(self, basename, numbered=False): + basetemp = self.getbasetemp() + if not numbered: + return basetemp.mkdir(basename) + else: + return py.path.local.make_numbered_dir(prefix=basename + "-", + keep=0, rootdir=basetemp, lock_timeout=None) + + def getcolitems(self): + return [self.getfsnode(arg) for arg in self.args] + + def getfsnode(self, path): + path = py.path.local(path) + if not path.check(): + raise self.Error("file not found: %s" %(path,)) + # we want our possibly custom collection tree to start at pkgroot + pkgpath = path.pypkgpath() + if pkgpath is None: + pkgpath = path.check(file=1) and path.dirpath() or path + Dir = self._conftest.rget("Directory", pkgpath) + col = Dir(pkgpath) + col.config = self + return col._getfsnode(path) + + def getconftest_pathlist(self, name, path=None): + """ return a matching value, which needs to be sequence + of filenames that will be returned as a list of Path + objects (they can be relative to the location + where they were found). + """ + try: + mod, relroots = self._conftest.rget_with_confmod(name, path) + except KeyError: + return None + modpath = py.path.local(mod.__file__).dirpath() + l = [] + for relroot in relroots: + relroot = relroot.replace("/", py.path.local.sep) + l.append(modpath.join(relroot, abs=True)) + return l + + def addoptions(self, groupname, *specs): + """ add a named group of options to the current testing session. + This function gets invoked during testing session initialization. + """ + py.log._apiwarn("1.0", "define plugins to add options", stacklevel=2) + group = self._parser.addgroup(groupname) + for opt in specs: + group._addoption_instance(opt) + return self.option + + def addoption(self, *optnames, **attrs): + return self._parser.addoption(*optnames, **attrs) + + def getvalueorskip(self, name, path=None): + """ return getvalue() or call py.test.skip if no value exists. """ + try: + val = self.getvalue(name, path) + if val is None: + raise KeyError(name) + return val + except KeyError: + py.test.skip("no %r value found" %(name,)) + + def getvalue(self, name, path=None): + """ return 'name' value looked up from the 'options' + and then from the first conftest file found up + the path (including the path itself). + if path is None, lookup the value in the initial + conftest modules found during command line parsing. + """ + try: + return getattr(self.option, name) + except AttributeError: + return self._conftest.rget(name, path) + + def setsessionclass(self, cls): + if self._sessionclass is not None: + raise ValueError("sessionclass already set to: %r" %( + self._sessionclass)) + self._sessionclass = cls + + def initsession(self): + """ return an initialized session object. """ + cls = self._sessionclass + if cls is None: + from py.impl.test.session import Session + cls = Session + session = cls(self) + self.trace("instantiated session %r" % session) + return session + + def _reparse(self, args): + """ this is used from tests that want to re-invoke parse(). """ + #assert args # XXX should not be empty + global config_per_process + oldconfig = py.test.config + try: + config_per_process = py.test.config = Config() + config_per_process.basetemp = self.mktemp("reparse", numbered=True) + config_per_process.parse(args) + return config_per_process + finally: + config_per_process = py.test.config = oldconfig + + def getxspecs(self): + xspeclist = [] + for xspec in self.getvalue("tx"): + i = xspec.find("*") + try: + num = int(xspec[:i]) + except ValueError: + xspeclist.append(xspec) + else: + xspeclist.extend([xspec[i+1:]] * num) + if not xspeclist: + raise self.Error("MISSING test execution (tx) nodes: please specify --tx") + import execnet + return [execnet.XSpec(x) for x in xspeclist] + + def getrsyncdirs(self): + config = self + roots = config.option.rsyncdir + conftestroots = config.getconftest_pathlist("rsyncdirs") + if conftestroots: + roots.extend(conftestroots) + pydirs = [x.realpath() for x in py._pydirs] + roots = [py.path.local(root) for root in roots] + for root in roots: + if not root.check(): + raise config.Error("rsyncdir doesn't exist: %r" %(root,)) + if pydirs is not None and root.basename in ("py", "_py"): + pydirs.remove(root) # otherwise it's a conflict + roots.extend(pydirs) + return roots + +# +# helpers +# + +def checkmarshal(name, value): + try: + py.std.marshal.dumps(value) + except ValueError: + raise ValueError("%s=%r is not marshallable" %(name, value)) + +def gettopdir(args): + """ return the top directory for the given paths. + if the common base dir resides in a python package + parent directory of the root package is returned. + """ + args = [py.path.local(arg) for arg in args] + p = args and args[0] or None + for x in args[1:]: + p = p.common(x) + assert p, "cannot determine common basedir of %s" %(args,) + pkgdir = p.pypkgpath() + if pkgdir is None: + if p.check(file=1): + p = p.dirpath() + return p + else: + return pkgdir.dirpath() + + +# this is the one per-process instance of py.test configuration +config_per_process = Config( + pluginmanager=py.test._PluginManager(py._com.comregistry) +) + Added: pypy/trunk/py/impl/test/conftesthandle.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/conftesthandle.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,82 @@ +import py +defaultconftestpath = py.path.local(__file__).dirpath("defaultconftest.py") + +class Conftest(object): + """ the single place for accessing values and interacting + towards conftest modules from py.test objects. + + Note that triggering Conftest instances to import + conftest.py files may result in added cmdline options. + XXX + """ + def __init__(self, path=None, onimport=None): + self._path2confmods = {} + self._onimport = onimport + if path is not None: + self.setinitial([path]) + + def setinitial(self, args): + """ try to find a first anchor path for looking up global values + from conftests. This function is usually called _before_ + argument parsing. conftest files may add command line options + and we thus have no completely safe way of determining + which parts of the arguments are actually related to options + and which are file system paths. We just try here to get + bootstrapped ... + """ + current = py.path.local() + for arg in args + [current]: + anchor = current.join(arg, abs=1) + if anchor.check(): # we found some file object + self._path2confmods[None] = self.getconftestmodules(anchor) + break + else: + assert 0, "no root of filesystem?" + + def getconftestmodules(self, path): + """ return a list of imported conftest modules for the given path. """ + try: + clist = self._path2confmods[path] + except KeyError: + if path is None: + raise ValueError("missing default conftest.") + dp = path.dirpath() + if dp == path: + return [self.importconftest(defaultconftestpath)] + clist = self.getconftestmodules(dp) + conftestpath = path.join("conftest.py") + if conftestpath.check(file=1): + clist.append(self.importconftest(conftestpath)) + self._path2confmods[path] = clist + # be defensive: avoid changes from caller side to + # affect us by always returning a copy of the actual list + return clist[:] + + def rget(self, name, path=None): + mod, value = self.rget_with_confmod(name, path) + return value + + def rget_with_confmod(self, name, path=None): + modules = self.getconftestmodules(path) + modules.reverse() + for mod in modules: + try: + return mod, getattr(mod, name) + except AttributeError: + continue + raise KeyError(name) + + def importconftest(self, conftestpath): + # Using caching here looks redundant since ultimately + # sys.modules caches already + assert conftestpath.check(), conftestpath + if not conftestpath.dirpath('__init__.py').check(file=1): + # HACK: we don't want any "globally" imported conftest.py, + # prone to conflicts and subtle problems + modname = str(conftestpath).replace('.', conftestpath.sep) + mod = conftestpath.pyimport(modname=modname) + else: + mod = conftestpath.pyimport() + if self._onimport: + self._onimport(mod) + return mod Added: pypy/trunk/py/impl/test/defaultconftest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/defaultconftest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,14 @@ +import py + +Module = py.test.collect.Module +Directory = py.test.collect.Directory +File = py.test.collect.File + +# python collectors +Class = py.test.collect.Class +Generator = py.test.collect.Generator +Function = py.test.collect.Function +Instance = py.test.collect.Instance + +pytest_plugins = "default runner capture terminal mark skipping tmpdir monkeypatch recwarn pdb pastebin unittest helpconfig nose assertion".split() + Added: pypy/trunk/py/impl/test/dist/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +# Added: pypy/trunk/py/impl/test/dist/dsession.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/dsession.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,280 @@ +""" + + EXPERIMENTAL dsession session (for dist/non-dist unification) + +""" + +import py +from py.impl.test.session import Session +from py.impl.test import outcome +from py.impl.test.dist.nodemanage import NodeManager +queue = py.builtin._tryimport('queue', 'Queue') + +debug_file = None # open('/tmp/loop.log', 'w') +def debug(*args): + if debug_file is not None: + s = " ".join(map(str, args)) + debug_file.write(s+"\n") + debug_file.flush() + +class LoopState(object): + def __init__(self, dsession, colitems): + self.dsession = dsession + self.colitems = colitems + self.exitstatus = None + # loopstate.dowork is False after reschedule events + # because otherwise we might very busily loop + # waiting for a host to become ready. + self.dowork = True + self.shuttingdown = False + self.testsfailed = False + + def __repr__(self): + return "" % ( + self.exitstatus, self.shuttingdown, len(self.colitems)) + + def pytest_runtest_logreport(self, report): + if report.item in self.dsession.item2nodes: + if report.when != "teardown": # otherwise we already managed it + self.dsession.removeitem(report.item, report.node) + if report.failed: + self.testsfailed = True + + def pytest_collectreport(self, report): + if report.passed: + self.colitems.extend(report.result) + + def pytest_testnodeready(self, node): + self.dsession.addnode(node) + + def pytest_testnodedown(self, node, error=None): + pending = self.dsession.removenode(node) + if pending: + if error: + crashitem = pending[0] + debug("determined crashitem", crashitem) + self.dsession.handle_crashitem(crashitem, node) + # XXX recovery handling for "each"? + # currently pending items are not retried + if self.dsession.config.option.dist == "load": + self.colitems.extend(pending[1:]) + + def pytest_rescheduleitems(self, items): + self.colitems.extend(items) + self.dowork = False # avoid busywait + +class DSession(Session): + """ + Session drives the collection and running of tests + and generates test events for reporters. + """ + MAXITEMSPERHOST = 15 + + def __init__(self, config): + self.queue = queue.Queue() + self.node2pending = {} + self.item2nodes = {} + super(DSession, self).__init__(config=config) + + #def pytest_configure(self, __multicall__, config): + # __multicall__.execute() + # try: + # config.getxspecs() + # except config.Error: + # print + # raise config.Error("dist mode %r needs test execution environments, " + # "none found." %(config.option.dist)) + + def main(self, colitems=None): + colitems = self.getinitialitems(colitems) + self.sessionstarts() + self.setup() + exitstatus = self.loop(colitems) + self.teardown() + self.sessionfinishes(exitstatus=exitstatus) + return exitstatus + + def loop_once(self, loopstate): + if loopstate.shuttingdown: + return self.loop_once_shutdown(loopstate) + colitems = loopstate.colitems + if loopstate.dowork and colitems: + self.triggertesting(loopstate.colitems) + colitems[:] = [] + # we use a timeout here so that control-C gets through + while 1: + try: + eventcall = self.queue.get(timeout=2.0) + break + except queue.Empty: + continue + loopstate.dowork = True + + callname, args, kwargs = eventcall + if callname is not None: + call = getattr(self.config.hook, callname) + assert not args + call(**kwargs) + + # termination conditions + if ((loopstate.testsfailed and self.config.option.exitfirst) or + (not self.item2nodes and not colitems and not self.queue.qsize())): + self.triggershutdown() + loopstate.shuttingdown = True + elif not self.node2pending: + loopstate.exitstatus = outcome.EXIT_NOHOSTS + + def loop_once_shutdown(self, loopstate): + # once we are in shutdown mode we dont send + # events other than HostDown upstream + eventname, args, kwargs = self.queue.get() + if eventname == "pytest_testnodedown": + self.config.hook.pytest_testnodedown(**kwargs) + self.removenode(kwargs['node']) + elif eventname == "pytest_runtest_logreport": + # might be some teardown report + self.config.hook.pytest_runtest_logreport(**kwargs) + if not self.node2pending: + # finished + if loopstate.testsfailed: + loopstate.exitstatus = outcome.EXIT_TESTSFAILED + else: + loopstate.exitstatus = outcome.EXIT_OK + #self.config.pluginmanager.unregister(loopstate) + + def _initloopstate(self, colitems): + loopstate = LoopState(self, colitems) + self.config.pluginmanager.register(loopstate) + return loopstate + + def loop(self, colitems): + try: + loopstate = self._initloopstate(colitems) + loopstate.dowork = False # first receive at least one HostUp events + while 1: + self.loop_once(loopstate) + if loopstate.exitstatus is not None: + exitstatus = loopstate.exitstatus + break + except KeyboardInterrupt: + excinfo = py.code.ExceptionInfo() + self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + exitstatus = outcome.EXIT_INTERRUPTED + except: + self.config.pluginmanager.notify_exception() + exitstatus = outcome.EXIT_INTERNALERROR + self.config.pluginmanager.unregister(loopstate) + if exitstatus == 0 and self._testsfailed: + exitstatus = outcome.EXIT_TESTSFAILED + return exitstatus + + def triggershutdown(self): + for node in self.node2pending: + node.shutdown() + + def addnode(self, node): + assert node not in self.node2pending + self.node2pending[node] = [] + + def removenode(self, node): + try: + pending = self.node2pending.pop(node) + except KeyError: + # this happens if we didn't receive a testnodeready event yet + return [] + for item in pending: + l = self.item2nodes[item] + l.remove(node) + if not l: + del self.item2nodes[item] + return pending + + def triggertesting(self, colitems): + colitems = self.filteritems(colitems) + senditems = [] + for next in colitems: + if isinstance(next, py.test.collect.Item): + senditems.append(next) + else: + self.config.hook.pytest_collectstart(collector=next) + colrep = self.config.hook.pytest_make_collect_report(collector=next) + self.queueevent("pytest_collectreport", report=colrep) + if self.config.option.dist == "each": + self.senditems_each(senditems) + else: + # XXX assert self.config.option.dist == "load" + self.senditems_load(senditems) + + def queueevent(self, eventname, **kwargs): + self.queue.put((eventname, (), kwargs)) + + def senditems_each(self, tosend): + if not tosend: + return + room = self.MAXITEMSPERHOST + for node, pending in self.node2pending.items(): + room = min(self.MAXITEMSPERHOST - len(pending), room) + sending = tosend[:room] + for node, pending in self.node2pending.items(): + node.sendlist(sending) + pending.extend(sending) + for item in sending: + nodes = self.item2nodes.setdefault(item, []) + assert node not in nodes + nodes.append(node) + self.config.hook.pytest_itemstart(item=item, node=node) + tosend[:] = tosend[room:] # update inplace + if tosend: + # we have some left, give it to the main loop + self.queueevent("pytest_rescheduleitems", items=tosend) + + def senditems_load(self, tosend): + if not tosend: + return + for node, pending in self.node2pending.items(): + room = self.MAXITEMSPERHOST - len(pending) + if room > 0: + sending = tosend[:room] + node.sendlist(sending) + for item in sending: + #assert item not in self.item2node, ( + # "sending same item %r to multiple " + # "not implemented" %(item,)) + self.item2nodes.setdefault(item, []).append(node) + self.config.hook.pytest_itemstart(item=item, node=node) + pending.extend(sending) + tosend[:] = tosend[room:] # update inplace + if not tosend: + break + if tosend: + # we have some left, give it to the main loop + self.queueevent("pytest_rescheduleitems", items=tosend) + + def removeitem(self, item, node): + if item not in self.item2nodes: + raise AssertionError(item, self.item2nodes) + nodes = self.item2nodes[item] + if node in nodes: # the node might have gone down already + nodes.remove(node) + if not nodes: + del self.item2nodes[item] + pending = self.node2pending[node] + pending.remove(item) + + def handle_crashitem(self, item, node): + runner = item.config.pluginmanager.getplugin("runner") + info = "!!! Node %r crashed during running of test %r" %(node, item) + rep = runner.ItemTestReport(item=item, excinfo=info, when="???") + rep.node = node + self.config.hook.pytest_runtest_logreport(report=rep) + + def setup(self): + """ setup any neccessary resources ahead of the test run. """ + self.nodemanager = NodeManager(self.config) + self.nodemanager.setup_nodes(putevent=self.queue.put) + if self.config.option.dist == "each": + self.nodemanager.wait_nodesready(5.0) + + def teardown(self): + """ teardown any resources after a test run. """ + self.nodemanager.teardown_nodes() Added: pypy/trunk/py/impl/test/dist/gwmanage.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/gwmanage.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,124 @@ +""" + instantiating, managing and rsyncing to test hosts +""" + +import py +import sys, os +import execnet +from execnet.gateway_base import RemoteError + +class GatewayManager: + RemoteError = RemoteError + def __init__(self, specs, hook, defaultchdir="pyexecnetcache"): + self.gateways = [] + self.specs = [] + self.hook = hook + for spec in specs: + if not isinstance(spec, execnet.XSpec): + spec = execnet.XSpec(spec) + if not spec.chdir and not spec.popen: + spec.chdir = defaultchdir + self.specs.append(spec) + + def makegateways(self): + assert not self.gateways + for spec in self.specs: + gw = execnet.makegateway(spec) + self.gateways.append(gw) + gw.id = "[%s]" % len(self.gateways) + self.hook.pytest_gwmanage_newgateway( + gateway=gw, platinfo=gw._rinfo()) + + def getgateways(self, remote=True, inplacelocal=True): + if not self.gateways and self.specs: + self.makegateways() + l = [] + for gw in self.gateways: + if gw.spec._samefilesystem(): + if inplacelocal: + l.append(gw) + else: + if remote: + l.append(gw) + return execnet.MultiGateway(gateways=l) + + def multi_exec(self, source, inplacelocal=True): + """ remote execute code on all gateways. + @param inplacelocal=False: don't send code to inplacelocal hosts. + """ + multigw = self.getgateways(inplacelocal=inplacelocal) + return multigw.remote_exec(source) + + def multi_chdir(self, basename, inplacelocal=True): + """ perform a remote chdir to the given path, may be relative. + @param inplacelocal=False: don't send code to inplacelocal hosts. + """ + self.multi_exec("import os ; os.chdir(%r)" % basename, + inplacelocal=inplacelocal).waitclose() + + def rsync(self, source, notify=None, verbose=False, ignores=None): + """ perform rsync to all remote hosts. + """ + rsync = HostRSync(source, verbose=verbose, ignores=ignores) + seen = py.builtin.set() + gateways = [] + for gateway in self.gateways: + spec = gateway.spec + if not spec._samefilesystem(): + if spec not in seen: + def finished(): + if notify: + notify("rsyncrootready", spec, source) + rsync.add_target_host(gateway, finished=finished) + seen.add(spec) + gateways.append(gateway) + if seen: + self.hook.pytest_gwmanage_rsyncstart( + source=source, + gateways=gateways, + ) + rsync.send() + self.hook.pytest_gwmanage_rsyncfinish( + source=source, + gateways=gateways, + ) + + def exit(self): + while self.gateways: + gw = self.gateways.pop() + gw.exit() + +class HostRSync(execnet.RSync): + """ RSyncer that filters out common files + """ + def __init__(self, sourcedir, *args, **kwargs): + self._synced = {} + ignores= None + if 'ignores' in kwargs: + ignores = kwargs.pop('ignores') + self._ignores = ignores or [] + super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs) + + def filter(self, path): + path = py.path.local(path) + if not path.ext in ('.pyc', '.pyo'): + if not path.basename.endswith('~'): + if path.check(dotfile=0): + for x in self._ignores: + if path == x: + break + else: + return True + + def add_target_host(self, gateway, finished=None): + remotepath = os.path.basename(self._sourcedir) + super(HostRSync, self).add_target(gateway, remotepath, + finishedcallback=finished, + delete=True,) + + def _report_send_file(self, gateway, modified_rel_path): + if self._verbose: + path = os.path.basename(self._sourcedir) + "/" + modified_rel_path + remotepath = gateway.spec.chdir + py.builtin.print_('%s:%s <= %s' % + (gateway.spec, remotepath, path)) Added: pypy/trunk/py/impl/test/dist/mypickle.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/mypickle.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,191 @@ +""" + + Pickling support for two processes that want to exchange + *immutable* object instances. Immutable in the sense + that the receiving side of an object can modify its + copy but when it sends it back the original sending + side will continue to see its unmodified version + (and no actual state will go over the wire). + + This module also implements an experimental + execnet pickling channel using this idea. + +""" + +import py +import sys, os, struct +#debug = open("log-mypickle-%d" % os.getpid(), 'w') + +if sys.version_info >= (3,0): + makekey = lambda x: x + fromkey = lambda x: x + from pickle import _Pickler as Pickler + from pickle import _Unpickler as Unpickler +else: + makekey = str + fromkey = int + from pickle import Pickler, Unpickler + + +class MyPickler(Pickler): + """ Pickler with a custom memoize() + to take care of unique ID creation. + See the usage in ImmutablePickler + XXX we could probably extend Pickler + and Unpickler classes to directly + update the other'S memos. + """ + def __init__(self, file, protocol, uneven): + Pickler.__init__(self, file, protocol) + self.uneven = uneven + + def memoize(self, obj): + if self.fast: + return + assert id(obj) not in self.memo + memo_len = len(self.memo) + key = memo_len * 2 + self.uneven + self.write(self.put(key)) + self.memo[id(obj)] = key, obj + + #if sys.version_info < (3,0): + # def save_string(self, obj, pack=struct.pack): + # obj = unicode(obj) + # self.save_unicode(obj, pack=pack) + # Pickler.dispatch[str] = save_string + +class ImmutablePickler: + def __init__(self, uneven, protocol=0): + """ ImmutablePicklers are instantiated in Pairs. + The two sides need to create unique IDs + while pickling their objects. This is + done by using either even or uneven + numbers, depending on the instantiation + parameter. + """ + self._picklememo = {} + self._unpicklememo = {} + self._protocol = protocol + self.uneven = uneven and 1 or 0 + + def selfmemoize(self, obj): + # this is for feeding objects to ourselfes + # which be the case e.g. if you want to pickle + # from a forked process back to the original + f = py.io.BytesIO() + pickler = MyPickler(f, self._protocol, uneven=self.uneven) + pickler.memo = self._picklememo + pickler.memoize(obj) + self._updateunpicklememo() + + def dumps(self, obj): + f = py.io.BytesIO() + pickler = MyPickler(f, self._protocol, uneven=self.uneven) + pickler.memo = self._picklememo + pickler.dump(obj) + if obj is not None: + self._updateunpicklememo() + #print >>debug, "dumped", obj + #print >>debug, "picklememo", self._picklememo + return f.getvalue() + + def loads(self, string): + f = py.io.BytesIO(string) + unpickler = Unpickler(f) + unpickler.memo = self._unpicklememo + res = unpickler.load() + self._updatepicklememo() + #print >>debug, "loaded", res + #print >>debug, "unpicklememo", self._unpicklememo + return res + + def _updatepicklememo(self): + for x, obj in self._unpicklememo.items(): + self._picklememo[id(obj)] = (fromkey(x), obj) + + def _updateunpicklememo(self): + for key,obj in self._picklememo.values(): + key = makekey(key) + if key in self._unpicklememo: + assert self._unpicklememo[key] is obj + self._unpicklememo[key] = obj + +NO_ENDMARKER_WANTED = object() + +class UnpickleError(Exception): + """ Problems while unpickling. """ + def __init__(self, formatted): + self.formatted = formatted + Exception.__init__(self, formatted) + def __str__(self): + return self.formatted + +class PickleChannel(object): + """ PickleChannels wrap execnet channels + and allow to send/receive by using + "immutable pickling". + """ + _unpicklingerror = None + def __init__(self, channel): + self._channel = channel + # we use the fact that each side of a + # gateway connection counts with uneven + # or even numbers depending on which + # side it is (for the purpose of creating + # unique ids - which is what we need it here for) + uneven = channel.gateway._channelfactory.count % 2 + self._ipickle = ImmutablePickler(uneven=uneven) + self.RemoteError = channel.RemoteError + + def send(self, obj): + from execnet.gateway_base import Channel + if not isinstance(obj, Channel): + pickled_obj = self._ipickle.dumps(obj) + self._channel.send(pickled_obj) + else: + self._channel.send(obj) + + def receive(self): + pickled_obj = self._channel.receive() + return self._unpickle(pickled_obj) + + def _unpickle(self, pickled_obj): + if isinstance(pickled_obj, self._channel.__class__): + return pickled_obj + return self._ipickle.loads(pickled_obj) + + def _getremoteerror(self): + return self._unpicklingerror or self._channel._getremoteerror() + + def close(self): + return self._channel.close() + + def isclosed(self): + return self._channel.isclosed() + + def waitclose(self, timeout=None): + return self._channel.waitclose(timeout=timeout) + + def setcallback(self, callback, endmarker=NO_ENDMARKER_WANTED): + if endmarker is NO_ENDMARKER_WANTED: + def unpickle_callback(pickled_obj): + obj = self._unpickle(pickled_obj) + callback(obj) + self._channel.setcallback(unpickle_callback) + return + uniqueendmarker = object() + def unpickle_callback(pickled_obj): + if pickled_obj is uniqueendmarker: + return callback(endmarker) + try: + obj = self._unpickle(pickled_obj) + except KeyboardInterrupt: + raise + except: + excinfo = py.code.ExceptionInfo() + formatted = str(excinfo.getrepr(showlocals=True,funcargs=True)) + self._unpicklingerror = UnpickleError(formatted) + callback(endmarker) + else: + callback(obj) + self._channel.setcallback(unpickle_callback, uniqueendmarker) Added: pypy/trunk/py/impl/test/dist/nodemanage.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/nodemanage.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,81 @@ +import py +import sys, os +from py.impl.test.dist.txnode import TXNode +from py.impl.test.dist.gwmanage import GatewayManager + + +class NodeManager(object): + def __init__(self, config, specs=None): + self.config = config + if specs is None: + specs = self.config.getxspecs() + self.roots = self.config.getrsyncdirs() + self.gwmanager = GatewayManager(specs, config.hook) + self.nodes = [] + self._nodesready = py.std.threading.Event() + + def trace(self, msg): + self.config.hook.pytest_trace(category="nodemanage", msg=msg) + + def config_getignores(self): + return self.config.getconftest_pathlist("rsyncignore") + + def rsync_roots(self): + """ make sure that all remote gateways + have the same set of roots in their + current directory. + """ + self.makegateways() + options = { + 'ignores': self.config_getignores(), + 'verbose': self.config.option.verbose, + } + if self.roots: + # send each rsync root + for root in self.roots: + self.gwmanager.rsync(root, **options) + else: + XXX # do we want to care for situations without explicit rsyncdirs? + # we transfer our topdir as the root + self.gwmanager.rsync(self.config.topdir, **options) + # and cd into it + self.gwmanager.multi_chdir(self.config.topdir.basename, inplacelocal=False) + + def makegateways(self): + # we change to the topdir sot that + # PopenGateways will have their cwd + # such that unpickling configs will + # pick it up as the right topdir + # (for other gateways this chdir is irrelevant) + self.trace("making gateways") + old = self.config.topdir.chdir() + try: + self.gwmanager.makegateways() + finally: + old.chdir() + + def setup_nodes(self, putevent): + self.rsync_roots() + self.trace("setting up nodes") + for gateway in self.gwmanager.gateways: + node = TXNode(gateway, self.config, putevent, slaveready=self._slaveready) + gateway.node = node # to keep node alive + self.trace("started node %r" % node) + + def _slaveready(self, node): + #assert node.gateway == node.gateway + #assert node.gateway.node == node + self.nodes.append(node) + self.trace("%s slave node ready %r" % (node.gateway.id, node)) + if len(self.nodes) == len(self.gwmanager.gateways): + self._nodesready.set() + + def wait_nodesready(self, timeout=None): + self._nodesready.wait(timeout) + if not self._nodesready.isSet(): + raise IOError("nodes did not get ready for %r secs" % timeout) + + def teardown_nodes(self): + # XXX do teardown nodes? + self.gwmanager.exit() + Added: pypy/trunk/py/impl/test/dist/txnode.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/dist/txnode.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,153 @@ +""" + Manage setup, running and local representation of remote nodes/processes. +""" +import py +from py.impl.test.dist.mypickle import PickleChannel + +class TXNode(object): + """ Represents a Test Execution environment in the controlling process. + - sets up a slave node through an execnet gateway + - manages sending of test-items and receival of results and events + - creates events when the remote side crashes + """ + ENDMARK = -1 + + def __init__(self, gateway, config, putevent, slaveready=None): + self.config = config + self.putevent = putevent + self.gateway = gateway + self.channel = install_slave(gateway, config) + self._sendslaveready = slaveready + self.channel.setcallback(self.callback, endmarker=self.ENDMARK) + self._down = False + + def __repr__(self): + id = self.gateway.id + status = self._down and 'true' or 'false' + return "" %(id, status) + + def notify(self, eventname, *args, **kwargs): + assert not args + self.putevent((eventname, args, kwargs)) + + def callback(self, eventcall): + """ this gets called for each object we receive from + the other side and if the channel closes. + + Note that channel callbacks run in the receiver + thread of execnet gateways - we need to + avoid raising exceptions or doing heavy work. + """ + try: + if eventcall == self.ENDMARK: + err = self.channel._getremoteerror() + if not self._down: + if not err: + err = "Not properly terminated" + self.notify("pytest_testnodedown", node=self, error=err) + self._down = True + return + eventname, args, kwargs = eventcall + if eventname == "slaveready": + if self._sendslaveready: + self._sendslaveready(self) + self.notify("pytest_testnodeready", node=self) + elif eventname == "slavefinished": + self._down = True + self.notify("pytest_testnodedown", error=None, node=self) + elif eventname == "pytest_runtest_logreport": + rep = kwargs['report'] + rep.node = self + self.notify("pytest_runtest_logreport", report=rep) + else: + self.notify(eventname, *args, **kwargs) + except KeyboardInterrupt: + # should not land in receiver-thread + raise + except: + excinfo = py.code.ExceptionInfo() + py.builtin.print_("!" * 20, excinfo) + self.config.pluginmanager.notify_exception(excinfo) + + def send(self, item): + assert item is not None + self.channel.send(item) + + def sendlist(self, itemlist): + self.channel.send(itemlist) + + def shutdown(self): + self.channel.send(None) + +# setting up slave code +def install_slave(gateway, config): + channel = gateway.remote_exec(source=""" + import os, sys + sys.path.insert(0, os.getcwd()) + from py.impl.test.dist.mypickle import PickleChannel + from py.impl.test.dist.txnode import SlaveNode + channel = PickleChannel(channel) + slavenode = SlaveNode(channel) + slavenode.run() + """) + channel = PickleChannel(channel) + basetemp = None + if gateway.spec.popen: + popenbase = config.ensuretemp("popen") + basetemp = py.path.local.make_numbered_dir(prefix="slave-", + keep=0, rootdir=popenbase) + basetemp = str(basetemp) + channel.send((config, basetemp)) + return channel + +class SlaveNode(object): + def __init__(self, channel): + self.channel = channel + + def __repr__(self): + return "<%s channel=%s>" %(self.__class__.__name__, self.channel) + + def sendevent(self, eventname, *args, **kwargs): + self.channel.send((eventname, args, kwargs)) + + def pytest_runtest_logreport(self, report): + self.sendevent("pytest_runtest_logreport", report=report) + + def run(self): + channel = self.channel + self.config, basetemp = channel.receive() + if basetemp: + self.config.basetemp = py.path.local(basetemp) + self.config.pluginmanager.do_configure(self.config) + self.config.pluginmanager.register(self) + self.runner = self.config.pluginmanager.getplugin("pytest_runner") + self.sendevent("slaveready") + try: + while 1: + task = channel.receive() + if task is None: + self.sendevent("slavefinished") + break + if isinstance(task, list): + for item in task: + self.run_single(item=item) + else: + self.run_single(item=task) + except KeyboardInterrupt: + raise + except: + er = py.code.ExceptionInfo().getrepr(funcargs=True, showlocals=True) + self.sendevent("pytest_internalerror", excrepr=er) + raise + + def run_single(self, item): + call = self.runner.CallInfo(item._checkcollectable, when='setup') + if call.excinfo: + # likely it is not collectable here because of + # platform/import-dependency induced skips + # we fake a setup-error report with the obtained exception + # and do not care about capturing or non-runner hooks + rep = self.runner.pytest_runtest_makereport(item=item, call=call) + self.pytest_runtest_logreport(rep) + return + item.config.hook.pytest_runtest_protocol(item=item) Added: pypy/trunk/py/impl/test/funcargs.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/funcargs.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,194 @@ +import py + +def getfuncargnames(function): + argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0] + startindex = py.std.inspect.ismethod(function) and 1 or 0 + defaults = getattr(function, 'func_defaults', + getattr(function, '__defaults__', None)) or () + numdefaults = len(defaults) + if numdefaults: + return argnames[startindex:-numdefaults] + return argnames[startindex:] + +def fillfuncargs(function): + """ fill missing funcargs. """ + request = FuncargRequest(pyfuncitem=function) + request._fillfuncargs() + + +_notexists = object() +class CallSpec: + def __init__(self, funcargs, id, param): + self.funcargs = funcargs + self.id = id + if param is not _notexists: + self.param = param + def __repr__(self): + return "" %( + self.id, getattr(self, 'param', '?'), self.funcargs) + +class Metafunc: + def __init__(self, function, config=None, cls=None, module=None): + self.config = config + self.module = module + self.function = function + self.funcargnames = getfuncargnames(function) + self.cls = cls + self.module = module + self._calls = [] + self._ids = py.builtin.set() + + def addcall(self, funcargs=None, id=_notexists, param=_notexists): + assert funcargs is None or isinstance(funcargs, dict) + if id is None: + raise ValueError("id=None not allowed") + if id is _notexists: + id = len(self._calls) + id = str(id) + if id in self._ids: + raise ValueError("duplicate id %r" % id) + self._ids.add(id) + self._calls.append(CallSpec(funcargs, id, param)) + +class FunctionCollector(py.test.collect.Collector): + def __init__(self, name, parent, calls): + super(FunctionCollector, self).__init__(name, parent) + self.calls = calls + self.obj = getattr(self.parent.obj, name) + + def collect(self): + l = [] + for callspec in self.calls: + name = "%s[%s]" %(self.name, callspec.id) + function = self.parent.Function(name=name, parent=self, + callspec=callspec, callobj=self.obj) + l.append(function) + return l + + def reportinfo(self): + try: + return self._fslineno, self.name + except AttributeError: + pass + fspath, lineno = py.code.getfslineno(self.obj) + self._fslineno = fspath, lineno + return fspath, lineno, self.name + + +class FuncargRequest: + _argprefix = "pytest_funcarg__" + _argname = None + + class Error(LookupError): + """ error on performing funcarg request. """ + + def __init__(self, pyfuncitem): + self._pyfuncitem = pyfuncitem + self.function = pyfuncitem.obj + self.module = pyfuncitem.getparent(py.test.collect.Module).obj + clscol = pyfuncitem.getparent(py.test.collect.Class) + self.cls = clscol and clscol.obj or None + self.instance = py.builtin._getimself(self.function) + self.config = pyfuncitem.config + self.fspath = pyfuncitem.fspath + if hasattr(pyfuncitem, '_requestparam'): + self.param = pyfuncitem._requestparam + self._plugins = self.config.pluginmanager.getplugins() + self._plugins.append(self.module) + if self.instance is not None: + self._plugins.append(self.instance) + self._funcargs = self._pyfuncitem.funcargs.copy() + self._name2factory = {} + self._currentarg = None + + def _fillfuncargs(self): + argnames = getfuncargnames(self.function) + if argnames: + assert not getattr(self._pyfuncitem, '_args', None), ( + "yielded functions cannot have funcargs") + for argname in argnames: + if argname not in self._pyfuncitem.funcargs: + self._pyfuncitem.funcargs[argname] = self.getfuncargvalue(argname) + + def cached_setup(self, setup, teardown=None, scope="module", extrakey=None): + """ cache and return result of calling setup(). + + The requested argument name, the scope and the ``extrakey`` + determine the cache key. The scope also determines when + teardown(result) will be called. valid scopes are: + scope == 'function': when the single test function run finishes. + scope == 'module': when tests in a different module are run + scope == 'session': when tests of the session have run. + """ + if not hasattr(self.config, '_setupcache'): + self.config._setupcache = {} # XXX weakref? + cachekey = (self._currentarg, self._getscopeitem(scope), extrakey) + cache = self.config._setupcache + try: + val = cache[cachekey] + except KeyError: + val = setup() + cache[cachekey] = val + if teardown is not None: + def finalizer(): + del cache[cachekey] + teardown(val) + self._addfinalizer(finalizer, scope=scope) + return val + + def getfuncargvalue(self, argname): + try: + return self._funcargs[argname] + except KeyError: + pass + if argname not in self._name2factory: + self._name2factory[argname] = self.config.pluginmanager.listattr( + plugins=self._plugins, + attrname=self._argprefix + str(argname) + ) + #else: we are called recursively + if not self._name2factory[argname]: + self._raiselookupfailed(argname) + funcargfactory = self._name2factory[argname].pop() + oldarg = self._currentarg + self._currentarg = argname + try: + self._funcargs[argname] = res = funcargfactory(request=self) + finally: + self._currentarg = oldarg + return res + + def _getscopeitem(self, scope): + if scope == "function": + return self._pyfuncitem + elif scope == "module": + return self._pyfuncitem.getparent(py.test.collect.Module) + elif scope == "session": + return None + raise ValueError("unknown finalization scope %r" %(scope,)) + + def _addfinalizer(self, finalizer, scope): + colitem = self._getscopeitem(scope) + self.config._setupstate.addfinalizer( + finalizer=finalizer, colitem=colitem) + + def addfinalizer(self, finalizer): + """ call the given finalizer after test function finished execution. """ + self._addfinalizer(finalizer, scope="function") + + def __repr__(self): + return "" %(self._pyfuncitem) + + def _raiselookupfailed(self, argname): + available = [] + for plugin in self._plugins: + for name in vars(plugin): + if name.startswith(self._argprefix): + name = name[len(self._argprefix):] + if name not in available: + available.append(name) + fspath, lineno, msg = self._pyfuncitem.reportinfo() + line = "%s:%s" %(fspath, lineno) + msg = "funcargument %r not found for: %s" %(argname, line) + msg += "\n available funcargs: %s" %(", ".join(available),) + raise self.Error(msg) Added: pypy/trunk/py/impl/test/looponfail/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/looponfail/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +# Added: pypy/trunk/py/impl/test/looponfail/remote.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/looponfail/remote.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,162 @@ +""" + LooponfailingSession and Helpers. + + NOTE that one really has to avoid loading and depending on + application modules within the controlling process + (the one that starts repeatedly test processes) + otherwise changes to source code can crash + the controlling process which should never happen. +""" +import py +import sys +import execnet +from py.impl.test.session import Session +from py.impl.test.dist.mypickle import PickleChannel +from py.impl.test.looponfail import util + +class LooponfailingSession(Session): + def __init__(self, config): + super(LooponfailingSession, self).__init__(config=config) + self.rootdirs = [self.config.topdir] # xxx dist_rsync_roots? + self.statrecorder = util.StatRecorder(self.rootdirs) + self.remotecontrol = RemoteControl(self.config) + self.out = py.io.TerminalWriter() + + def main(self, initialitems=None): + try: + self.loopstate = loopstate = LoopState(initialitems) + self.remotecontrol.setup() + while 1: + self.loop_once(loopstate) + if not loopstate.colitems and loopstate.wasfailing: + continue # the last failures passed, let's rerun all + self.statrecorder.waitonchange(checkinterval=2.0) + except KeyboardInterrupt: + print + + def loop_once(self, loopstate): + colitems = loopstate.colitems + loopstate.wasfailing = colitems and len(colitems) + loopstate.colitems = self.remotecontrol.runsession(colitems or ()) + self.remotecontrol.setup() + +class LoopState: + def __init__(self, colitems=None): + self.colitems = colitems + +class RemoteControl(object): + def __init__(self, config): + self.config = config + + def trace(self, *args): + if self.config.option.debug: + msg = " ".join([str(x) for x in args]) + py.builtin.print_("RemoteControl:", msg) + + def initgateway(self): + return execnet.PopenGateway() + + def setup(self, out=None): + if out is None: + out = py.io.TerminalWriter() + if hasattr(self, 'gateway'): + raise ValueError("already have gateway %r" % self.gateway) + self.trace("setting up slave session") + old = self.config.topdir.chdir() + try: + self.gateway = self.initgateway() + finally: + old.chdir() + channel = self.gateway.remote_exec(source=""" + from py.impl.test.dist.mypickle import PickleChannel + from py.impl.test.looponfail.remote import slave_runsession + outchannel = channel.gateway.newchannel() + channel.send(outchannel) + channel = PickleChannel(channel) + config, fullwidth, hasmarkup = channel.receive() + import sys + sys.stdout = sys.stderr = outchannel.makefile('w') + slave_runsession(channel, config, fullwidth, hasmarkup) + """) + remote_outchannel = channel.receive() + def write(s): + out._file.write(s) + out._file.flush() + remote_outchannel.setcallback(write) + channel = self.channel = PickleChannel(channel) + channel.send((self.config, out.fullwidth, out.hasmarkup)) + self.trace("set up of slave session complete") + + def ensure_teardown(self): + if hasattr(self, 'channel'): + if not self.channel.isclosed(): + self.trace("closing", self.channel) + self.channel.close() + del self.channel + if hasattr(self, 'gateway'): + self.trace("exiting", self.gateway) + self.gateway.exit() + del self.gateway + + def runsession(self, colitems=()): + try: + self.trace("sending", colitems) + trails = colitems + self.channel.send(trails) + try: + return self.channel.receive() + except self.channel.RemoteError: + e = sys.exc_info()[1] + self.trace("ERROR", e) + raise + finally: + self.ensure_teardown() + +def slave_runsession(channel, config, fullwidth, hasmarkup): + """ we run this on the other side. """ + if config.option.debug: + def DEBUG(*args): + print(" ".join(map(str, args))) + else: + def DEBUG(*args): pass + + DEBUG("SLAVE: received configuration, using topdir:", config.topdir) + #config.option.session = None + config.option.looponfail = False + config.option.usepdb = False + trails = channel.receive() + config.pluginmanager.do_configure(config) + DEBUG("SLAVE: initsession()") + session = config.initsession() + # XXX configure the reporter object's terminal writer more directly + # XXX and write a test for this remote-terminal setting logic + config.pytest_terminal_hasmarkup = hasmarkup + config.pytest_terminal_fullwidth = fullwidth + if trails: + colitems = [] + for trail in trails: + try: + colitem = py.test.collect.Collector._fromtrail(trail, config) + except AssertionError: + #XXX send info for "test disappeared" or so + continue + colitems.append(colitem) + else: + colitems = None + session.shouldclose = channel.isclosed + + class Failures(list): + def pytest_runtest_logreport(self, report): + if report.failed: + self.append(report) + pytest_collectreport = pytest_runtest_logreport + + failreports = Failures() + session.pluginmanager.register(failreports) + + DEBUG("SLAVE: starting session.main()") + session.main(colitems) + session.config.hook.pytest_looponfailinfo( + failreports=list(failreports), + rootdirs=[config.topdir]) + channel.send([rep.getnode()._totrail() for rep in failreports]) Added: pypy/trunk/py/impl/test/looponfail/util.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/looponfail/util.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,53 @@ +import py + +class StatRecorder: + def __init__(self, rootdirlist): + self.rootdirlist = rootdirlist + self.statcache = {} + self.check() # snapshot state + + def fil(self, p): + return p.ext in ('.py', '.txt', '.c', '.h') + def rec(self, p): + return p.check(dotfile=0) + + def waitonchange(self, checkinterval=1.0): + while 1: + changed = self.check() + if changed: + return + py.std.time.sleep(checkinterval) + + def check(self, removepycfiles=True): + changed = False + statcache = self.statcache + newstat = {} + for rootdir in self.rootdirlist: + for path in rootdir.visit(self.fil, self.rec): + oldstat = statcache.get(path, None) + if oldstat is not None: + del statcache[path] + try: + newstat[path] = curstat = path.stat() + except py.error.ENOENT: + if oldstat: + del statcache[path] + changed = True + else: + if oldstat: + if oldstat.mtime != curstat.mtime or \ + oldstat.size != curstat.size: + changed = True + py.builtin.print_("# MODIFIED", path) + if removepycfiles and path.ext == ".py": + pycfile = path + "c" + if pycfile.check(): + pycfile.remove() + + else: + changed = True + if statcache: + changed = True + self.statcache = newstat + return changed + Added: pypy/trunk/py/impl/test/outcome.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/outcome.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,122 @@ +""" + Test OutcomeExceptions and helpers for creating them. + py.test.skip|fail|raises helper implementations + +""" + +import py +import sys + +class OutcomeException(Exception): + """ OutcomeException and its subclass instances indicate and + contain info about test and collection outcomes. + """ + def __init__(self, msg=None, excinfo=None): + self.msg = msg + self.excinfo = excinfo + + def __repr__(self): + if self.msg: + return repr(self.msg) + return "<%s instance>" %(self.__class__.__name__,) + __str__ = __repr__ + +class Passed(OutcomeException): + pass + +class Skipped(OutcomeException): + # XXX slighly hackish: on 3k we fake to live in the builtins + # in order to have Skipped exception printing shorter/nicer + __module__ = 'builtins' + +class Failed(OutcomeException): + pass + +class ExceptionFailure(Failed): + def __init__(self, expr, expected, msg=None, excinfo=None): + Failed.__init__(self, msg=msg, excinfo=excinfo) + self.expr = expr + self.expected = expected + +class Exit(KeyboardInterrupt): + """ for immediate program exits without tracebacks and reporter/summary. """ + def __init__(self, msg="unknown reason"): + self.msg = msg + KeyboardInterrupt.__init__(self, msg) + +# exposed helper methods + +def exit(msg): + """ exit testing process immediately. """ + __tracebackhide__ = True + raise Exit(msg) + +def skip(msg=""): + """ skip with the given message. """ + __tracebackhide__ = True + raise Skipped(msg=msg) + +def fail(msg="unknown failure"): + """ fail with the given Message. """ + __tracebackhide__ = True + raise Failed(msg=msg) + +def raises(ExpectedException, *args, **kwargs): + """ raise AssertionError, if target code does not raise the expected + exception. + """ + __tracebackhide__ = True + assert args + if isinstance(args[0], str): + code, = args + assert isinstance(code, str) + frame = sys._getframe(1) + loc = frame.f_locals.copy() + loc.update(kwargs) + #print "raises frame scope: %r" % frame.f_locals + try: + code = py.code.Source(code).compile() + py.builtin.exec_(code, frame.f_globals, loc) + # XXX didn'T mean f_globals == f_locals something special? + # this is destroyed here ... + except ExpectedException: + return py.code.ExceptionInfo() + else: + func = args[0] + try: + func(*args[1:], **kwargs) + except ExpectedException: + return py.code.ExceptionInfo() + k = ", ".join(["%s=%r" % x for x in kwargs.items()]) + if k: + k = ', ' + k + expr = '%s(%r%s)' %(func.__name__, args, k) + raise ExceptionFailure(msg="DID NOT RAISE", + expr=args, expected=ExpectedException) + +def importorskip(modname, minversion=None): + """ return imported module or perform a dynamic skip() """ + compile(modname, '', 'eval') # to catch syntaxerrors + try: + mod = __import__(modname, None, None, ['__doc__']) + except ImportError: + py.test.skip("could not import %r" %(modname,)) + if minversion is None: + return mod + verattr = getattr(mod, '__version__', None) + if isinstance(minversion, str): + minver = minversion.split(".") + else: + minver = list(minversion) + if verattr is None or verattr.split(".") < minver: + py.test.skip("module %r has __version__ %r, required is: %r" %( + modname, verattr, minversion)) + return mod + + +# exitcodes for the command line +EXIT_OK = 0 +EXIT_TESTSFAILED = 1 +EXIT_INTERRUPTED = 2 +EXIT_INTERNALERROR = 3 +EXIT_NOHOSTS = 4 Added: pypy/trunk/py/impl/test/parseopt.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/parseopt.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,103 @@ +""" +thin wrapper around Python's optparse.py +adding some extra checks and ways to systematically +have Environment variables provide default values +for options. basic usage: + + >>> parser = Parser() + >>> parser.addoption("--hello", action="store_true", dest="hello") + >>> option, args = parser.parse(['--hello']) + >>> option.hello + True + >>> args + [] + +""" +import py +import optparse + +class Parser: + """ Parser for command line arguments. """ + + def __init__(self, usage=None, processopt=None): + self._anonymous = OptionGroup("custom options", parser=self) + self._groups = [] + self._processopt = processopt + self._usage = usage + self.epilog = "" + + def processoption(self, option): + if self._processopt: + if option.dest: + self._processopt(option) + + def addnote(self, note): + self._notes.append(note) + + def getgroup(self, name, description="", after=None): + for group in self._groups: + if group.name == name: + return group + group = OptionGroup(name, description, parser=self) + i = 0 + for i, grp in enumerate(self._groups): + if grp.name == after: + break + self._groups.insert(i+1, group) + return group + + addgroup = getgroup + def addgroup(self, name, description=""): + py.log._apiwarn("1.1", "use getgroup() which gets-or-creates") + return self.getgroup(name, description) + + def addoption(self, *opts, **attrs): + """ add an optparse-style option. """ + self._anonymous.addoption(*opts, **attrs) + + def parse(self, args): + optparser = optparse.OptionParser(usage=self._usage) + # make sure anaonymous group is at the end + optparser.epilog = self.epilog + groups = self._groups + [self._anonymous] + for group in groups: + if group.options: + desc = group.description or group.name + optgroup = optparse.OptionGroup(optparser, desc) + optgroup.add_options(group.options) + optparser.add_option_group(optgroup) + return optparser.parse_args([str(x) for x in args]) + + def parse_setoption(self, args, option): + parsedoption, args = self.parse(args) + for name, value in parsedoption.__dict__.items(): + setattr(option, name, value) + return args + + +class OptionGroup: + def __init__(self, name, description="", parser=None): + self.name = name + self.description = description + self.options = [] + self.parser = parser + + def addoption(self, *optnames, **attrs): + """ add an option to this group. """ + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=False) + + def _addoption(self, *optnames, **attrs): + option = optparse.Option(*optnames, **attrs) + self._addoption_instance(option, shortupper=True) + + def _addoption_instance(self, option, shortupper=False): + if not shortupper: + for opt in option._short_opts: + if opt[0] == '-' and opt[1].islower(): + raise ValueError("lowercase shortoptions reserved") + if self.parser: + self.parser.processoption(option) + self.options.append(option) + + Added: pypy/trunk/py/impl/test/pluginmanager.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/pluginmanager.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,300 @@ +""" +managing loading and interacting with pytest plugins. +""" +import py +from py.plugin import hookspec +from py.impl.test.outcome import Skipped + +def check_old_use(mod, modname): + clsname = modname[len('pytest_'):].capitalize() + "Plugin" + assert not hasattr(mod, clsname), (mod, clsname) + +class PluginManager(object): + class Error(Exception): + """signals a plugin specific error.""" + def __init__(self, comregistry=None): + if comregistry is None: + comregistry = py._com.Registry() + self.comregistry = comregistry + self._name2plugin = {} + + self.hook = py._com.HookRelay( + hookspecs=hookspec, + registry=self.comregistry) + + def _getpluginname(self, plugin, name): + if name is None: + if hasattr(plugin, '__name__'): + name = plugin.__name__.split(".")[-1] + else: + name = id(plugin) + return name + + def register(self, plugin, name=None): + assert not self.isregistered(plugin) + name = self._getpluginname(plugin, name) + if name in self._name2plugin: + return False + self._name2plugin[name] = plugin + self.hook.pytest_plugin_registered(plugin=plugin) + self._checkplugin(plugin) + self.comregistry.register(plugin) + return True + + def unregister(self, plugin): + self.hook.pytest_plugin_unregistered(plugin=plugin) + self.comregistry.unregister(plugin) + for name, value in list(self._name2plugin.items()): + if value == plugin: + del self._name2plugin[name] + + def isregistered(self, plugin, name=None): + if self._getpluginname(plugin, name) in self._name2plugin: + return True + for val in self._name2plugin.values(): + if plugin == val: + return True + + def getplugins(self): + return list(self.comregistry) + + def getplugin(self, name): + try: + return self._name2plugin[name] + except KeyError: + impname = canonical_importname(name) + return self._name2plugin[impname] + + # API for bootstrapping + # + def _envlist(self, varname): + val = py.std.os.environ.get(varname, None) + if val is not None: + return val.split(',') + return () + + def consider_env(self): + for spec in self._envlist("PYTEST_PLUGINS"): + self.import_plugin(spec) + + def consider_preparse(self, args): + for opt1,opt2 in zip(args, args[1:]): + if opt1 == "-p": + self.import_plugin(opt2) + + def consider_conftest(self, conftestmodule): + cls = getattr(conftestmodule, 'ConftestPlugin', None) + if cls is not None: + raise ValueError("%r: 'ConftestPlugins' only existed till 1.0.0b1, " + "were removed in 1.0.0b2" % (cls,)) + if self.register(conftestmodule, name=conftestmodule.__file__): + self.consider_module(conftestmodule) + + def consider_module(self, mod): + attr = getattr(mod, "pytest_plugins", ()) + if attr: + if not isinstance(attr, (list, tuple)): + attr = (attr,) + for spec in attr: + self.import_plugin(spec) + + def import_plugin(self, spec): + assert isinstance(spec, str) + modname = canonical_importname(spec) + if modname in self._name2plugin: + return + try: + mod = importplugin(modname) + except KeyboardInterrupt: + raise + except Skipped: + e = py.std.sys.exc_info()[1] + self._warn("could not import plugin %r, reason: %r" %( + (modname, e.msg))) + else: + check_old_use(mod, modname) + self.register(mod) + self.consider_module(mod) + + def _warn(self, msg): + print ("===WARNING=== %s" % (msg,)) + + def _checkplugin(self, plugin): + # ===================================================== + # check plugin hooks + # ===================================================== + methods = collectattr(plugin) + hooks = collectattr(hookspec) + stringio = py.io.TextIO() + def Print(*args): + if args: + stringio.write(" ".join(map(str, args))) + stringio.write("\n") + + fail = False + while methods: + name, method = methods.popitem() + #print "checking", name + if isgenerichook(name): + continue + if name not in hooks: + Print("found unknown hook:", name) + fail = True + else: + method_args = getargs(method) + if '__multicall__' in method_args: + method_args.remove('__multicall__') + hook = hooks[name] + hookargs = getargs(hook) + for arg in method_args: + if arg not in hookargs: + Print("argument %r not available" %(arg, )) + Print("actual definition: %s" %(formatdef(method))) + Print("available hook arguments: %s" % + ", ".join(hookargs)) + fail = True + break + #if not fail: + # print "matching hook:", formatdef(method) + if fail: + name = getattr(plugin, '__name__', plugin) + raise self.Error("%s:\n%s" %(name, stringio.getvalue())) + # + # + # API for interacting with registered and instantiated plugin objects + # + # + def listattr(self, attrname, plugins=None, extra=()): + return self.comregistry.listattr(attrname, plugins=plugins, extra=extra) + + def notify_exception(self, excinfo=None): + if excinfo is None: + excinfo = py.code.ExceptionInfo() + excrepr = excinfo.getrepr(funcargs=True, showlocals=True) + return self.hook.pytest_internalerror(excrepr=excrepr) + + def do_addoption(self, parser): + mname = "pytest_addoption" + methods = self.comregistry.listattr(mname, reverse=True) + mc = py._com.MultiCall(methods, {'parser': parser}) + mc.execute() + + def pytest_plugin_registered(self, plugin): + if hasattr(self, '_config'): + self.call_plugin(plugin, "pytest_addoption", + {'parser': self._config._parser}) + self.call_plugin(plugin, "pytest_configure", + {'config': self._config}) + #dic = self.call_plugin(plugin, "pytest_namespace") + #self._updateext(dic) + + def call_plugin(self, plugin, methname, kwargs): + return py._com.MultiCall( + methods=self.listattr(methname, plugins=[plugin]), + kwargs=kwargs, firstresult=True).execute() + + def _updateext(self, dic): + if dic: + for name, value in dic.items(): + setattr(py.test, name, value) + + def do_configure(self, config): + assert not hasattr(self, '_config') + config.pluginmanager.register(self) + self._config = config + config.hook.pytest_configure(config=self._config) + for dic in config.hook.pytest_namespace() or []: + self._updateext(dic) + + def do_unconfigure(self, config): + config = self._config + del self._config + config.hook.pytest_unconfigure(config=config) + config.pluginmanager.unregister(self) + +# +# XXX old code to automatically load classes +# +def canonical_importname(name): + name = name.lower() + modprefix = "pytest_" + if not name.startswith(modprefix): + name = modprefix + name + return name + +def importplugin(importspec): + try: + return __import__(importspec) + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + try: + return __import__("py.plugin.%s" %(importspec), + None, None, '__doc__') + except ImportError: + e = py.std.sys.exc_info()[1] + if str(e).find(importspec) == -1: + raise + #print "syspath:", py.std.sys.path + #print "curdir:", py.std.os.getcwd() + return __import__(importspec) # show the original exception + + + +def isgenerichook(name): + return name == "pytest_plugins" or \ + name.startswith("pytest_funcarg__") + +def getargs(func): + args = py.std.inspect.getargs(py.code.getrawcode(func))[0] + startindex = py.std.inspect.ismethod(func) and 1 or 0 + return args[startindex:] + +def collectattr(obj, prefixes=("pytest_",)): + methods = {} + for apiname in dir(obj): + for prefix in prefixes: + if apiname.startswith(prefix): + methods[apiname] = getattr(obj, apiname) + return methods + +def formatdef(func): + return "%s%s" %( + func.__name__, + py.std.inspect.formatargspec(*py.std.inspect.getargspec(func)) + ) + +if __name__ == "__main__": + import py.plugin + basedir = py._dir.join('_plugin') + name2text = {} + for p in basedir.listdir("pytest_*"): + if p.ext == ".py" or ( + p.check(dir=1) and p.join("__init__.py").check()): + impname = p.purebasename + if impname.find("__") != -1: + continue + try: + plugin = importplugin(impname) + except (ImportError, py.impl.test.outcome.Skipped): + name2text[impname] = "IMPORT ERROR" + else: + doc = plugin.__doc__ or "" + doc = doc.strip() + name2text[impname] = doc + + for name in sorted(name2text.keys()): + text = name2text[name] + if name[0] == "_": + continue + print ("%-20s %s" % (name, text.split("\n")[0])) + + #text = py.std.textwrap.wrap(name2text[name], + # width = 80, + # initial_indent="%s: " % name, + # replace_whitespace = False) + #for line in text: + # print line + + Added: pypy/trunk/py/impl/test/pycollect.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/pycollect.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,371 @@ +""" +Python related collection nodes. Here is an example of +a tree of collectors and test items that this modules provides:: + + Module # File + Class + Instance + Function + Generator + ... + Function + Generator + Function + + DoctestFile # File + DoctestFileContent # acts as Item + +""" +import py +import inspect +from py.impl.test.collect import configproperty, warnoldcollect +from py.impl.test import funcargs + +class PyobjMixin(object): + def obj(): + def fget(self): + try: + return self._obj + except AttributeError: + self._obj = obj = self._getobj() + return obj + def fset(self, value): + self._obj = value + return property(fget, fset, None, "underlying python object") + obj = obj() + + def _getobj(self): + return getattr(self.parent.obj, self.name) + + def getmodpath(self, stopatmodule=True, includemodule=False): + """ return python path relative to the containing module. """ + chain = self.listchain() + chain.reverse() + parts = [] + for node in chain: + if isinstance(node, Instance): + continue + name = node.name + if isinstance(node, Module): + assert name.endswith(".py") + name = name[:-3] + if stopatmodule: + if includemodule: + parts.append(name) + break + parts.append(name) + parts.reverse() + s = ".".join(parts) + return s.replace(".[", "[") + + def _getfslineno(self): + try: + return self._fslineno + except AttributeError: + pass + obj = self.obj + # xxx let decorators etc specify a sane ordering + if hasattr(obj, 'place_as'): + obj = obj.place_as + + self._fslineno = py.code.getfslineno(obj) + return self._fslineno + + def reportinfo(self): + fspath, lineno = self._getfslineno() + modpath = self.getmodpath() + return fspath, lineno, modpath + +class PyCollectorMixin(PyobjMixin, py.test.collect.Collector): + Class = configproperty('Class') + Instance = configproperty('Instance') + Function = configproperty('Function') + Generator = configproperty('Generator') + + def funcnamefilter(self, name): + return name.startswith('test') + def classnamefilter(self, name): + return name.startswith('Test') + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + name2items = self._buildname2items() + colitems = list(name2items.values()) + colitems.sort(key=lambda item: item.reportinfo()[:2]) + return colitems + + def _buildname2items(self): + # NB. we avoid random getattrs and peek in the __dict__ instead + d = {} + dicts = [getattr(self.obj, '__dict__', {})] + for basecls in inspect.getmro(self.obj.__class__): + dicts.append(basecls.__dict__) + seen = {} + for dic in dicts: + for name, obj in dic.items(): + if name in seen: + continue + seen[name] = True + if name[0] != "_": + res = self.makeitem(name, obj) + if res is not None: + d[name] = res + return d + + def _deprecated_join(self, name): + if self.__class__.join != py.test.collect.Collector.join: + warnoldcollect() + return self.join(name) + + def makeitem(self, name, obj): + return self.config.hook.pytest_pycollect_makeitem( + collector=self, name=name, obj=obj) + + def _istestclasscandidate(self, name, obj): + if self.classnamefilter(name) and \ + inspect.isclass(obj): + if hasinit(obj): + # XXX WARN + return False + return True + + def _genfunctions(self, name, funcobj): + module = self.getparent(Module).obj + clscol = self.getparent(Class) + cls = clscol and clscol.obj or None + metafunc = funcargs.Metafunc(funcobj, config=self.config, + cls=cls, module=module) + gentesthook = self.config.hook._makecall( + "pytest_generate_tests", extralookup=module) + gentesthook(metafunc=metafunc) + if not metafunc._calls: + return self.Function(name, parent=self) + return funcargs.FunctionCollector(name=name, + parent=self, calls=metafunc._calls) + + +class Module(py.test.collect.File, PyCollectorMixin): + def _getobj(self): + return self._memoizedcall('_obj', self._importtestmodule) + + def _importtestmodule(self): + # we assume we are only called once per module + mod = self.fspath.pyimport() + #print "imported test module", mod + self.config.pluginmanager.consider_module(mod) + return mod + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.test.skip("%r is disabled" %(self.obj,)) + if hasattr(self.obj, 'setup_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.setup_module)[0]: + self.obj.setup_module(self.obj) + else: + self.obj.setup_module() + + def teardown(self): + if hasattr(self.obj, 'teardown_module'): + #XXX: nose compat hack, move to nose plugin + # if it takes a positional arg, its probably a py.test style one + # so we pass the current module object + if inspect.getargspec(self.obj.teardown_module)[0]: + self.obj.teardown_module(self.obj) + else: + self.obj.teardown_module() + +class Class(PyCollectorMixin, py.test.collect.Collector): + + def collect(self): + l = self._deprecated_collect() + if l is not None: + return l + return [self.Instance(name="()", parent=self)] + + def setup(self): + if getattr(self.obj, 'disabled', 0): + py.test.skip("%r is disabled" %(self.obj,)) + setup_class = getattr(self.obj, 'setup_class', None) + if setup_class is not None: + setup_class = getattr(setup_class, 'im_func', setup_class) + setup_class(self.obj) + + def teardown(self): + teardown_class = getattr(self.obj, 'teardown_class', None) + if teardown_class is not None: + teardown_class = getattr(teardown_class, 'im_func', teardown_class) + teardown_class(self.obj) + +class Instance(PyCollectorMixin, py.test.collect.Collector): + def _getobj(self): + return self.parent.obj() + def Function(self): + return getattr(self.obj, 'Function', + PyCollectorMixin.Function.__get__(self)) # XXX for python 2.2 + def _keywords(self): + return [] + Function = property(Function) + + #def __repr__(self): + # return "<%s of '%s'>" %(self.__class__.__name__, + # self.parent.obj.__name__) + + def newinstance(self): + self.obj = self._getobj() + return self.obj + +class FunctionMixin(PyobjMixin): + """ mixin for the code common to Function and Generator. + """ + + def setup(self): + """ perform setup for this test function. """ + if inspect.ismethod(self.obj): + name = 'setup_method' + else: + name = 'setup_function' + if isinstance(self.parent, Instance): + obj = self.parent.newinstance() + self.obj = self._getobj() + else: + obj = self.parent.obj + setup_func_or_method = getattr(obj, name, None) + if setup_func_or_method is not None: + setup_func_or_method(self.obj) + + def teardown(self): + """ perform teardown for this test function. """ + if inspect.ismethod(self.obj): + name = 'teardown_method' + else: + name = 'teardown_function' + obj = self.parent.obj + teardown_func_or_meth = getattr(obj, name, None) + if teardown_func_or_meth is not None: + teardown_func_or_meth(self.obj) + + def _prunetraceback(self, traceback): + if hasattr(self, '_obj') and not self.config.option.fulltrace: + code = py.code.Code(self.obj) + path, firstlineno = code.path, code.firstlineno + ntraceback = traceback.cut(path=path, firstlineno=firstlineno) + if ntraceback == traceback: + ntraceback = ntraceback.cut(path=path) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=py._dir) + traceback = ntraceback.filter() + return traceback + + def repr_failure(self, excinfo, outerr=None): + assert outerr is None, "XXX outerr usage is deprecated" + return self._repr_failure_py(excinfo) + + shortfailurerepr = "F" + +class Generator(FunctionMixin, PyCollectorMixin, py.test.collect.Collector): + def collect(self): + # test generators are seen as collectors but they also + # invoke setup/teardown on popular request + # (induced by the common "test_*" naming shared with normal tests) + self.config._setupstate.prepare(self) + l = [] + seen = {} + for i, x in enumerate(self.obj()): + name, call, args = self.getcallargs(x) + if not py.builtin.callable(call): + raise TypeError("%r yielded non callable test %r" %(self.obj, call,)) + if name is None: + name = "[%d]" % i + else: + name = "['%s']" % name + if name in seen: + raise ValueError("%r generated tests with non-unique name %r" %(self, name)) + seen[name] = True + l.append(self.Function(name, self, args=args, callobj=call)) + return l + + def getcallargs(self, obj): + if not isinstance(obj, (tuple, list)): + obj = (obj,) + # explict naming + if isinstance(obj[0], py.builtin._basestring): + name = obj[0] + obj = obj[1:] + else: + name = None + call, args = obj[0], obj[1:] + return name, call, args + + +# +# Test Items +# +_dummy = object() +class Function(FunctionMixin, py.test.collect.Item): + """ a Function Item is responsible for setting up + and executing a Python callable test object. + """ + _genid = None + def __init__(self, name, parent=None, args=None, + callspec=None, callobj=_dummy): + super(Function, self).__init__(name, parent) + self._args = args + if self._isyieldedfunction(): + assert not callspec, "yielded functions (deprecated) cannot have funcargs" + else: + if callspec is not None: + self.funcargs = callspec.funcargs or {} + self._genid = callspec.id + if hasattr(callspec, "param"): + self._requestparam = callspec.param + else: + self.funcargs = {} + if callobj is not _dummy: + self._obj = callobj + + def _isyieldedfunction(self): + return self._args is not None + + def readkeywords(self): + d = super(Function, self).readkeywords() + d.update(py.builtin._getfuncdict(self.obj)) + return d + + def runtest(self): + """ execute the underlying test function. """ + self.config.hook.pytest_pyfunc_call(pyfuncitem=self) + + def setup(self): + super(Function, self).setup() + if hasattr(self, 'funcargs'): + funcargs.fillfuncargs(self) + + def __eq__(self, other): + try: + return (self.name == other.name and + self._args == other._args and + self.parent == other.parent and + self.obj == other.obj and + getattr(self, '_genid', None) == + getattr(other, '_genid', None) + ) + except AttributeError: + pass + return False + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.parent, self.name)) + +def hasinit(obj): + init = getattr(obj, '__init__', None) + if init: + if not isinstance(init, type(object.__init__)): + return True Added: pypy/trunk/py/impl/test/session.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/test/session.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,126 @@ +""" basic test session implementation. + +* drives collection of tests +* triggers executions of tests +* produces events used by reporting +""" + +import py +from py.impl.test import outcome + +# imports used for genitems() +Item = py.test.collect.Item +Collector = py.test.collect.Collector + +class Session(object): + """ + Session drives the collection and running of tests + and generates test events for reporters. + """ + def __init__(self, config): + self.config = config + self.pluginmanager = config.pluginmanager # shortcut + self.pluginmanager.register(self) + self._testsfailed = False + self._nomatch = False + self.shouldstop = False + + def genitems(self, colitems, keywordexpr=None): + """ yield Items from iterating over the given colitems. """ + while colitems: + next = colitems.pop(0) + if isinstance(next, (tuple, list)): + colitems[:] = list(next) + colitems + continue + assert self.pluginmanager is next.config.pluginmanager + if isinstance(next, Item): + remaining = self.filteritems([next]) + if remaining: + self.config.hook.pytest_itemstart(item=next) + yield next + else: + assert isinstance(next, Collector) + self.config.hook.pytest_collectstart(collector=next) + rep = self.config.hook.pytest_make_collect_report(collector=next) + if rep.passed: + for x in self.genitems(rep.result, keywordexpr): + yield x + self.config.hook.pytest_collectreport(report=rep) + if self.shouldstop: + break + + def filteritems(self, colitems): + """ return items to process (some may be deselected)""" + keywordexpr = self.config.option.keyword + if not keywordexpr or self._nomatch: + return colitems + if keywordexpr[-1] == ":": + keywordexpr = keywordexpr[:-1] + remaining = [] + deselected = [] + for colitem in colitems: + if isinstance(colitem, Item): + if colitem._skipbykeyword(keywordexpr): + deselected.append(colitem) + continue + remaining.append(colitem) + if deselected: + self.config.hook.pytest_deselected(items=deselected) + if self.config.option.keyword.endswith(":"): + self._nomatch = True + return remaining + + def collect(self, colitems): + keyword = self.config.option.keyword + for x in self.genitems(colitems, keyword): + yield x + + def sessionstarts(self): + """ setup any neccessary resources ahead of the test run. """ + self.config.hook.pytest_sessionstart(session=self) + + def pytest_runtest_logreport(self, report): + if report.failed: + self._testsfailed = True + if self.config.option.exitfirst: + self.shouldstop = True + pytest_collectreport = pytest_runtest_logreport + + def sessionfinishes(self, exitstatus): + """ teardown any resources after a test run. """ + self.config.hook.pytest_sessionfinish( + session=self, + exitstatus=exitstatus, + ) + + def getinitialitems(self, colitems): + if colitems is None: + colitems = [self.config.getfsnode(arg) + for arg in self.config.args] + return colitems + + def main(self, colitems=None): + """ main loop for running tests. """ + colitems = self.getinitialitems(colitems) + self.shouldstop = False + self.sessionstarts() + exitstatus = outcome.EXIT_OK + captured_excinfo = None + try: + for item in self.collect(colitems): + if self.shouldstop: + break + if not self.config.option.collectonly: + item.config.hook.pytest_runtest_protocol(item=item) + except KeyboardInterrupt: + excinfo = py.code.ExceptionInfo() + self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + exitstatus = outcome.EXIT_INTERRUPTED + except: + excinfo = py.code.ExceptionInfo() + self.config.pluginmanager.notify_exception(captured_excinfo) + exitstatus = outcome.EXIT_INTERNALERROR + if exitstatus == 0 and self._testsfailed: + exitstatus = outcome.EXIT_TESTSFAILED + self.sessionfinishes(exitstatus=exitstatus) + return exitstatus Added: pypy/trunk/py/impl/xmlgen.py ============================================================================== --- (empty file) +++ pypy/trunk/py/impl/xmlgen.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,243 @@ +""" +module for generating and serializing xml and html structures +by using simple python objects. + +(c) holger krekel, holger at merlinux eu. 2009 +""" +import py +import sys, re + +if sys.version_info >= (3,0): + def u(s): + return s + def unicode(x): + if hasattr(x, '__unicode__'): + return x.__unicode__() + return str(x) +else: + def u(s): + return unicode(s) + unicode = unicode + + +class NamespaceMetaclass(type): + def __getattr__(self, name): + if name[:1] == '_': + raise AttributeError(name) + if self == Namespace: + raise ValueError("Namespace class is abstract") + tagspec = self.__tagspec__ + if tagspec is not None and name not in tagspec: + raise AttributeError(name) + classattr = {} + if self.__stickyname__: + classattr['xmlname'] = name + cls = type(name, (self.__tagclass__,), classattr) + setattr(self, name, cls) + return cls + +class Tag(list): + class Attr(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __init__(self, *args, **kwargs): + super(Tag, self).__init__(args) + self.attr = self.Attr(**kwargs) + + def __unicode__(self): + return self.unicode(indent=0) + __str__ = __unicode__ + + def unicode(self, indent=2): + l = [] + SimpleUnicodeVisitor(l.append, indent).visit(self) + return "".join(l) + + def __repr__(self): + name = self.__class__.__name__ + return "<%r tag object %d>" % (name, id(self)) + +Namespace = NamespaceMetaclass('Namespace', (object, ), { + '__tagspec__': None, + '__tagclass__': Tag, + '__stickyname__': False, +}) + +class HtmlTag(Tag): + def unicode(self, indent=2): + l = [] + HtmlVisitor(l.append, indent, shortempty=False).visit(self) + return u("").join(l) + +# exported plain html namespace +class html(Namespace): + __tagclass__ = HtmlTag + __stickyname__ = True + __tagspec__ = dict([(x,1) for x in ( + 'a,abbr,acronym,address,applet,area,b,bdo,big,blink,' + 'blockquote,body,br,button,caption,center,cite,code,col,' + 'colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed,' + 'fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html,' + 'i,iframe,img,input,ins,kbd,label,legend,li,link,listing,' + 'map,marquee,menu,meta,multicol,nobr,noembed,noframes,' + 'noscript,object,ol,optgroup,option,p,pre,q,s,script,' + 'select,small,span,strike,strong,style,sub,sup,table,' + 'tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp,' + 'base,basefont,frame,hr,isindex,param,samp,var' + ).split(',') if x]) + + class Style(object): + def __init__(self, **kw): + for x, y in kw.items(): + x = x.replace('_', '-') + setattr(self, x, y) + + +class raw(object): + """just a box that can contain a unicode string that will be + included directly in the output""" + def __init__(self, uniobj): + self.uniobj = uniobj + +class SimpleUnicodeVisitor(object): + """ recursive visitor to write unicode. """ + def __init__(self, write, indent=0, curindent=0, shortempty=True): + self.write = write + self.cache = {} + self.visited = {} # for detection of recursion + self.indent = indent + self.curindent = curindent + self.parents = [] + self.shortempty = shortempty # short empty tags or not + + def visit(self, node): + """ dispatcher on node's class/bases name. """ + cls = node.__class__ + try: + visitmethod = self.cache[cls] + except KeyError: + for subclass in cls.__mro__: + visitmethod = getattr(self, subclass.__name__, None) + if visitmethod is not None: + break + else: + visitmethod = self.object + self.cache[cls] = visitmethod + visitmethod(node) + + def object(self, obj): + #self.write(obj) + self.write(escape(unicode(obj))) + + def raw(self, obj): + self.write(obj.uniobj) + + def list(self, obj): + assert id(obj) not in self.visited + self.visited[id(obj)] = 1 + map(self.visit, obj) + + def Tag(self, tag): + assert id(tag) not in self.visited + try: + tag.parent = self.parents[-1] + except IndexError: + tag.parent = None + self.visited[id(tag)] = 1 + tagname = getattr(tag, 'xmlname', tag.__class__.__name__) + if self.curindent and not self._isinline(tagname): + self.write("\n" + u(' ') * self.curindent) + if tag: + self.curindent += self.indent + self.write(u('<%s%s>') % (tagname, self.attributes(tag))) + self.parents.append(tag) + for x in tag: + self.visit(x) + self.parents.pop() + self.write(u('') % tagname) + self.curindent -= self.indent + else: + nameattr = tagname+self.attributes(tag) + if self._issingleton(tagname): + self.write(u('<%s/>') % (nameattr,)) + else: + self.write(u('<%s>') % (nameattr, tagname)) + + def attributes(self, tag): + # serialize attributes + attrlist = dir(tag.attr) + attrlist.sort() + l = [] + for name in attrlist: + res = self.repr_attribute(tag.attr, name) + if res is not None: + l.append(res) + l.extend(self.getstyle(tag)) + return u("").join(l) + + def repr_attribute(self, attrs, name): + if name[:2] != '__': + value = getattr(attrs, name) + if name.endswith('_'): + name = name[:-1] + return ' %s="%s"' % (name, escape(unicode(value))) + + def getstyle(self, tag): + """ return attribute list suitable for styling. """ + try: + styledict = tag.style.__dict__ + except AttributeError: + return [] + else: + stylelist = [x+': ' + y for x,y in styledict.items()] + return [u(' style="%s"') % u('; ').join(stylelist)] + + def _issingleton(self, tagname): + """can (and will) be overridden in subclasses""" + return self.shortempty + + def _isinline(self, tagname): + """can (and will) be overridden in subclasses""" + return False + +class HtmlVisitor(SimpleUnicodeVisitor): + + single = dict([(x, 1) for x in + ('br,img,area,param,col,hr,meta,link,base,' + 'input,frame').split(',')]) + inline = dict([(x, 1) for x in + ('a abbr acronym b basefont bdo big br cite code dfn em font ' + 'i img input kbd label q s samp select small span strike ' + 'strong sub sup textarea tt u var'.split(' '))]) + + def repr_attribute(self, attrs, name): + if name == 'class_': + value = getattr(attrs, name) + if value is None: + return + return super(HtmlVisitor, self).repr_attribute(attrs, name) + + def _issingleton(self, tagname): + return tagname in self.single + + def _isinline(self, tagname): + return tagname in self.inline + + +class _escape: + def __init__(self): + self.escape = { + u('"') : u('"'), u('<') : u('<'), u('>') : u('>'), + u('&') : u('&'), u("'") : u('''), + } + self.charef_rex = re.compile(u("|").join(self.escape.keys())) + + def _replacer(self, match): + return self.escape[match.group(0)] + + def __call__(self, ustring): + """ xml-escape the given unicode string. """ + return self.charef_rex.sub(self._replacer, ustring) + +escape = _escape() Added: pypy/trunk/py/plugin/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/__init__.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1 @@ +# Added: pypy/trunk/py/plugin/hookspec.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/hookspec.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,175 @@ +""" +hook specifications for py.test plugins +""" + +# ------------------------------------------------------------------------- +# Command line and configuration +# ------------------------------------------------------------------------- + +def pytest_addoption(parser): + """ called before commandline parsing. """ + +def pytest_namespace(): + """ return dict of name->object which will get stored at py.test. namespace""" + +def pytest_configure(config): + """ called after command line options have been parsed. + and all plugins and initial conftest files been loaded. + """ + +def pytest_unconfigure(config): + """ called before test process is exited. """ + +# ------------------------------------------------------------------------- +# collection hooks +# ------------------------------------------------------------------------- + +def pytest_collect_directory(path, parent): + """ return Collection node or None for the given path. """ + +def pytest_collect_file(path, parent): + """ return Collection node or None for the given path. """ + +def pytest_collectstart(collector): + """ collector starts collecting. """ + +def pytest_collectreport(report): + """ collector finished collecting. """ + +def pytest_deselected(items): + """ called for test items deselected by keyword. """ + +def pytest_make_collect_report(collector): + """ perform a collection and return a collection. """ +pytest_make_collect_report.firstresult = True + +# XXX rename to item_collected()? meaning in distribution context? +def pytest_itemstart(item, node=None): + """ test item gets collected. """ + +# ------------------------------------------------------------------------- +# Python test function related hooks +# ------------------------------------------------------------------------- + +def pytest_pycollect_makeitem(collector, name, obj): + """ return custom item/collector for a python object in a module, or None. """ +pytest_pycollect_makeitem.firstresult = True + +def pytest_pyfunc_call(pyfuncitem): + """ perform function call to the with the given function arguments. """ +pytest_pyfunc_call.firstresult = True + +def pytest_generate_tests(metafunc): + """ generate (multiple) parametrized calls to a test function.""" + +# ------------------------------------------------------------------------- +# generic runtest related hooks +# ------------------------------------------------------------------------- + +def pytest_runtest_protocol(item): + """ implement fixture, run and report protocol. """ +pytest_runtest_protocol.firstresult = True + +def pytest_runtest_setup(item): + """ called before pytest_runtest_call(). """ + +def pytest_runtest_call(item): + """ execute test item. """ + +def pytest_runtest_teardown(item): + """ called after pytest_runtest_call(). """ + +def pytest_runtest_makereport(item, call): + """ make ItemTestReport for the given item and call outcome. """ +pytest_runtest_makereport.firstresult = True + +def pytest_runtest_logreport(report): + """ process item test report. """ + +# special handling for final teardown - somewhat internal for now +def pytest__teardown_final(session): + """ called before test session finishes. """ +pytest__teardown_final.firstresult = True + +def pytest__teardown_final_logerror(report): + """ called if runtest_teardown_final failed. """ + +# ------------------------------------------------------------------------- +# test session related hooks +# ------------------------------------------------------------------------- + +def pytest_sessionstart(session): + """ before session.main() is called. """ + +def pytest_sessionfinish(session, exitstatus): + """ whole test run finishes. """ + +# ------------------------------------------------------------------------- +# hooks for influencing reporting (invoked from pytest_terminal) +# ------------------------------------------------------------------------- + +def pytest_report_teststatus(report): + """ return result-category, shortletter and verbose word for reporting.""" +pytest_report_teststatus.firstresult = True + +def pytest_terminal_summary(terminalreporter): + """ add additional section in terminal summary reporting. """ + +def pytest_report_iteminfo(item): + """ return (fspath, lineno, name) for the item. + the information is used for result display and to sort tests + """ +pytest_report_iteminfo.firstresult = True + +# ------------------------------------------------------------------------- +# doctest hooks +# ------------------------------------------------------------------------- + +def pytest_doctest_prepare_content(content): + """ return processed content for a given doctest""" +pytest_doctest_prepare_content.firstresult = True + +# ------------------------------------------------------------------------- +# distributed testing +# ------------------------------------------------------------------------- + +def pytest_gwmanage_newgateway(gateway, platinfo): + """ called on new raw gateway creation. """ + +def pytest_gwmanage_rsyncstart(source, gateways): + """ called before rsyncing a directory to remote gateways takes place. """ + +def pytest_gwmanage_rsyncfinish(source, gateways): + """ called after rsyncing a directory to remote gateways takes place. """ + +def pytest_testnodeready(node): + """ Test Node is ready to operate. """ + +def pytest_testnodedown(node, error): + """ Test Node is down. """ + +def pytest_rescheduleitems(items): + """ reschedule Items from a node that went down. """ + +def pytest_looponfailinfo(failreports, rootdirs): + """ info for repeating failing tests. """ + + +# ------------------------------------------------------------------------- +# error handling and internal debugging hooks +# ------------------------------------------------------------------------- + +def pytest_plugin_registered(plugin): + """ a new py lib plugin got registered. """ + +def pytest_plugin_unregistered(plugin): + """ a py lib plugin got unregistered. """ + +def pytest_internalerror(excrepr): + """ called for internal errors. """ + +def pytest_keyboard_interrupt(excinfo): + """ called for keyboard interrupt. """ + +def pytest_trace(category, msg): + """ called for debug info. """ Added: pypy/trunk/py/plugin/pytest__pytest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest__pytest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,101 @@ +import py + +def pytest_funcarg___pytest(request): + return PytestArg(request) + +class PytestArg: + def __init__(self, request): + self.request = request + self.monkeypatch = self.request.getfuncargvalue("monkeypatch") + self.comregistry = py._com.Registry() + self.monkeypatch.setattr(py._com, 'comregistry', self.comregistry) + + def gethookrecorder(self, hookspecs, registry=None): + if registry is not None: + self.monkeypatch.setattr(py._com, 'comregistry', registry) + self.comregistry = registry + hookrecorder = HookRecorder(self.comregistry) + hookrecorder.start_recording(hookspecs) + self.request.addfinalizer(hookrecorder.finish_recording) + return hookrecorder + +class ParsedCall: + def __init__(self, name, locals): + assert '_name' not in locals + self.__dict__.update(locals) + self.__dict__.pop('self') + self._name = name + + def __repr__(self): + d = self.__dict__.copy() + del d['_name'] + return "" %(self._name, d) + +class HookRecorder: + def __init__(self, comregistry): + self._comregistry = comregistry + self.calls = [] + self._recorders = {} + + def start_recording(self, hookspecs): + assert hookspecs not in self._recorders + class RecordCalls: + _recorder = self + for name, method in vars(hookspecs).items(): + if name[0] != "_": + setattr(RecordCalls, name, self._makecallparser(method)) + recorder = RecordCalls() + self._recorders[hookspecs] = recorder + self._comregistry.register(recorder) + self.hook = py._com.HookRelay(hookspecs, registry=self._comregistry) + + def finish_recording(self): + for recorder in self._recorders.values(): + self._comregistry.unregister(recorder) + self._recorders.clear() + + def _makecallparser(self, method): + name = method.__name__ + args, varargs, varkw, default = py.std.inspect.getargspec(method) + if not args or args[0] != "self": + args.insert(0, 'self') + fspec = py.std.inspect.formatargspec(args, varargs, varkw, default) + # we use exec because we want to have early type + # errors on wrong input arguments, using + # *args/**kwargs delays this and gives errors + # elsewhere + exec (py.code.compile(""" + def %(name)s%(fspec)s: + self._recorder.calls.append( + ParsedCall(%(name)r, locals())) + """ % locals())) + return locals()[name] + + def getcalls(self, names): + if isinstance(names, str): + names = names.split() + for name in names: + for cls in self._recorders: + if name in vars(cls): + break + else: + raise ValueError("callname %r not found in %r" %( + name, self._recorders.keys())) + l = [] + for call in self.calls: + if call._name in names: + l.append(call) + return l + + def popcall(self, name): + for i, call in enumerate(self.calls): + if call._name == name: + del self.calls[i] + return call + raise ValueError("could not find call %r" %(name, )) + + def getcall(self, name): + l = self.getcalls(name) + assert len(l) == 1, (name, l) + return l[0] + Added: pypy/trunk/py/plugin/pytest_assertion.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_assertion.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,31 @@ +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup("debugconfig") + group._addoption('--no-assert', action="store_true", default=False, + dest="noassert", + help="disable python assert expression reinterpretation."), + +def pytest_configure(config): + #if sys.platform.startswith("java"): + # return # XXX assertions don't work yet with jython 2.5.1 + + if not config.getvalue("noassert") and not config.getvalue("nomagic"): + warn_about_missing_assertion() + config._oldassertion = py.builtin.builtins.AssertionError + py.builtin.builtins.AssertionError = py.code._AssertionError + +def pytest_unconfigure(config): + if hasattr(config, '_oldassertion'): + py.builtin.builtins.AssertionError = config._oldassertion + del config._oldassertion + +def warn_about_missing_assertion(): + try: + assert False + except AssertionError: + pass + else: + py.std.warnings.warn("Assertions are turned off!" + " (are you using python -O?)") Added: pypy/trunk/py/plugin/pytest_capture.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_capture.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,277 @@ +""" +configurable per-test stdout/stderr capturing mechanisms. + +This plugin captures stdout/stderr output for each test separately. +In case of test failures this captured output is shown grouped +togtther with the test. + +The plugin also provides test function arguments that help to +assert stdout/stderr output from within your tests, see the +`funcarg example`_. + + +Capturing of input/output streams during tests +--------------------------------------------------- + +By default ``sys.stdout`` and ``sys.stderr`` are substituted with +temporary streams during the execution of tests and setup/teardown code. +During the whole testing process it will re-use the same temporary +streams allowing to play well with the logging module which easily +takes ownership on these streams. + +Also, 'sys.stdin' is substituted with a file-like "null" object that +does not return any values. This is to immediately error out +on tests that wait on reading something from stdin. + +You can influence output capturing mechanisms from the command line:: + + py.test -s # disable all capturing + py.test --capture=sys # replace sys.stdout/stderr with in-mem files + py.test --capture=fd # point filedescriptors 1 and 2 to temp file + +If you set capturing values in a conftest file like this:: + + # conftest.py + option_capture = 'fd' + +then all tests in that directory will execute with "fd" style capturing. + +sys-level capturing +------------------------------------------ + +Capturing on 'sys' level means that ``sys.stdout`` and ``sys.stderr`` +will be replaced with in-memory files (``py.io.TextIO`` to be precise) +that capture writes and decode non-unicode strings to a unicode object +(using a default, usually, UTF-8, encoding). + +FD-level capturing and subprocesses +------------------------------------------ + +The ``fd`` based method means that writes going to system level files +based on the standard file descriptors will be captured, for example +writes such as ``os.write(1, 'hello')`` will be captured properly. +Capturing on fd-level will include output generated from +any subprocesses created during a test. + +.. _`funcarg example`: + +Example Usage of the capturing Function arguments +--------------------------------------------------- + +You can use the `capsys funcarg`_ and `capfd funcarg`_ to +capture writes to stdout and stderr streams. Using the +funcargs frees your test from having to care about setting/resetting +the old streams and also interacts well with py.test's own +per-test capturing. Here is an example test function: + +.. sourcecode:: python + + def test_myoutput(capsys): + print ("hello") + sys.stderr.write("world\\n") + out, err = capsys.readouterr() + assert out == "hello\\n" + assert err == "world\\n" + print "next" + out, err = capsys.readouterr() + assert out == "next\\n" + +The ``readouterr()`` call snapshots the output so far - +and capturing will be continued. After the test +function finishes the original streams will +be restored. If you want to capture on +the filedescriptor level you can use the ``capfd`` function +argument which offers the same interface. +""" + +import py +import os + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--capture', action="store", default=None, + metavar="method", type="choice", choices=['fd', 'sys', 'no'], + help="set capturing method during tests: fd (default)|sys|no.") + group._addoption('-s', action="store_const", const="no", dest="capture", + help="shortcut for --capture=no.") + +def addouterr(rep, outerr): + repr = getattr(rep, 'longrepr', None) + if not hasattr(repr, 'addsection'): + return + for secname, content in zip(["out", "err"], outerr): + if content: + repr.addsection("Captured std%s" % secname, content.rstrip()) + +def pytest_configure(config): + config.pluginmanager.register(CaptureManager(), 'capturemanager') + +class CaptureManager: + def __init__(self): + self._method2capture = {} + + def _maketempfile(self): + f = py.std.tempfile.TemporaryFile() + newf = py.io.dupfile(f, encoding="UTF-8") + return newf + + def _makestringio(self): + return py.io.TextIO() + + def _startcapture(self, method): + if method == "fd": + return py.io.StdCaptureFD( + out=self._maketempfile(), err=self._maketempfile() + ) + elif method == "sys": + return py.io.StdCapture( + out=self._makestringio(), err=self._makestringio() + ) + else: + raise ValueError("unknown capturing method: %r" % method) + + def _getmethod(self, config, fspath): + if config.option.capture: + method = config.option.capture + else: + try: + method = config._conftest.rget("option_capture", path=fspath) + except KeyError: + method = "fd" + if method == "fd" and not hasattr(os, 'dup'): # e.g. jython + method = "sys" + return method + + def resumecapture_item(self, item): + method = self._getmethod(item.config, item.fspath) + if not hasattr(item, 'outerr'): + item.outerr = ('', '') # we accumulate outerr on the item + return self.resumecapture(method) + + def resumecapture(self, method): + if hasattr(self, '_capturing'): + raise ValueError("cannot resume, already capturing with %r" % + (self._capturing,)) + if method != "no": + cap = self._method2capture.get(method) + if cap is None: + cap = self._startcapture(method) + self._method2capture[method] = cap + else: + cap.resume() + self._capturing = method + + def suspendcapture(self): + self.deactivate_funcargs() + method = self._capturing + if method != "no": + cap = self._method2capture[method] + outerr = cap.suspend() + else: + outerr = "", "" + del self._capturing + return outerr + + def activate_funcargs(self, pyfuncitem): + if not hasattr(pyfuncitem, 'funcargs'): + return + assert not hasattr(self, '_capturing_funcargs') + l = [] + for name, obj in pyfuncitem.funcargs.items(): + if name in ('capsys', 'capfd'): + obj._start() + l.append(obj) + if l: + self._capturing_funcargs = l + + def deactivate_funcargs(self): + if hasattr(self, '_capturing_funcargs'): + for capfuncarg in self._capturing_funcargs: + capfuncarg._finalize() + del self._capturing_funcargs + + def pytest_make_collect_report(self, __multicall__, collector): + method = self._getmethod(collector.config, collector.fspath) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + addouterr(rep, outerr) + return rep + + def pytest_runtest_setup(self, item): + self.resumecapture_item(item) + + def pytest_runtest_call(self, item): + self.resumecapture_item(item) + self.activate_funcargs(item) + + def pytest_runtest_teardown(self, item): + self.resumecapture_item(item) + + def pytest_runtest_teardown(self, item): + self.resumecapture_item(item) + + def pytest__teardown_final(self, __multicall__, session): + method = self._getmethod(session.config, None) + self.resumecapture(method) + try: + rep = __multicall__.execute() + finally: + outerr = self.suspendcapture() + if rep: + addouterr(rep, outerr) + return rep + + def pytest_keyboard_interrupt(self, excinfo): + if hasattr(self, '_capturing'): + self.suspendcapture() + + def pytest_runtest_makereport(self, __multicall__, item, call): + self.deactivate_funcargs() + rep = __multicall__.execute() + outerr = self.suspendcapture() + outerr = (item.outerr[0] + outerr[0], item.outerr[1] + outerr[1]) + if not rep.passed: + addouterr(rep, outerr) + if not rep.passed or rep.when == "teardown": + outerr = ('', '') + item.outerr = outerr + return rep + +def pytest_funcarg__capsys(request): + """captures writes to sys.stdout/sys.stderr and makes + them available successively via a ``capsys.readouterr()`` method + which returns a ``(out, err)`` tuple of captured snapshot strings. + """ + return CaptureFuncarg(request, py.io.StdCapture) + +def pytest_funcarg__capfd(request): + """captures writes to file descriptors 1 and 2 and makes + snapshotted ``(out, err)`` string tuples available + via the ``capsys.readouterr()`` method. + """ + return CaptureFuncarg(request, py.io.StdCaptureFD) + + +class CaptureFuncarg: + def __init__(self, request, captureclass): + self._cclass = captureclass + #request.addfinalizer(self._finalize) + + def _start(self): + self.capture = self._cclass() + + def _finalize(self): + if hasattr(self, 'capture'): + self.capture.reset() + del self.capture + + def readouterr(self): + return self.capture.readouterr() + + def close(self): + self.capture.reset() + del self.capture + Added: pypy/trunk/py/plugin/pytest_default.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_default.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,149 @@ +""" default hooks and general py.test options. """ + +import sys +import py + +try: + import execnet +except ImportError: + execnet = None + +def pytest_pyfunc_call(__multicall__, pyfuncitem): + if not __multicall__.execute(): + testfunction = pyfuncitem.obj + if pyfuncitem._isyieldedfunction(): + testfunction(*pyfuncitem._args) + else: + funcargs = pyfuncitem.funcargs + testfunction(**funcargs) + +def pytest_collect_file(path, parent): + ext = path.ext + pb = path.purebasename + if pb.startswith("test_") or pb.endswith("_test") or \ + path in parent.config.args: + if ext == ".py": + return parent.Module(path, parent=parent) + +def pytest_collect_directory(path, parent): + # XXX reconsider the following comment + # not use parent.Directory here as we generally + # want dir/conftest.py to be able to + # define Directory(dir) already + if not parent.recfilter(path): # by default special ".cvs", ... + # check if cmdline specified this dir or a subdir directly + for arg in parent.config.args: + if path == arg or arg.relto(path): + break + else: + return + Directory = parent.config.getvalue('Directory', path) + return Directory(path, parent=parent) + +def pytest_report_iteminfo(item): + return item.reportinfo() + +def pytest_addoption(parser): + group = parser.getgroup("general", "running and selection options") + group._addoption('-x', '--exitfirst', + action="store_true", dest="exitfirst", default=False, + help="exit instantly on first error or failed test."), + group._addoption('-k', + action="store", dest="keyword", default='', + help="only run test items matching the given " + "space separated keywords. precede a keyword with '-' to negate. " + "Terminate the expression with ':' to treat a match as a signal " + "to run all subsequent tests. ") + group._addoption('-p', action="append", dest="plugins", default = [], + help=("load the specified plugin after command line parsing. ")) + if execnet: + group._addoption('-f', '--looponfail', + action="store_true", dest="looponfail", default=False, + help="run tests, re-run failing test set until all pass.") + + group = parser.getgroup("debugconfig", + "test process debugging and configuration") + group.addoption('--basetemp', dest="basetemp", default=None, metavar="dir", + help="base temporary directory for this test run.") + + if execnet: + add_dist_options(parser) + else: + parser.epilog = ( + "'execnet' package required for --looponfailing / distributed testing.") + +def add_dist_options(parser): + # see http://pytest.org/help/dist") + group = parser.getgroup("dist", "distributed testing") + group._addoption('--dist', metavar="distmode", + action="store", choices=['load', 'each', 'no'], + type="choice", dest="dist", default="no", + help=("set mode for distributing tests to exec environments.\n\n" + "each: send each test to each available environment.\n\n" + "load: send each test to available environment.\n\n" + "(default) no: run tests inprocess, don't distribute.")) + group._addoption('--tx', dest="tx", action="append", default=[], metavar="xspec", + help=("add a test execution environment. some examples: " + "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 " + "--tx ssh=user at codespeak.net//chdir=testcache")) + group._addoption('-d', + action="store_true", dest="distload", default=False, + help="load-balance tests. shortcut for '--dist=load'") + group._addoption('-n', dest="numprocesses", metavar="numprocesses", + action="store", type="int", + help="shortcut for '--dist=load --tx=NUM*popen'") + group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", + help="add directory for rsyncing to remote tx nodes.") + +def pytest_configure(config): + fixoptions(config) + setsession(config) + +def fixoptions(config): + if execnet: + if config.option.numprocesses: + config.option.dist = "load" + config.option.tx = ['popen'] * int(config.option.numprocesses) + if config.option.distload: + config.option.dist = "load" + +def setsession(config): + val = config.getvalue + if val("collectonly"): + from py.impl.test.session import Session + config.setsessionclass(Session) + elif execnet: + if val("looponfail"): + from py.impl.test.looponfail.remote import LooponfailingSession + config.setsessionclass(LooponfailingSession) + elif val("dist") != "no": + from py.impl.test.dist.dsession import DSession + config.setsessionclass(DSession) + +# pycollect related hooks and code, should move to pytest_pycollect.py + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + res = __multicall__.execute() + if res is not None: + return res + if collector._istestclasscandidate(name, obj): + res = collector._deprecated_join(name) + if res is not None: + return res + return collector.Class(name, parent=collector) + elif collector.funcnamefilter(name) and hasattr(obj, '__call__'): + res = collector._deprecated_join(name) + if res is not None: + return res + if is_generator(obj): + # XXX deprecation warning + return collector.Generator(name, parent=collector) + else: + return collector._genfunctions(name, obj) + +def is_generator(func): + try: + return py.code.getrawcode(func).co_flags & 32 # generator function + except AttributeError: # builtin functions have no bytecode + # assume them to not be generators + return False Added: pypy/trunk/py/plugin/pytest_doctest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_doctest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,86 @@ +""" +collect and execute doctests from modules and test files. + +Usage +------------- + +By default all files matching the ``test_*.txt`` pattern will +be run with the ``doctest`` module. If you issue:: + + py.test --doctest-modules + +all python files in your projects will be doctest-run +as well. +""" + +import py +from py.impl.code.code import TerminalRepr, ReprFileLocation +import doctest + +def pytest_addoption(parser): + group = parser.getgroup("doctest options") + group.addoption("--doctest-modules", + action="store_true", default=False, + help="search all python files for doctests", + dest="doctestmodules") + +def pytest_collect_file(path, parent): + if path.ext == ".py": + if parent.config.getvalue("doctestmodules"): + return DoctestModule(path, parent) + if path.check(fnmatch="test_*.txt"): + return DoctestTextfile(path, parent) + +class ReprFailDoctest(TerminalRepr): + def __init__(self, reprlocation, lines): + self.reprlocation = reprlocation + self.lines = lines + def toterminal(self, tw): + for line in self.lines: + tw.line(line) + self.reprlocation.toterminal(tw) + +class DoctestItem(py.test.collect.Item): + def __init__(self, path, parent): + name = self.__class__.__name__ + ":" + path.basename + super(DoctestItem, self).__init__(name=name, parent=parent) + self.fspath = path + + def repr_failure(self, excinfo): + if excinfo.errisinstance(doctest.DocTestFailure): + doctestfailure = excinfo.value + example = doctestfailure.example + test = doctestfailure.test + filename = test.filename + lineno = test.lineno + example.lineno + 1 + message = excinfo.type.__name__ + reprlocation = ReprFileLocation(filename, lineno, message) + checker = doctest.OutputChecker() + REPORT_UDIFF = doctest.REPORT_UDIFF + filelines = py.path.local(filename).readlines(cr=0) + i = max(test.lineno, max(0, lineno - 10)) # XXX? + lines = [] + for line in filelines[i:lineno]: + lines.append("%03d %s" % (i+1, line)) + i += 1 + lines += checker.output_difference(example, + doctestfailure.got, REPORT_UDIFF).split("\n") + return ReprFailDoctest(reprlocation, lines) + elif excinfo.errisinstance(doctest.UnexpectedException): + excinfo = py.code.ExceptionInfo(excinfo.value.exc_info) + return super(DoctestItem, self).repr_failure(excinfo) + else: + return super(DoctestItem, self).repr_failure(excinfo) + +class DoctestTextfile(DoctestItem): + def runtest(self): + if not self._deprecated_testexecution(): + failed, tot = doctest.testfile( + str(self.fspath), module_relative=False, + raise_on_error=True, verbose=0) + +class DoctestModule(DoctestItem): + def runtest(self): + module = self.fspath.pyimport() + failed, tot = doctest.testmod( + module, raise_on_error=True, verbose=0) Added: pypy/trunk/py/plugin/pytest_figleaf.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_figleaf.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,51 @@ +""" +write and report coverage data with 'figleaf'. + +""" +import py + +py.test.importorskip("figleaf.annotate_html") +import figleaf + +def pytest_addoption(parser): + group = parser.getgroup('figleaf options') + group.addoption('-F', action='store_true', default=False, + dest = 'figleaf', + help=('trace python coverage with figleaf and write HTML ' + 'for files below the current working dir')) + group.addoption('--figleaf-data', action='store', default='.figleaf', + dest='figleafdata', + help='path to coverage tracing file.') + group.addoption('--figleaf-html', action='store', default='html', + dest='figleafhtml', + help='path to the coverage html dir.') + +def pytest_configure(config): + figleaf.start() + +def pytest_terminal_summary(terminalreporter): + config = terminalreporter.config + datafile = py.path.local(config.getvalue('figleafdata')) + tw = terminalreporter._tw + tw.sep('-', 'figleaf') + tw.line('Writing figleaf data to %s' % (datafile)) + figleaf.stop() + figleaf.write_coverage(str(datafile)) + coverage = get_coverage(datafile, config) + reportdir = py.path.local(config.getvalue('figleafhtml')) + tw.line('Writing figleaf html to file://%s' % (reportdir)) + figleaf.annotate_html.prepare_reportdir(str(reportdir)) + exclude = [] + figleaf.annotate_html.report_as_html(coverage, + str(reportdir), exclude, {}) + +def get_coverage(datafile, config): + # basepath = config.topdir + basepath = py.path.local() + data = figleaf.read_coverage(str(datafile)) + d = {} + coverage = figleaf.combine_coverage(d, data) + for path in coverage.keys(): + if not py.path.local(path).relto(basepath): + del coverage[path] + return coverage Added: pypy/trunk/py/plugin/pytest_helpconfig.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_helpconfig.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,63 @@ +""" provide version info, conftest/environment config names. +""" +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup('debugconfig') + group.addoption("--help-config", action="store_true", dest="helpconfig", + help="show available conftest.py and ENV-variable names.") + group.addoption('--version', action="store_true", + help="display py lib version and import information.") + +def pytest_configure(__multicall__, config): + if config.option.version: + p = py.path.local(py.__file__).dirpath() + sys.stderr.write("This is py.test version %s, imported from %s\n" % + (py.__version__, p)) + sys.exit(0) + if not config.option.helpconfig: + return + __multicall__.execute() + options = [] + for group in config._parser._groups: + options.extend(group.options) + widths = [0] * 10 + tw = py.io.TerminalWriter() + tw.sep("-") + tw.line("%-13s | %-18s | %-25s | %s" %( + "cmdline name", "conftest.py name", "ENV-variable name", "help")) + tw.sep("-") + + options = [opt for opt in options if opt._long_opts] + options.sort(key=lambda x: x._long_opts) + for opt in options: + if not opt._long_opts: + continue + optstrings = list(opt._long_opts) # + list(opt._short_opts) + optstrings = filter(None, optstrings) + optstring = "|".join(optstrings) + line = "%-13s | %-18s | %-25s | %s" %( + optstring, + "option_%s" % opt.dest, + "PYTEST_OPTION_%s" % opt.dest.upper(), + opt.help and opt.help or "", + ) + tw.line(line[:tw.fullwidth]) + for name, help in conftest_options: + line = "%-13s | %-18s | %-25s | %s" %( + "", + name, + "", + help, + ) + tw.line(line[:tw.fullwidth]) + + tw.sep("-") + sys.exit(0) + +conftest_options = ( + ('pytest_plugins', 'list of plugin names to load'), + ('collect_ignore', '(relative) paths ignored during collection'), + ('rsyncdirs', 'to-be-rsynced directories for dist-testing'), +) Added: pypy/trunk/py/plugin/pytest_hooklog.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_hooklog.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,33 @@ +""" log invocations of extension hooks to a file. """ +import py + +def pytest_addoption(parser): + parser.addoption("--hooklog", dest="hooklog", default=None, + help="write hook calls to the given file.") + +def pytest_configure(config): + hooklog = config.getvalue("hooklog") + if hooklog: + config._hooklogfile = open(hooklog, 'w') + config._hooklog_oldperformcall = config.hook._performcall + config.hook._performcall = (lambda name, multicall: + logged_call(name=name, multicall=multicall, config=config)) + +def logged_call(name, multicall, config): + f = config._hooklogfile + f.write("%s(**%s)\n" % (name, multicall.kwargs)) + try: + res = config._hooklog_oldperformcall(name=name, multicall=multicall) + except: + f.write("-> exception") + raise + f.write("-> %r" % (res,)) + return res + +def pytest_unconfigure(config): + try: + del config.hook.__dict__['_performcall'] + except KeyError: + pass + else: + config._hooklogfile.close() Added: pypy/trunk/py/plugin/pytest_mark.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_mark.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,148 @@ +""" +generic mechanism for marking python functions. + +By using the ``py.test.mark`` helper you can instantiate +decorators that will set named meta data on test functions. + +Marking a single function +---------------------------------------------------- + +You can "mark" a test function with meta data like this:: + + @py.test.mark.webtest + def test_send_http(): + ... + +This will set a "Marker" instance as a function attribute named "webtest". +You can also specify parametrized meta data like this:: + + @py.test.mark.webtest(firefox=30) + def test_receive(): + ... + +The named marker can be accessed like this later:: + + test_receive.webtest.kwargs['firefox'] == 30 + +In addition to set key-value pairs you can also use positional arguments:: + + @py.test.mark.webtest("triangular") + def test_receive(): + ... + +and later access it with ``test_receive.webtest.args[0] == 'triangular``. + +.. _`scoped-marking`: + +Marking classes or modules +---------------------------------------------------- + +To mark all methods of a class set a ``pytestmark`` attribute like this:: + + import py + + class TestClass: + pytestmark = py.test.mark.webtest + +You can re-use the same markers that you would use for decorating +a function - in fact this marker decorator will be applied +to all test methods of the class. + +You can also set a module level marker:: + + import py + pytestmark = py.test.mark.webtest + +in which case then the marker decorator will be applied to all functions and +methods defined in the module. + +The order in which marker functions are called is this:: + + per-function (upon import of module already) + per-class + per-module + +Later called markers may overwrite previous key-value settings. +Positional arguments are all appended to the same 'args' list +of the Marker object. + +Using "-k MARKNAME" to select tests +---------------------------------------------------- + +You can use the ``-k`` command line option to select +tests:: + + py.test -k webtest # will only run tests marked as webtest + +""" +import py + +def pytest_namespace(): + return {'mark': Mark()} + + +class Mark(object): + def __getattr__(self, name): + if name[0] == "_": + raise AttributeError(name) + return MarkerDecorator(name) + +class MarkerDecorator: + """ decorator for setting function attributes. """ + def __init__(self, name): + self.markname = name + self.kwargs = {} + self.args = [] + + def __repr__(self): + d = self.__dict__.copy() + name = d.pop('markname') + return "" %(name, d) + + def __call__(self, *args, **kwargs): + if args: + if len(args) == 1 and hasattr(args[0], '__call__'): + func = args[0] + holder = getattr(func, self.markname, None) + if holder is None: + holder = Marker(self.markname, self.args, self.kwargs) + setattr(func, self.markname, holder) + else: + holder.kwargs.update(self.kwargs) + holder.args.extend(self.args) + return func + else: + self.args.extend(args) + self.kwargs.update(kwargs) + return self + +class Marker: + def __init__(self, name, args, kwargs): + self._name = name + self.args = args + self.kwargs = kwargs + + def __getattr__(self, name): + if name[0] != '_' and name in self.kwargs: + py.log._apiwarn("1.1", "use .kwargs attribute to access key-values") + return self.kwargs[name] + raise AttributeError(name) + + def __repr__(self): + return "" % ( + self._name, self.args, self.kwargs) + + +def pytest_pycollect_makeitem(__multicall__, collector, name, obj): + item = __multicall__.execute() + if isinstance(item, py.test.collect.Function): + cls = collector.getparent(py.test.collect.Class) + mod = collector.getparent(py.test.collect.Module) + func = item.obj + func = getattr(func, '__func__', func) # py3 + func = getattr(func, 'im_func', func) # py2 + for parent in [x for x in (mod, cls) if x]: + marker = getattr(parent.obj, 'pytestmark', None) + if isinstance(marker, MarkerDecorator): + marker(func) + return item Added: pypy/trunk/py/plugin/pytest_monkeypatch.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_monkeypatch.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,142 @@ +""" +safely patch object attributes, dicts and environment variables. + +Usage +---------------- + +Use the `monkeypatch funcarg`_ to tweak your global test environment +for running a particular test. You can safely set/del an attribute, +dictionary item or environment variable by respective methods +on the monkeypatch funcarg. If you want e.g. to set an ENV1 variable +and have os.path.expanduser return a particular directory, you can +write it down like this: + +.. sourcecode:: python + + def test_mytest(monkeypatch): + monkeypatch.setenv('ENV1', 'myval') + monkeypatch.setattr(os.path, 'expanduser', lambda x: '/tmp/xyz') + ... # your test code that uses those patched values implicitely + +After the test function finished all modifications will be undone, +because the ``monkeypatch.undo()`` method is registered as a finalizer. + +``monkeypatch.setattr/delattr/delitem/delenv()`` all +by default raise an Exception if the target does not exist. +Pass ``raising=False`` if you want to skip this check. + +prepending to PATH or other environment variables +--------------------------------------------------------- + +To prepend a value to an already existing environment parameter: + +.. sourcecode:: python + + def test_mypath_finding(monkeypatch): + monkeypatch.setenv('PATH', 'x/y', prepend=":") + # in bash language: export PATH=x/y:$PATH + +calling "undo" finalization explicitely +----------------------------------------- + +At the end of function execution py.test invokes +a teardown hook which undoes all monkeypatch changes. +If you do not want to wait that long you can call +finalization explicitely:: + + monkeypatch.undo() + +This will undo previous changes. This call consumes the +undo stack. Calling it a second time has no effect unless +you start monkeypatching after the undo call. + +.. _`monkeypatch blog post`: http://tetamap.wordpress.com/2009/03/03/monkeypatching-in-unit-tests-done-right/ +""" + +import py, os, sys + +def pytest_funcarg__monkeypatch(request): + """The returned ``monkeypatch`` funcarg provides these + helper methods to modify objects, dictionaries or os.environ:: + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=False) + monkeypatch.delenv(name, value, raising=True) + monkeypatch.syspath_prepend(path) + + All modifications will be undone when the requesting + test function finished its execution. For the ``del`` + methods the ``raising`` parameter determines if a + KeyError or AttributeError will be raised if the + deletion has no target. + """ + monkeypatch = MonkeyPatch() + request.addfinalizer(monkeypatch.undo) + return monkeypatch + +notset = object() + +class MonkeyPatch: + def __init__(self): + self._setattr = [] + self._setitem = [] + + def setattr(self, obj, name, value, raising=True): + oldval = getattr(obj, name, notset) + if raising and oldval is notset: + raise AttributeError("%r has no attribute %r" %(obj, name)) + self._setattr.insert(0, (obj, name, oldval)) + setattr(obj, name, value) + + def delattr(self, obj, name, raising=True): + if not hasattr(obj, name): + if raising: + raise AttributeError(name) + else: + self._setattr.insert(0, (obj, name, getattr(obj, name, notset))) + delattr(obj, name) + + def setitem(self, dic, name, value): + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + dic[name] = value + + def delitem(self, dic, name, raising=True): + if name not in dic: + if raising: + raise KeyError(name) + else: + self._setitem.insert(0, (dic, name, dic.get(name, notset))) + del dic[name] + + def setenv(self, name, value, prepend=None): + value = str(value) + if prepend and name in os.environ: + value = value + prepend + os.environ[name] + self.setitem(os.environ, name, value) + + def delenv(self, name, raising=True): + self.delitem(os.environ, name, raising=raising) + + def syspath_prepend(self, path): + if not hasattr(self, '_savesyspath'): + self._savesyspath = sys.path[:] + sys.path.insert(0, str(path)) + + def undo(self): + for obj, name, value in self._setattr: + if value is not notset: + setattr(obj, name, value) + else: + delattr(obj, name) + self._setattr[:] = [] + for dictionary, name, value in self._setitem: + if value is notset: + del dictionary[name] + else: + dictionary[name] = value + self._setitem[:] = [] + if hasattr(self, '_savesyspath'): + sys.path[:] = self._savesyspath Added: pypy/trunk/py/plugin/pytest_nose.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_nose.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,98 @@ +"""nose-compatibility plugin: allow to run nose test suites natively. + +This is an experimental plugin for allowing to run tests written +in 'nosetests style with py.test. + +Usage +------------- + +type:: + + py.test # instead of 'nosetests' + +and you should be able to run nose style tests and at the same +time can make full use of py.test's capabilities. + +Supported nose Idioms +---------------------- + +* setup and teardown at module/class/method level +* SkipTest exceptions and markers +* setup/teardown decorators +* yield-based tests and their setup +* general usage of nose utilities + +Unsupported idioms / issues +---------------------------------- + +- nose-style doctests are not collected and executed correctly, + also fixtures don't work. + +- no nose-configuration is recognized + +If you find other issues or have suggestions please run:: + + py.test --pastebin=all + +and send the resulting URL to a py.test contact channel, +at best to the mailing list. +""" +import py +import inspect +import sys + +def pytest_runtest_makereport(__multicall__, item, call): + SkipTest = getattr(sys.modules.get('nose', None), 'SkipTest', None) + if SkipTest: + if call.excinfo and call.excinfo.errisinstance(SkipTest): + # let's substitute the excinfo with a py.test.skip one + call2 = call.__class__(lambda: py.test.skip(str(call.excinfo.value)), call.when) + call.excinfo = call2.excinfo + +def pytest_report_iteminfo(item): + # nose 0.11.1 uses decorators for "raises" and other helpers. + # for reporting progress by filename we fish for the filename + if isinstance(item, py.test.collect.Function): + obj = item.obj + if hasattr(obj, 'compat_co_firstlineno'): + fn = sys.modules[obj.__module__].__file__ + if fn.endswith(".pyc"): + fn = fn[:-1] + #assert 0 + #fn = inspect.getsourcefile(obj) or inspect.getfile(obj) + lineno = obj.compat_co_firstlineno + return py.path.local(fn), lineno, obj.__module__ + +def pytest_runtest_setup(item): + if isinstance(item, (py.test.collect.Function)): + if isinstance(item.parent, py.test.collect.Generator): + gen = item.parent + if not hasattr(gen, '_nosegensetup'): + call_optional(gen.obj, 'setup') + if isinstance(gen.parent, py.test.collect.Instance): + call_optional(gen.parent.obj, 'setup') + gen._nosegensetup = True + if not call_optional(item.obj, 'setup'): + # call module level setup if there is no object level one + call_optional(item.parent.obj, 'setup') + +def pytest_runtest_teardown(item): + if isinstance(item, py.test.collect.Function): + if not call_optional(item.obj, 'teardown'): + call_optional(item.parent.obj, 'teardown') + #if hasattr(item.parent, '_nosegensetup'): + # #call_optional(item._nosegensetup, 'teardown') + # del item.parent._nosegensetup + +def pytest_make_collect_report(collector): + if isinstance(collector, py.test.collect.Generator): + call_optional(collector.obj, 'setup') + +def call_optional(obj, name): + method = getattr(obj, name, None) + if method: + ismethod = inspect.ismethod(method) + rawcode = py.code.getrawcode(method) + if not rawcode.co_varnames[ismethod:]: + method() + return True Added: pypy/trunk/py/plugin/pytest_pastebin.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_pastebin.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,83 @@ +""" +submit failure or test session information to a pastebin service. + +Usage +---------- + +**Creating a URL for each test failure**:: + + py.test --pastebin=failed + +This will submit test run information to a remote Paste service and +provide a URL for each failure. You may select tests as usual or add +for example ``-x`` if you only want to send one particular failure. + +**Creating a URL for a whole test session log**:: + + py.test --pastebin=all + +Currently only pasting to the http://paste.pocoo.org service is implemented. + +""" +import py, sys + +class url: + base = "http://paste.pocoo.org" + xmlrpc = base + "/xmlrpc/" + show = base + "/show/" + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--pastebin', metavar="mode", + action='store', dest="pastebin", default=None, + type="choice", choices=['failed', 'all'], + help="send failed|all info to Pocoo pastebin service.") + +def pytest_configure(__multicall__, config): + import tempfile + __multicall__.execute() + if config.option.pastebin == "all": + config._pastebinfile = tempfile.TemporaryFile('w+') + tr = config.pluginmanager.getplugin('terminalreporter') + oldwrite = tr._tw.write + def tee_write(s, **kwargs): + oldwrite(s, **kwargs) + config._pastebinfile.write(str(s)) + tr._tw.write = tee_write + +def pytest_unconfigure(config): + if hasattr(config, '_pastebinfile'): + config._pastebinfile.seek(0) + sessionlog = config._pastebinfile.read() + config._pastebinfile.close() + del config._pastebinfile + proxyid = getproxy().newPaste("python", sessionlog) + pastebinurl = "%s%s" % (url.show, proxyid) + sys.stderr.write("pastebin session-log: %s\n" % pastebinurl) + tr = config.pluginmanager.getplugin('terminalreporter') + del tr._tw.__dict__['write'] + +def getproxy(): + return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes + +def pytest_terminal_summary(terminalreporter): + if terminalreporter.config.option.pastebin != "failed": + return + tr = terminalreporter + if 'failed' in tr.stats: + terminalreporter.write_sep("=", "Sending information to Paste Service") + if tr.config.option.debug: + terminalreporter.write_line("xmlrpcurl: %s" %(url.xmlrpc,)) + serverproxy = getproxy() + for rep in terminalreporter.stats.get('failed'): + try: + msg = rep.longrepr.reprtraceback.reprentries[-1].reprfileloc + except AttributeError: + msg = tr._getfailureheadline(rep) + tw = py.io.TerminalWriter(stringio=True) + rep.toterminal(tw) + s = tw.stringio.getvalue() + assert len(s) + proxyid = serverproxy.newPaste("python", s) + pastebinurl = "%s%s" % (url.show, proxyid) + tr.write_line("%s --> %s" %(msg, pastebinurl)) Added: pypy/trunk/py/plugin/pytest_pdb.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_pdb.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,114 @@ +""" +interactive debugging with the Python Debugger. +""" +import py +import pdb, sys, linecache +from py.impl.test.outcome import Skipped +try: + import execnet +except ImportError: + execnet = None + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption('--pdb', + action="store_true", dest="usepdb", default=False, + help="start pdb (the Python debugger) on errors.") + + +def pytest_configure(config): + if config.option.usepdb: + if execnet: + if config.getvalue("looponfail"): + raise config.Error("--pdb incompatible with --looponfail.") + if config.option.dist != "no": + raise config.Error("--pdb incompatible with distributing tests.") + config.pluginmanager.register(PdbInvoke()) + +class PdbInvoke: + def pytest_runtest_makereport(self, item, call): + if call.excinfo and not call.excinfo.errisinstance(Skipped): + # play well with capturing, slightly hackish + capman = item.config.pluginmanager.getplugin('capturemanager') + capman.suspendcapture() + + tw = py.io.TerminalWriter() + repr = call.excinfo.getrepr() + repr.toterminal(tw) + post_mortem(call.excinfo._excinfo[2]) + + capman.resumecapture_item(item) + +class Pdb(py.std.pdb.Pdb): + def do_list(self, arg): + self.lastcmd = 'list' + last = None + if arg: + try: + x = eval(arg, {}, {}) + if type(x) == type(()): + first, last = x + first = int(first) + last = int(last) + if last < first: + # Assume it's a count + last = first + last + else: + first = max(1, int(x) - 5) + except: + print ('*** Error in argument: %s' % repr(arg)) + return + elif self.lineno is None: + first = max(1, self.curframe.f_lineno - 5) + else: + first = self.lineno + 1 + if last is None: + last = first + 10 + filename = self.curframe.f_code.co_filename + breaklist = self.get_file_breaks(filename) + try: + for lineno in range(first, last+1): + # start difference from normal do_line + line = self._getline(filename, lineno) + # end difference from normal do_line + if not line: + print ('[EOF]') + break + else: + s = repr(lineno).rjust(3) + if len(s) < 4: s = s + ' ' + if lineno in breaklist: s = s + 'B' + else: s = s + ' ' + if lineno == self.curframe.f_lineno: + s = s + '->' + sys.stdout.write(s + '\t' + line) + self.lineno = lineno + except KeyboardInterrupt: + pass + do_l = do_list + + def _getline(self, filename, lineno): + if hasattr(filename, "__source__"): + try: + return filename.__source__.lines[lineno - 1] + "\n" + except IndexError: + return None + return linecache.getline(filename, lineno) + + def get_stack(self, f, t): + # Modified from bdb.py to be able to walk the stack beyond generators, + # which does not work in the normal pdb :-( + stack, i = pdb.Pdb.get_stack(self, f, t) + if f is None: + i = max(0, len(stack) - 1) + return stack, i + +def post_mortem(t): + # modified from pdb.py for the new get_stack() implementation + p = Pdb() + p.reset() + p.interaction(None, t) + +def set_trace(): + # again, a copy of the version in pdb.py + Pdb().set_trace(sys._getframe().f_back) Added: pypy/trunk/py/plugin/pytest_pylint.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_pylint.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,36 @@ +"""pylint plugin + +XXX: Currently in progress, NOT IN WORKING STATE. +""" +import py + +pylint = py.test.importorskip("pylint.lint") + +def pytest_addoption(parser): + group = parser.getgroup('pylint options') + group.addoption('--pylint', action='store_true', + default=False, dest='pylint', + help='run pylint on python files.') + +def pytest_collect_file(path, parent): + if path.ext == ".py": + if parent.config.getvalue('pylint'): + return PylintItem(path, parent) + +#def pytest_terminal_summary(terminalreporter): +# print 'placeholder for pylint output' + +class PylintItem(py.test.collect.Item): + def runtest(self): + capture = py.io.StdCaptureFD() + try: + linter = pylint.lint.PyLinter() + linter.check(str(self.fspath)) + finally: + out, err = capture.reset() + rating = out.strip().split('\n')[-1] + sys.stdout.write(">>>") + print(rating) + assert 0 + + Added: pypy/trunk/py/plugin/pytest_pytester.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_pytester.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,459 @@ +""" +funcargs and support code for testing py.test's own functionality. +""" + +import py +import sys, os +import inspect +from py.impl.test.config import Config as pytestConfig +from py.plugin import hookspec +from py.builtin import print_ + +pytest_plugins = '_pytest' + +def pytest_funcarg__linecomp(request): + return LineComp() + +def pytest_funcarg__LineMatcher(request): + return LineMatcher + +def pytest_funcarg__testdir(request): + tmptestdir = TmpTestdir(request) + return tmptestdir + +def pytest_funcarg__reportrecorder(request): + reprec = ReportRecorder(py._com.comregistry) + request.addfinalizer(lambda: reprec.comregistry.unregister(reprec)) + return reprec + +class RunResult: + def __init__(self, ret, outlines, errlines): + self.ret = ret + self.outlines = outlines + self.errlines = errlines + self.stdout = LineMatcher(outlines) + self.stderr = LineMatcher(errlines) + +class TmpTestdir: + def __init__(self, request): + self.request = request + self._pytest = request.getfuncargvalue("_pytest") + # XXX remove duplication with tmpdir plugin + basetmp = request.config.ensuretemp("testdir") + name = request.function.__name__ + for i in range(100): + try: + tmpdir = basetmp.mkdir(name + str(i)) + except py.error.EEXIST: + continue + break + # we need to create another subdir + # because Directory.collect() currently loads + # conftest.py from sibling directories + self.tmpdir = tmpdir.mkdir(name) + self.plugins = [] + self._syspathremove = [] + self.chdir() # always chdir + assert hasattr(self, '_olddir') + self.request.addfinalizer(self.finalize) + + def __repr__(self): + return "" % (self.tmpdir,) + + def Config(self, comregistry=None, topdir=None): + if topdir is None: + topdir = self.tmpdir.dirpath() + return pytestConfig(comregistry, topdir=topdir) + + def finalize(self): + for p in self._syspathremove: + py.std.sys.path.remove(p) + if hasattr(self, '_olddir'): + self._olddir.chdir() + # delete modules that have been loaded from tmpdir + for name, mod in list(sys.modules.items()): + if mod: + fn = getattr(mod, '__file__', None) + if fn and fn.startswith(str(self.tmpdir)): + del sys.modules[name] + + def getreportrecorder(self, obj): + if isinstance(obj, py._com.Registry): + registry = obj + elif hasattr(obj, 'comregistry'): + registry = obj.comregistry + elif hasattr(obj, 'pluginmanager'): + registry = obj.pluginmanager.comregistry + elif hasattr(obj, 'config'): + registry = obj.config.pluginmanager.comregistry + else: + raise ValueError("obj %r provides no comregistry" %(obj,)) + assert isinstance(registry, py._com.Registry) + reprec = ReportRecorder(registry) + reprec.hookrecorder = self._pytest.gethookrecorder(hookspec, registry) + reprec.hook = reprec.hookrecorder.hook + return reprec + + def chdir(self): + old = self.tmpdir.chdir() + if not hasattr(self, '_olddir'): + self._olddir = old + + def _makefile(self, ext, args, kwargs): + items = list(kwargs.items()) + if args: + source = "\n".join(map(str, args)) + basename = self.request.function.__name__ + items.insert(0, (basename, source)) + ret = None + for name, value in items: + p = self.tmpdir.join(name).new(ext=ext) + source = py.code.Source(value) + p.write(str(py.code.Source(value)).lstrip()) + if ret is None: + ret = p + return ret + + + def makefile(self, ext, *args, **kwargs): + return self._makefile(ext, args, kwargs) + + def makeconftest(self, source): + return self.makepyfile(conftest=source) + + def makepyfile(self, *args, **kwargs): + return self._makefile('.py', args, kwargs) + + def maketxtfile(self, *args, **kwargs): + return self._makefile('.txt', args, kwargs) + + def syspathinsert(self, path=None): + if path is None: + path = self.tmpdir + py.std.sys.path.insert(0, str(path)) + self._syspathremove.append(str(path)) + + def mkdir(self, name): + return self.tmpdir.mkdir(name) + + def mkpydir(self, name): + p = self.mkdir(name) + p.ensure("__init__.py") + return p + + def genitems(self, colitems): + return list(self.session.genitems(colitems)) + + def inline_genitems(self, *args): + #config = self.parseconfig(*args) + config = self.parseconfig(*args) + session = config.initsession() + rec = self.getreportrecorder(config) + colitems = [config.getfsnode(arg) for arg in config.args] + items = list(session.genitems(colitems)) + return items, rec + + def runitem(self, source): + # used from runner functional tests + item = self.getitem(source) + # the test class where we are called from wants to provide the runner + testclassinstance = py.builtin._getimself(self.request.function) + runner = testclassinstance.getrunner() + return runner(item) + + def inline_runsource(self, source, *cmdlineargs): + p = self.makepyfile(source) + l = list(cmdlineargs) + [p] + return self.inline_run(*l) + + def inline_runsource1(self, *args): + args = list(args) + source = args.pop() + p = self.makepyfile(source) + l = list(args) + [p] + reprec = self.inline_run(*l) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 1, reports + return reports[0] + + def inline_run(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + session = config.initsession() + reprec = self.getreportrecorder(config) + session.main() + config.pluginmanager.do_unconfigure(config) + return reprec + + def config_preparse(self): + config = self.Config() + for plugin in self.plugins: + if isinstance(plugin, str): + config.pluginmanager.import_plugin(plugin) + else: + if isinstance(plugin, dict): + plugin = PseudoPlugin(plugin) + if not config.pluginmanager.isregistered(plugin): + config.pluginmanager.register(plugin) + return config + + def parseconfig(self, *args): + if not args: + args = (self.tmpdir,) + config = self.config_preparse() + args = list(args) + ["--basetemp=%s" % self.tmpdir.dirpath('basetemp')] + config.parse(args) + return config + + def parseconfigure(self, *args): + config = self.parseconfig(*args) + config.pluginmanager.do_configure(config) + return config + + def getitem(self, source, funcname="test_func"): + modcol = self.getmodulecol(source) + moditems = modcol.collect() + for item in modcol.collect(): + if item.name == funcname: + return item + else: + assert 0, "%r item not found in module:\n%s" %(funcname, source) + + def getitems(self, source): + modcol = self.getmodulecol(source) + return list(modcol.config.initsession().genitems([modcol])) + #assert item is not None, "%r item not found in module:\n%s" %(funcname, source) + #return item + + def getfscol(self, path, configargs=()): + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + return self.config.getfsnode(path) + + def getmodulecol(self, source, configargs=(), withinit=False): + kw = {self.request.function.__name__: py.code.Source(source).strip()} + path = self.makepyfile(**kw) + if withinit: + self.makepyfile(__init__ = "#") + self.config = self.parseconfig(path, *configargs) + self.session = self.config.initsession() + #self.config.pluginmanager.do_configure(config=self.config) + # XXX + self.config.pluginmanager.import_plugin("runner") + plugin = self.config.pluginmanager.getplugin("runner") + plugin.pytest_configure(config=self.config) + + return self.config.getfsnode(path) + + def prepare(self): + p = self.tmpdir.join("conftest.py") + if not p.check(): + plugins = [x for x in self.plugins if isinstance(x, str)] + if not plugins: + return + p.write("import py ; pytest_plugins = %r" % plugins) + else: + if self.plugins: + print ("warning, ignoring reusing existing %s" % p) + + def popen(self, cmdargs, stdout, stderr, **kw): + if not hasattr(py.std, 'subprocess'): + py.test.skip("no subprocess module") + env = os.environ.copy() + env['PYTHONPATH'] = ":".join(filter(None, [ + str(os.getcwd()), env.get('PYTHONPATH', '')])) + kw['env'] = env + #print "env", env + return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw) + + def run(self, *cmdargs): + self.prepare() + old = self.tmpdir.chdir() + #print "chdir", self.tmpdir + try: + return self._run(*cmdargs) + finally: + old.chdir() + + def _run(self, *cmdargs): + cmdargs = [str(x) for x in cmdargs] + p1 = py.path.local("stdout") + p2 = py.path.local("stderr") + print_("running", cmdargs, "curdir=", py.path.local()) + f1 = p1.open("w") + f2 = p2.open("w") + popen = self.popen(cmdargs, stdout=f1, stderr=f2, + close_fds=(sys.platform != "win32")) + ret = popen.wait() + f1.close() + f2.close() + out, err = p1.readlines(cr=0), p2.readlines(cr=0) + if err: + for line in err: + py.builtin.print_(line, file=sys.stderr) + if out: + for line in out: + py.builtin.print_(line, file=sys.stdout) + return RunResult(ret, out, err) + + def runpybin(self, scriptname, *args): + fullargs = self._getpybinargs(scriptname) + args + return self.run(*fullargs) + + def _getpybinargs(self, scriptname): + bindir = py._dir.dirpath('bin') + if not bindir.check(): + script = py.path.local.sysfind(scriptname) + else: + script = bindir.join(scriptname) + assert script.check() + return py.std.sys.executable, script + + def runpython(self, script): + return self.run(py.std.sys.executable, script) + + def runpytest(self, *args): + p = py.path.local.make_numbered_dir(prefix="runpytest-", + keep=None, rootdir=self.tmpdir) + args = ('--basetemp=%s' % p, ) + args + return self.runpybin("py.test", *args) + + def spawn_pytest(self, string, expect_timeout=10.0): + pexpect = py.test.importorskip("pexpect", "2.3") + basetemp = self.tmpdir.mkdir("pexpect") + invoke = "%s %s" % self._getpybinargs("py.test") + cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string) + child = pexpect.spawn(cmd, logfile=basetemp.join("spawn.out").open("w")) + child.timeout = expect_timeout + return child + +class PseudoPlugin: + def __init__(self, vars): + self.__dict__.update(vars) + +class ReportRecorder(object): + def __init__(self, comregistry): + self.comregistry = comregistry + comregistry.register(self) + + def getcall(self, name): + return self.hookrecorder.getcall(name) + + def popcall(self, name): + return self.hookrecorder.popcall(name) + + def getcalls(self, names): + """ return list of ParsedCall instances matching the given eventname. """ + return self.hookrecorder.getcalls(names) + + # functionality for test reports + + def getreports(self, names="pytest_runtest_logreport pytest_collectreport"): + return [x.report for x in self.getcalls(names)] + + def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport"): + """ return a testreport whose dotted import path matches """ + l = [] + for rep in self.getreports(names=names): + colitem = rep.getnode() + if not inamepart or inamepart in colitem.listnames(): + l.append(rep) + if not l: + raise ValueError("could not find test report matching %r: no test reports at all!" % + (inamepart,)) + if len(l) > 1: + raise ValueError("found more than one testreport matching %r: %s" %( + inamepart, l)) + return l[0] + + def getfailures(self, names='pytest_runtest_logreport pytest_collectreport'): + return [rep for rep in self.getreports(names) if rep.failed] + + def getfailedcollections(self): + return self.getfailures('pytest_collectreport') + + def listoutcomes(self): + passed = [] + skipped = [] + failed = [] + for rep in self.getreports("pytest_runtest_logreport"): + if rep.passed: + if rep.when == "call": + passed.append(rep) + elif rep.skipped: + skipped.append(rep) + elif rep.failed: + failed.append(rep) + return passed, skipped, failed + + def countoutcomes(self): + return [len(x) for x in self.listoutcomes()] + + def assertoutcome(self, passed=0, skipped=0, failed=0): + realpassed, realskipped, realfailed = self.listoutcomes() + assert passed == len(realpassed) + assert skipped == len(realskipped) + assert failed == len(realfailed) + + def clear(self): + self.hookrecorder.calls[:] = [] + + def unregister(self): + self.comregistry.unregister(self) + self.hookrecorder.finish_recording() + +class LineComp: + def __init__(self): + self.stringio = py.io.TextIO() + + def assert_contains_lines(self, lines2): + """ assert that lines2 are contained (linearly) in lines1. + return a list of extralines found. + """ + __tracebackhide__ = True + val = self.stringio.getvalue() + self.stringio.truncate(0) # remove what we got + lines1 = val.split("\n") + return LineMatcher(lines1).fnmatch_lines(lines2) + +class LineMatcher: + def __init__(self, lines): + self.lines = lines + + def str(self): + return "\n".join(self.lines) + + def fnmatch_lines(self, lines2): + if isinstance(lines2, str): + lines2 = py.code.Source(lines2) + if isinstance(lines2, py.code.Source): + lines2 = lines2.strip().lines + + from fnmatch import fnmatch + __tracebackhide__ = True + lines1 = self.lines[:] + nextline = None + extralines = [] + for line in lines2: + nomatchprinted = False + while lines1: + nextline = lines1.pop(0) + if line == nextline: + print_("exact match:", repr(line)) + break + elif fnmatch(nextline, line): + print_("fnmatch:", repr(line)) + print_(" with:", repr(nextline)) + break + else: + if not nomatchprinted: + print_("nomatch:", repr(line)) + nomatchprinted = True + print_(" and:", repr(nextline)) + extralines.append(nextline) + else: + if line != nextline: + #__tracebackhide__ = True + raise AssertionError("expected line not found: %r" % line) + extralines.extend(lines1) + return extralines Added: pypy/trunk/py/plugin/pytest_recwarn.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_recwarn.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,120 @@ +""" +helpers for asserting deprecation and other warnings. + +Example usage +--------------------- + +You can use the ``recwarn`` funcarg to track +warnings within a test function: + +.. sourcecode:: python + + def test_hello(recwarn): + from warnings import warn + warn("hello", DeprecationWarning) + w = recwarn.pop(DeprecationWarning) + assert issubclass(w.category, DeprecationWarning) + assert 'hello' in str(w.message) + assert w.filename + assert w.lineno + +You can also call a global helper for checking +taht a certain function call yields a Deprecation +warning: + +.. sourcecode:: python + + import py + + def test_global(): + py.test.deprecated_call(myfunction, 17) + + +""" + +import py +import os + +def pytest_funcarg__recwarn(request): + """Return a WarningsRecorder instance that provides these methods: + + * ``pop(category=None)``: return last warning matching the category. + * ``clear()``: clear list of warnings + """ + warnings = WarningsRecorder() + request.addfinalizer(warnings.finalize) + return warnings + +def pytest_namespace(): + return {'deprecated_call': deprecated_call} + +def deprecated_call(func, *args, **kwargs): + """ assert that calling func(*args, **kwargs) + triggers a DeprecationWarning. + """ + warningmodule = py.std.warnings + l = [] + oldwarn_explicit = getattr(warningmodule, 'warn_explicit') + def warn_explicit(*args, **kwargs): + l.append(args) + oldwarn_explicit(*args, **kwargs) + oldwarn = getattr(warningmodule, 'warn') + def warn(*args, **kwargs): + l.append(args) + oldwarn(*args, **kwargs) + + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + try: + ret = func(*args, **kwargs) + finally: + warningmodule.warn_explicit = warn_explicit + warningmodule.warn = warn + if not l: + #print warningmodule + raise AssertionError("%r did not produce DeprecationWarning" %(func,)) + return ret + + +class RecordedWarning: + def __init__(self, message, category, filename, lineno, line): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.line = line + +class WarningsRecorder: + def __init__(self): + warningmodule = py.std.warnings + self.list = [] + def showwarning(message, category, filename, lineno, line=0): + self.list.append(RecordedWarning( + message, category, filename, lineno, line)) + try: + self.old_showwarning(message, category, + filename, lineno, line=line) + except TypeError: + # < python2.6 + self.old_showwarning(message, category, filename, lineno) + self.old_showwarning = warningmodule.showwarning + warningmodule.showwarning = showwarning + + def pop(self, cls=Warning): + """ pop the first recorded warning, raise exception if not exists.""" + for i, w in enumerate(self.list): + if issubclass(w.category, cls): + return self.list.pop(i) + __tracebackhide__ = True + assert 0, "%r not found in %r" %(cls, self.list) + + #def resetregistry(self): + # import warnings + # warnings.onceregistry.clear() + # warnings.__warningregistry__.clear() + + def clear(self): + self.list[:] = [] + + def finalize(self): + py.std.warnings.showwarning = self.old_showwarning Added: pypy/trunk/py/plugin/pytest_restdoc.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_restdoc.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,432 @@ +""" +perform ReST syntax, local and remote reference tests on .rst/.txt files. +""" +import py +import sys, os, re + +def pytest_addoption(parser): + group = parser.getgroup("ReST", "ReST documentation check options") + group.addoption('-R', '--urlcheck', + action="store_true", dest="urlcheck", default=False, + help="urlopen() remote links found in ReST text files.") + group.addoption('--urltimeout', action="store", metavar="secs", + type="int", dest="urlcheck_timeout", default=5, + help="timeout in seconds for remote urlchecks") + group.addoption('--forcegen', + action="store_true", dest="forcegen", default=False, + help="force generation of html files.") + +def pytest_collect_file(path, parent): + if path.ext in (".txt", ".rst"): + project = getproject(path) + if project is not None: + return ReSTFile(path, parent=parent, project=project) + +def getproject(path): + for parent in path.parts(reverse=True): + confrest = parent.join("confrest.py") + if confrest.check(): + Project = confrest.pyimport().Project + return Project(parent) + +class ReSTFile(py.test.collect.File): + def __init__(self, fspath, parent, project=None): + super(ReSTFile, self).__init__(fspath=fspath, parent=parent) + if project is None: + project = getproject(fspath) + assert project is not None + self.project = project + + def collect(self): + return [ + ReSTSyntaxTest(self.project, "ReSTSyntax", parent=self), + LinkCheckerMaker("checklinks", parent=self), + DoctestText("doctest", parent=self), + ] + +def deindent(s, sep='\n'): + leastspaces = -1 + lines = s.split(sep) + for line in lines: + if not line.strip(): + continue + spaces = len(line) - len(line.lstrip()) + if leastspaces == -1 or spaces < leastspaces: + leastspaces = spaces + if leastspaces == -1: + return s + for i, line in enumerate(lines): + if not line.strip(): + lines[i] = '' + else: + lines[i] = line[leastspaces:] + return sep.join(lines) + +class ReSTSyntaxTest(py.test.collect.Item): + def __init__(self, project, *args, **kwargs): + super(ReSTSyntaxTest, self).__init__(*args, **kwargs) + self.project = project + + def reportinfo(self): + return self.fspath, None, "syntax check" + + def runtest(self): + self.restcheck(py.path.svnwc(self.fspath)) + + def restcheck(self, path): + py.test.importorskip("docutils") + self.register_linkrole() + from docutils.utils import SystemMessage + try: + self._checkskip(path, self.project.get_htmloutputpath(path)) + self.project.process(path) + except KeyboardInterrupt: + raise + except SystemMessage: + # we assume docutils printed info on stdout + py.test.fail("docutils processing failed, see captured stderr") + + def register_linkrole(self): + #directive.register_linkrole('api', self.resolve_linkrole) + #directive.register_linkrole('source', self.resolve_linkrole) +# +# # XXX fake sphinx' "toctree" and refs +# directive.register_linkrole('ref', self.resolve_linkrole) + + from docutils.parsers.rst import directives + def toctree_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + toctree_directive.content = 1 + toctree_directive.options = {'maxdepth': int, 'glob': directives.flag, + 'hidden': directives.flag} + directives.register_directive('toctree', toctree_directive) + self.register_pygments() + + def register_pygments(self): + # taken from pygments-main/external/rst-directive.py + from docutils.parsers.rst import directives + try: + from pygments.formatters import HtmlFormatter + except ImportError: + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + return [] + pygments_directive.options = {} + else: + # The default formatter + DEFAULT = HtmlFormatter(noclasses=True) + # Add name -> formatter pairs for every variant you want to use + VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), + } + + from docutils import nodes + + from pygments import highlight + from pygments.lexers import get_lexer_by_name, TextLexer + + def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + try: + lexer = get_lexer_by_name(arguments[0]) + except ValueError: + # no lexer found - use the text one instead of an exception + lexer = TextLexer() + # take an arbitrary option if more than one is given + formatter = options and VARIANTS[options.keys()[0]] or DEFAULT + parsed = highlight('\n'.join(content), lexer, formatter) + return [nodes.raw('', parsed, format='html')] + + pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS]) + + pygments_directive.arguments = (1, 0, 1) + pygments_directive.content = 1 + directives.register_directive('sourcecode', pygments_directive) + + def resolve_linkrole(self, name, text, check=True): + apigen_relpath = self.project.apigen_relpath + + if name == 'api': + if text == 'py': + return ('py', apigen_relpath + 'api/index.html') + else: + assert text.startswith('py.'), ( + 'api link "%s" does not point to the py package') % (text,) + dotted_name = text + if dotted_name.find('(') > -1: + dotted_name = dotted_name[:text.find('(')] + # remove pkg root + path = dotted_name.split('.')[1:] + dotted_name = '.'.join(path) + obj = py + if check: + for chunk in path: + try: + obj = getattr(obj, chunk) + except AttributeError: + raise AssertionError( + 'problem with linkrole :api:`%s`: can not resolve ' + 'dotted name %s' % (text, dotted_name,)) + return (text, apigen_relpath + 'api/%s.html' % (dotted_name,)) + elif name == 'source': + assert text.startswith('py/'), ('source link "%s" does not point ' + 'to the py package') % (text,) + relpath = '/'.join(text.split('/')[1:]) + if check: + pkgroot = py._impldir + abspath = pkgroot.join(relpath) + assert pkgroot.join(relpath).check(), ( + 'problem with linkrole :source:`%s`: ' + 'path %s does not exist' % (text, relpath)) + if relpath.endswith('/') or not relpath: + relpath += 'index.html' + else: + relpath += '.html' + return (text, apigen_relpath + 'source/%s' % (relpath,)) + elif name == 'ref': + return ("", "") + + def _checkskip(self, lpath, htmlpath=None): + if not self.config.getvalue("forcegen"): + lpath = py.path.local(lpath) + if htmlpath is not None: + htmlpath = py.path.local(htmlpath) + if lpath.ext == '.txt': + htmlpath = htmlpath or lpath.new(ext='.html') + if htmlpath.check(file=1) and htmlpath.mtime() >= lpath.mtime(): + py.test.skip("html file is up to date, use --forcegen to regenerate") + #return [] # no need to rebuild + +class DoctestText(py.test.collect.Item): + def reportinfo(self): + return self.fspath, None, "doctest" + + def runtest(self): + content = self._normalize_linesep() + newcontent = self.config.hook.pytest_doctest_prepare_content(content=content) + if newcontent is not None: + content = newcontent + s = content + l = [] + prefix = '.. >>> ' + mod = py.std.types.ModuleType(self.fspath.purebasename) + skipchunk = False + for line in deindent(s).split('\n'): + stripped = line.strip() + if skipchunk and line.startswith(skipchunk): + py.builtin.print_("skipping", line) + continue + skipchunk = False + if stripped.startswith(prefix): + try: + py.builtin.exec_(py.code.Source( + stripped[len(prefix):]).compile(), mod.__dict__) + except ValueError: + e = sys.exc_info()[1] + if e.args and e.args[0] == "skipchunk": + skipchunk = " " * (len(line) - len(line.lstrip())) + else: + raise + else: + l.append(line) + docstring = "\n".join(l) + mod.__doc__ = docstring + failed, tot = py.std.doctest.testmod(mod, verbose=1) + if failed: + py.test.fail("doctest %s: %s failed out of %s" %( + self.fspath, failed, tot)) + + def _normalize_linesep(self): + # XXX quite nasty... but it works (fixes win32 issues) + s = self.fspath.read() + linesep = '\n' + if '\r' in s: + if '\n' not in s: + linesep = '\r' + else: + linesep = '\r\n' + s = s.replace(linesep, '\n') + return s + +class LinkCheckerMaker(py.test.collect.Collector): + def collect(self): + return list(self.genlinkchecks()) + + def genlinkchecks(self): + path = self.fspath + # generating functions + args as single tests + timeout = self.config.getvalue("urlcheck_timeout") + for lineno, line in enumerate(path.readlines()): + line = line.strip() + if line.startswith('.. _'): + if line.startswith('.. _`'): + delim = '`:' + else: + delim = ':' + l = line.split(delim, 1) + if len(l) != 2: + continue + tryfn = l[1].strip() + name = "%s:%d" %(tryfn, lineno) + if tryfn.startswith('http:') or tryfn.startswith('https'): + if self.config.getvalue("urlcheck"): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno, timeout), checkfunc=urlcheck) + elif tryfn.startswith('webcal:'): + continue + else: + i = tryfn.find('#') + if i != -1: + checkfn = tryfn[:i] + else: + checkfn = tryfn + if checkfn.strip() and (1 or checkfn.endswith('.html')): + yield CheckLink(name, parent=self, + args=(tryfn, path, lineno), checkfunc=localrefcheck) + +class CheckLink(py.test.collect.Item): + def __init__(self, name, parent, args, checkfunc): + super(CheckLink, self).__init__(name, parent) + self.args = args + self.checkfunc = checkfunc + + def runtest(self): + return self.checkfunc(*self.args) + + def reportinfo(self, basedir=None): + return (self.fspath, self.args[2], "checklink: %s" % self.args[0]) + +def urlcheck(tryfn, path, lineno, TIMEOUT_URLOPEN): + old = py.std.socket.getdefaulttimeout() + py.std.socket.setdefaulttimeout(TIMEOUT_URLOPEN) + try: + try: + py.builtin.print_("trying remote", tryfn) + py.std.urllib2.urlopen(tryfn) + finally: + py.std.socket.setdefaulttimeout(old) + except (py.std.urllib2.URLError, py.std.urllib2.HTTPError): + e = sys.exc_info()[1] + if getattr(e, 'code', None) in (401, 403): # authorization required, forbidden + py.test.skip("%s: %s" %(tryfn, str(e))) + else: + py.test.fail("remote reference error %r in %s:%d\n%s" %( + tryfn, path.basename, lineno+1, e)) + +def localrefcheck(tryfn, path, lineno): + # assume it should be a file + i = tryfn.find('#') + if tryfn.startswith('javascript:'): + return # don't check JS refs + if i != -1: + anchor = tryfn[i+1:] + tryfn = tryfn[:i] + else: + anchor = '' + fn = path.dirpath(tryfn) + ishtml = fn.ext == '.html' + fn = ishtml and fn.new(ext='.txt') or fn + py.builtin.print_("filename is", fn) + if not fn.check(): # not ishtml or not fn.check(): + if not py.path.local(tryfn).check(): # the html could be there + py.test.fail("reference error %r in %s:%d" %( + tryfn, path.basename, lineno+1)) + if anchor: + source = unicode(fn.read(), 'latin1') + source = source.lower().replace('-', ' ') # aehem + + anchor = anchor.replace('-', ' ') + match2 = ".. _`%s`:" % anchor + match3 = ".. _%s:" % anchor + candidates = (anchor, match2, match3) + py.builtin.print_("candidates", repr(candidates)) + for line in source.split('\n'): + line = line.strip() + if line in candidates: + break + else: + py.test.fail("anchor reference error %s#%s in %s:%d" %( + tryfn, anchor, path.basename, lineno+1)) + +if hasattr(sys.stdout, 'fileno') and os.isatty(sys.stdout.fileno()): + def log(msg): + print(msg) +else: + def log(msg): + pass + +def convert_rest_html(source, source_path, stylesheet=None, encoding='latin1'): + """ return html latin1-encoded document for the given input. + source a ReST-string + sourcepath where to look for includes (basically) + stylesheet path (to be used if any) + """ + from docutils.core import publish_string + kwargs = { + 'stylesheet' : stylesheet, + 'stylesheet_path': None, + 'traceback' : 1, + 'embed_stylesheet': 0, + 'output_encoding' : encoding, + #'halt' : 0, # 'info', + 'halt_level' : 2, + } + # docutils uses os.getcwd() :-( + source_path = os.path.abspath(str(source_path)) + prevdir = os.getcwd() + try: + #os.chdir(os.path.dirname(source_path)) + return publish_string(source, source_path, writer_name='html', + settings_overrides=kwargs) + finally: + os.chdir(prevdir) + +def process(txtpath, encoding='latin1'): + """ process a textfile """ + log("processing %s" % txtpath) + assert txtpath.check(ext='.txt') + if isinstance(txtpath, py.path.svnwc): + txtpath = txtpath.localpath + htmlpath = txtpath.new(ext='.html') + #svninfopath = txtpath.localpath.new(ext='.svninfo') + + style = txtpath.dirpath('style.css') + if style.check(): + stylesheet = style.basename + else: + stylesheet = None + content = unicode(txtpath.read(), encoding) + doc = convert_rest_html(content, txtpath, stylesheet=stylesheet, encoding=encoding) + htmlpath.open('wb').write(doc) + #log("wrote %r" % htmlpath) + #if txtpath.check(svnwc=1, versioned=1): + # info = txtpath.info() + # svninfopath.dump(info) + +if sys.version_info > (3, 0): + def _uni(s): return s +else: + def _uni(s): + return unicode(s) + +rex1 = re.compile(r'.*(.*).*', re.MULTILINE | re.DOTALL) +rex2 = re.compile(r'.*
(.*)
.*', re.MULTILINE | re.DOTALL) + +def strip_html_header(string, encoding='utf8'): + """ return the content of the body-tag """ + uni = unicode(string, encoding) + for rex in rex1,rex2: + match = rex.search(uni) + if not match: + break + uni = match.group(1) + return uni + +class Project: # used for confrest.py files + def __init__(self, sourcepath): + self.sourcepath = sourcepath + def process(self, path): + return process(path) + def get_htmloutputpath(self, path): + return path.new(ext='html') Added: pypy/trunk/py/plugin/pytest_resultlog.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_resultlog.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,94 @@ +"""resultlog plugin for machine-readable logging of test results. + Useful for buildbot integration code. +""" + +import py +from py.builtin import print_ + +def pytest_addoption(parser): + group = parser.getgroup("resultlog", "resultlog plugin options") + group.addoption('--resultlog', action="store", dest="resultlog", metavar="path", default=None, + help="path for machine-readable result log.") + +def pytest_configure(config): + resultlog = config.option.resultlog + if resultlog: + logfile = open(resultlog, 'w', 1) # line buffered + config._resultlog = ResultLog(config, logfile) + config.pluginmanager.register(config._resultlog) + +def pytest_unconfigure(config): + resultlog = getattr(config, '_resultlog', None) + if resultlog: + resultlog.logfile.close() + del config._resultlog + config.pluginmanager.unregister(resultlog) + +def generic_path(item): + chain = item.listchain() + gpath = [chain[0].name] + fspath = chain[0].fspath + fspart = False + for node in chain[1:]: + newfspath = node.fspath + if newfspath == fspath: + if fspart: + gpath.append(':') + fspart = False + else: + gpath.append('.') + else: + gpath.append('/') + fspart = True + name = node.name + if name[0] in '([': + gpath.pop() + gpath.append(name) + fspath = newfspath + return ''.join(gpath) + +class ResultLog(object): + def __init__(self, config, logfile): + self.config = config + self.logfile = logfile # preferably line buffered + + def write_log_entry(self, testpath, shortrepr, longrepr): + print_("%s %s" % (shortrepr, testpath), file=self.logfile) + for line in longrepr.splitlines(): + print_(" %s" % line, file=self.logfile) + + def log_outcome(self, node, shortrepr, longrepr): + testpath = generic_path(node) + self.write_log_entry(testpath, shortrepr, longrepr) + + def pytest_runtest_logreport(self, report): + res = self.config.hook.pytest_report_teststatus(report=report) + if res is not None: + code = res[1] + else: + code = report.shortrepr + if code == 'x': + longrepr = str(report.longrepr) + elif code == 'P': + longrepr = '' + elif report.passed: + longrepr = "" + elif report.failed: + longrepr = str(report.longrepr) + elif report.skipped: + longrepr = str(report.longrepr.reprcrash.message) + self.log_outcome(report.item, code, longrepr) + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + code = "F" + else: + assert report.skipped + code = "S" + longrepr = str(report.longrepr.reprcrash) + self.log_outcome(report.collector, code, longrepr) + + def pytest_internalerror(self, excrepr): + path = excrepr.reprcrash.path + self.write_log_entry(path, '!', str(excrepr)) Added: pypy/trunk/py/plugin/pytest_runner.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_runner.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,294 @@ +""" +collect and run test items and create reports. +""" + +import py +from py.impl.test.outcome import Skipped + +# +# pytest plugin hooks + +def pytest_addoption(parser): + group = parser.getgroup("general") + group.addoption('--boxed', + action="store_true", dest="boxed", default=False, + help="box each test run in a separate process") + +# XXX move to pytest_sessionstart and fix py.test owns tests +def pytest_configure(config): + config._setupstate = SetupState() + +def pytest_sessionfinish(session, exitstatus): + if hasattr(session.config, '_setupstate'): + hook = session.config.hook + rep = hook.pytest__teardown_final(session=session) + if rep: + hook.pytest__teardown_final_logerror(report=rep) + +def pytest_make_collect_report(collector): + result = excinfo = None + try: + result = collector._memocollect() + except KeyboardInterrupt: + raise + except: + excinfo = py.code.ExceptionInfo() + return CollectReport(collector, result, excinfo) + +def pytest_runtest_protocol(item): + if item.config.getvalue("boxed"): + reports = forked_run_report(item) + for rep in reports: + item.config.hook.pytest_runtest_logreport(report=rep) + else: + runtestprotocol(item) + return True + +def runtestprotocol(item, log=True): + rep = call_and_report(item, "setup", log) + reports = [rep] + if rep.passed: + reports.append(call_and_report(item, "call", log)) + reports.append(call_and_report(item, "teardown", log)) + return reports + +def pytest_runtest_setup(item): + item.config._setupstate.prepare(item) + +def pytest_runtest_call(item): + if not item._deprecated_testexecution(): + item.runtest() + +def pytest_runtest_makereport(item, call): + return ItemTestReport(item, call.excinfo, call.when) + +def pytest_runtest_teardown(item): + item.config._setupstate.teardown_exact(item) + +def pytest__teardown_final(session): + call = CallInfo(session.config._setupstate.teardown_all, when="teardown") + if call.excinfo: + rep = TeardownErrorReport(call.excinfo) + return rep + +def pytest_report_teststatus(report): + if report.when in ("setup", "teardown"): + if report.failed: + # category, shortletter, verbose-word + return "error", "E", "ERROR" + elif report.skipped: + return "skipped", "s", "SKIPPED" + else: + return "", "", "" +# +# Implementation + +def call_and_report(item, when, log=True): + call = call_runtest_hook(item, when) + hook = item.config.hook + report = hook.pytest_runtest_makereport(item=item, call=call) + if log and (when == "call" or not report.passed): + hook.pytest_runtest_logreport(report=report) + return report + +def call_runtest_hook(item, when): + hookname = "pytest_runtest_" + when + hook = getattr(item.config.hook, hookname) + return CallInfo(lambda: hook(item=item), when=when) + +class CallInfo: + excinfo = None + def __init__(self, func, when): + self.when = when + try: + self.result = func() + except KeyboardInterrupt: + raise + except: + self.excinfo = py.code.ExceptionInfo() + + def __repr__(self): + if self.excinfo: + status = "exception: %s" % str(self.excinfo.value) + else: + status = "result: %r" % (self.result,) + return "" % (self.when, status) + +def forked_run_report(item): + # for now, we run setup/teardown in the subprocess + # XXX optionally allow sharing of setup/teardown + EXITSTATUS_TESTEXIT = 4 + from py.impl.test.dist.mypickle import ImmutablePickler + ipickle = ImmutablePickler(uneven=0) + ipickle.selfmemoize(item.config) + # XXX workaround the issue that 2.6 cannot pickle + # instances of classes defined in global conftest.py files + ipickle.selfmemoize(item) + def runforked(): + try: + reports = runtestprotocol(item, log=False) + except KeyboardInterrupt: + py.std.os._exit(EXITSTATUS_TESTEXIT) + return ipickle.dumps(reports) + + ff = py.process.ForkedFunc(runforked) + result = ff.waitfinish() + if result.retval is not None: + return ipickle.loads(result.retval) + else: + if result.exitstatus == EXITSTATUS_TESTEXIT: + py.test.exit("forked test item %s raised Exit" %(item,)) + return [report_process_crash(item, result)] + +def report_process_crash(item, result): + path, lineno = item._getfslineno() + info = "%s:%s: running the test CRASHED with signal %d" %( + path, lineno, result.signal) + return ItemTestReport(item, excinfo=info, when="???") + +class BaseReport(object): + def __repr__(self): + l = ["%s=%s" %(key, value) + for key, value in self.__dict__.items()] + return "<%s %s>" %(self.__class__.__name__, " ".join(l),) + + def toterminal(self, out): + longrepr = self.longrepr + if hasattr(longrepr, 'toterminal'): + longrepr.toterminal(out) + else: + out.line(str(longrepr)) + +class ItemTestReport(BaseReport): + failed = passed = skipped = False + + def __init__(self, item, excinfo=None, when=None): + self.item = item + self.when = when + if item and when != "setup": + self.keywords = item.readkeywords() + else: + # if we fail during setup it might mean + # we are not able to access the underlying object + # this might e.g. happen if we are unpickled + # and our parent collector did not collect us + # (because it e.g. skipped for platform reasons) + self.keywords = {} + if not excinfo: + self.passed = True + self.shortrepr = "." + else: + if not isinstance(excinfo, py.code.ExceptionInfo): + self.failed = True + shortrepr = "?" + longrepr = excinfo + elif excinfo.errisinstance(Skipped): + self.skipped = True + shortrepr = "s" + longrepr = self.item._repr_failure_py(excinfo) + else: + self.failed = True + shortrepr = self.item.shortfailurerepr + if self.when == "call": + longrepr = self.item.repr_failure(excinfo) + else: # exception in setup or teardown + longrepr = self.item._repr_failure_py(excinfo) + shortrepr = shortrepr.lower() + self.shortrepr = shortrepr + self.longrepr = longrepr + + def __repr__(self): + status = (self.passed and "passed" or + self.skipped and "skipped" or + self.failed and "failed" or + "CORRUPT") + l = [repr(self.item.name), "when=%r" % self.when, "outcome %r" % status,] + if hasattr(self, 'node'): + l.append("txnode=%s" % self.node.gateway.id) + info = " " .join(map(str, l)) + return "" % info + + def getnode(self): + return self.item + +class CollectReport(BaseReport): + skipped = failed = passed = False + + def __init__(self, collector, result, excinfo=None): + self.collector = collector + if not excinfo: + self.passed = True + self.result = result + else: + self.longrepr = self.collector._repr_failure_py(excinfo) + if excinfo.errisinstance(Skipped): + self.skipped = True + self.reason = str(excinfo.value) + else: + self.failed = True + + def getnode(self): + return self.collector + +class TeardownErrorReport(BaseReport): + skipped = passed = False + failed = True + when = "teardown" + def __init__(self, excinfo): + self.longrepr = excinfo.getrepr(funcargs=True) + +class SetupState(object): + """ shared state for setting up/tearing down test items or collectors. """ + def __init__(self): + self.stack = [] + self._finalizers = {} + + def addfinalizer(self, finalizer, colitem): + """ attach a finalizer to the given colitem. + if colitem is None, this will add a finalizer that + is called at the end of teardown_all(). + """ + assert hasattr(finalizer, '__call__') + #assert colitem in self.stack + self._finalizers.setdefault(colitem, []).append(finalizer) + + def _pop_and_teardown(self): + colitem = self.stack.pop() + self._teardown_with_finalization(colitem) + + def _callfinalizers(self, colitem): + finalizers = self._finalizers.pop(colitem, None) + while finalizers: + fin = finalizers.pop() + fin() + + def _teardown_with_finalization(self, colitem): + self._callfinalizers(colitem) + if colitem: + colitem.teardown() + for colitem in self._finalizers: + assert colitem is None or colitem in self.stack + + def teardown_all(self): + while self.stack: + self._pop_and_teardown() + self._teardown_with_finalization(None) + assert not self._finalizers + + def teardown_exact(self, item): + if self.stack and item == self.stack[-1]: + self._pop_and_teardown() + else: + self._callfinalizers(item) + + def prepare(self, colitem): + """ setup objects along the collector chain to the test-method + and teardown previously setup objects.""" + needed_collectors = colitem.listchain() + while self.stack: + if self.stack == needed_collectors[:len(self.stack)]: + break + self._pop_and_teardown() + for col in needed_collectors[len(self.stack):]: + col.setup() + self.stack.append(col) Added: pypy/trunk/py/plugin/pytest_skipping.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_skipping.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,232 @@ +""" +advanced skipping for python test functions, classes or modules. + +With this plugin you can mark test functions for conditional skipping +or as "xfail", expected-to-fail. Skipping a test will avoid running it +at all while xfail-marked tests will run and result in an inverted outcome: +a pass becomes a failure and a fail becomes a semi-passing one. + +The need for skipping a test is usually connected to a condition. +If a test fails under all conditions then it's probably better +to mark your test as 'xfail'. + +By passing ``--report=xfailed,skipped`` to the terminal reporter +you will see summary information on skips and xfail-run tests +at the end of a test run. + +.. _skipif: + +Skipping a single function +------------------------------------------- + +Here is an example for marking a test function to be skipped +when run on a Python3 interpreter:: + + @py.test.mark.skipif("sys.version_info >= (3,0)") + def test_function(): + ... + +During test function setup the skipif condition is +evaluated by calling ``eval(expr, namespace)``. The namespace +contains the ``sys`` and ``os`` modules and the test +``config`` object. The latter allows you to skip based +on a test configuration value e.g. like this:: + + @py.test.mark.skipif("not config.getvalue('db')") + def test_function(...): + ... + +Create a shortcut for your conditional skip decorator +at module level like this:: + + win32only = py.test.mark.skipif("sys.platform != 'win32'") + + @win32only + def test_function(): + ... + + +skip groups of test functions +-------------------------------------- + +As with all metadata function marking you can do it at +`whole class- or module level`_. Here is an example +for skipping all methods of a test class based on platform:: + + class TestPosixCalls: + pytestmark = py.test.mark.skipif("sys.platform == 'win32'") + + def test_function(self): + # will not be setup or run under 'win32' platform + # + +The ``pytestmark`` decorator will be applied to each test function. + +.. _`whole class- or module level`: mark.html#scoped-marking + + +mark a test function as **expected to fail** +------------------------------------------------------- + +You can use the ``xfail`` marker to indicate that you +expect the test to fail:: + + @py.test.mark.xfail + def test_function(): + ... + +This test will be run but no traceback will be reported +when it fails. Instead terminal reporting will list it in the +"expected to fail" or "unexpectedly passing" sections. + +Same as with skipif_ you can also selectively expect a failure +depending on platform:: + + @py.test.mark.xfail("sys.version_info >= (3,0)") + + def test_function(): + ... + + +skipping on a missing import dependency +-------------------------------------------------- + +You can use the following import helper at module level +or within a test or test setup function:: + + docutils = py.test.importorskip("docutils") + +If ``docutils`` cannot be imported here, this will lead to a +skip outcome of the test. You can also skip dependeing if +if a library does not come with a high enough version:: + + docutils = py.test.importorskip("docutils", minversion="0.3") + +The version will be read from the specified module's ``__version__`` attribute. + +imperative skip from within a test or setup function +------------------------------------------------------ + +If for some reason you cannot declare skip-conditions +you can also imperatively produce a Skip-outcome from +within test or setup code. Example:: + + def test_function(): + if not valid_config(): + py.test.skip("unsuppored configuration") + +""" +# XXX py.test.skip, .importorskip and the Skipped class +# should also be defined in this plugin, requires thought/changes + +import py + +def pytest_runtest_setup(item): + expr, result = evalexpression(item, 'skipif') + if result: + py.test.skip(expr) + +def pytest_runtest_makereport(__multicall__, item, call): + if call.when != "call": + return + expr, result = evalexpression(item, 'xfail') + rep = __multicall__.execute() + if result: + if call.excinfo: + rep.skipped = True + rep.failed = rep.passed = False + else: + rep.skipped = rep.passed = False + rep.failed = True + rep.keywords['xfail'] = expr + else: + if 'xfail' in rep.keywords: + del rep.keywords['xfail'] + return rep + +# called by terminalreporter progress reporting +def pytest_report_teststatus(report): + if 'xfail' in report.keywords: + if report.skipped: + return "xfailed", "x", "xfail" + elif report.failed: + return "xpassed", "P", "xpass" + +# called by the terminalreporter instance/plugin +def pytest_terminal_summary(terminalreporter): + show_xfailed(terminalreporter) + show_skipped(terminalreporter) + +def show_xfailed(terminalreporter): + tr = terminalreporter + xfailed = tr.stats.get("xfailed") + if xfailed: + if not tr.hasopt('xfailed'): + if tr.config.getvalue("verbose"): + tr.write_line( + "%d expected failures, use --report=xfailed for more info" % + len(xfailed)) + return + tr.write_sep("_", "expected failures") + for rep in xfailed: + entry = rep.longrepr.reprcrash + modpath = rep.item.getmodpath(includemodule=True) + pos = "%s %s:%d: " %(modpath, entry.path, entry.lineno) + reason = rep.longrepr.reprcrash.message + i = reason.find("\n") + if i != -1: + reason = reason[:i] + tr._tw.line("%s %s" %(pos, reason)) + + xpassed = terminalreporter.stats.get("xpassed") + if xpassed: + tr.write_sep("_", "UNEXPECTEDLY PASSING TESTS") + for rep in xpassed: + fspath, lineno, modpath = rep.item.reportinfo() + pos = "%s %s:%d: unexpectedly passing" %(modpath, fspath, lineno) + tr._tw.line(pos) + + +def evalexpression(item, keyword): + if isinstance(item, py.test.collect.Function): + markholder = getattr(item.obj, keyword, None) + result = False + if markholder: + d = {'os': py.std.os, 'sys': py.std.sys, 'config': item.config} + expr, result = None, True + for expr in markholder.args: + if isinstance(expr, str): + result = eval(expr, d) + else: + result = expr + if not result: + break + return expr, result + return None, False + +def folded_skips(skipped): + d = {} + for event in skipped: + entry = event.longrepr.reprcrash + key = entry.path, entry.lineno, entry.message + d.setdefault(key, []).append(event) + l = [] + for key, events in d.items(): + l.append((len(events),) + key) + return l + +def show_skipped(terminalreporter): + tr = terminalreporter + skipped = tr.stats.get('skipped', []) + if skipped: + if not tr.hasopt('skipped'): + if tr.config.getvalue("verbose"): + tr.write_line( + "%d skipped tests, use --report=skipped for more info" % + len(skipped)) + return + fskips = folded_skips(skipped) + if fskips: + tr.write_sep("_", "skipped test summary") + for num, fspath, lineno, reason in fskips: + tr._tw.line("%s:%d: [%d] %s" %(fspath, lineno, num, reason)) Added: pypy/trunk/py/plugin/pytest_terminal.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_terminal.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,468 @@ +""" +Implements terminal reporting of the full testing process. + +This is a good source for looking at the various reporting hooks. +""" +import py +import sys + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", after="general") + group._addoption('-v', '--verbose', action="count", + dest="verbose", default=0, help="increase verbosity."), + group._addoption('-l', '--showlocals', + action="store_true", dest="showlocals", default=False, + help="show locals in tracebacks (disabled by default).") + group.addoption('--report', + action="store", dest="report", default=None, metavar="opts", + help="comma separated reporting options") + group._addoption('--tb', metavar="style", + action="store", dest="tbstyle", default='long', + type="choice", choices=['long', 'short', 'no'], + help="traceback verboseness (long/short/no).") + group._addoption('--fulltrace', + action="store_true", dest="fulltrace", default=False, + help="don't cut any tracebacks (default is to cut).") + + group = parser.getgroup("debugconfig") + group.addoption('--collectonly', + action="store_true", dest="collectonly", + help="only collect tests, don't execute them."), + group.addoption('--traceconfig', + action="store_true", dest="traceconfig", default=False, + help="trace considerations of conftest.py files."), + group._addoption('--nomagic', + action="store_true", dest="nomagic", default=False, + help="don't reinterpret asserts, no traceback cutting. ") + group.addoption('--debug', + action="store_true", dest="debug", default=False, + help="generate and show internal debugging information.") + + +def pytest_configure(config): + if config.option.collectonly: + reporter = CollectonlyReporter(config) + else: + reporter = TerminalReporter(config) + # XXX see remote.py's XXX + for attr in 'pytest_terminal_hasmarkup', 'pytest_terminal_fullwidth': + if hasattr(config, attr): + #print "SETTING TERMINAL OPTIONS", attr, getattr(config, attr) + name = attr.split("_")[-1] + assert hasattr(self.reporter._tw, name), name + setattr(reporter._tw, name, getattr(config, attr)) + config.pluginmanager.register(reporter, 'terminalreporter') + +def getreportopt(optvalue): + d = {} + if optvalue: + for setting in optvalue.split(","): + setting = setting.strip() + val = True + if setting.startswith("no"): + val = False + setting = setting[2:] + d[setting] = val + return d + +class TerminalReporter: + def __init__(self, config, file=None): + self.config = config + self.stats = {} + self.curdir = py.path.local() + if file is None: + file = py.std.sys.stdout + self._tw = py.io.TerminalWriter(file) + self.currentfspath = None + self.gateway2info = {} + self._reportopt = getreportopt(config.getvalue('report')) + + def hasopt(self, name): + return self._reportopt.get(name, False) + + def write_fspath_result(self, fspath, res): + fspath = self.curdir.bestrelpath(fspath) + if fspath != self.currentfspath: + self._tw.line() + relpath = self.curdir.bestrelpath(fspath) + self._tw.write(relpath + " ") + self.currentfspath = fspath + self._tw.write(res) + + def write_ensure_prefix(self, prefix, extra="", **kwargs): + if self.currentfspath != prefix: + self._tw.line() + self.currentfspath = prefix + self._tw.write(prefix) + if extra: + self._tw.write(extra, **kwargs) + self.currentfspath = -2 + + def ensure_newline(self): + if self.currentfspath: + self._tw.line() + self.currentfspath = None + + def write_line(self, line, **markup): + line = str(line) + self.ensure_newline() + self._tw.line(line, **markup) + + def write_sep(self, sep, title=None, **markup): + self.ensure_newline() + self._tw.sep(sep, title, **markup) + + def getcategoryletterword(self, rep): + res = self.config.hook.pytest_report_teststatus(report=rep) + if res: + return res + for cat in 'skipped failed passed ???'.split(): + if getattr(rep, cat, None): + break + return cat, self.getoutcomeletter(rep), self.getoutcomeword(rep) + + def getoutcomeletter(self, rep): + return rep.shortrepr + + def getoutcomeword(self, rep): + if rep.passed: + return "PASS", dict(green=True) + elif rep.failed: + return "FAIL", dict(red=True) + elif rep.skipped: + return "SKIP" + else: + return "???", dict(red=True) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.write_line("INTERNALERROR> " + line) + + def pytest_gwmanage_newgateway(self, gateway, platinfo): + #self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec)) + d = {} + d['version'] = repr_pythonversion(platinfo.version_info) + d['id'] = gateway.id + d['spec'] = gateway.spec._spec + d['platform'] = platinfo.platform + if self.config.option.verbose: + d['extra'] = "- " + platinfo.executable + else: + d['extra'] = "" + d['cwd'] = platinfo.cwd + infoline = ("%(id)s %(spec)s -- platform %(platform)s, " + "Python %(version)s " + "cwd: %(cwd)s" + "%(extra)s" % d) + self.write_line(infoline) + self.gateway2info[gateway] = infoline + + def pytest_gwmanage_rsyncstart(self, source, gateways): + targets = ", ".join([gw.id for gw in gateways]) + msg = "rsyncstart: %s -> %s" %(source, targets) + if not self.config.option.verbose: + msg += " # use --verbose to see rsync progress" + self.write_line(msg) + + def pytest_gwmanage_rsyncfinish(self, source, gateways): + targets = ", ".join([gw.id for gw in gateways]) + self.write_line("rsyncfinish: %s -> %s" %(source, targets)) + + def pytest_plugin_registered(self, plugin): + if self.config.option.traceconfig: + msg = "PLUGIN registered: %s" %(plugin,) + # XXX this event may happen during setup/teardown time + # which unfortunately captures our output here + # which garbles our output if we use self.write_line + self.write_line(msg) + + def pytest_testnodeready(self, node): + self.write_line("%s txnode ready to receive tests" %(node.gateway.id,)) + + def pytest_testnodedown(self, node, error): + if error: + self.write_line("%s node down, error: %s" %(node.gateway.id, error)) + + def pytest_trace(self, category, msg): + if self.config.option.debug or \ + self.config.option.traceconfig and category.find("config") != -1: + self.write_line("[%s] %s" %(category, msg)) + + def pytest_rescheduleitems(self, items): + if self.config.option.debug: + self.write_sep("!", "RESCHEDULING %s " %(items,)) + + def pytest_deselected(self, items): + self.stats.setdefault('deselected', []).append(items) + + def pytest_itemstart(self, item, node=None): + if getattr(self.config.option, 'dist', 'no') != "no": + # for dist-testing situations itemstart means we + # queued the item for sending, not interesting (unless debugging) + if self.config.option.debug: + line = self._reportinfoline(item) + extra = "" + if node: + extra = "-> " + str(node.gateway.id) + self.write_ensure_prefix(line, extra) + else: + if self.config.option.verbose: + line = self._reportinfoline(item) + self.write_ensure_prefix(line, "") + else: + # ensure that the path is printed before the + # 1st test of a module starts running + + self.write_fspath_result(self._getfspath(item), "") + + def pytest__teardown_final_logerror(self, report): + self.stats.setdefault("error", []).append(report) + + def pytest_runtest_logreport(self, report): + rep = report + cat, letter, word = self.getcategoryletterword(rep) + if not letter and not word: + # probably passed setup/teardown + return + if isinstance(word, tuple): + word, markup = word + else: + markup = {} + self.stats.setdefault(cat, []).append(rep) + if not self.config.option.verbose: + self.write_fspath_result(self._getfspath(rep.item), letter) + else: + line = self._reportinfoline(rep.item) + if not hasattr(rep, 'node'): + self.write_ensure_prefix(line, word, **markup) + else: + self.ensure_newline() + if hasattr(rep, 'node'): + self._tw.write("%s " % rep.node.gateway.id) + self._tw.write(word, **markup) + self._tw.write(" " + line) + self.currentfspath = -2 + + def pytest_collectreport(self, report): + if not report.passed: + if report.failed: + self.stats.setdefault("error", []).append(report) + msg = report.longrepr.reprcrash.message + self.write_fspath_result(report.collector.fspath, "E") + elif report.skipped: + self.stats.setdefault("skipped", []).append(report) + self.write_fspath_result(report.collector.fspath, "S") + + def pytest_sessionstart(self, session): + self.write_sep("=", "test session starts", bold=True) + self._sessionstarttime = py.std.time.time() + + verinfo = ".".join(map(str, sys.version_info[:3])) + msg = "python: platform %s -- Python %s" % (sys.platform, verinfo) + msg += " -- pytest-%s" % (py.__version__) + if self.config.option.verbose or self.config.option.debug or getattr(self.config.option, 'pastebin', None): + msg += " -- " + str(sys.executable) + self.write_line(msg) + + if self.config.option.debug or self.config.option.traceconfig: + self.write_line("using py lib: %s" % (py.path.local(py.__file__).dirpath())) + if self.config.option.traceconfig: + self.write_line("active plugins:") + plugins = [] + items = self.config.pluginmanager._name2plugin.items() + for name, plugin in items: + repr_plugin = repr(plugin) + fullwidth = getattr(self._tw, 'fullwidth', 65000) + if len(repr_plugin)+26 > fullwidth: + repr_plugin = repr_plugin[:(fullwidth-30)] + '...' + self.write_line(" %-20s: %s" %(name, repr_plugin)) + for i, testarg in enumerate(self.config.args): + self.write_line("test object %d: %s" %(i+1, testarg)) + + def pytest_sessionfinish(self, exitstatus, __multicall__): + __multicall__.execute() + self._tw.line("") + if exitstatus in (0, 1, 2): + self.summary_errors() + self.summary_failures() + self.config.hook.pytest_terminal_summary(terminalreporter=self) + if exitstatus == 2: + self._report_keyboardinterrupt() + self.summary_deselected() + self.summary_stats() + + def pytest_keyboard_interrupt(self, excinfo): + self._keyboardinterrupt_memo = excinfo.getrepr() + + def _report_keyboardinterrupt(self): + self.write_sep("!", "KEYBOARD INTERRUPT") + excrepr = self._keyboardinterrupt_memo + if self.config.option.verbose: + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) + + def pytest_looponfailinfo(self, failreports, rootdirs): + if failreports: + self.write_sep("#", "LOOPONFAILING", red=True) + for report in failreports: + try: + loc = report.longrepr.reprcrash + except AttributeError: + loc = str(report.longrepr)[:50] + self.write_line(loc, red=True) + self.write_sep("#", "waiting for changes") + for rootdir in rootdirs: + self.write_line("### Watching: %s" %(rootdir,), bold=True) + + def _reportinfoline(self, item): + collect_fspath = self._getfspath(item) + fspath, lineno, msg = self._getreportinfo(item) + if fspath and fspath != collect_fspath: + fspath = "%s <- %s" % ( + self.curdir.bestrelpath(collect_fspath), + self.curdir.bestrelpath(fspath)) + elif fspath: + fspath = self.curdir.bestrelpath(fspath) + if lineno is not None: + lineno += 1 + if fspath and lineno and msg: + line = "%(fspath)s:%(lineno)s: %(msg)s" + elif fspath and msg: + line = "%(fspath)s: %(msg)s" + elif fspath and lineno: + line = "%(fspath)s:%(lineno)s %(extrapath)s" + else: + line = "[noreportinfo]" + return line % locals() + " " + + def _getfailureheadline(self, rep): + if hasattr(rep, "collector"): + return str(rep.collector.fspath) + elif hasattr(rep, 'item'): + fspath, lineno, msg = self._getreportinfo(rep.item) + return msg + else: + return "test session" + + def _getreportinfo(self, item): + try: + return item.__reportinfo + except AttributeError: + pass + reportinfo = item.config.hook.pytest_report_iteminfo(item=item) + # cache on item + item.__reportinfo = reportinfo + return reportinfo + + def _getfspath(self, item): + try: + return item.fspath + except AttributeError: + fspath, lineno, msg = self._getreportinfo(item) + return fspath + + # + # summaries for sessionfinish + # + + def summary_failures(self): + if 'failed' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "FAILURES") + for rep in self.stats['failed']: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def summary_errors(self): + if 'error' in self.stats and self.config.option.tbstyle != "no": + self.write_sep("=", "ERRORS") + for rep in self.stats['error']: + msg = self._getfailureheadline(rep) + if not hasattr(rep, 'when'): + # collect + msg = "ERROR during collection " + msg + elif rep.when == "setup": + msg = "ERROR at setup of " + msg + elif rep.when == "teardown": + msg = "ERROR at teardown of " + msg + self.write_sep("_", msg) + self.write_platinfo(rep) + rep.toterminal(self._tw) + + def write_platinfo(self, rep): + if hasattr(rep, 'node'): + self.write_line(self.gateway2info.get( + rep.node.gateway, + "node %r (platinfo not found? strange)") + [:self._tw.fullwidth-1]) + + def summary_stats(self): + session_duration = py.std.time.time() - self._sessionstarttime + + keys = "failed passed skipped deselected".split() + for key in self.stats.keys(): + if key not in keys: + keys.append(key) + parts = [] + for key in keys: + val = self.stats.get(key, None) + if val: + parts.append("%d %s" %(len(val), key)) + line = ", ".join(parts) + # XXX coloring + self.write_sep("=", "%s in %.2f seconds" %(line, session_duration)) + + def summary_deselected(self): + if 'deselected' in self.stats: + self.write_sep("=", "%d tests deselected by %r" %( + len(self.stats['deselected']), self.config.option.keyword), bold=True) + + +class CollectonlyReporter: + INDENT = " " + + def __init__(self, config, out=None): + self.config = config + if out is None: + out = py.std.sys.stdout + self.out = py.io.TerminalWriter(out) + self.indent = "" + self._failed = [] + + def outindent(self, line): + self.out.line(self.indent + str(line)) + + def pytest_internalerror(self, excrepr): + for line in str(excrepr).split("\n"): + self.out.line("INTERNALERROR> " + line) + + def pytest_collectstart(self, collector): + self.outindent(collector) + self.indent += self.INDENT + + def pytest_itemstart(self, item, node=None): + self.outindent(item) + + def pytest_collectreport(self, report): + if not report.passed: + self.outindent("!!! %s !!!" % report.longrepr.reprcrash.message) + self._failed.append(report) + self.indent = self.indent[:-len(self.INDENT)] + + def pytest_sessionfinish(self, session, exitstatus): + if self._failed: + self.out.sep("!", "collection failures") + for rep in self._failed: + rep.toterminal(self.out) + + +def repr_pythonversion(v=None): + if v is None: + v = sys.version_info + try: + return "%s.%s.%s-%s-%s" % v + except (TypeError, ValueError): + return str(v) + Added: pypy/trunk/py/plugin/pytest_tmpdir.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_tmpdir.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,21 @@ +"""provide temporary directories to test functions. + +usage example:: + + def test_plugin(tmpdir): + tmpdir.join("hello").write("hello") + +.. _`py.path.local`: ../../path.html + +""" +import py + +def pytest_funcarg__tmpdir(request): + """return a temporary directory path object + unique to each test function invocation, + created as a sub directory of the base temporary + directory. The returned object is a `py.path.local`_ + path object. + """ + name = request.function.__name__ + return request.config.mktemp(name, numbered=True) Added: pypy/trunk/py/plugin/pytest_unittest.py ============================================================================== --- (empty file) +++ pypy/trunk/py/plugin/pytest_unittest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,79 @@ +""" +automatically discover and run traditional "unittest.py" style tests. + +Usage +---------------- + +This plugin collects and runs Python `unittest.py style`_ tests. +It will automatically collect ``unittest.TestCase`` subclasses +and their ``test`` methods from the test modules of a project +(usually following the ``test_*.py`` pattern). + +This plugin is enabled by default. + +.. _`unittest.py style`: http://docs.python.org/library/unittest.html +""" +import py +import sys + +def pytest_pycollect_makeitem(collector, name, obj): + if 'unittest' not in sys.modules: + return # nobody derived unittest.TestCase + try: + isunit = issubclass(obj, py.std.unittest.TestCase) + except TypeError: + pass + else: + if isunit: + return UnitTestCase(name, parent=collector) + +class UnitTestCase(py.test.collect.Class): + def collect(self): + return [UnitTestCaseInstance("()", self)] + + def setup(self): + pass + + def teardown(self): + pass + +_dummy = object() +class UnitTestCaseInstance(py.test.collect.Instance): + def collect(self): + loader = py.std.unittest.TestLoader() + names = loader.getTestCaseNames(self.obj.__class__) + l = [] + for name in names: + callobj = getattr(self.obj, name) + if py.builtin.callable(callobj): + l.append(UnitTestFunction(name, parent=self)) + return l + + def _getobj(self): + x = self.parent.obj + return self.parent.obj(methodName='run') + +class UnitTestFunction(py.test.collect.Function): + def __init__(self, name, parent, args=(), obj=_dummy, sort_value=None): + super(UnitTestFunction, self).__init__(name, parent) + self._args = args + if obj is not _dummy: + self._obj = obj + self._sort_value = sort_value + if hasattr(self.parent, 'newinstance'): + self.parent.newinstance() + self.obj = self._getobj() + + def runtest(self): + target = self.obj + args = self._args + target(*args) + + def setup(self): + instance = py.builtin._getimself(self.obj) + instance.setUp() + + def teardown(self): + instance = py.builtin._getimself(self.obj) + instance.tearDown() + Modified: pypy/trunk/pypy/annotation/test/autopath.py ============================================================================== --- pypy/trunk/pypy/annotation/test/autopath.py (original) +++ pypy/trunk/pypy/annotation/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/bin/autopath.py ============================================================================== --- pypy/trunk/pypy/bin/autopath.py (original) +++ pypy/trunk/pypy/bin/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/bin/py.py ============================================================================== --- pypy/trunk/pypy/bin/py.py (original) +++ pypy/trunk/pypy/bin/py.py Wed Nov 11 18:54:49 2009 @@ -12,7 +12,7 @@ pass from pypy.tool import option -from py.compat.optparse import make_option +from optparse import make_option from pypy.interpreter import main, interactive, error, gateway from pypy.config.config import OptionDescription, BoolOption, StrOption from pypy.config.config import Config, to_optparse Modified: pypy/trunk/pypy/config/autopath.py ============================================================================== --- pypy/trunk/pypy/config/autopath.py (original) +++ pypy/trunk/pypy/config/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/config/config.py ============================================================================== --- pypy/trunk/pypy/config/config.py (original) +++ pypy/trunk/pypy/config/config.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py -from py.compat import optparse +import optparse from pypy.tool.pairtype import extendabletype SUPPRESS_USAGE = optparse.SUPPRESS_USAGE Modified: pypy/trunk/pypy/config/makerestdoc.py ============================================================================== --- pypy/trunk/pypy/config/makerestdoc.py (original) +++ pypy/trunk/pypy/config/makerestdoc.py Wed Nov 11 18:54:49 2009 @@ -1,13 +1,13 @@ import py -from py.__.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link -from py.__.rest.rst import Directive, Em, Quote, Text +from pypy.tool.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link +from pypy.tool.rest.rst import Directive, Em, Quote, Text from pypy.config.config import ChoiceOption, BoolOption, StrOption, IntOption from pypy.config.config import FloatOption, OptionDescription, Option, Config from pypy.config.config import ArbitraryOption, DEFAULT_OPTION_NAME from pypy.config.config import _getnegation -configdocdir = py.magic.autopath().dirpath().dirpath().join("doc", "config") +configdocdir = py.path.local(__file__).dirpath().dirpath().join("doc", "config") def get_fullpath(opt, path): if path: @@ -212,7 +212,7 @@ """ register a :config: ReST link role for use in documentation. """ try: from docutils.parsers.rst import directives, states, roles - from py.__.rest.directive import register_linkrole + from pypy.tool.rest.directive import register_linkrole except ImportError: return # enable :config: link role Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Wed Nov 11 18:54:49 2009 @@ -5,7 +5,7 @@ from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConflictConfigError -modulepath = py.magic.autopath().dirpath().dirpath().join("module") +modulepath = py.path.local(__file__).dirpath().dirpath().join("module") all_modules = [p.basename for p in modulepath.listdir() if p.check(dir=True, dotfile=False) and p.join('__init__.py').check()] Modified: pypy/trunk/pypy/config/test/test_makerestdoc.py ============================================================================== --- pypy/trunk/pypy/config/test/test_makerestdoc.py (original) +++ pypy/trunk/pypy/config/test/test_makerestdoc.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ from pypy.config.config import * from pypy.config.makerestdoc import make_cmdline_overview -from py.__.misc.rest import process as restcheck +from pypy.tool.rest.rest import process as restcheck tempdir = py.test.ensuretemp('config') Modified: pypy/trunk/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/test/test_pypyoption.py (original) +++ pypy/trunk/pypy/config/test/test_pypyoption.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ from pypy.config.config import Config, ConfigError from pypy.config.translationoption import set_opt_level -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() def test_required(): conf = get_pypy_config() Modified: pypy/trunk/pypy/conftest.py ============================================================================== --- pypy/trunk/pypy/conftest.py (original) +++ pypy/trunk/pypy/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py, sys, os -from py.__.test.outcome import Failed +from py.impl.test.outcome import Failed from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport @@ -9,18 +9,14 @@ from pypy.tool.udir import udir from pypy.tool.autopath import pypydir -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() # pytest settings pytest_plugins = "resultlog", rsyncdirs = ['.', '../lib-python', '../demo'] rsyncignore = ['_cache'] -# XXX workaround for a py.test bug clashing with lib/py symlink -# do we really need the latter? -empty_conftest = type(sys)('conftest') -empty_conftest.__file__ = "?" -sys.modules['pypy.lib.py.conftest'] = empty_conftest +collect_ignore = ['./lib/py'] # PyPy's command line extra options (these are added # to py.test's standard options) @@ -36,7 +32,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("pypy options") + group = parser.getgroup("pypy options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") group.addoption('-A', '--runappdirect', action="store_true", Modified: pypy/trunk/pypy/doc/config/autopath.py ============================================================================== --- pypy/trunk/pypy/doc/config/autopath.py (original) +++ pypy/trunk/pypy/doc/config/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/doc/config/generate.py ============================================================================== --- pypy/trunk/pypy/doc/config/generate.py (original) +++ pypy/trunk/pypy/doc/config/generate.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ from pypy.config import pypyoption, translationoption, config from pypy.doc.config.confrest import all_optiondescrs -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() for descr in all_optiondescrs: prefix = descr._name Modified: pypy/trunk/pypy/doc/config/makemodules.py ============================================================================== --- pypy/trunk/pypy/doc/config/makemodules.py (original) +++ pypy/trunk/pypy/doc/config/makemodules.py Wed Nov 11 18:54:49 2009 @@ -2,7 +2,7 @@ import py from pypy.config import pypyoption, translationoption, config -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() if __name__ == '__main__': c = config.Config(pypyoption.pypy_optiondescription).usemodules Modified: pypy/trunk/pypy/doc/confrest.py ============================================================================== --- pypy/trunk/pypy/doc/confrest.py (original) +++ pypy/trunk/pypy/doc/confrest.py Wed Nov 11 18:54:49 2009 @@ -34,7 +34,7 @@ class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' Modified: pypy/trunk/pypy/doc/confrest_oldpy.py ============================================================================== --- pypy/trunk/pypy/doc/confrest_oldpy.py (original) +++ pypy/trunk/pypy/doc/confrest_oldpy.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import py -from py.__.misc.rest import convert_rest_html, strip_html_header -from py.__.misc.difftime import worded_time +from pypy.tool.rest.rest import convert_rest_html, strip_html_header +from pypy.tool.difftime import worded_time html = py.xml.html @@ -104,7 +104,7 @@ class Project: - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "py lib" prefix_title = "" # we have a logo already containing "py lib" encoding = 'latin1' @@ -186,7 +186,9 @@ id = 'docinfoline')) page.contentspace.append(py.xml.raw(content)) - outputpath.ensure().write(page.unicode().encode(encoding)) + f = outputpath.open('w') + f.write(page.unicode().encode(encoding)) + f.close() # XXX this function comes from apigen/linker.py, put it # somewhere in py lib Modified: pypy/trunk/pypy/doc/conftest.py ============================================================================== --- pypy/trunk/pypy/doc/conftest.py (original) +++ pypy/trunk/pypy/doc/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,12 +1,12 @@ import py from pypy.config.makerestdoc import register_config_role -docdir = py.magic.autopath().dirpath() +docdir = py.path.local(__file__).dirpath() pytest_plugins = "pytest_restdoc" def pytest_addoption(parser): - group = parser.addgroup("pypy-doc options") + group = parser.getgroup("pypy-doc options") group.addoption('--pypy-doctests', action="store_true", dest="pypy_doctests", default=False, help="enable doctests in .txt files") Modified: pypy/trunk/pypy/doc/statistic/confrest.py ============================================================================== --- pypy/trunk/pypy/doc/statistic/confrest.py (original) +++ pypy/trunk/pypy/doc/statistic/confrest.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py -from py.__.doc.confrest import * +from pypy.doc.confrest import * class PyPyPage(Page): def fill_menubar(self): @@ -17,7 +17,7 @@ " ", id="menubar") class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' encoding = 'latin1' Modified: pypy/trunk/pypy/doc/test_redirections.py ============================================================================== --- pypy/trunk/pypy/doc/test_redirections.py (original) +++ pypy/trunk/pypy/doc/test_redirections.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import py -redir = py.magic.autopath().dirpath('redirections') +redir = py.path.local(__file__).dirpath('redirections') def checkexist(path): print "checking", path Modified: pypy/trunk/pypy/doc/tool/makeref.py ============================================================================== --- pypy/trunk/pypy/doc/tool/makeref.py (original) +++ pypy/trunk/pypy/doc/tool/makeref.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import py -py.magic.autopath() +py.path.local(__file__) import pypy pypydir = py.path.local(pypy.__file__).dirpath() distdir = pypydir.dirpath() Modified: pypy/trunk/pypy/doc/tool/mydot.py ============================================================================== --- pypy/trunk/pypy/doc/tool/mydot.py (original) +++ pypy/trunk/pypy/doc/tool/mydot.py Wed Nov 11 18:54:49 2009 @@ -62,7 +62,7 @@ if __name__ == '__main__': - from py.compat import optparse + import optparse parser = optparse.OptionParser() parser.add_option("-T", dest="format", help="output format") Modified: pypy/trunk/pypy/jit/backend/autopath.py ============================================================================== --- pypy/trunk/pypy/jit/backend/autopath.py (original) +++ pypy/trunk/pypy/jit/backend/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/jit/backend/test/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/conftest.py (original) +++ pypy/trunk/pypy/jit/backend/test/conftest.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup('random test options') + group = parser.getgroup('random test options') group.addoption('--random-seed', action="store", type="int", default=random.randrange(0, 10000), dest="randomseed", Modified: pypy/trunk/pypy/jit/backend/x86/autopath.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/autopath.py (original) +++ pypy/trunk/pypy/jit/backend/x86/autopath.py Wed Nov 11 18:54:49 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/trunk/pypy/jit/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/conftest.py (original) +++ pypy/trunk/pypy/jit/conftest.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("JIT options") + group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", default=False, dest="run_slow_tests", help="run all the compiled tests (instead of just a few)") Modified: pypy/trunk/pypy/jit/tl/autopath.py ============================================================================== --- pypy/trunk/pypy/jit/tl/autopath.py (original) +++ pypy/trunk/pypy/jit/tl/autopath.py Wed Nov 11 18:54:49 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/trunk/pypy/jit/tl/conftest.py ============================================================================== --- pypy/trunk/pypy/jit/tl/conftest.py (original) +++ pypy/trunk/pypy/jit/tl/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit.py options") + group = parser.getgroup("pypyjit.py options") group.addoption('--ootype', action="store_true", dest="ootype", default=False, help="use ootype") Modified: pypy/trunk/pypy/jit/tl/spli/autopath.py ============================================================================== --- pypy/trunk/pypy/jit/tl/spli/autopath.py (original) +++ pypy/trunk/pypy/jit/tl/spli/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/trunk/pypy/jit/tl/targettlc.py (original) +++ pypy/trunk/pypy/jit/tl/targettlc.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import time import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/trunk/pypy/jit/tl/targettlr.py ============================================================================== --- pypy/trunk/pypy/jit/tl/targettlr.py (original) +++ pypy/trunk/pypy/jit/tl/targettlr.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlr import interpret from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/trunk/pypy/jit/tl/test/test_pypyjit.py ============================================================================== --- pypy/trunk/pypy/jit/tl/test/test_pypyjit.py (original) +++ pypy/trunk/pypy/jit/tl/test/test_pypyjit.py Wed Nov 11 18:54:49 2009 @@ -21,7 +21,7 @@ def check_crasher(func_name): try: JIT_EXECUTABLE.sysexec(CRASH_FILE, func_name) - except py.__.process.cmdexec.ExecutionFailed, e: + except py.impl.process.cmdexec.ExecutionFailed, e: print "stderr" print "------" print e.err Modified: pypy/trunk/pypy/jit/tl/tla/targettla.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/targettla.py (original) +++ pypy/trunk/pypy/jit/tl/tla/targettla.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla import tla Modified: pypy/trunk/pypy/jit/tl/tla/tla_assembler.py ============================================================================== --- pypy/trunk/pypy/jit/tl/tla/tla_assembler.py (original) +++ pypy/trunk/pypy/jit/tl/tla/tla_assembler.py Wed Nov 11 18:54:49 2009 @@ -2,7 +2,7 @@ import sys import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla.test_tla import assemble def usage(): Modified: pypy/trunk/pypy/jit/tool/autopath.py ============================================================================== --- pypy/trunk/pypy/jit/tool/autopath.py (original) +++ pypy/trunk/pypy/jit/tool/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py (original) +++ pypy/trunk/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py Wed Nov 11 18:54:49 2009 @@ -11,7 +11,7 @@ # ------------------------------------------------------------------------------ -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" # filename = ROM_PATH + "/rom9/rom9.gb" filename = "/home/tverwaes/roms/SuperMarioLand.gb" SOCKET_PORT = 55682 @@ -83,7 +83,7 @@ # ------------------------------------------------------------------------------ -MARIO_DIR = str(py.magic.autopath().dirpath().dirpath()\ +MARIO_DIR = str(py.path.local(__file__).dirpath().dirpath()\ .dirpath().dirpath()\ .dirpath().dirpath()) + "/mario" Modified: pypy/trunk/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (original) +++ pypy/trunk/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Wed Nov 11 18:54:49 2009 @@ -8,7 +8,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/trunk/pypy/lang/gameboy/profiling/gameboyTest.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/profiling/gameboyTest.py (original) +++ pypy/trunk/pypy/lang/gameboy/profiling/gameboyTest.py Wed Nov 11 18:54:49 2009 @@ -7,7 +7,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/trunk/pypy/lang/gameboy/test/test_cartridge.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/test/test_cartridge.py (original) +++ pypy/trunk/pypy/lang/gameboy/test/test_cartridge.py Wed Nov 11 18:54:49 2009 @@ -8,7 +8,7 @@ def mapToByte(value): return ord(value) & 0xFF -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" CONTENT = "abcdefghijklmnopqrstuvwxyz1234567890" MAPPED_CONTENT = map_to_byte(CONTENT) Modified: pypy/trunk/pypy/lang/gameboy/test/test_rom.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/test/test_rom.py (original) +++ pypy/trunk/pypy/lang/gameboy/test/test_rom.py Wed Nov 11 18:54:49 2009 @@ -6,7 +6,7 @@ from pypy.lang.gameboy.gameboy import * -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" EMULATION_CYCLES = 64 # ------------------------------------------------------------------------------ Modified: pypy/trunk/pypy/lang/gameboy/tool/autopath.py ============================================================================== --- pypy/trunk/pypy/lang/gameboy/tool/autopath.py (original) +++ pypy/trunk/pypy/lang/gameboy/tool/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lang/js/autopath.py ============================================================================== --- pypy/trunk/pypy/lang/js/autopath.py (original) +++ pypy/trunk/pypy/lang/js/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lang/js/jsparser.py ============================================================================== --- pypy/trunk/pypy/lang/js/jsparser.py (original) +++ pypy/trunk/pypy/lang/js/jsparser.py Wed Nov 11 18:54:49 2009 @@ -2,7 +2,7 @@ from pypy.rlib.parsing.parsing import ParseError, Rule import py -GFILE = py.magic.autopath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/trunk/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/trunk/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/trunk/pypy/lang/js/test/ecma/conftest.py Wed Nov 11 18:54:49 2009 @@ -2,13 +2,13 @@ from pypy.lang.js.interpreter import * from pypy.lang.js.jsobj import W_Array, JsBaseExcept from pypy.rlib.parsing.parsing import ParseError -from py.__.test.outcome import Failed, ExceptionFailure +from py.impl.test.outcome import Failed, ExceptionFailure import pypy.lang.js as js from pypy.lang.js import interpreter interpreter.TEST = True -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() exclusionlist = ['shell.js', 'browser.js'] def pytest_addoption(parser): Modified: pypy/trunk/pypy/lang/js/test/test_interactive.py ============================================================================== --- pypy/trunk/pypy/lang/js/test/test_interactive.py (original) +++ pypy/trunk/pypy/lang/js/test/test_interactive.py Wed Nov 11 18:54:49 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv): - return self._spawn(str(py.magic.autopath().dirpath().dirpath().join('js_interactive.py')), argv) + return self._spawn(str(py.path.local(__file__).dirpath().dirpath().join('js_interactive.py')), argv) def prompt_send(self, message): self.child.expect('js>') Modified: pypy/trunk/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/trunk/pypy/lang/js/test/test_parser.py (original) +++ pypy/trunk/pypy/lang/js/test/test_parser.py Wed Nov 11 18:54:49 2009 @@ -9,7 +9,7 @@ from pypy import conftest import sys -GFILE = py.magic.autopath().dirpath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/trunk/pypy/lang/prolog/interpreter/autopath.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/autopath.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lang/prolog/interpreter/conftest.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/conftest.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import py, sys -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() Option = py.test.config.Option Modified: pypy/trunk/pypy/lang/prolog/interpreter/interactive.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/interactive.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/interactive.py Wed Nov 11 18:54:49 2009 @@ -7,7 +7,7 @@ import py import sys -#sys.path.append(str(py.magic.autopath().dirpath().dirpath())) +#sys.path.append(str(py.path.local(__file__).dirpath().dirpath())) from pypy.rlib.parsing.parsing import ParseError from pypy.rlib.parsing.deterministic import LexerError Modified: pypy/trunk/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/parsing.py Wed Nov 11 18:54:49 2009 @@ -3403,7 +3403,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/trunk/pypy/lang/scheme/autopath.py ============================================================================== --- pypy/trunk/pypy/lang/scheme/autopath.py (original) +++ pypy/trunk/pypy/lang/scheme/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lang/scheme/execution.py ============================================================================== --- pypy/trunk/pypy/lang/scheme/execution.py (original) +++ pypy/trunk/pypy/lang/scheme/execution.py Wed Nov 11 18:54:49 2009 @@ -19,7 +19,7 @@ except (TypeError, AttributeError): pass -de_file = py.magic.autopath().dirpath().join("r5rs_derived_expr.ss") +de_file = py.path.local(__file__).dirpath().join("r5rs_derived_expr.ss") de_code = de_file.read() de_expr_lst = parse(de_code) Modified: pypy/trunk/pypy/lang/scheme/test/test_interactive.py ============================================================================== --- pypy/trunk/pypy/lang/scheme/test/test_interactive.py (original) +++ pypy/trunk/pypy/lang/scheme/test/test_interactive.py Wed Nov 11 18:54:49 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv=[]): - path = py.magic.autopath()/".."/".."/"interactive.py" + path = py.path.local(__file__)/".."/".."/"interactive.py" return self._spawn(str(path), argv) def test_interactive(self): Modified: pypy/trunk/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/trunk/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/trunk/pypy/lang/smalltalk/test/test_miniimage.py Wed Nov 11 18:54:49 2009 @@ -12,7 +12,7 @@ def setup_module(module, filename='mini.image'): space = objspace.ObjSpace() - module.mini_image = py.magic.autopath().dirpath().dirpath().join(filename) + module.mini_image = py.path.local(__file__).dirpath().dirpath().join(filename) module.reader = open_miniimage(space) reader.initialize() module.image = squeakimage.SqueakImage() Modified: pypy/trunk/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/trunk/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/trunk/pypy/lang/smalltalk/tool/analyseimage.py Wed Nov 11 18:54:49 2009 @@ -6,7 +6,7 @@ from pypy.lang.smalltalk import interpreter import sys -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') +mini_image = py.path.local(__file__).dirpath().dirpath().join('mini.image') def get_miniimage(space): return squeakimage.ImageReader(space, squeakimage.Stream(mini_image.open())) Modified: pypy/trunk/pypy/lang/smalltalk/tool/autopath.py ============================================================================== --- pypy/trunk/pypy/lang/smalltalk/tool/autopath.py (original) +++ pypy/trunk/pypy/lang/smalltalk/tool/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py (original) +++ pypy/trunk/pypy/lib/app_test/ctypes_tests/conftest.py Wed Nov 11 18:54:49 2009 @@ -11,7 +11,7 @@ from pypy.translator.platform import platform from pypy.translator.tool.cbuild import ExternalCompilationInfo udir = py.test.ensuretemp('_ctypes_test') - cfile = py.magic.autopath().dirpath().join("_ctypes_test.c") + cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': libraries = ['oleaut32'] Modified: pypy/trunk/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/trunk/pypy/lib/distributed/socklayer.py (original) +++ pypy/trunk/pypy/lib/distributed/socklayer.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ import py from socket import socket -from py.__.green.msgstruct import decodemessage, message +from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal import sys Modified: pypy/trunk/pypy/lib/test2/autopath.py ============================================================================== --- pypy/trunk/pypy/lib/test2/autopath.py (original) +++ pypy/trunk/pypy/lib/test2/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/module/__builtin__/test/autopath.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/autopath.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_import.py Wed Nov 11 18:54:49 2009 @@ -76,7 +76,7 @@ code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) s2 = struct.pack("i", os.stat(str(p.join("x.py")))[stat.ST_MTIME]) - p.join("x.pyc").write(imp.get_magic() + s2 + s3) + p.join("x.pyc").write(imp.get_magic() + s2 + s3, mode='wb') else: w = space.wrap w_modname = w("compiled.x") @@ -92,7 +92,7 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read()) + p.join('lone.pyc').write(p.join('x.pyc').read(), mode='wb') return str(root) Modified: pypy/trunk/pypy/module/_codecs/test/autopath.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/test/autopath.py (original) +++ pypy/trunk/pypy/module/_codecs/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/trunk/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/trunk/pypy/module/_file/test/test_file_extra.py Wed Nov 11 18:54:49 2009 @@ -19,7 +19,7 @@ def setup_module(mod): - udir.join('sample').write(SAMPLE) + udir.join('sample').write(SAMPLE, 'wb') class BaseROTests: Modified: pypy/trunk/pypy/module/_sre/test/autopath.py ============================================================================== --- pypy/trunk/pypy/module/_sre/test/autopath.py (original) +++ pypy/trunk/pypy/module/_sre/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/trunk/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/trunk/pypy/module/_sre/test/test_app_sre.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option -from py.__.test.outcome import Skipped +from py.impl.test.outcome import Skipped def init_globals_hack(space): space.appexec([space.wrap(autopath.this_dir)], """(this_dir): Modified: pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/trunk/pypy/module/bz2/test/test_bz2_compdecomp.py Wed Nov 11 18:54:49 2009 @@ -25,7 +25,7 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.BUGGY_DATA = py.magic.autopath().dirpath().join('data.bz2').read() + mod.BUGGY_DATA = py.path.local(__file__).dirpath().join('data.bz2').read() mod.decompress = decompress class AppTestBZ2Compressor(CheckAllocation): Modified: pypy/trunk/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/trunk/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/trunk/pypy/module/bz2/test/test_bz2_file.py Wed Nov 11 18:54:49 2009 @@ -15,12 +15,12 @@ def create_temp_file(crlf=False): f = py.test.ensuretemp("bz2").join("foo") data = (DATA, DATA_CRLF)[crlf] - f.write(data) + f.write(data, 'wb') def create_broken_temp_file(): f = py.test.ensuretemp("bz2").join("foo") data = DATA[:100] - f.write(data) + f.write(data, 'wb') def decompress(data): import popen2 Modified: pypy/trunk/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit options") + group = parser.getgroup("pypyjit options") group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/trunk/pypy/module/sys/test/autopath.py ============================================================================== --- pypy/trunk/pypy/module/sys/test/autopath.py (original) +++ pypy/trunk/pypy/module/sys/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_complexobject.py Wed Nov 11 18:54:49 2009 @@ -67,7 +67,7 @@ sys.path.append(%r) import helper return helper - """ % (str(py.magic.autopath().dirpath()))) + """ % (str(py.path.local(__file__).dirpath()))) def test_div(self): h = self.helper Modified: pypy/trunk/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/trunk/pypy/rlib/parsing/ebnfparse.py Wed Nov 11 18:54:49 2009 @@ -2125,7 +2125,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/trunk/pypy/rlib/parsing/makepackrat.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/makepackrat.py (original) +++ pypy/trunk/pypy/rlib/parsing/makepackrat.py Wed Nov 11 18:54:49 2009 @@ -718,7 +718,7 @@ def test_generate(): - f = py.magic.autopath().dirpath().join("pypackrat.py") + f = py.path.local(__file__).dirpath().join("pypackrat.py") from pypackrat import PyPackratSyntaxParser p = PyPackratSyntaxParser(syntax) t = p.file() Modified: pypy/trunk/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/regexparse.py (original) +++ pypy/trunk/pypy/rlib/parsing/regexparse.py Wed Nov 11 18:54:49 2009 @@ -1966,7 +1966,7 @@ def test_generate(): - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/trunk/pypy/rlib/parsing/test/autopath.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/test/autopath.py (original) +++ pypy/trunk/pypy/rlib/parsing/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py (original) +++ pypy/trunk/pypy/rlib/parsing/test/test_pythonlexer.py Wed Nov 11 18:54:49 2009 @@ -230,7 +230,7 @@ assert tokens[i * 3].name == 'String' def test_self(): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlexer.tokenize(s) print tokens @@ -263,7 +263,7 @@ "op": "operator", } import tokenize, token - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlex(s) print [t.name for t in tokens][:20] tokens2 = list(tokenize.generate_tokens(iter(s.splitlines(True)).next)) Modified: pypy/trunk/pypy/rlib/parsing/test/test_pythonparse.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/test/test_pythonparse.py (original) +++ pypy/trunk/pypy/rlib/parsing/test/test_pythonparse.py Wed Nov 11 18:54:49 2009 @@ -7,7 +7,7 @@ from pypy.rlib.parsing.parsing import PackratParser, Symbol, ParseError, Rule from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function -grammar = py.magic.autopath().dirpath().join("pygrammar.txt").read(mode='rt') +grammar = py.path.local(__file__).dirpath().join("pygrammar.txt").read(mode='rt') def test_parse_grammar(): @@ -240,12 +240,12 @@ t = self.ToAST.transform(t) def test_parse_this(self): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() t = self.parse(s) t = self.ToAST.transform(t) def test_parsing(self): - s = py.magic.autopath().dirpath().dirpath().join("parsing.py").read() + s = py.path.local(__file__).dirpath().dirpath().join("parsing.py").read() t = self.parse(s) t = self.ToAST.transform(t) Modified: pypy/trunk/pypy/rlib/rsdl/eci.py ============================================================================== --- pypy/trunk/pypy/rlib/rsdl/eci.py (original) +++ pypy/trunk/pypy/rlib/rsdl/eci.py Wed Nov 11 18:54:49 2009 @@ -9,7 +9,7 @@ includes = ['SDL.h'], include_dirs = ['/Library/Frameworks/SDL.framework/Headers'], link_files = [ - str(py.magic.autopath().dirpath().join('macosx-sdl-main/SDLMain.m')), + str(py.path.local(__file__).dirpath().join('macosx-sdl-main/SDLMain.m')), ], frameworks = ['SDL', 'Cocoa'] ) Modified: pypy/trunk/pypy/rlib/rsdl/test/autopath.py ============================================================================== --- pypy/trunk/pypy/rlib/rsdl/test/autopath.py (original) +++ pypy/trunk/pypy/rlib/rsdl/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/rlib/test/test_listsort.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_listsort.py (original) +++ pypy/trunk/pypy/rlib/test/test_listsort.py Wed Nov 11 18:54:49 2009 @@ -35,7 +35,7 @@ sorttest(lst1) def test_file(): - for fn in py.magic.autopath().dirpath().listdir(): + for fn in py.path.local(__file__).dirpath().listdir(): if fn.ext == '.py': lines1 = fn.readlines() sorttest(lines1) Modified: pypy/trunk/pypy/rpython/microbench/autopath.py ============================================================================== --- pypy/trunk/pypy/rpython/microbench/autopath.py (original) +++ pypy/trunk/pypy/rpython/microbench/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/rpython/module/test/test_ll_os_path.py ============================================================================== --- pypy/trunk/pypy/rpython/module/test/test_ll_os_path.py (original) +++ pypy/trunk/pypy/rpython/module/test/test_ll_os_path.py Wed Nov 11 18:54:49 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir def test_exists(): - filename = impl.to_rstr(str(py.magic.autopath())) + filename = impl.to_rstr(str(py.path.local(__file__))) assert impl.ll_os_path_exists(filename) == True assert not impl.ll_os_path_exists(impl.to_rstr( "strange_filename_that_looks_improbable.sde")) Modified: pypy/trunk/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/trunk/pypy/rpython/test/test_rbuiltin.py Wed Nov 11 18:54:49 2009 @@ -294,7 +294,7 @@ def f(fn): fn = hlstr(fn) return os.path.exists(fn) - filename = self.string_to_ll(str(py.magic.autopath())) + filename = self.string_to_ll(str(py.path.local(__file__))) assert self.interpret(f, [filename]) == True #assert self.interpret(f, [ # self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False @@ -308,7 +308,7 @@ fn = hlstr(fn) return os.path.isdir(fn) assert self.interpret(f, [self.string_to_ll("/")]) == True - assert self.interpret(f, [self.string_to_ll(str(py.magic.autopath()))]) == False + assert self.interpret(f, [self.string_to_ll(str(py.path.local(__file__)))]) == False assert self.interpret(f, [self.string_to_ll("another/unlikely/directory/name")]) == False def test_pbc_isTrue(self): Modified: pypy/trunk/pypy/test_all.py ============================================================================== --- pypy/trunk/pypy/test_all.py (original) +++ pypy/trunk/pypy/test_all.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ #! /usr/bin/env python -if __name__ == '__main__': +if __name__ == '__main__': import tool.autopath - import py - py.test.cmdline.main() + import py + py.cmdline.pytest() Modified: pypy/trunk/pypy/tool/algo/test/autopath.py ============================================================================== --- pypy/trunk/pypy/tool/algo/test/autopath.py (original) +++ pypy/trunk/pypy/tool/algo/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/trunk/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/trunk/pypy/tool/ansi_mandelbrot.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ import sys -from py.__.io.terminalwriter import ansi_print, get_terminal_width +from py.impl.io.terminalwriter import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/trunk/pypy/tool/ansi_print.py ============================================================================== --- pypy/trunk/pypy/tool/ansi_print.py (original) +++ pypy/trunk/pypy/tool/ansi_print.py Wed Nov 11 18:54:49 2009 @@ -4,7 +4,7 @@ import sys -from py.__.io.terminalwriter import ansi_print +from py.impl.io.terminalwriter import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: Modified: pypy/trunk/pypy/tool/autopath.py ============================================================================== --- pypy/trunk/pypy/tool/autopath.py (original) +++ pypy/trunk/pypy/tool/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/tool/bench/pypyresult.py ============================================================================== --- pypy/trunk/pypy/tool/bench/pypyresult.py (original) +++ pypy/trunk/pypy/tool/bench/pypyresult.py Wed Nov 11 18:54:49 2009 @@ -52,7 +52,7 @@ if __name__ == "__main__": - x = py.magic.autopath().dirpath("bench-unix.benchmark_result") + x = py.path.local(__file__).dirpath("bench-unix.benchmark_result") db = ResultDB() db.parsepickle(x) Added: pypy/trunk/pypy/tool/difftime.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/difftime.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,32 @@ +import py + +_time_desc = { + 1 : 'second', 60 : 'minute', 3600 : 'hour', 86400 : 'day', + 2628000 : 'month', 31536000 : 'year', } + +def worded_diff_time(ctime): + difftime = py.std.time.time() - ctime + keys = _time_desc.keys() + keys.sort() + for i, key in py.builtin.enumerate(keys): + if key >=difftime: + break + l = [] + keylist = keys[:i] + + keylist.reverse() + for key in keylist[:1]: + div = int(difftime / key) + if div==0: + break + difftime -= div * key + plural = div > 1 and 's' or '' + l.append('%d %s%s' %(div, _time_desc[key], plural)) + return ", ".join(l) + " ago " + +_months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +def worded_time(ctime): + tm = py.std.time.gmtime(ctime) + return "%s %d, %d" % (_months[tm.tm_mon-1], tm.tm_mday, tm.tm_year) Modified: pypy/trunk/pypy/tool/genstatistic.py ============================================================================== --- pypy/trunk/pypy/tool/genstatistic.py (original) +++ pypy/trunk/pypy/tool/genstatistic.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ import autopath import py -from py.__.misc.cmdline import countloc +from py.impl.misc.cmdline import countloc from py.xml import raw pypydir = py.path.local(autopath.pypydir) Modified: pypy/trunk/pypy/tool/option.py ============================================================================== --- pypy/trunk/pypy/tool/option.py (original) +++ pypy/trunk/pypy/tool/option.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ import os from pypy.config.pypyoption import get_pypy_config from pypy.config.config import Config, OptionDescription, to_optparse -from py.compat import optparse +import optparse extra_useage = """For detailed descriptions of all the options see http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" Modified: pypy/trunk/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/appsupport.py (original) +++ pypy/trunk/pypy/tool/pytest/appsupport.py Wed Nov 11 18:54:49 2009 @@ -1,9 +1,10 @@ import autopath import py -from py.__.magic import exprinfo +import py.impl.code.assertion +from py.impl.code import _assertionold as exprinfo from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from py.__.test.outcome import ExceptionFailure +from py.impl.test.outcome import ExceptionFailure # ____________________________________________________________ Modified: pypy/trunk/pypy/tool/pytest/autopath.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/autopath.py (original) +++ pypy/trunk/pypy/tool/pytest/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/tool/pytest/genreportdata.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/genreportdata.py (original) +++ pypy/trunk/pypy/tool/pytest/genreportdata.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ import py import sys -mydir = py.magic.autopath().dirpath().realpath() +mydir = py.path.local(__file__).dirpath().realpath() from pypy.tool.pytest import htmlreport from pypy.tool.pytest import confpath Modified: pypy/trunk/pypy/tool/pytest/htmlreport.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/htmlreport.py (original) +++ pypy/trunk/pypy/tool/pytest/htmlreport.py Wed Nov 11 18:54:49 2009 @@ -232,7 +232,7 @@ t = self.rep.render_latest_table(self.rep.results) assert unicode(t) -mydir = py.magic.autopath().dirpath() +mydir = py.path.local(__file__).dirpath() def getpicklepath(): return mydir.join('.htmlreport.pickle') Modified: pypy/trunk/pypy/tool/pytest/test/test_new_count.py ============================================================================== --- pypy/trunk/pypy/tool/pytest/test/test_new_count.py (original) +++ pypy/trunk/pypy/tool/pytest/test/test_new_count.py Wed Nov 11 18:54:49 2009 @@ -2,7 +2,7 @@ import py #from pypy.tool.pytest.confpath import testresultdir from pypy.tool.pytest.result import ResultFromMime -testpath = py.magic.autopath().dirpath('data') +testpath = py.path.local(__file__).dirpath('data') class TestResultCache: Added: pypy/trunk/pypy/tool/rest/__init__.py ============================================================================== Added: pypy/trunk/pypy/tool/rest/convert.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/rest/convert.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,163 @@ +import py + +ExecutionFailed = py.process.cmdexec.Error +# utility functions to convert between various formats + +format_to_dotargument = {"png": "png", + "eps": "ps", + "ps": "ps", + "pdf": "ps", + } + +def ps2eps(ps): + # XXX write a pure python version + if not py.path.local.sysfind("ps2epsi") and \ + not py.path.local.sysfind("ps2eps"): + raise SystemExit("neither ps2eps nor ps2epsi found") + try: + eps = ps.new(ext=".eps") + py.process.cmdexec('ps2epsi "%s" "%s"' % (ps, eps)) + except ExecutionFailed: + py.process.cmdexec('ps2eps -l -f "%s"' % ps) + +def ps2pdf(ps, compat_level="1.2"): + if not py.path.local.sysfind("gs"): + raise SystemExit("ERROR: gs not found") + pdf = ps.new(ext=".pdf") + options = dict(OPTIONS="-dSAFER -dCompatibilityLevel=%s" % compat_level, + infile=ps, outfile=pdf) + cmd = ('gs %(OPTIONS)s -q -dNOPAUSE -dBATCH -sDEVICE=pdfwrite ' + '"-sOutputFile=%(outfile)s" %(OPTIONS)s -c .setpdfwrite ' + '-f "%(infile)s"') % options + py.process.cmdexec(cmd) + return pdf + +def eps2pdf(eps): + # XXX write a pure python version + if not py.path.local.sysfind("epstopdf"): + raise SystemExit("ERROR: epstopdf not found") + py.process.cmdexec('epstopdf "%s"' % eps) + +def dvi2eps(dvi, dest=None): + if dest is None: + dest = eps.new(ext=".eps") + command = 'dvips -q -E -n 1 -D 600 -p 1 -o "%s" "%s"' % (dest, dvi) + if not py.path.local.sysfind("dvips"): + raise SystemExit("ERROR: dvips not found") + py.process.cmdexec(command) + +def convert_dot(fn, new_extension): + if not py.path.local.sysfind("dot"): + raise SystemExit("ERROR: dot not found") + result = fn.new(ext=new_extension) + print result + arg = "-T%s" % (format_to_dotargument[new_extension], ) + py.std.os.system('dot "%s" "%s" > "%s"' % (arg, fn, result)) + if new_extension == "eps": + ps = result.new(ext="ps") + result.move(ps) + ps2eps(ps) + ps.remove() + elif new_extension == "pdf": + # convert to eps file first, to get the bounding box right + eps = result.new(ext="eps") + ps = result.new(ext="ps") + result.move(ps) + ps2eps(ps) + eps2pdf(eps) + ps.remove() + eps.remove() + return result + + +class latexformula2png(object): + def __init__(self, formula, dest, temp=None): + self.formula = formula + try: + import Image + self.Image = Image + self.scale = 2 # create a larger image + self.upscale = 5 # create the image upscale times larger, then scale it down + except ImportError: + self.scale = 2 + self.upscale = 1 + self.Image = None + self.output_format = ('pngmono', 'pnggray', 'pngalpha')[2] + if temp is None: + temp = py.test.ensuretemp("latexformula") + self.temp = temp + self.latex = self.temp.join('formula.tex') + self.dvi = self.temp.join('formula.dvi') + self.eps = self.temp.join('formula.eps') + self.png = self.temp.join('formula.png') + self.saveas(dest) + + def saveas(self, dest): + self.gen_latex() + self.gen_dvi() + dvi2eps(self.dvi, self.eps) + self.gen_png() + self.scale_image() + self.png.copy(dest) + + def gen_latex(self): + self.latex.write (""" + \\documentclass{article} + \\pagestyle{empty} + \\begin{document} + + %s + \\pagebreak + + \\end{document} + """ % (self.formula)) + + def gen_dvi(self): + origdir = py.path.local() + self.temp.chdir() + py.process.cmdexec('latex "%s"' % (self.latex)) + origdir.chdir() + + def gen_png(self): + tempdir = py.path.local.mkdtemp() + + re_bbox = py.std.re.compile('%%BoundingBox:\s*(\d+) (\d+) (\d+) (\d+)') + eps = self.eps.read() + x1, y1, x2, y2 = [int(i) for i in re_bbox.search(eps).groups()] + X = x2 - x1 + 2 + Y = y2 - y1 + 2 + mx = -x1 + my = -y1 + ps = self.temp.join('temp.ps') + source = self.eps + ps.write(""" + 1 1 1 setrgbcolor + newpath + -1 -1 moveto + %(X)d -1 lineto + %(X)d %(Y)d lineto + -1 %(Y)d lineto + closepath + fill + %(mx)d %(my)d translate + 0 0 0 setrgbcolor + (%(source)s) run + + """ % locals()) + + sx = int((x2 - x1) * self.scale * self.upscale) + sy = int((y2 - y1) * self.scale * self.upscale) + res = 72 * self.scale * self.upscale + command = ('gs -q -g%dx%d -r%dx%d -sDEVICE=%s -sOutputFile="%s" ' + '-dNOPAUSE -dBATCH "%s"') % ( + sx, sy, res, res, self.output_format, self.png, ps) + py.process.cmdexec(command) + + def scale_image(self): + if self.Image is None: + return + image = self.Image.open(str(self.png)) + image.resize((image.size[0] / self.upscale, + image.size[1] / self.upscale), + self.Image.ANTIALIAS).save(str(self.png)) + Added: pypy/trunk/pypy/tool/rest/directive.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/rest/directive.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,115 @@ +# XXX this file is messy since it tries to deal with several docutils versions +import py + +from pypy.tool.rest.convert import convert_dot, latexformula2png + +import sys +import docutils +from docutils import nodes +from docutils.parsers.rst import directives, states, roles +from docutils.parsers.rst.directives import images + +if hasattr(images, "image"): + directives_are_functions = True +else: + directives_are_functions = False + +try: + from docutils.utils import unescape # docutils version > 0.3.5 +except ImportError: + from docutils.parsers.rst.states import unescape # docutils 0.3.5 + +if not directives_are_functions: + ImageClass = images.Image + +else: + class ImageClass(object): + option_spec = images.image.options + def run(self): + return images.image(u'image', + self.arguments, + self.options, + self.content, + self.lineno, + self.content_offset, + self.block_text, + self.state, + self.state_machine) + + +backend_to_image_format = {"html": "png", "latex": "pdf"} + +class GraphvizDirective(ImageClass): + def convert(self, fn, path): + path = py.path.local(path).dirpath() + dot = path.join(fn) + result = convert_dot(dot, backend_to_image_format[_backend]) + return result.relto(path) + + def run(self): + newname = self.convert(self.arguments[0], + self.state.document.settings._source) + text = self.block_text.replace("graphviz", "image", 1) + self.block_text = text.replace(self.arguments[0], newname, 1) + self.name = u'image' + self.arguments = [newname] + return ImageClass.run(self) + + def old_interface(self): + def f(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + for arg in "name arguments options content lineno " \ + "content_offset block_text state state_machine".split(): + setattr(self, arg, locals()[arg]) + return self.run() + f.arguments = (1, 0, 1) + f.options = self.option_spec + return f + + +_backend = None +def set_backend_and_register_directives(backend): + #XXX this is only used to work around the inflexibility of docutils: + # a directive does not know the target format + global _backend + _backend = backend + if not directives_are_functions: + directives.register_directive("graphviz", GraphvizDirective) + else: + directives.register_directive("graphviz", + GraphvizDirective().old_interface()) + roles.register_canonical_role("latexformula", latexformula_role) + +def latexformula_role(name, rawtext, text, lineno, inliner, + options={}, content=[]): + if _backend == 'latex': + options['format'] = 'latex' + return roles.raw_role(name, rawtext, text, lineno, inliner, + options, content) + else: + # XXX: make the place of the image directory configurable + sourcedir = py.path.local(inliner.document.settings._source).dirpath() + imagedir = sourcedir.join("img") + if not imagedir.check(): + imagedir.mkdir() + # create halfway senseful imagename: + # use hash of formula + alphanumeric characters of it + # could + imagename = "%s_%s.png" % ( + hash(text), "".join([c for c in text if c.isalnum()])) + image = imagedir.join(imagename) + latexformula2png(unescape(text, True), image) + imagenode = nodes.image(image.relto(sourcedir), uri=image.relto(sourcedir)) + return [imagenode], [] +latexformula_role.content = True +latexformula_role.options = {} + +def register_linkrole(role_name, callback): + def source_role(name, rawtext, text, lineno, inliner, options={}, + content=[]): + text, target = callback(name, text) + reference_node = nodes.reference(rawtext, text, name=text, refuri=target) + return [reference_node], [] + source_role.content = True + source_role.options = {} + roles.register_canonical_role(role_name, source_role) Added: pypy/trunk/pypy/tool/rest/rest.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/rest/rest.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,81 @@ +import py +import sys, os, traceback +import re + +if hasattr(sys.stdout, 'fileno') and os.isatty(sys.stdout.fileno()): + def log(msg): + print msg +else: + def log(msg): + pass + +def convert_rest_html(source, source_path, stylesheet=None, encoding='latin1'): + from pypy.tool.rest import directive + """ return html latin1-encoded document for the given input. + source a ReST-string + sourcepath where to look for includes (basically) + stylesheet path (to be used if any) + """ + from docutils.core import publish_string + directive.set_backend_and_register_directives("html") + kwargs = { + 'stylesheet' : stylesheet, + 'stylesheet_path': None, + 'traceback' : 1, + 'embed_stylesheet': 0, + 'output_encoding' : encoding, + #'halt' : 0, # 'info', + 'halt_level' : 2, + } + # docutils uses os.getcwd() :-( + source_path = os.path.abspath(str(source_path)) + prevdir = os.getcwd() + try: + #os.chdir(os.path.dirname(source_path)) + return publish_string(source, source_path, writer_name='html', + settings_overrides=kwargs) + finally: + os.chdir(prevdir) + +def process(txtpath, encoding='latin1'): + """ process a textfile """ + log("processing %s" % txtpath) + assert txtpath.check(ext='.txt') + if isinstance(txtpath, py.path.svnwc): + txtpath = txtpath.localpath + htmlpath = txtpath.new(ext='.html') + #svninfopath = txtpath.localpath.new(ext='.svninfo') + + style = txtpath.dirpath('style.css') + if style.check(): + stylesheet = style.basename + else: + stylesheet = None + content = unicode(txtpath.read(), encoding) + doc = convert_rest_html(content, txtpath, stylesheet=stylesheet, encoding=encoding) + htmlpath.write(doc) + #log("wrote %r" % htmlpath) + #if txtpath.check(svnwc=1, versioned=1): + # info = txtpath.info() + # svninfopath.dump(info) + +rex1 = re.compile(ur'.*(.*).*', re.MULTILINE | re.DOTALL) +rex2 = re.compile(ur'.*
(.*)
.*', re.MULTILINE | re.DOTALL) + +def strip_html_header(string, encoding='utf8'): + """ return the content of the body-tag """ + uni = unicode(string, encoding) + for rex in rex1,rex2: + match = rex.search(uni) + if not match: + break + uni = match.group(1) + return uni + +class Project: # used for confrest.py files + def __init__(self, sourcepath): + self.sourcepath = sourcepath + def process(self, path): + return process(path) + def get_htmloutputpath(self, path): + return path.new(ext='html') Added: pypy/trunk/pypy/tool/rest/rst.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/rest/rst.py Wed Nov 11 18:54:49 2009 @@ -0,0 +1,417 @@ + +""" reStructuredText generation tools + + provides an api to build a tree from nodes, which can be converted to + ReStructuredText on demand + + note that not all of ReST is supported, a usable subset is offered, but + certain features aren't supported, and also certain details (like how links + are generated, or how escaping is done) can not be controlled +""" + +from __future__ import generators + +import py + +def escape(txt): + """escape ReST markup""" + if not isinstance(txt, str) and not isinstance(txt, unicode): + txt = str(txt) + # XXX this takes a very naive approach to escaping, but it seems to be + # sufficient... + for c in '\\*`|:_': + txt = txt.replace(c, '\\%s' % (c,)) + return txt + +class RestError(Exception): + """ raised on containment errors (wrong parent) """ + +class AbstractMetaclass(type): + def __new__(cls, *args): + obj = super(AbstractMetaclass, cls).__new__(cls, *args) + parent_cls = obj.parentclass + if parent_cls is None: + return obj + if not isinstance(parent_cls, list): + class_list = [parent_cls] + else: + class_list = parent_cls + if obj.allow_nesting: + class_list.append(obj) + + for _class in class_list: + if not _class.allowed_child: + _class.allowed_child = {obj:True} + else: + _class.allowed_child[obj] = True + return obj + +class AbstractNode(object): + """ Base class implementing rest generation + """ + sep = '' + __metaclass__ = AbstractMetaclass + parentclass = None # this exists to allow parent to know what + # children can exist + allow_nesting = False + allowed_child = {} + defaults = {} + + _reg_whitespace = py.std.re.compile('\s+') + + def __init__(self, *args, **kwargs): + self.parent = None + self.children = [] + for child in args: + self._add(child) + for arg in kwargs: + setattr(self, arg, kwargs[arg]) + + def join(self, *children): + """ add child nodes + + returns a reference to self + """ + for child in children: + self._add(child) + return self + + def add(self, child): + """ adds a child node + + returns a reference to the child + """ + self._add(child) + return child + + def _add(self, child): + if child.__class__ not in self.allowed_child: + raise RestError("%r cannot be child of %r" % \ + (child.__class__, self.__class__)) + self.children.append(child) + child.parent = self + + def __getitem__(self, item): + return self.children[item] + + def __setitem__(self, item, value): + self.children[item] = value + + def text(self): + """ return a ReST string representation of the node """ + return self.sep.join([child.text() for child in self.children]) + + def wordlist(self): + """ return a list of ReST strings for this node and its children """ + return [self.text()] + +class Rest(AbstractNode): + """ Root node of a document """ + + sep = "\n\n" + def __init__(self, *args, **kwargs): + AbstractNode.__init__(self, *args, **kwargs) + self.links = {} + + def render_links(self, check=False): + """render the link attachments of the document""" + assert not check, "Link checking not implemented" + if not self.links: + return "" + link_texts = [] + # XXX this could check for duplicates and remove them... + for link, target in self.links.iteritems(): + link_texts.append(".. _`%s`: %s" % (escape(link), target)) + return "\n" + "\n".join(link_texts) + "\n\n" + + def text(self): + outcome = [] + if (isinstance(self.children[0], Transition) or + isinstance(self.children[-1], Transition)): + raise ValueError, ('document must not begin or end with a ' + 'transition') + for child in self.children: + outcome.append(child.text()) + + # always a trailing newline + text = self.sep.join([i for i in outcome if i]) + "\n" + return text + self.render_links() + +class Transition(AbstractNode): + """ a horizontal line """ + parentclass = Rest + + def __init__(self, char='-', width=80, *args, **kwargs): + self.char = char + self.width = width + super(Transition, self).__init__(*args, **kwargs) + + def text(self): + return (self.width - 1) * self.char + +class Paragraph(AbstractNode): + """ simple paragraph """ + + parentclass = Rest + sep = " " + indent = "" + width = 80 + + def __init__(self, *args, **kwargs): + # make shortcut + args = list(args) + for num, arg in py.builtin.enumerate(args): + if isinstance(arg, str): + args[num] = Text(arg) + super(Paragraph, self).__init__(*args, **kwargs) + + def text(self): + texts = [] + for child in self.children: + texts += child.wordlist() + + buf = [] + outcome = [] + lgt = len(self.indent) + + def grab(buf): + outcome.append(self.indent + self.sep.join(buf)) + + texts.reverse() + while texts: + next = texts[-1] + if not next: + texts.pop() + continue + if lgt + len(self.sep) + len(next) <= self.width or not buf: + buf.append(next) + lgt += len(next) + len(self.sep) + texts.pop() + else: + grab(buf) + lgt = len(self.indent) + buf = [] + grab(buf) + return "\n".join(outcome) + +class SubParagraph(Paragraph): + """ indented sub paragraph """ + + indent = " " + +class Title(Paragraph): + """ title element """ + + parentclass = Rest + belowchar = "=" + abovechar = "" + + def text(self): + txt = self._get_text() + lines = [] + if self.abovechar: + lines.append(self.abovechar * len(txt)) + lines.append(txt) + if self.belowchar: + lines.append(self.belowchar * len(txt)) + return "\n".join(lines) + + def _get_text(self): + txt = [] + for node in self.children: + txt += node.wordlist() + return ' '.join(txt) + +class AbstractText(AbstractNode): + parentclass = [Paragraph, Title] + start = "" + end = "" + def __init__(self, _text): + self._text = _text + + def text(self): + text = self.escape(self._text) + return self.start + text + self.end + + def escape(self, text): + if not isinstance(text, str) and not isinstance(text, unicode): + text = str(text) + if self.start: + text = text.replace(self.start, '\\%s' % (self.start,)) + if self.end and self.end != self.start: + text = text.replace(self.end, '\\%s' % (self.end,)) + return text + +class Text(AbstractText): + def wordlist(self): + text = escape(self._text) + return self._reg_whitespace.split(text) + +class LiteralBlock(AbstractText): + parentclass = Rest + start = '::\n\n' + + def text(self): + if not self._text.strip(): + return '' + text = self.escape(self._text).split('\n') + for i, line in py.builtin.enumerate(text): + if line.strip(): + text[i] = ' %s' % (line,) + return self.start + '\n'.join(text) + +class Em(AbstractText): + start = "*" + end = "*" + +class Strong(AbstractText): + start = "**" + end = "**" + +class Quote(AbstractText): + start = '``' + end = '``' + +class Anchor(AbstractText): + start = '_`' + end = '`' + +class Footnote(AbstractText): + def __init__(self, note, symbol=False): + raise NotImplemented('XXX') + +class Citation(AbstractText): + def __init__(self, text, cite): + raise NotImplemented('XXX') + +class ListItem(Paragraph): + allow_nesting = True + item_chars = '*+-' + + def text(self): + idepth = self.get_indent_depth() + indent = self.indent + (idepth + 1) * ' ' + txt = '\n\n'.join(self.render_children(indent)) + ret = [] + item_char = self.item_chars[idepth] + ret += [indent[len(item_char)+1:], item_char, ' ', txt[len(indent):]] + return ''.join(ret) + + def render_children(self, indent): + txt = [] + buffer = [] + def render_buffer(fro, to): + if not fro: + return + p = Paragraph(indent=indent, *fro) + p.parent = self.parent + to.append(p.text()) + for child in self.children: + if isinstance(child, AbstractText): + buffer.append(child) + else: + if buffer: + render_buffer(buffer, txt) + buffer = [] + txt.append(child.text()) + + render_buffer(buffer, txt) + return txt + + def get_indent_depth(self): + depth = 0 + current = self + while (current.parent is not None and + isinstance(current.parent, ListItem)): + depth += 1 + current = current.parent + return depth + +class OrderedListItem(ListItem): + item_chars = ["#."] * 5 + +class DListItem(ListItem): + item_chars = None + def __init__(self, term, definition, *args, **kwargs): + self.term = term + super(DListItem, self).__init__(definition, *args, **kwargs) + + def text(self): + idepth = self.get_indent_depth() + indent = self.indent + (idepth + 1) * ' ' + txt = '\n\n'.join(self.render_children(indent)) + ret = [] + ret += [indent[2:], self.term, '\n', txt] + return ''.join(ret) + +class Link(AbstractText): + start = '`' + end = '`_' + + def __init__(self, _text, target): + self._text = _text + self.target = target + self.rest = None + + def text(self): + if self.rest is None: + self.rest = self.find_rest() + if self.rest.links.get(self._text, self.target) != self.target: + raise ValueError('link name %r already in use for a different ' + 'target' % (self.target,)) + self.rest.links[self._text] = self.target + return AbstractText.text(self) + + def find_rest(self): + # XXX little overkill, but who cares... + next = self + while next.parent is not None: + next = next.parent + return next + +class InternalLink(AbstractText): + start = '`' + end = '`_' + +class LinkTarget(Paragraph): + def __init__(self, name, target): + self.name = name + self.target = target + + def text(self): + return ".. _`%s`:%s\n" % (self.name, self.target) + +class Substitution(AbstractText): + def __init__(self, text, **kwargs): + raise NotImplemented('XXX') + +class Directive(Paragraph): + indent = ' ' + def __init__(self, name, *args, **options): + self.name = name + self.content = options.pop('content', []) + children = list(args) + super(Directive, self).__init__(*children) + self.options = options + + def text(self): + # XXX not very pretty... + namechunksize = len(self.name) + 2 + self.children.insert(0, Text('X' * namechunksize)) + txt = super(Directive, self).text() + txt = '.. %s::%s' % (self.name, txt[namechunksize + 3:],) + options = '\n'.join([' :%s: %s' % (k, v) for (k, v) in + self.options.iteritems()]) + if options: + txt += '\n%s' % (options,) + + if self.content: + txt += '\n' + for item in self.content: + assert item.parentclass == Rest, 'only top-level items allowed' + assert not item.indent + item.indent = ' ' + txt += '\n' + item.text() + + return txt + Modified: pypy/trunk/pypy/tool/statistic_over_time.py ============================================================================== --- pypy/trunk/pypy/tool/statistic_over_time.py (original) +++ pypy/trunk/pypy/tool/statistic_over_time.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import py -from py.__.misc.cmdline.countloc import get_loccount +from py.impl.misc.cmdline.countloc import get_loccount import datetime import time Modified: pypy/trunk/pypy/tool/test/autopath.py ============================================================================== --- pypy/trunk/pypy/tool/test/autopath.py (original) +++ pypy/trunk/pypy/tool/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/tool/test/test_conftest1.py ============================================================================== --- pypy/trunk/pypy/tool/test/test_conftest1.py (original) +++ pypy/trunk/pypy/tool/test/test_conftest1.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ import py -innertest = py.magic.autopath().dirpath('conftest1_innertest.py') +innertest = py.path.local(__file__).dirpath('conftest1_innertest.py') pytest_plugins = "pytest_pytester" class TestPyPyTests: Modified: pypy/trunk/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/trunk/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/trunk/pypy/tool/test/test_pytestsupport.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ import autopath -from py.__.magic import exprinfo +from py.impl.code import _assertionold as exprinfo from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.argument import Arguments Modified: pypy/trunk/pypy/tool/udir.py ============================================================================== --- pypy/trunk/pypy/tool/udir.py (original) +++ pypy/trunk/pypy/tool/udir.py Wed Nov 11 18:54:49 2009 @@ -35,7 +35,7 @@ dir = local(dir) if basename is None: try: - p = py.magic.autopath().dirpath() + p = py.path.local(__file__).dirpath() basename = svn_info(py.path.svnwc(p).info().url) except: basename = '' Modified: pypy/trunk/pypy/translator/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/autopath.py (original) +++ pypy/trunk/pypy/translator/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/benchmark/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/autopath.py (original) +++ pypy/trunk/pypy/translator/benchmark/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/benchmark/benchmarks.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/benchmarks.py (original) +++ pypy/trunk/pypy/translator/benchmark/benchmarks.py Wed Nov 11 18:54:49 2009 @@ -34,7 +34,7 @@ def external_dependency(dirname, svnurl, revision): """Check out (if necessary) a given fixed revision of a svn url.""" - dirpath = py.magic.autopath().dirpath().join(dirname) + dirpath = py.path.local(__file__).dirpath().join(dirname) revtag = dirpath.join('-svn-rev-') if dirpath.check(): if not revtag.check() or int(revtag.read()) != revision: @@ -70,13 +70,13 @@ return get_result(txt, PYSTONE_PATTERN) def run_richards(executable='/usr/local/bin/python', n=5): - richards = py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py') + richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') txt = run_cmd('"%s" %s %s' % (executable, richards, n)) return get_result(txt, RICHARDS_PATTERN) def run_translate(executable='/usr/local/bin/python'): - translate = py.magic.autopath().dirpath().dirpath().join('goal').join('translate.py') - target = py.magic.autopath().dirpath().dirpath().join('goal').join('targetrpystonedalone.py') + translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') + target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' T = time.time() status = os.system(argstr%(executable, translate, target)) @@ -87,7 +87,7 @@ def run_docutils(executable='/usr/local/bin/python'): docutilssvnpath = 'docutils' # subdir of the local dir - translatetxt = py.magic.autopath().dirpath().dirpath().dirpath().join('doc').join('translation.txt') + translatetxt = py.path.local(__file__).dirpath().dirpath().dirpath().join('doc').join('translation.txt') command = """import sys sys.modules['unicodedata'] = sys # docutils need 'import unicodedata' to work, but no more... sys.path[0:0] = ['%s', '%s/extras'] @@ -123,7 +123,7 @@ templess is some simple templating language, to check out use 'svn co -r100 http://johnnydebris.net/templess/trunk templess' """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() pypath = os.path.dirname(os.path.dirname(py.__file__)) templessdir = here.join('templess') testscript = templessdir.join('test/oneshot.py') @@ -143,7 +143,7 @@ def run_gadfly(executable='/usr/local/bin/python'): """ run some tests in the gadfly pure Python database """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() gadfly = here.join('gadfly') testscript = gadfly.join('test', 'testsubset.py') command = 'PYTHONPATH="%s" "%s" "%s"' % (gadfly, executable, testscript) @@ -167,7 +167,7 @@ def run_mako(executable='/usr/local/bin/python'): """ run some tests in the mako templating system """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() mako = here.join('mako') testscript = mako.join('examples', 'bench', 'basic.py') command = 'PYTHONPATH="%s" "%s" "%s" mako' % (mako.join('lib'), Modified: pypy/trunk/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/trunk/pypy/translator/benchmark/jitbench.py (original) +++ pypy/trunk/pypy/translator/benchmark/jitbench.py Wed Nov 11 18:54:49 2009 @@ -14,7 +14,7 @@ response.read() def run_richards(executable='python'): - richards = str(py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py')) + richards = str(py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py')) pipe = subprocess.Popen([executable, richards], stdout=subprocess.PIPE, stderr=subprocess.PIPE) return pipe.communicate() Modified: pypy/trunk/pypy/translator/c/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/c/autopath.py (original) +++ pypy/trunk/pypy/translator/c/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/c/test/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/autopath.py (original) +++ pypy/trunk/pypy/translator/c/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_extfunc.py Wed Nov 11 18:54:49 2009 @@ -141,7 +141,7 @@ os.unlink(filename) def test_os_access(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_access(path, mode): return os.access(path, mode) f = compile(call_access, [str, int]) @@ -150,7 +150,7 @@ def test_os_stat(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) has_blksize = hasattr(os.stat_result, 'st_blksize') has_blocks = hasattr(os.stat_result, 'st_blocks') def call_stat(): @@ -189,7 +189,7 @@ def test_os_fstat(): if os.environ.get('PYPY_CC', '').startswith('tcc'): py.test.skip("segfault with tcc :-(") - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_fstat(): fd = os.open(filename, os.O_RDONLY, 0777) st = os.fstat(fd) Modified: pypy/trunk/pypy/translator/cli/conftest.py ============================================================================== --- pypy/trunk/pypy/translator/cli/conftest.py (original) +++ pypy/trunk/pypy/translator/cli/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-cli options") + group = parser.getgroup("pypy-cli options") group.addoption('--source', action="store_true", dest="source", default=False, help="only generate IL source, don't compile") group.addoption('--wd', action="store_true", dest="wd", default=False, Modified: pypy/trunk/pypy/translator/cli/test/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/cli/test/autopath.py (original) +++ pypy/trunk/pypy/translator/cli/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Wed Nov 11 18:54:49 2009 @@ -9,7 +9,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings from pypy.annotation import policy as annpolicy -from py.compat import optparse +import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS Modified: pypy/trunk/pypy/translator/goal/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/goal/autopath.py (original) +++ pypy/trunk/pypy/translator/goal/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py (original) +++ pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import GameBoyProfiler -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" def entry_point(argv=None): if argv is not None and len(argv) > 1: Modified: pypy/trunk/pypy/translator/goal/targetgbimplementation.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetgbimplementation.py (original) +++ pypy/trunk/pypy/translator/goal/targetgbimplementation.py Wed Nov 11 18:54:49 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.gameboy_implementation import GameBoyImplementation -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" print ROM_PATH Modified: pypy/trunk/pypy/translator/goal/targetgbrom4.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetgbrom4.py (original) +++ pypy/trunk/pypy/translator/goal/targetgbrom4.py Wed Nov 11 18:54:49 2009 @@ -4,7 +4,7 @@ from pypy.lang.gameboy.gameboy import GameBoy -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" EMULATION_CYCLES = 1<<24 Modified: pypy/trunk/pypy/translator/goal/targetpreimportedpypy.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetpreimportedpypy.py (original) +++ pypy/trunk/pypy/translator/goal/targetpreimportedpypy.py Wed Nov 11 18:54:49 2009 @@ -23,7 +23,7 @@ "random", ] -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/trunk/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/trunk/pypy/translator/goal/targetpypystandalone.py Wed Nov 11 18:54:49 2009 @@ -11,7 +11,7 @@ from pypy.tool.option import make_objspace from pypy.translator.goal.nanos import setup_nanos -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/trunk/pypy/translator/goal/test2/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/goal/test2/autopath.py (original) +++ pypy/trunk/pypy/translator/goal/test2/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/goal/translate.py ============================================================================== --- pypy/trunk/pypy/translator/goal/translate.py (original) +++ pypy/trunk/pypy/translator/goal/translate.py Wed Nov 11 18:54:49 2009 @@ -84,7 +84,7 @@ import py # we want 2.4 expand_default functionality -optparse = py.compat.optparse +import optparse from pypy.tool.ansi_print import ansi_log log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) Modified: pypy/trunk/pypy/translator/interactive.py ============================================================================== --- pypy/trunk/pypy/translator/interactive.py (original) +++ pypy/trunk/pypy/translator/interactive.py Wed Nov 11 18:54:49 2009 @@ -1,4 +1,4 @@ -from py.compat import optparse +import optparse import autopath from pypy.translator.translator import TranslationContext Modified: pypy/trunk/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/conftest.py (original) +++ pypy/trunk/pypy/translator/jvm/conftest.py Wed Nov 11 18:54:49 2009 @@ -1,6 +1,6 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-jvm options") + group = parser.getgroup("pypy-jvm options") group.addoption('--java', action='store', dest='java', default='java', help='Define the java executable to use') group.addoption('--javac', action='store', dest='javac', Modified: pypy/trunk/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/genjvm.py (original) +++ pypy/trunk/pypy/translator/jvm/genjvm.py Wed Nov 11 18:54:49 2009 @@ -83,7 +83,7 @@ self.jasmin_files = None # Determine various paths: - self.thisdir = py.magic.autopath().dirpath() + self.thisdir = py.path.local(__file__).dirpath() self.rootdir = self.thisdir.join('src') self.srcdir = self.rootdir.join('pypy') self.jnajar = self.rootdir.join('jna.jar') Modified: pypy/trunk/pypy/translator/microbench/pybench/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/microbench/pybench/autopath.py (original) +++ pypy/trunk/pypy/translator/microbench/pybench/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Wed Nov 11 18:54:49 2009 @@ -6,7 +6,7 @@ import sys, py, os from pypy.tool.ansi_print import ansi_log -from py.__.code import safe_repr +from py.impl.code.code import safe_repr log = py.log.Producer("platform") py.log.setconsumer("platform", ansi_log) @@ -33,9 +33,9 @@ def __repr__(self): if self.err: - return "" % safe_repr._repr(self.err) + return "" % safe_repr(self.err) else: - return "" % safe_repr._repr(self.out) + return "" % safe_repr(self.out) __str__ = __repr__ Modified: pypy/trunk/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_darwin.py Wed Nov 11 18:54:49 2009 @@ -32,7 +32,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(frameworks=('Cocoa',), include_dirs=(includedir,)) executable = self.platform.compile([objcfile], eci) Modified: pypy/trunk/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_maemo.py Wed Nov 11 18:54:49 2009 @@ -26,7 +26,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(include_dirs=(includedir,)) executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) Modified: pypy/trunk/pypy/translator/sandbox/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/sandbox/autopath.py (original) +++ pypy/trunk/pypy/translator/sandbox/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/sandbox/test/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/sandbox/test/autopath.py (original) +++ pypy/trunk/pypy/translator/sandbox/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/test/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/test/autopath.py (original) +++ pypy/trunk/pypy/translator/test/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/trunk/pypy/translator/test/test_driver.py ============================================================================== --- pypy/trunk/pypy/translator/test/test_driver.py (original) +++ pypy/trunk/pypy/translator/test/test_driver.py Wed Nov 11 18:54:49 2009 @@ -1,7 +1,7 @@ import py from pypy.translator.driver import TranslationDriver -from py.compat import optparse +import optparse def test_ctr(): td = TranslationDriver() Modified: pypy/trunk/pypy/translator/tool/autopath.py ============================================================================== --- pypy/trunk/pypy/translator/tool/autopath.py (original) +++ pypy/trunk/pypy/translator/tool/autopath.py Wed Nov 11 18:54:49 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break From hpk at codespeak.net Wed Nov 11 19:01:46 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 19:01:46 +0100 (CET) Subject: [pypy-svn] r69195 - pypy/trunk/py/bin Message-ID: <20091111180146.34FEF16802D@codespeak.net> Author: hpk Date: Wed Nov 11 19:01:45 2009 New Revision: 69195 Modified: pypy/trunk/py/bin/py.cleanup (props changed) pypy/trunk/py/bin/py.convert_unittest (props changed) pypy/trunk/py/bin/py.countloc (props changed) pypy/trunk/py/bin/py.lookup (props changed) pypy/trunk/py/bin/py.svnwcrevert (props changed) pypy/trunk/py/bin/py.test (props changed) pypy/trunk/py/bin/py.which (props changed) Log: adding executable bits which strangely got lost somewhere. From hpk at codespeak.net Wed Nov 11 19:05:02 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Wed, 11 Nov 2009 19:05:02 +0100 (CET) Subject: [pypy-svn] r69196 - pypy/trunk/py/bin Message-ID: <20091111180502.5BFF916802D@codespeak.net> Author: hpk Date: Wed Nov 11 19:05:01 2009 New Revision: 69196 Modified: pypy/trunk/py/bin/py.cleanup (props changed) pypy/trunk/py/bin/py.convert_unittest (props changed) pypy/trunk/py/bin/py.countloc (props changed) pypy/trunk/py/bin/py.lookup (props changed) pypy/trunk/py/bin/py.svnwcrevert (props changed) pypy/trunk/py/bin/py.test (props changed) pypy/trunk/py/bin/py.which (props changed) Log: another try at setting executable bits (coming from using mercurial for a few months svn appears a bit clunky in places) From antocuni at codespeak.net Wed Nov 11 19:10:42 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 11 Nov 2009 19:10:42 +0100 (CET) Subject: [pypy-svn] r69197 - pypy/trunk/pypy/translator Message-ID: <20091111181042.2576916802F@codespeak.net> Author: antocuni Date: Wed Nov 11 19:10:41 2009 New Revision: 69197 Modified: pypy/trunk/pypy/translator/driver.py Log: aaargh, we use backslashes inside Modified: pypy/trunk/pypy/translator/driver.py ============================================================================== --- pypy/trunk/pypy/translator/driver.py (original) +++ pypy/trunk/pypy/translator/driver.py Wed Nov 11 19:10:41 2009 @@ -599,7 +599,7 @@ shutil.copy(os.path.join(usession_path, 'main.il'), dirname) newexename = basename f = file(newexename, 'w') - f.write("""#!/bin/bash + f.write(r"""#!/bin/bash LEDIT=`type -p ledit` EXE=`readlink $0` if [ -z $EXE ] From arigo at codespeak.net Wed Nov 11 19:22:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 19:22:23 +0100 (CET) Subject: [pypy-svn] r69199 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091111182223.A509016802A@codespeak.net> Author: arigo Date: Wed Nov 11 19:22:22 2009 New Revision: 69199 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Optimize a rather obscure case that we saw in pypy-c-jit: "oois" cannot return True, because the classes of the arguments are known to not match. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Wed Nov 11 19:22:22 2009 @@ -688,6 +688,14 @@ elif value0.is_null(): self._optimize_nullness(op, op.args[1], expect_isnot) else: + cls0 = value0.get_constant_class(self.cpu) + if cls0 is not None: + cls1 = value1.get_constant_class(self.cpu) + if cls1 is not None and not cls0.same_constant(cls1): + # cannot be the same object, as we know that their + # class is different + self.make_constant_int(op.result, expect_isnot) + return self.optimize_default(op) def optimize_OOISNOT(self, op): Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Wed Nov 11 19:22:22 2009 @@ -1523,6 +1523,21 @@ """ self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + def test_guard_class_oois(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable2)) [] + i = ooisnot(ConstPtr(myptr), p1) + guard_true(i) [] + jump(p1) + """ + expected = """ + [p1] + guard_class(p1, ConstClass(node_vtable2)) [] + jump(p1) + """ + self.optimize_loop(ops, "Not", expected) + # ---------- def make_fail_descr(self): From arigo at codespeak.net Wed Nov 11 19:27:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 11 Nov 2009 19:27:43 +0100 (CET) Subject: [pypy-svn] r69200 - pypy/branch/faster-raise/pypy/tool Message-ID: <20091111182743.75F3816802A@codespeak.net> Author: arigo Date: Wed Nov 11 19:27:42 2009 New Revision: 69200 Removed: pypy/branch/faster-raise/pypy/tool/_enum_exceptions_broken.py Log: Kill old, not-working-since-forever code. From fijal at codespeak.net Wed Nov 11 19:59:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 19:59:14 +0100 (CET) Subject: [pypy-svn] r69201 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091111185914.5AC0A168032@codespeak.net> Author: fijal Date: Wed Nov 11 19:59:13 2009 New Revision: 69201 Added: pypy/branch/faster-raise/pypy/module/tempexceptions/ (props changed) pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py (contents, props changed) pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (contents, props changed) pypy/branch/faster-raise/pypy/module/tempexceptions/test/ pypy/branch/faster-raise/pypy/module/tempexceptions/test/__init__.py (contents, props changed) pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (contents, props changed) Log: (arigo, fijal) Start implementing exceptions module in RPython, to avoid geninterp mess Added: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py Wed Nov 11 19:59:13 2009 @@ -0,0 +1,11 @@ + +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevel_name = 'tempexceptions' + + appleveldefs = {} + + interpleveldefs = { + 'BaseException' : 'interp_exceptions.W_BaseException', + } Added: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- (empty file) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Wed Nov 11 19:59:13 2009 @@ -0,0 +1,127 @@ + +"""Python's standard exception class hierarchy. + +Before Python 1.5, the standard exceptions were all simple string objects. +In Python 1.5, the standard exceptions were converted to classes organized +into a relatively flat hierarchy. String-based standard exceptions were +optional, or used as a fallback if some problem occurred while importing +the exception module. With Python 1.6, optional string-based standard +exceptions were removed (along with the -X command line flag). + +The class exceptions were implemented in such a way as to be almost +completely backward compatible. Some tricky uses of IOError could +potentially have broken, but by Python 1.6, all of these should have +been fixed. As of Python 1.6, the class-based standard exceptions are +now implemented in C, and are guaranteed to exist in the Python +interpreter. + +Here is a rundown of the class hierarchy. The classes found here are +inserted into both the exceptions module and the `built-in' module. It is +recommended that user defined class based exceptions be derived from the +`Exception' class, although this is currently not enforced. + +BaseException + +-- SystemExit + +-- KeyboardInterrupt + +-- Exception + +-- GeneratorExit + +-- StopIteration + +-- StandardError + | +-- ArithmeticError + | | +-- FloatingPointError + | | +-- OverflowError + | | +-- ZeroDivisionError + | +-- AssertionError + | +-- AttributeError + | +-- EnvironmentError + | | +-- IOError + | | +-- OSError + | | +-- WindowsError (Windows) + | | +-- VMSError (VMS) + | +-- EOFError + | +-- ImportError + | +-- LookupError + | | +-- IndexError + | | +-- KeyError + | +-- MemoryError + | +-- NameError + | | +-- UnboundLocalError + | +-- ReferenceError + | +-- RuntimeError + | | +-- NotImplementedError + | +-- SyntaxError + | | +-- IndentationError + | | +-- TabError + | +-- SystemError + | +-- TypeError + | +-- ValueError + | | +-- UnicodeError + | | +-- UnicodeDecodeError + | | +-- UnicodeEncodeError + | | +-- UnicodeTranslateError + +-- Warning + +-- DeprecationWarning + +-- PendingDeprecationWarning + +-- RuntimeWarning + +-- SyntaxWarning + +-- UserWarning + +-- FutureWarning + +-- ImportWarning + +-- UnicodeWarning +""" + +from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ + GetSetProperty +from pypy.interpreter.gateway import interp2app + +class W_BaseException(Wrappable): + """Superclass representing the base of the exception hierarchy. + + The __getitem__ method is provided for backwards-compatibility + and will be deprecated at some point. + """ + + def __init__(self, space, args_w): + self.args_w = args_w + if len(args_w) == 1: + self.w_message = args_w[0] + else: + self.w_message = space.wrap("") + + def descr_str(self, space): + lgt = len(self.args_w) + if lgt == 0: + return space.wrap('') + elif lgt == 1: + return space.str(self.w_message) + else: + return space.str(space.newtuple(self.args_w)) + descr_str.unwrap_spec = ['self', ObjSpace] + + def descr_repr(self, space): + if self.args_w: + args_repr = space.str_w(space.repr(space.newtuple(self.args_w))) + else: + args_repr = "()" + clsname = self.getclass(space).getname(space, '?') + return space.wrap(clsname + args_repr) + descr_repr.unwrap_spec = ['self', ObjSpace] + + def descr_getargs(space, self): + return space.newtuple(self.args_w) + +def descr_new_base_exception(space, w_subtype, args_w): + exc = space.allocate_instance(W_BaseException, w_subtype) + W_BaseException.__init__(exc, space, args_w) + return space.wrap(exc) +descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] + +W_BaseException.typedef = TypeDef( + 'BaseException', + __new__ = interp2app(descr_new_base_exception), + __str__ = interp2app(W_BaseException.descr_str), + __repr__ = interp2app(W_BaseException.descr_repr), + message = interp_attrproperty_w('w_message', W_BaseException), + args = GetSetProperty(W_BaseException.descr_getargs), +) Added: pypy/branch/faster-raise/pypy/module/tempexceptions/test/__init__.py ============================================================================== Added: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- (empty file) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Wed Nov 11 19:59:13 2009 @@ -0,0 +1,22 @@ + +from pypy.conftest import gettestobjspace + +class AppTestExc(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('tempexceptions',)) + + def test_str(self): + from tempexceptions import BaseException + + assert str(BaseException()) == '' + assert repr(BaseException()) == 'BaseException()' + assert BaseException().message == '' + assert BaseException(3).message == 3 + assert repr(BaseException(3)) == 'BaseException(3,)' + assert str(BaseException(3)) == '3' + assert BaseException().args == () + assert BaseException(3).args == (3,) + assert BaseException(3, "x").args == (3, "x") + assert repr(BaseException(3, "x")) == "BaseException(3, 'x')" + assert str(BaseException(3, "x")) == "(3, 'x')" + assert BaseException(3, "x").message == '' From fijal at codespeak.net Wed Nov 11 20:07:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 11 Nov 2009 20:07:03 +0100 (CET) Subject: [pypy-svn] r69202 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091111190703.BD8B516802D@codespeak.net> Author: fijal Date: Wed Nov 11 20:07:02 2009 New Revision: 69202 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: Add Exception Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py Wed Nov 11 20:07:02 2009 @@ -8,4 +8,5 @@ interpleveldefs = { 'BaseException' : 'interp_exceptions.W_BaseException', + 'Exception' : 'interp_exceptions.W_Exception', } Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Wed Nov 11 20:07:02 2009 @@ -111,17 +111,31 @@ def descr_getargs(space, self): return space.newtuple(self.args_w) -def descr_new_base_exception(space, w_subtype, args_w): - exc = space.allocate_instance(W_BaseException, w_subtype) - W_BaseException.__init__(exc, space, args_w) - return space.wrap(exc) -descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] +def _new(cls): + def descr_new_base_exception(space, w_subtype, args_w): + exc = space.allocate_instance(cls, w_subtype) + cls.__init__(exc, space, args_w) + return space.wrap(exc) + descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] + descr_new_base_exception.func_name = 'descr_new_' + cls.__name__ + return interp2app(descr_new_base_exception) W_BaseException.typedef = TypeDef( 'BaseException', - __new__ = interp2app(descr_new_base_exception), + __doc__ = W_BaseException.__doc__, + __new__ = _new(W_BaseException), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), message = interp_attrproperty_w('w_message', W_BaseException), args = GetSetProperty(W_BaseException.descr_getargs), ) + +class W_Exception(W_BaseException): + """Common base class for all non-exit exceptions.""" + +W_Exception.typedef = TypeDef( + 'Exception', + W_BaseException.typedef, + __doc__ = W_Exception.__doc__, + __new__ = _new(W_Exception), +) Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Wed Nov 11 20:07:02 2009 @@ -5,7 +5,7 @@ def setup_class(cls): cls.space = gettestobjspace(usemodules=('tempexceptions',)) - def test_str(self): + def test_baseexc(self): from tempexceptions import BaseException assert str(BaseException()) == '' @@ -20,3 +20,11 @@ assert repr(BaseException(3, "x")) == "BaseException(3, 'x')" assert str(BaseException(3, "x")) == "(3, 'x')" assert BaseException(3, "x").message == '' + + def test_exc(self): + from tempexceptions import Exception, BaseException + + assert issubclass(Exception, BaseException) + assert isinstance(Exception(), Exception) + assert isinstance(Exception(), BaseException) + assert repr(Exception(3, "x")) == "Exception(3, 'x')" From afa at codespeak.net Thu Nov 12 02:30:06 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 12 Nov 2009 02:30:06 +0100 (CET) Subject: [pypy-svn] r69208 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: . gcc Message-ID: <20091112013006.87A3416802D@codespeak.net> Author: afa Date: Thu Nov 12 02:30:05 2009 New Revision: 69208 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Log: Finally fix test_callback_simple and test_callback_with_collect. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Nov 12 02:30:05 2009 @@ -791,7 +791,7 @@ r_gcroot_marker = re.compile(r"$1") # never matches r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot") - r_bottom_marker = re.compile(r"\tcall\t_pypy_asm_stack_bottom\s*") + r_bottom_marker = re.compile(r"; .+\tpypy_asm_stack_bottom\(\);") r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") @@ -1228,7 +1228,7 @@ def _offset(name): if self.format == 'msvc': - return "DWORD PTR [%s]" % _globalname(name) + return "OFFSET %s" % _globalname(name) else: return "$%s" % _globalname(name) Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/genc.py Thu Nov 12 02:30:05 2009 @@ -491,7 +491,8 @@ trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] if self.translator.platform.name == 'msvc': trackgcfiles = [f for f in trackgcfiles - if f.startswith(('implement', 'testing'))] + if f.startswith(('implement', 'testing', + '../module_cache/module'))] sfiles = ['%s.s' % (c,) for c in trackgcfiles] lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] From arigo at codespeak.net Thu Nov 12 10:17:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 10:17:13 +0100 (CET) Subject: [pypy-svn] r69210 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091112091713.3A0D816802F@codespeak.net> Author: arigo Date: Thu Nov 12 10:17:12 2009 New Revision: 69210 Modified: pypy/trunk/pypy/jit/metainterp/test/test_list.py Log: Also test ll_list_contains, which works. Modified: pypy/trunk/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_list.py Thu Nov 12 10:17:12 2009 @@ -399,6 +399,9 @@ n += a.x n = lst.pop() lst.append(n - 10 + a.x) + if a.x in lst: + pass + a.x = a.x + 1 - 1 a = lst.pop() b = lst.pop() return a * b From afa at codespeak.net Thu Nov 12 10:31:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 12 Nov 2009 10:31:25 +0100 (CET) Subject: [pypy-svn] r69211 - in pypy/branch/msvc-asmgcroot/pypy/translator/c: gcc test Message-ID: <20091112093125.4949416802F@codespeak.net> Author: afa Date: Thu Nov 12 10:31:23 2009 New Revision: 69211 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py Log: Don't hardcode the name of the libc inside the generated executable This fixes the case when the CPython used to run the tests was built with a different compiler. Also remove PUBLIC for one more symbol, found with some compile options. Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Nov 12 10:31:23 2009 @@ -1111,6 +1111,8 @@ line = '; ' + line elif line.startswith("PUBLIC\t??_C@"): line = '; ' + line + elif line == "PUBLIC\t__$ArrayPad$\n": + line = '; ' + line # Because we insert labels in the code, some "SHORT" jumps # are now longer than 127 bytes. We turn them all into Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/test/test_newgc.py Thu Nov 12 10:31:23 2009 @@ -622,7 +622,7 @@ import gc slong = cast_type_to_ffitype(rffi.LONG) - from pypy.rpython.lltypesystem.ll2ctypes import libc_name + from pypy.rlib.libffi import get_libc_name def callback(ll_args, ll_res, stuff): gc.collect() @@ -637,7 +637,7 @@ res[0] = -1 def f(): - libc = CDLL(libc_name) + libc = CDLL(get_libc_name()) qsort = libc.getpointer('qsort', [ffi_type_pointer, slong, slong, ffi_type_pointer], ffi_type_void) From cfbolz at codespeak.net Thu Nov 12 10:38:10 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 12 Nov 2009 10:38:10 +0100 (CET) Subject: [pypy-svn] r69212 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091112093810.1219C168030@codespeak.net> Author: cfbolz Date: Thu Nov 12 10:38:09 2009 New Revision: 69212 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: (all): planning for today Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Thu Nov 12 10:38:09 2009 @@ -10,19 +10,21 @@ TASKS ====== - - make the size of ResOperation instances less insane SEMI-DONE - - set up the buildbot on the Mac Mini (Samuele, Carl Friedrich) + - set up the buildbot on the Mac Mini DONE - directly call assembler for residual portal calls - - making the CLI backend working with logging + - making the CLI backend working with logging DONE + - set up nightly run on Mac (Samuele, Carl Friedrich) + - debug CLI crashes (Anto) - compress the virtuals part of resume data more (Samuele, Carl Friedrich) - - try to do something non-insane about Python-level exceptions (Maciek, Armin) + - try to do something non-insane about Python-level exceptions IN-PROGRESS + - improve operations logging DONE? + - rewrite exceptions module to be a mixed module (Armin, Maciek) - make the assembler produced by generate_failure smaller - - merge guard_nonnull(x, ...), guard_class(x, ...) DONE, cli needs fixing + - merge guard_nonnull(x, ...), guard_class(x, ...) DONE - put the class into the structure to get only one promote when using an instance - look into failing pypy-c-jit apptests - - port writeanalyze to ootype DONE - - fix buildbot on py.test 1.1 branch (Holger, Samuele around) + - fix buildbot on py.test 1.1 branch BRANCH MERGED From arigo at codespeak.net Thu Nov 12 10:42:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 10:42:08 +0100 (CET) Subject: [pypy-svn] r69213 - in pypy/trunk/pypy: config module/_minimal_curses Message-ID: <20091112094208.82A58168030@codespeak.net> Author: arigo Date: Thu Nov 12 10:42:07 2009 New Revision: 69213 Modified: pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/module/_minimal_curses/fficurses.py Log: Reported by 'ot' on #pypy: crash early if curses.h and term.h are not on the system. Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Thu Nov 12 10:42:07 2009 @@ -77,6 +77,7 @@ "bz2" : ["pypy.module.bz2.interp_bz2"], "pyexpat" : ["pypy.module.pyexpat.interp_pyexpat"], "_ssl" : ["pypy.module._ssl.interp_ssl"], + "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], } def get_module_validator(modname): Modified: pypy/trunk/pypy/module/_minimal_curses/fficurses.py ============================================================================== --- pypy/trunk/pypy/module/_minimal_curses/fficurses.py (original) +++ pypy/trunk/pypy/module/_minimal_curses/fficurses.py Thu Nov 12 10:42:07 2009 @@ -5,6 +5,7 @@ import sys from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype +from pypy.rpython.tool import rffi_platform from pypy.rpython.extfunc import register_external from pypy.rpython.extregistry import ExtRegistryEntry from pypy.module._minimal_curses import interp_curses @@ -15,6 +16,7 @@ libraries = ['curses'], ) +rffi_platform.verify_eci(eci) INT = rffi.INT From hpk at codespeak.net Thu Nov 12 11:42:05 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Thu, 12 Nov 2009 11:42:05 +0100 (CET) Subject: [pypy-svn] r69214 - pypy/branch/py11 Message-ID: <20091112104205.4E53B16802F@codespeak.net> Author: hpk Date: Thu Nov 12 11:42:04 2009 New Revision: 69214 Removed: pypy/branch/py11/ Log: removing merged branch From fijal at codespeak.net Thu Nov 12 11:45:43 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 11:45:43 +0100 (CET) Subject: [pypy-svn] r69215 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112104543.21DA916802F@codespeak.net> Author: fijal Date: Thu Nov 12 11:45:42 2009 New Revision: 69215 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: (arigo, fijal) Slight progress Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py Thu Nov 12 11:45:42 2009 @@ -7,6 +7,51 @@ appleveldefs = {} interpleveldefs = { + 'ArithmeticError' : 'interp_exceptions.W_ArithmeticError', + 'AssertionError' : 'interp_exceptions.W_AssertionError', + 'AttributeError' : 'interp_exceptions.W_AttributeError', 'BaseException' : 'interp_exceptions.W_BaseException', + 'DeprecationWarning' : 'interp_exceptions.W_DeprecationWarning', + 'EOFError' : 'interp_exceptions.W_EOFError', + 'EnvironmentError' : 'interp_exceptions.W_EnvironmentError', 'Exception' : 'interp_exceptions.W_Exception', + 'FloatingPointError' : 'interp_exceptions.W_FloatingPointError', + 'FutureWarning' : 'interp_exceptions.W_FutureWarning', + 'GeneratorExit' : 'interp_exceptions.W_GeneratorExit', + 'IOError' : 'interp_exceptions.W_IOError', + 'ImportError' : 'interp_exceptions.W_ImportError', + 'ImportWarning' : 'interp_exceptions.W_ImportWarning', + 'IndentationError' : 'interp_exceptions.W_IndentationError', + 'IndexError' : 'interp_exceptions.W_IndexError', + 'KeyError' : 'interp_exceptions.W_KeyError', + 'KeyboardInterrupt' : 'interp_exceptions.W_KeyboardInterrupt', + 'LookupError' : 'interp_exceptions.W_LookupError', + 'MemoryError' : 'interp_exceptions.W_MemoryError', + 'NameError' : 'interp_exceptions.W_NameError', + 'NotImplementedError' : 'interp_exceptions.W_NotImplementedError', + 'OSError' : 'interp_exceptions.W_OSError', + 'OverflowError' : 'interp_exceptions.W_OverflowError', + 'PendingDeprecationWarning' : 'interp_exceptions.W_PendingDeprecationWarning', + 'ReferenceError' : 'interp_exceptions.W_ReferenceError', + 'RuntimeError' : 'interp_exceptions.W_RuntimeError', + 'RuntimeWarning' : 'interp_exceptions.W_RuntimeWarning', + 'StandardError' : 'interp_exceptions.W_StandardError', + 'StopIteration' : 'interp_exceptions.W_StopIteration', + 'SyntaxError' : 'interp_exceptions.W_SyntaxError', + 'SyntaxWarning' : 'interp_exceptions.W_SyntaxWarning', + 'SystemError' : 'interp_exceptions.W_SystemError', + 'SystemExit' : 'interp_exceptions.W_SystemExit', + 'TabError' : 'interp_exceptions.W_TabError', + 'TypeError' : 'interp_exceptions.W_TypeError', + 'UnboundLocalError' : 'interp_exceptions.W_UnboundLocalError', + 'UnicodeDecodeError' : 'interp_exceptions.W_UnicodeDecodeError', + 'UnicodeEncodeError' : 'interp_exceptions.W_UnicodeEncodeError', + 'UnicodeError' : 'interp_exceptions.W_UnicodeError', + 'UnicodeTranslateError' : 'interp_exceptions.W_UnicodeTranslateError', + 'UnicodeWarning' : 'interp_exceptions.W_UnicodeWarning', + 'UserWarning' : 'interp_exceptions.W_UserWarning', + 'ValueError' : 'interp_exceptions.W_ValueError', + 'Warning' : 'interp_exceptions.W_Warning', + 'ZeroDivisionError' : 'interp_exceptions.W_ZeroDivisionError', + } Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Thu Nov 12 11:45:42 2009 @@ -72,7 +72,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ - GetSetProperty + GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict from pypy.interpreter.gateway import interp2app class W_BaseException(Wrappable): @@ -81,9 +81,11 @@ The __getitem__ method is provided for backwards-compatibility and will be deprecated at some point. """ + w_dict = None def __init__(self, space, args_w): self.args_w = args_w + self.space = space if len(args_w) == 1: self.w_message = args_w[0] else: @@ -111,6 +113,17 @@ def descr_getargs(space, self): return space.newtuple(self.args_w) + def getdict(self): + if self.w_dict is None: + self.w_dict = self.space.newdict() + return self.w_dict + + def setdict(self, space, w_dict): + if not space.is_true(space.isinstance( w_dict, space.w_dict )): + raise OperationError( space.w_TypeError, space.wrap("setting exceptions's dictionary to a non-dict") ) + self.w_dict = w_dict + + def _new(cls): def descr_new_base_exception(space, w_subtype, args_w): exc = space.allocate_instance(cls, w_subtype) @@ -126,16 +139,111 @@ __new__ = _new(W_BaseException), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), + __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, + cls=W_BaseException), message = interp_attrproperty_w('w_message', W_BaseException), args = GetSetProperty(W_BaseException.descr_getargs), ) -class W_Exception(W_BaseException): - """Common base class for all non-exit exceptions.""" +def _new_exception(name, base, docstring, **kwargs): + class W_Exception(base): + __doc__ = docstring + + W_Exception.__name__ = 'W_' + name + + for k, v in kwargs.items(): + kwargs[k] = interp2app(v.__get__(None, W_Exception)) + W_Exception.typedef = TypeDef( + name, + base.typedef, + __doc__ = W_Exception.__doc__, + __new__ = _new(W_Exception), + **kwargs + ) + return W_Exception + +W_Exception = _new_exception('Exception', W_BaseException, + """Common base class for all non-exit exceptions.""") + +W_GeneratorExit = _new_exception('GeneratorExit', W_Exception, + """Request that a generator exit.""") + +W_StandardError = _new_exception('StandardError', W_Exception, + """Base class for all standard Python exceptions.""") + +W_ValueError = _new_exception('ValueError', W_StandardError, + """Inappropriate argument value (of correct type).""") + +W_ImportError = _new_exception('ImportError', W_StandardError, + """Import can't find module, or can't find name in module.""") + +W_RuntimeError = _new_exception('RuntimeError', W_StandardError, + """Unspecified run-time error.""") + +W_UnicodeError = _new_exception('UnicodeError', W_ValueError, + """Unicode related error.""") + + +class W_UnicodeTranslateError(W_UnicodeError): + """Unicode translation error.""" + def __init__(self, space, w_obj, w_start, w_end, w_reason): + self.object = space.unicode_w(w_obj) + self.start = space.int_w(w_start) + self.end = space.int_w(w_end) + self.reason = space.str_w(w_reason) + W_BaseException.__init__(self, space, [w_obj, w_start, w_end, w_reason]) -W_Exception.typedef = TypeDef( - 'Exception', - W_BaseException.typedef, - __doc__ = W_Exception.__doc__, - __new__ = _new(W_Exception), + def descr_str(self, space): + return space.appexec([space.wrap(self)], """(self): + if self.end == self.start + 1: + badchar = ord(self.object[self.start]) + if badchar <= 0xff: + return "can't translate character u'\\\\x%02x' in position %d: %s" % (badchar, self.start, self.reason) + if badchar <= 0xffff: + return "can't translate character u'\\\\u%04x' in position %d: %s"%(badchar, self.start, self.reason) + return "can't translate character u'\\\\U%08x' in position %d: %s"%(badchar, self.start, self.reason) + return "can't translate characters in position %d-%d: %s" % (self.start, self.end - 1, self.reason) + """) + descr_str.unwrap_spec = ['self', ObjSpace] + +def descr_new_unicode_translate_error(space, w_subtype, w_obj, w_start, w_end, + w_reason): + exc = space.allocate_instance(W_UnicodeTranslateError, w_subtype) + W_UnicodeTranslateError.__init__(exc, space, w_obj, w_start, + w_end, w_reason) + return space.wrap(exc) + +def readwrite_attrproperty(name, cls, unwrapname): + def fget(space, obj): + return space.wrap(getattr(obj, name)) + def fset(space, obj, w_val): + setattr(obj, name, getattr(space, unwrapname)(w_val)) + return GetSetProperty(fget, fset, cls=cls) + +W_UnicodeTranslateError.typedef = TypeDef( + 'UnicodeTranslateError', + W_UnicodeError.typedef, + __doc__ = W_UnicodeTranslateError.__doc__, + __new__ = interp2app(descr_new_unicode_translate_error), + __str__ = interp2app(W_UnicodeTranslateError.descr_str), + object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), + start = readwrite_attrproperty('start', W_UnicodeTranslateError, 'int_w'), + end = readwrite_attrproperty('end', W_UnicodeTranslateError, 'int_w'), + reason = readwrite_attrproperty('reason', W_UnicodeTranslateError, 'str_w'), ) + +W_LookupError = _new_exception('LookupError', W_StandardError, + """Base class for lookup errors.""") + +def key_error_str(self, space): + if len(self.args_w) == 0: + return space.wrap('') + elif len(self.args_w) == 1: + return space.repr(self.args_w[0]) + else: + return space.str(space.newtuple(self.args_w)) +key_error_str.unwrap_spec = ['self', ObjSpace] + +W_KeyError = _new_exception('KeyError', W_LookupError, + """Mapping key not found.""", + __str__ = key_error_str) Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Thu Nov 12 11:45:42 2009 @@ -20,6 +20,9 @@ assert repr(BaseException(3, "x")) == "BaseException(3, 'x')" assert str(BaseException(3, "x")) == "(3, 'x')" assert BaseException(3, "x").message == '' + x = BaseException() + x.xyz = 3 + assert x.xyz == 3 def test_exc(self): from tempexceptions import Exception, BaseException @@ -28,3 +31,34 @@ assert isinstance(Exception(), Exception) assert isinstance(Exception(), BaseException) assert repr(Exception(3, "x")) == "Exception(3, 'x')" + + def test_custom_class(self): + from tempexceptions import Exception + + class MyException(Exception): + def __init__(self, x): + self.x = x + + def __str__(self): + return self.x + + assert issubclass(MyException, Exception) + assert str(MyException("x")) == "x" + + def test_unicode_translate_error(self): + from tempexceptions import UnicodeTranslateError + ut = UnicodeTranslateError(u"x", 1, 5, "bah") + assert ut.object == u'x' + assert ut.start == 1 + assert ut.end == 5 + assert ut.reason == 'bah' + assert ut.args == (u'x', 1, 5, 'bah') + assert ut.message == '' + ut.object = u'y' + assert ut.object == u'y' + assert str(ut) == "can't translate characters in position 1-4: bah" + + def test_key_error(self): + from tempexceptions import KeyError + + assert str(KeyError('s')) == "'s'" From arigo at codespeak.net Thu Nov 12 12:02:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 12:02:21 +0100 (CET) Subject: [pypy-svn] r69216 - pypy/trunk/pypy/tool/rest Message-ID: <20091112110221.9670216802F@codespeak.net> Author: arigo Date: Thu Nov 12 12:02:20 2009 New Revision: 69216 Modified: pypy/trunk/pypy/tool/rest/ (props changed) pypy/trunk/pypy/tool/rest/__init__.py (props changed) pypy/trunk/pypy/tool/rest/convert.py (props changed) pypy/trunk/pypy/tool/rest/directive.py (props changed) pypy/trunk/pypy/tool/rest/rest.py (props changed) pypy/trunk/pypy/tool/rest/rst.py (props changed) Log: fixeol From arigo at codespeak.net Thu Nov 12 12:55:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 12:55:50 +0100 (CET) Subject: [pypy-svn] r69217 - in pypy/trunk/pypy: doc doc/config jit/backend/x86 jit/metainterp jit/metainterp/test lang/gameboy lang/gameboy/debug lib lib/app_test/ctypes_tests rlib rlib/rsdl rlib/rstruct rlib/test rpython/module/test tool tool/pytest/test translator/goal Message-ID: <20091112115550.4F13316802F@codespeak.net> Author: arigo Date: Thu Nov 12 12:55:49 2009 New Revision: 69217 Modified: pypy/trunk/pypy/doc/config/objspace.std.immutable_builtintypes.txt (contents, props changed) pypy/trunk/pypy/doc/config/objspace.usemodules._locale.txt (props changed) pypy/trunk/pypy/doc/config/objspace.usemodules._winreg.txt (contents, props changed) pypy/trunk/pypy/doc/index.txt (contents, props changed) pypy/trunk/pypy/jit/backend/x86/detect_sse2.py (props changed) pypy/trunk/pypy/jit/backend/x86/valgrind.py (props changed) pypy/trunk/pypy/jit/metainterp/optimizeutil.py (props changed) pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (props changed) pypy/trunk/pypy/lang/gameboy/debug/gameboy_debug_parts.py (props changed) pypy/trunk/pypy/lang/gameboy/video_meta.py (props changed) pypy/trunk/pypy/lib/_csv.py (props changed) pypy/trunk/pypy/lib/_sha256.py (props changed) pypy/trunk/pypy/lib/_sha512.py (props changed) pypy/trunk/pypy/lib/app_test/ctypes_tests/test_win32.py (contents, props changed) pypy/trunk/pypy/rlib/rsdl/RMix_helper.py (props changed) pypy/trunk/pypy/rlib/rstruct/__init__.py (props changed) pypy/trunk/pypy/rlib/rwinreg.py (contents, props changed) pypy/trunk/pypy/rlib/test/test_timer.py (props changed) pypy/trunk/pypy/rlib/timer.py (props changed) pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py (contents, props changed) pypy/trunk/pypy/tool/difftime.py (props changed) pypy/trunk/pypy/tool/pytest/test/test_appsupport.py (props changed) pypy/trunk/pypy/tool/watchdog_nt.py (props changed) pypy/trunk/pypy/translator/goal/targetgbfullprofiling.py (props changed) pypy/trunk/pypy/translator/goal/targetgbprofiling.py (props changed) pypy/trunk/pypy/translator/goal/targetpreimportedpypy.py (props changed) Log: fixeol Modified: pypy/trunk/pypy/doc/config/objspace.std.immutable_builtintypes.txt ============================================================================== --- pypy/trunk/pypy/doc/config/objspace.std.immutable_builtintypes.txt (original) +++ pypy/trunk/pypy/doc/config/objspace.std.immutable_builtintypes.txt Thu Nov 12 12:55:49 2009 @@ -1 +1 @@ -Disallow modification of builtin types. Enabled by default. +Disallow modification of builtin types. Enabled by default. Modified: pypy/trunk/pypy/doc/config/objspace.usemodules._winreg.txt ============================================================================== --- pypy/trunk/pypy/doc/config/objspace.usemodules._winreg.txt (original) +++ pypy/trunk/pypy/doc/config/objspace.usemodules._winreg.txt Thu Nov 12 12:55:49 2009 @@ -1,2 +1,2 @@ -Use the built-in '_winreg' module, provides access to the Windows registry. -This module is expected to be working and is included by default on Windows. +Use the built-in '_winreg' module, provides access to the Windows registry. +This module is expected to be working and is included by default on Windows. Modified: pypy/trunk/pypy/doc/index.txt ============================================================================== --- pypy/trunk/pypy/doc/index.txt (original) +++ pypy/trunk/pypy/doc/index.txt Thu Nov 12 12:55:49 2009 @@ -1,59 +1,331 @@ +================================================= +PyPy - a Python_ implementation written in Python +================================================= -The PyPy project aims at producing a flexible and fast Python_ -implementation. The guiding idea is to translate a Python-level -description of the Python language itself to lower level languages. -Rumors have it that the secret goal is being faster-than-C which is -nonsense, isn't it? `more...`_ +.. _Python: http://www.python.org/dev/doc/maint24/ref/ref.html -Getting into PyPy ... -============================================= +.. sectnum:: +.. contents:: :depth: 1 -* `Release 1.1`_: the latest official release -* `PyPy Blog`_: news and status info about PyPy +PyPy User Documentation +=============================================== -* `Documentation`_: extensive documentation and papers_ about PyPy. +`getting started`_ provides hands-on instructions +including a two-liner to run the PyPy Python interpreter +on your system, examples on advanced features and +entry points for using PyPy's translation tool chain. -* `Getting Started`_: Getting started and playing with PyPy. +`FAQ`_ contains some frequently asked questions. -Mailing lists, bug tracker, IRC channel -============================================= +New features of PyPy's Python Interpreter and +Translation Framework: -* `Development mailing list`_: development and conceptual - discussions. + * `What PyPy can do for your objects`_ + * `Stackless and coroutines`_ + * `JIT Generation in PyPy`_ + * `Sandboxing Python code`_ -* `Subversion commit mailing list`_: updates to code and - documentation. +`PyPy Prolog Interpreter`_ describes an implementation of +Prolog that makes use of our Translation Tool chain. -* `Development bug/feature tracker`_: filing bugs and feature requests. +Status_ of the project. -* `Sprint mailing list`_: mailing list for organising upcoming sprints. -* **IRC channel #pypy on freenode**: Many of the core developers are hanging out - at #pypy on irc.freenode.net. You are welcome to join and ask questions - (if they are not already developed in the FAQ_). - You can find logs of the channel here_. +Project Documentation +===================================== -.. XXX play1? +PyPy was funded by the EU for several years. See the `web site of the EU +project`_ for more details. + +.. _`web site of the EU project`: http://pypy.org + +architecture_ gives a complete view of PyPy's basic design. + +`coding guide`_ helps you to write code for PyPy (especially also describes +coding in RPython a bit). + +`sprint reports`_ lists reports written at most of our sprints, from +2003 to the present. + +`talks and related projects`_ lists presentations +and related projects. + +`ideas for PyPy related projects`_ which might be a good way to get +into PyPy. + +`PyPy video documentation`_ is a page linking to the videos (e.g. of talks and +introductions) that are available. + +`Technical reports`_ is a page that contains links to the +reports that we submitted to the European Union. + +`development methodology`_ describes our sprint-driven approach. + +`license`_ contains licensing details (basically a straight MIT-license). + +`Glossary`_ of PyPy words to help you align your inner self with +the PyPy universe. + + +Status +=================================== + +PyPy can be used to run Python programs on Linux, OS/X, +Windows, on top of .NET, and on top of Java. +It is recommended to try out the current Subversion HEAD, +which contains `major improvements`__ since the last release. + +.. __: http://codespeak.net/pipermail/pypy-dev/2007q4/004103.html + +PyPy is mainly developed on Linux and Mac OS X. Windows is supported, +but platform-specific bugs tend to take longer before we notice and fix +them. Linux 64-bit machines are supported (though it may also take some +time before we notice and fix bugs). + +PyPy's own tests `summary`_, daily updated, run through BuildBot infrastructure. +You can also find CPython's compliance tests run with compiled ``pypy-c`` +exeuctables there. + +information dating from early 2007: + +`PyPy LOC statistics`_ shows LOC statistics about PyPy. + +`PyPy statistics`_ is a page with various statistics about the PyPy project. + +`compatibility matrix`_ is a diagram that shows which of the various features +of the PyPy interpreter work together with which other features. + + +Source Code Documentation +=============================================== + +`object spaces`_ discusses the object space interface +and several implementations. + +`bytecode interpreter`_ explains the basic mechanisms +of the bytecode interpreter and virtual machine. + +`interpreter optimizations`_ describes our various strategies for +improving the performance of our interpreter, including alternative +object implementations (for strings, dictionaries and lists) in the +standard object space. + +`translation`_ is a detailed overview of our translation process. The +rtyper_ is the largest component of our translation process. + +`dynamic-language translation`_ is a paper that describes +the translation process, especially the flow object space +and the annotator in detail. This document is also part +of the `EU reports`_. + +`low-level encapsulation`_ describes how our approach hides +away a lot of low level details. This document is also part +of the `EU reports`_. + +`translation aspects`_ describes how we weave different +properties into our interpreter during the translation +process. This document is also part of the `EU reports`_. + +`garbage collector`_ strategies that can be used by the virtual +machines produced by the translation process. + +`parser`_ contains (outdated, unfinished) documentation about +the parser. + +`rlib`_ describes some modules that can be used when implementing programs in +RPython. + +`configuration documentation`_ describes the various configuration options that +allow you to customize PyPy. + +`CLI backend`_ describes the details of the .NET backend. + +`JIT Generation in PyPy`_ describes how we produce the Python Just-in-time Compiler +from our Python interpreter. -Meeting PyPy developers -======================= -The PyPy developers are organising sprints and presenting results at -conferences all year round. They will be happy to meet in person with -anyone interested in the project. Watch out for sprint announcements -on the `development mailing list`_. -.. _Python: http://docs.python.org/index.html -.. _`more...`: architecture.html#mission-statement -.. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ -.. _here: http://tismerysoft.de/pypy/irc-logs/pypy -.. _`sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint -.. _`subversion commit mailing list`: http://codespeak.net/mailman/listinfo/pypy-svn -.. _`development mailing list`: http://codespeak.net/mailman/listinfo/pypy-dev .. _`FAQ`: faq.html -.. _`Documentation`: docindex.html -.. _`Getting Started`: getting-started.html -.. _papers: extradoc.html -.. _`Release 1.1`: release-1.1.0.html +.. _Glossary: glossary.html +.. _`PyPy video documentation`: video-index.html +.. _parser: parser.html +.. _`development methodology`: dev_method.html +.. _`sprint reports`: sprint-reports.html +.. _`talks and related projects`: extradoc.html +.. _`license`: ../../LICENSE +.. _`PyPy LOC statistics`: http://codespeak.net/~hpk/pypy-stat/ +.. _`PyPy statistics`: http://codespeak.net/pypy/trunk/pypy/doc/statistic +.. _`object spaces`: objspace.html +.. _`interpreter optimizations`: interpreter-optimizations.html +.. _`translation`: translation.html +.. _`dynamic-language translation`: dynamic-language-translation.html +.. _`low-level encapsulation`: low-level-encapsulation.html +.. _`translation aspects`: translation-aspects.html +.. _`configuration documentation`: config/ +.. _`coding guide`: coding-guide.html +.. _`architecture`: architecture.html +.. _`getting started`: getting-started.html +.. _`theory`: theory.html +.. _`bytecode interpreter`: interpreter.html +.. _`EU reports`: index-report.html +.. _`Technical reports`: index-report.html +.. _`summary`: http://codespeak.net:8099/summary +.. _`ideas for PyPy related projects`: project-ideas.html +.. _`Nightly builds and benchmarks`: http://tuatara.cs.uni-duesseldorf.de/benchmark.html +.. _`directory reference`: +.. _`rlib`: rlib.html +.. _`Sandboxing Python code`: sandbox.html + +PyPy directory cross-reference +------------------------------ + +Here is a fully referenced alphabetical two-level deep +directory overview of PyPy: + +============================ =========================================== +Directory explanation/links +============================ =========================================== +`annotation/`_ `type inferencing code`_ for `RPython`_ programs + +`bin/`_ command-line scripts, mainly `py.py`_ and `translatorshell.py`_ + +`config/`_ handles the numerous options for building and running PyPy + +`doc/`_ text versions of PyPy developer documentation + +`doc/config/`_ documentation for the numerous translation options + +`doc/discussion/`_ drafts of ideas and documentation + +``doc/*/`` other specific documentation topics or tools + +`interpreter/`_ `bytecode interpreter`_ and related objects + (frames, functions, modules,...) + +`interpreter/pyparser/`_ interpreter-level Python source parser + +`interpreter/astcompiler/`_ interpreter-level bytecode compiler, via an AST + representation + +`lang/`_ interpreters for non-Python languages, written in RPython_ + +`lang/js/`_ a JavaScript interpreter (in-progress) + +`lang/prolog/`_ a `Prolog interpreter`_ + +`lib/`_ PyPy's wholesale reimplementations of CPython modules_ + and experimental new application-level modules + +`lib/app_test/`_ tests for the reimplementations, running on top of CPython + +`lib/distributed/`_ distributed execution prototype, based on `transparent proxies`_ + +`module/`_ contains `mixed modules`_ implementing core modules with + both application and interpreter level code. + Not all are finished and working. Use the ``--withmod-xxx`` + or ``--allworkingmodules`` translation options. + +`objspace/`_ `object space`_ implementations + +`objspace/trace.py`_ the `trace object space`_ monitoring bytecode and space operations + +`objspace/dump.py`_ the dump object space saves a large, searchable log file + with all operations + +`objspace/taint.py`_ the `taint object space`_, providing object tainting + +`objspace/thunk.py`_ the `thunk object space`_, providing unique object features + +`objspace/flow/`_ the FlowObjSpace_ implementing `abstract interpretation` + +`objspace/std/`_ the StdObjSpace_ implementing CPython's objects and types + +`rlib/`_ a `"standard library"`_ for RPython_ programs + +`rpython/`_ the `RPython Typer`_ + +`rpython/lltypesystem/`_ the `low-level type system`_ for C-like backends + +`rpython/ootypesystem/`_ the `object-oriented type system`_ for OO backends + +`rpython/memory/`_ the `garbage collector`_ construction framework + +`tool/`_ various utilities and hacks used from various places + +`tool/algo/`_ general-purpose algorithmic and mathematic + tools + +`tool/pytest/`_ support code for our `testing methods`_ + +`translator/`_ translation_ backends and support code + +`translator/backendopt/`_ general optimizations that run before a backend generates code + +`translator/c/`_ the `GenC backend`_, producing C code from an + RPython program (generally via the rtyper_) + +`translator/cli/`_ the `CLI backend`_ for `.NET`_ (Microsoft CLR or Mono_) + +`translator/goal/`_ our `main PyPy-translation scripts`_ live here + +`translator/jvm/`_ the Java backend + +`translator/stackless/`_ the `Stackless Transform`_ + +`translator/tool/`_ helper tools for translation, including the Pygame + `graph viewer`_ + +``*/test/`` many directories have a test subdirectory containing test + modules (see `Testing in PyPy`_) + +``_cache/`` holds cache files from internally `translating application + level to interpreterlevel`_ code. +============================ =========================================== + +.. _`bytecode interpreter`: interpreter.html +.. _`translating application level to interpreterlevel`: geninterp.html +.. _documentation: index.html +.. _`Testing in PyPy`: coding-guide.html#testing-in-pypy +.. _`mixed modules`: coding-guide.html#mixed-modules +.. _`modules`: coding-guide.html#modules +.. _`basil`: http://people.cs.uchicago.edu/~jriehl/BasilTalk.pdf +.. _`object space`: objspace.html +.. _FlowObjSpace: objspace.html#the-flow-object-space +.. _`trace object space`: objspace.html#the-trace-object-space +.. _`taint object space`: objspace-proxies.html#taint +.. _`thunk object space`: objspace-proxies.html#thunk +.. _`transparent proxies`: objspace-proxies.html#tproxy +.. _`What PyPy can do for your objects`: objspace-proxies.html +.. _`Stackless and coroutines`: stackless.html +.. _StdObjSpace: objspace.html#the-standard-object-space +.. _`abstract interpretation`: theory.html#abstract-interpretation +.. _`rpython`: coding-guide.html#rpython +.. _`type inferencing code`: translation.html#the-annotation-pass +.. _`RPython Typer`: translation.html#rpython-typer +.. _`testing methods`: coding-guide.html#testing-in-pypy +.. _`translation`: translation.html +.. _`GenC backend`: translation.html#genc +.. _`CLI backend`: cli-backend.html +.. _`py.py`: getting-started.html#main-entry-point +.. _`translatorshell.py`: getting-started.html#try-out-the-translator +.. _JIT: jit/index.html +.. _`JIT Generation in PyPy`: jit/index.html +.. _`just-in-time compiler generator`: jit/index.html +.. _rtyper: rtyper.html +.. _`low-level type system`: rtyper.html#low-level-type +.. _`object-oriented type system`: rtyper.html#oo-type +.. _`garbage collector`: garbage_collection.html +.. _`Stackless Transform`: translation.html#the-stackless-transform +.. _`PyPy Prolog Interpreter`: prolog-interpreter.html +.. _`Prolog Interpreter`: prolog-interpreter.html +.. _`main PyPy-translation scripts`: getting-started.html#translating-the-pypy-python-interpreter +.. _`translate.py`: getting-started.html#translating-the-pypy-python-interpreter +.. _`.NET`: http://www.microsoft.com/net/ +.. _Mono: http://www.mono-project.com/ +.. _`"standard library"`: rlib.html +.. _`graph viewer`: getting-started.html#try-out-the-translator +.. _`compatibility matrix`: image/compat-matrix.png + +.. include:: _ref.txt + Modified: pypy/trunk/pypy/lib/app_test/ctypes_tests/test_win32.py ============================================================================== --- pypy/trunk/pypy/lib/app_test/ctypes_tests/test_win32.py (original) +++ pypy/trunk/pypy/lib/app_test/ctypes_tests/test_win32.py Thu Nov 12 12:55:49 2009 @@ -1,80 +1,80 @@ -# Windows specific tests - -from ctypes import * -from ctypes.test import is_resource_enabled -from support import BaseCTypesTestChecker - -import py -import sys - -if sys.platform != "win32": - py.test.skip("win32-only tests") - -class TestWindows(BaseCTypesTestChecker): - def test_callconv_1(self): - # Testing stdcall function - - IsWindow = windll.user32.IsWindow - # ValueError: Procedure probably called with not enough arguments (4 bytes missing) - py.test.raises(ValueError, IsWindow) - - # This one should succeeed... - assert IsWindow(0) == 0 - - # ValueError: Procedure probably called with too many arguments (8 bytes in excess) - py.test.raises(ValueError, IsWindow, 0, 0, 0) - - def test_callconv_2(self): - # Calling stdcall function as cdecl - - IsWindow = cdll.user32.IsWindow - - # ValueError: Procedure called with not enough arguments (4 bytes missing) - # or wrong calling convention - py.test.raises(ValueError, IsWindow, None) - - if is_resource_enabled("SEH"): - def test_SEH(self): - # Call functions with invalid arguments, and make sure that access violations - # are trapped and raise an exception. - py.test.raises(WindowsError, windll.kernel32.GetModuleHandleA, 32) - -class TestWintypes(BaseCTypesTestChecker): - - def test_COMError(self): - import _ctypes - from _ctypes import COMError - assert COMError.__doc__ == "Raised when a COM method call failed." - - ex = COMError(-1, "text", ("details",)) - assert ex.hresult == -1 - assert ex.text == "text" - assert ex.details == ("details",) - assert (ex.hresult, ex.text, ex.details) == ex[:] - - def test_VARIANT(self): - from ctypes import wintypes - a = wintypes.VARIANT_BOOL() - assert a.value is False - b = wintypes.VARIANT_BOOL(3) - assert b.value is True - -class TestStructures(BaseCTypesTestChecker): - - def test_struct_by_value(self): - class POINT(Structure): - _fields_ = [("x", c_long), - ("y", c_long)] - - class RECT(Structure): - _fields_ = [("left", c_long), - ("top", c_long), - ("right", c_long), - ("bottom", c_long)] - - import conftest - dll = CDLL(str(conftest.sofile)) - - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - assert dll.PointInRect(byref(rect), pt) == 1 +# Windows specific tests + +from ctypes import * +from ctypes.test import is_resource_enabled +from support import BaseCTypesTestChecker + +import py +import sys + +if sys.platform != "win32": + py.test.skip("win32-only tests") + +class TestWindows(BaseCTypesTestChecker): + def test_callconv_1(self): + # Testing stdcall function + + IsWindow = windll.user32.IsWindow + # ValueError: Procedure probably called with not enough arguments (4 bytes missing) + py.test.raises(ValueError, IsWindow) + + # This one should succeeed... + assert IsWindow(0) == 0 + + # ValueError: Procedure probably called with too many arguments (8 bytes in excess) + py.test.raises(ValueError, IsWindow, 0, 0, 0) + + def test_callconv_2(self): + # Calling stdcall function as cdecl + + IsWindow = cdll.user32.IsWindow + + # ValueError: Procedure called with not enough arguments (4 bytes missing) + # or wrong calling convention + py.test.raises(ValueError, IsWindow, None) + + if is_resource_enabled("SEH"): + def test_SEH(self): + # Call functions with invalid arguments, and make sure that access violations + # are trapped and raise an exception. + py.test.raises(WindowsError, windll.kernel32.GetModuleHandleA, 32) + +class TestWintypes(BaseCTypesTestChecker): + + def test_COMError(self): + import _ctypes + from _ctypes import COMError + assert COMError.__doc__ == "Raised when a COM method call failed." + + ex = COMError(-1, "text", ("details",)) + assert ex.hresult == -1 + assert ex.text == "text" + assert ex.details == ("details",) + assert (ex.hresult, ex.text, ex.details) == ex[:] + + def test_VARIANT(self): + from ctypes import wintypes + a = wintypes.VARIANT_BOOL() + assert a.value is False + b = wintypes.VARIANT_BOOL(3) + assert b.value is True + +class TestStructures(BaseCTypesTestChecker): + + def test_struct_by_value(self): + class POINT(Structure): + _fields_ = [("x", c_long), + ("y", c_long)] + + class RECT(Structure): + _fields_ = [("left", c_long), + ("top", c_long), + ("right", c_long), + ("bottom", c_long)] + + import conftest + dll = CDLL(str(conftest.sofile)) + + pt = POINT(10, 10) + rect = RECT(0, 0, 20, 20) + assert dll.PointInRect(byref(rect), pt) == 1 Modified: pypy/trunk/pypy/rlib/rwinreg.py ============================================================================== --- pypy/trunk/pypy/rlib/rwinreg.py (original) +++ pypy/trunk/pypy/rlib/rwinreg.py Thu Nov 12 12:55:49 2009 @@ -1,134 +1,134 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.tool import rffi_platform as platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib import rwin32 - -eci = ExternalCompilationInfo( - includes = ['windows.h', - ], - libraries = ('Advapi32',) - ) -class CConfig: - _compilation_info_ = eci - - -constant_names = ''' -KEY_QUERY_VALUE KEY_SET_VALUE KEY_CREATE_SUB_KEY KEY_ENUMERATE_SUB_KEYS -KEY_NOTIFY KEY_CREATE_LINK KEY_READ KEY_WRITE KEY_EXECUTE KEY_ALL_ACCESS -KEY_WOW64_64KEY KEY_WOW64_32KEY REG_OPTION_RESERVED REG_OPTION_NON_VOLATILE -REG_OPTION_VOLATILE REG_OPTION_CREATE_LINK REG_OPTION_BACKUP_RESTORE -REG_OPTION_OPEN_LINK REG_LEGAL_OPTION REG_CREATED_NEW_KEY -REG_OPENED_EXISTING_KEY REG_WHOLE_HIVE_VOLATILE REG_REFRESH_HIVE -REG_NO_LAZY_FLUSH REG_NOTIFY_CHANGE_NAME REG_NOTIFY_CHANGE_ATTRIBUTES -REG_NOTIFY_CHANGE_LAST_SET REG_NOTIFY_CHANGE_SECURITY REG_LEGAL_CHANGE_FILTER -REG_NONE REG_SZ REG_EXPAND_SZ REG_BINARY REG_DWORD REG_DWORD_LITTLE_ENDIAN -REG_DWORD_BIG_ENDIAN REG_LINK REG_MULTI_SZ REG_RESOURCE_LIST -REG_FULL_RESOURCE_DESCRIPTOR REG_RESOURCE_REQUIREMENTS_LIST - -HKEY_LOCAL_MACHINE HKEY_CLASSES_ROOT HKEY_CURRENT_CONFIG HKEY_CURRENT_USER -HKEY_DYN_DATA HKEY_LOCAL_MACHINE HKEY_PERFORMANCE_DATATA HKEY_USERS -'''.split() -for name in constant_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -constants = {} -cConfig = platform.configure(CConfig) -constants.update(cConfig) -globals().update(cConfig) - -def external(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv='win') - -HKEY = rwin32.HANDLE -PHKEY = rffi.CArrayPtr(HKEY) -REGSAM = rwin32.DWORD - -RegSetValue = external( - 'RegSetValueA', - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) - -RegSetValueEx = external( - 'RegSetValueExA', - [HKEY, rffi.CCHARP, rwin32.DWORD, - rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) - -RegQueryValue = external( - 'RegQueryValueA', - [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], - rffi.LONG) - -RegQueryValueEx = external( - 'RegQueryValueExA', - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) - -RegCreateKey = external( - 'RegCreateKeyA', - [HKEY, rffi.CCHARP, PHKEY], - rffi.LONG) - -RegDeleteValue = external( - 'RegDeleteValueA', - [HKEY, rffi.CCHARP], - rffi.LONG) - -RegDeleteKey = external( - 'RegDeleteKeyA', - [HKEY, rffi.CCHARP], - rffi.LONG) - -RegOpenKeyEx = external( - 'RegOpenKeyExA', - [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], - rffi.LONG) - -RegEnumValue = external( - 'RegEnumValueA', - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) - -RegEnumKeyEx = external( - 'RegEnumKeyExA', - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) - -RegQueryInfoKey = external( - 'RegQueryInfoKeyA', - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) - -RegCloseKey = external( - 'RegCloseKey', - [HKEY], - rffi.LONG) - -RegFlushKey = external( - 'RegFlushKey', - [HKEY], - rffi.LONG) - -RegLoadKey = external( - 'RegLoadKeyA', - [HKEY, rffi.CCHARP, rffi.CCHARP], - rffi.LONG) - -RegSaveKey = external( - 'RegSaveKeyA', - [HKEY, rffi.CCHARP, rffi.VOIDP], - rffi.LONG) - -RegConnectRegistry = external( - 'RegConnectRegistryA', - [rffi.CCHARP, HKEY, PHKEY], - rffi.LONG) +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.tool import rffi_platform as platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib import rwin32 + +eci = ExternalCompilationInfo( + includes = ['windows.h', + ], + libraries = ('Advapi32',) + ) +class CConfig: + _compilation_info_ = eci + + +constant_names = ''' +KEY_QUERY_VALUE KEY_SET_VALUE KEY_CREATE_SUB_KEY KEY_ENUMERATE_SUB_KEYS +KEY_NOTIFY KEY_CREATE_LINK KEY_READ KEY_WRITE KEY_EXECUTE KEY_ALL_ACCESS +KEY_WOW64_64KEY KEY_WOW64_32KEY REG_OPTION_RESERVED REG_OPTION_NON_VOLATILE +REG_OPTION_VOLATILE REG_OPTION_CREATE_LINK REG_OPTION_BACKUP_RESTORE +REG_OPTION_OPEN_LINK REG_LEGAL_OPTION REG_CREATED_NEW_KEY +REG_OPENED_EXISTING_KEY REG_WHOLE_HIVE_VOLATILE REG_REFRESH_HIVE +REG_NO_LAZY_FLUSH REG_NOTIFY_CHANGE_NAME REG_NOTIFY_CHANGE_ATTRIBUTES +REG_NOTIFY_CHANGE_LAST_SET REG_NOTIFY_CHANGE_SECURITY REG_LEGAL_CHANGE_FILTER +REG_NONE REG_SZ REG_EXPAND_SZ REG_BINARY REG_DWORD REG_DWORD_LITTLE_ENDIAN +REG_DWORD_BIG_ENDIAN REG_LINK REG_MULTI_SZ REG_RESOURCE_LIST +REG_FULL_RESOURCE_DESCRIPTOR REG_RESOURCE_REQUIREMENTS_LIST + +HKEY_LOCAL_MACHINE HKEY_CLASSES_ROOT HKEY_CURRENT_CONFIG HKEY_CURRENT_USER +HKEY_DYN_DATA HKEY_LOCAL_MACHINE HKEY_PERFORMANCE_DATATA HKEY_USERS +'''.split() +for name in constant_names: + setattr(CConfig, name, platform.DefinedConstantInteger(name)) + +constants = {} +cConfig = platform.configure(CConfig) +constants.update(cConfig) +globals().update(cConfig) + +def external(name, args, result): + return rffi.llexternal(name, args, result, compilation_info=eci, + calling_conv='win') + +HKEY = rwin32.HANDLE +PHKEY = rffi.CArrayPtr(HKEY) +REGSAM = rwin32.DWORD + +RegSetValue = external( + 'RegSetValueA', + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) + +RegSetValueEx = external( + 'RegSetValueExA', + [HKEY, rffi.CCHARP, rwin32.DWORD, + rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) + +RegQueryValue = external( + 'RegQueryValueA', + [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], + rffi.LONG) + +RegQueryValueEx = external( + 'RegQueryValueExA', + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) + +RegCreateKey = external( + 'RegCreateKeyA', + [HKEY, rffi.CCHARP, PHKEY], + rffi.LONG) + +RegDeleteValue = external( + 'RegDeleteValueA', + [HKEY, rffi.CCHARP], + rffi.LONG) + +RegDeleteKey = external( + 'RegDeleteKeyA', + [HKEY, rffi.CCHARP], + rffi.LONG) + +RegOpenKeyEx = external( + 'RegOpenKeyExA', + [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], + rffi.LONG) + +RegEnumValue = external( + 'RegEnumValueA', + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) + +RegEnumKeyEx = external( + 'RegEnumKeyExA', + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) + +RegQueryInfoKey = external( + 'RegQueryInfoKeyA', + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) + +RegCloseKey = external( + 'RegCloseKey', + [HKEY], + rffi.LONG) + +RegFlushKey = external( + 'RegFlushKey', + [HKEY], + rffi.LONG) + +RegLoadKey = external( + 'RegLoadKeyA', + [HKEY, rffi.CCHARP, rffi.CCHARP], + rffi.LONG) + +RegSaveKey = external( + 'RegSaveKeyA', + [HKEY, rffi.CCHARP, rffi.VOIDP], + rffi.LONG) + +RegConnectRegistry = external( + 'RegConnectRegistryA', + [rffi.CCHARP, HKEY, PHKEY], + rffi.LONG) Modified: pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/trunk/pypy/rpython/module/test/test_ll_os_stat.py Thu Nov 12 12:55:49 2009 @@ -1,17 +1,17 @@ -from pypy.rpython.module import ll_os_stat -import sys, os -import py - -class TestWin32Implementation: - def setup_class(cls): - if sys.platform != 'win32': - py.test.skip("win32 specific tests") - - def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl - def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime - - check('c:/') - check('c:/temp') - check('c:/pagefile.sys') +from pypy.rpython.module import ll_os_stat +import sys, os +import py + +class TestWin32Implementation: + def setup_class(cls): + if sys.platform != 'win32': + py.test.skip("win32 specific tests") + + def test_stat(self): + stat = ll_os_stat.win32_stat_llimpl + def check(f): + assert stat(f).st_mtime == os.stat(f).st_mtime + + check('c:/') + check('c:/temp') + check('c:/pagefile.sys') From arigo at codespeak.net Thu Nov 12 13:04:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 13:04:24 +0100 (CET) Subject: [pypy-svn] r69218 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091112120424.01F3516802F@codespeak.net> Author: arigo Date: Thu Nov 12 13:04:24 2009 New Revision: 69218 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/logger.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py pypy/trunk/pypy/jit/metainterp/test/test_logger.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Log: Merge the branch 'jit-log-class-func': a list that maps the address of functions and classes to their name, for the logger. Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Thu Nov 12 13:04:24 2009 @@ -65,6 +65,8 @@ self.counter = 0 self.class_sizes = [] self._class_sizes_seen = {} + self.list_of_addr2name = [] + self._functions_addr_seen = {} # set later with .start() self.metainterp_sd = None @@ -172,7 +174,9 @@ portal_code = self.make_portal_bytecode(portal_graph) self.metainterp_sd.info_from_codewriter(portal_code, leave_code, - self.class_sizes) + self.class_sizes, + self.list_of_addr2name, + portal_runner_ptr) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -306,6 +310,8 @@ methdescr.setup(jitcodes) def getcalldescr(self, v_func, args, result, consider_effects_of=None): + if isinstance(v_func, Constant): + self.register_known_function(v_func.value) non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -330,6 +336,8 @@ self._class_sizes_seen[key] = True sizedescr = self.cpu.sizeof(STRUCT) self.class_sizes.append((vtable, sizedescr)) + vtable_addr = llmemory.cast_ptr_to_adr(vtable) + self.list_of_addr2name.append((vtable_addr, STRUCT.__name__)) def register_known_ooclass(self, cls, CLASS): # ootype only @@ -338,6 +346,16 @@ typedescr = self.cpu.typedescrof(CLASS) self.class_sizes.append((cls, typedescr)) + def register_known_function(self, func): + if self.rtyper.type_system.name == 'lltypesystem': + try: + obj = func._obj + except lltype.DelayedPointer: # probably ll_portal_runner + return + if obj not in self._functions_addr_seen: + self._functions_addr_seen[obj] = True + func_addr = llmemory.cast_ptr_to_adr(func) + self.list_of_addr2name.append((func_addr, obj._name)) class BytecodeMaker(object): Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Thu Nov 12 13:04:24 2009 @@ -7,9 +7,10 @@ class Logger(object): - def __init__(self, ts, guard_number=False): - self.ts = ts - self.guard_number=guard_number + def __init__(self, metainterp_sd, guard_number=False): + self.metainterp_sd = metainterp_sd + self.ts = metainterp_sd.cpu.ts + self.guard_number = guard_number def log_loop(self, inputargs, operations, number=0, type=None): if type is None: @@ -57,7 +58,11 @@ elif isinstance(arg, BoxFloat): return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): - return 'ConstClass(cls' + str(mv) + ')' + addr = arg.getaddr(self.metainterp_sd.cpu) + name = self.metainterp_sd.get_name_from_address(addr) + if not name: + name = 'cls' + str(mv) + return 'ConstClass(' + name + ')' else: return '?' Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Thu Nov 12 13:04:24 2009 @@ -996,8 +996,8 @@ self.cpu = cpu self.stats = stats self.options = options - self.logger_noopt = Logger(cpu.ts) - self.logger_ops = Logger(cpu.ts, guard_number=True) + self.logger_noopt = Logger(self) + self.logger_ops = Logger(self, guard_number=True) RESULT = portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) @@ -1022,15 +1022,21 @@ self.portal_code = None self.leave_code = None - self._class_sizes = None + self._class_sizes = None + self._addr2name_keys = [] + self._addr2name_values = [] def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes): + def info_from_codewriter(self, portal_code, leave_code, class_sizes, + list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes + self._addr2name_keys = [key for key, value in list_of_addr2name] + self._addr2name_values = [value for key, value in list_of_addr2name] + self._portal_runner_ptr = portal_runner_ptr def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1063,6 +1069,28 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) + def get_name_from_address(self, addr): + # for debugging only + if we_are_translated(): + d = self.globaldata.addr2name + if d is None: + # Build the dictionary at run-time. This is needed + # because the keys are function/class addresses, so they + # can change from run to run. + k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr) + d = {k: 'recursive call'} + keys = self._addr2name_keys + values = self._addr2name_values + for i in range(len(keys)): + d[keys[i]] = values[i] + self.globaldata.addr2name = d + return d.get(addr, '') + else: + for i in range(len(self._addr2name_keys)): + if addr == self._addr2name_keys[i]: + return self._addr2name_values[i] + return '' + def bytecode_for_address(self, fnaddress): if we_are_translated(): d = self.globaldata.indirectcall_dict @@ -1115,6 +1143,7 @@ def __init__(self, staticdata, prebuilt_fail_descr_list): self.initialized = False self.indirectcall_dict = None + self.addr2name = None self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # Modified: pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_codewriter.py Thu Nov 12 13:04:24 2009 @@ -121,12 +121,14 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): return FakeMethDescr(CLASS, methname) + def sizeof(self, STRUCT): + return ('sizeof', STRUCT) if type_system == 'lltype': FakeCPU.ts = typesystem.llhelper @@ -366,6 +368,24 @@ assert jitcode._source.count('oononnull') == 2 assert jitcode._source.count('ooisnull') == 2 + def test_list_of_addr2name(self): + class A1: + def g(self): + self.x = 123 + return 5 + def f(): + a = A1() + a.y = a.g() + return a + graphs = self.make_graphs(f, []) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert len(cw.list_of_addr2name) == 2 + assert cw.list_of_addr2name[0][1].endswith('.A1') + assert cw.list_of_addr2name[1][1] == 'A1.g' + class ImmutableFieldsTests: def test_fields(self): Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Thu Nov 12 13:04:24 2009 @@ -34,6 +34,14 @@ class TestLogger(object): ts = llhelper + def make_metainterp_sd(self): + class FakeMetaInterpSd: + class cpu: + ts = self.ts + def get_name_from_address(self, addr): + return 'Name' + return FakeMetaInterpSd() + def reparse(self, inp, namespace=None, check_equal=True): """ parse loop once, then log it and parse again. Checks that we get the same thing. @@ -41,7 +49,7 @@ if namespace is None: namespace = {} loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop, namespace) oloop = pure_parse(output, namespace=namespace) if check_equal: @@ -99,7 +107,7 @@ jump(i0, descr=target) ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop) assert output.splitlines()[-1] == "jump(i0, descr=)" pure_parse(output) @@ -111,7 +119,7 @@ guard_true(i0, descr=fdescr) [i0] ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts, guard_number=True) + logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) @@ -119,18 +127,34 @@ def boom(): raise Exception namespace['fdescr'].get_index = boom - logger = Logger(self.ts, guard_number=False) + logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + def test_class_name(self): + from pypy.rpython.lltypesystem import lltype + AbcVTable = lltype.Struct('AbcVTable') + abcvtable = lltype.malloc(AbcVTable, immortal=True) + namespace = {'Name': abcvtable} + inp = ''' + [i0] + p = new_with_vtable(ConstClass(Name)) + ''' + loop = pure_parse(inp, namespace=namespace) + logger = Logger(self.make_metainterp_sd()) + output = logger.log_loop(loop) + assert output.splitlines()[-1].endswith( + " = new_with_vtable(ConstClass(Name))") + pure_parse(output, namespace=namespace) + def test_intro_loop(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_loop, [], [], 1, "foo") assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_bridge, [], [], 3) assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Thu Nov 12 13:04:24 2009 @@ -142,3 +142,14 @@ assert equaloplists(metainterp.history.operations, [ ResOperation(rop.SAME_AS, [b2], boxes[1]), ]) + +def test_get_name_from_address(): + class FakeMetaInterpSd(pyjitpl.MetaInterpStaticData): + def __init__(self): + pass + metainterp_sd = FakeMetaInterpSd() + metainterp_sd.info_from_codewriter(None, None, None, + [(123, "a"), (456, "b")]) + assert metainterp_sd.get_name_from_address(123) == 'a' + assert metainterp_sd.get_name_from_address(456) == 'b' + assert metainterp_sd.get_name_from_address(789) == '' From arigo at codespeak.net Thu Nov 12 13:06:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 13:06:32 +0100 (CET) Subject: [pypy-svn] r69219 - pypy/branch/jit-log-class-func Message-ID: <20091112120632.BD4D516802F@codespeak.net> Author: arigo Date: Thu Nov 12 13:06:32 2009 New Revision: 69219 Removed: pypy/branch/jit-log-class-func/ Log: Remove merged branch. From fijal at codespeak.net Thu Nov 12 13:35:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 13:35:45 +0100 (CET) Subject: [pypy-svn] r69220 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112123545.B3FAD16802C@codespeak.net> Author: fijal Date: Thu Nov 12 13:35:45 2009 New Revision: 69220 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: (arigo, fijal) slow progres... Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Thu Nov 12 13:35:45 2009 @@ -75,6 +75,20 @@ GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict from pypy.interpreter.gateway import interp2app +def readwrite_attrproperty(name, cls, unwrapname): + def fget(space, obj): + return space.wrap(getattr(obj, name)) + def fset(space, obj, w_val): + setattr(obj, name, getattr(space, unwrapname)(w_val)) + return GetSetProperty(fget, fset, cls=cls) + +def readwrite_attrproperty_w(name, cls): + def fget(space, obj): + return getattr(obj, name) + def fset(space, obj, w_val): + setattr(obj, name, w_val) + return GetSetProperty(fget, fset, cls=cls) + class W_BaseException(Wrappable): """Superclass representing the base of the exception hierarchy. @@ -213,13 +227,6 @@ w_end, w_reason) return space.wrap(exc) -def readwrite_attrproperty(name, cls, unwrapname): - def fget(space, obj): - return space.wrap(getattr(obj, name)) - def fset(space, obj, w_val): - setattr(obj, name, getattr(space, unwrapname)(w_val)) - return GetSetProperty(fget, fset, cls=cls) - W_UnicodeTranslateError.typedef = TypeDef( 'UnicodeTranslateError', W_UnicodeError.typedef, @@ -247,3 +254,151 @@ W_KeyError = _new_exception('KeyError', W_LookupError, """Mapping key not found.""", __str__ = key_error_str) + +W_StopIteration = _new_exception('StopIteration', W_Exception, + """Signal the end from iterator.next().""") + +W_Warning = _new_exception('Warning', W_Exception, + """Base class for warning categories.""") + +W_PendingDeprecationWarning = _new_exception('PendingDeprecationWarning', + W_Warning, + """Base class for warnings about features which will be deprecated in the future.""") + +class W_EnvironmentError(W_StandardError): + """Base class for I/O related errors.""" + + def __init__(self, space, args_w): + W_BaseException.__init__(self, space, args_w) + self.w_errno = space.w_None + self.w_strerror = space.w_None + self.w_filename = space.w_None + if 2 <= len(args_w) <= 3: + self.w_errno = args_w[0] + self.w_strerror = args_w[1] + if len(args_w) == 3: + self.w_filename = args_w[2] + self.args_w = [args_w[0], args_w[1]] + + def descr_str(self, space): + if not space.is_w(self.w_filename, space.w_None): + return space.wrap("[Errno %d] %s: %s" % (space.int_w(self.w_errno), + space.str_w(self.w_strerror), + space.str_w(self.w_filename))) + if (not space.is_w(self.w_errno, space.w_None) and + not space.is_w(self.w_errno, space.w_None)): + return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), + space.str_w(self.w_strerror))) + return W_BaseException.__str__(self, space) + descr_str.unwrap_spec = ['self', ObjSpace] + +W_EnvironmentError.typedef = TypeDef( + 'EnvironmentError', + W_StandardError.typedef, + __new__ = _new(W_EnvironmentError), + __str__ = interp2app(W_EnvironmentError.descr_str) + ) + +W_OSError = _new_exception('OSError', W_EnvironmentError, + """OS system call failed.""") + +W_DeprecationWarning = _new_exception('DeprecationWarning', W_Warning, + """Base class for warnings about deprecated features.""") + +W_ArithmeticError = _new_exception('ArithmeticError', W_StandardError, + """Base class for arithmetic errors.""") + +W_FloatingPointError = _new_exception('FloatingPointError', W_ArithmeticError, + """Floating point operation failed.""") + +W_ReferenceError = _new_exception('ReferenceError', W_StandardError, + """Weak ref proxy used after referent went away.""") + +W_NameError = _new_exception('NameError', W_StandardError, + """Name not found globally.""") + +W_IOError = _new_exception('IOError', W_EnvironmentError, + """I/O operation failed.""") + + +class W_SyntaxError(W_StandardError): + """Invalid syntax.""" + + def __init__(self, space, args_w): + W_BaseException.__init__(self, space, args_w) + # that's not a self.w_message!!! + if len(args_w) > 0: + self.w_msg = args_w[0] + else: + self.w_msg = space.wrap('') + if len(args_w) == 2: + values_w = space.viewiterable(args_w[1], 4) + self.w_filename = values_w[0] + self.w_lineno = values_w[1] + self.w_offset = values_w[2] + self.w_text = values_w[3] + else: + self.w_filename = space.w_None + self.w_lineno = space.w_None + self.w_offset = space.w_None + self.w_text = space.w_None + + def descr_str(self, space): + return space.appexec([self], """(self): + if type(self.msg) is not str: + return str(self.msg) + + buffer = self.msg + have_filename = type(self.filename) is str + have_lineno = type(self.lineno) is int + if have_filename: + import os + fname = os.path.basename(self.filename or "???") + if have_lineno: + buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + else: + buffer ="%s (%s)" % (self.msg, fname) + elif have_lineno: + buffer = "%s (line %ld)" % (self.msg, self.lineno) + return buffer + """) + + descr_str.unwrap_spec = ['self', ObjSpace] + +W_SyntaxError.typedef = TypeDef( + 'SyntaxError', + W_StandardError.typedef, + __new__ = _new(W_SyntaxError), + __str__ = interp2app(W_SyntaxError.descr_str), + __doc__ = W_SyntaxError.__doc__, + msg = readwrite_attrproperty_w('w_msg', W_SyntaxError), + filename = readwrite_attrproperty_w('w_filename', W_SyntaxError), + lineno = readwrite_attrproperty_w('w_lineno', W_SyntaxError), + offset = readwrite_attrproperty_w('w_offset', W_SyntaxError), + text = readwrite_attrproperty_w('w_text', W_SyntaxError), +) + +W_FutureWarning = _new_exception('FutureWarning', W_Warning, + """Base class for warnings about constructs that will change semantically in the future.""") + +class W_SystemExit(W_BaseException): + """Request to exit from the interpreter.""" + + def __init__(self, space, args_w): + W_BaseException.__init__(self, space, args_w) + if len(args_w) == 0: + self.w_code = space.w_None + elif len(args_w) == 1: + self.w_code = args_w[0] + else: + self.w_code = space.newtuple(args_w) + +W_SystemExit.typedef = TypeDef( + 'SystemExit', + W_BaseException.typedef, + __new__ = _new(W_SystemExit), + __doc__ = W_SystemExit.__doc__, + code = readwrite_attrproperty_w('w_code', W_SystemExit) +) + + Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Thu Nov 12 13:35:45 2009 @@ -62,3 +62,26 @@ from tempexceptions import KeyError assert str(KeyError('s')) == "'s'" + + def test_environment_error(self): + from tempexceptions import EnvironmentError + ee = EnvironmentError(3, "x", "y") + assert str(ee) == "[Errno 3] x: y" + assert str(EnvironmentError(3, "x")) == "[Errno 3] x" + + def test_syntax_error(self): + from tempexceptions import SyntaxError + s = SyntaxError(3) + assert str(s) == '3' + assert str(SyntaxError("a", "b", 123)) == "a" + assert str(SyntaxError("a", (1, 2, 3, 4))) == "a (line 2)" + s = SyntaxError("a", (1, 2, 3, 4)) + assert s.msg == "a" + assert s.filename == 1 + assert str(SyntaxError("msg", ("file.py", 2, 3, 4))) == "msg (file.py, line 2)" + + def test_system_exit(self): + from tempexceptions import SystemExit + assert SystemExit().code is None + assert SystemExit("x").code == "x" + assert SystemExit(1, 2).code == (1, 2) From david at codespeak.net Thu Nov 12 13:51:21 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 12 Nov 2009 13:51:21 +0100 (CET) Subject: [pypy-svn] r69221 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20091112125121.3198316802C@codespeak.net> Author: david Date: Thu Nov 12 13:51:20 2009 New Revision: 69221 Added: pypy/branch/io-lang/pypy/lang/io/parser.py pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Log: io lexer Added: pypy/branch/io-lang/pypy/lang/io/parser.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/parser.py Thu Nov 12 13:51:20 2009 @@ -0,0 +1,90 @@ +from pypy.rlib.parsing.regexparse import parse_regex +from pypy.rlib.parsing.lexer import Lexer, Token, SourcePos + +# taken from rlib/parsing/test/python_lexer.py +# reg exp helper methods +def group(*choices): + return '(' + '|'.join(choices) + ')' +def any(*choices): + return group(*choices) + '*' +def maybe(*choices): + return group(*choices) + '?' + +# io token definitions +# Identifiers +Names = r'[a-zA-Z_][a-zA-Z0-9_]*' +Operators = r'(\:|\.|\'|\~|!|@|\$|%|\^|&|\*|\-|\+|/|=|\||\\|\<|\>|\?)+' +Identifiers = group(Names, Operators) + +# Numbers +Decnumber = r'[1-9][0-9]*' +Exponent = r'[eE][\-\+]?[0-9]+' +Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent) +Expfloat = r'[0-9]+' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Number = group(Floatnumber, Decnumber) + +# Hexnuber +Hexnumber = r'0[xX][0-9a-fA-F]*' + +# Comments and Whitespace, the ignored stuff +Whitespace = r'[ \f\t\n]*' +slashStarComment = r'/\*[^*/]*\*/' +slashSlashComment = r'//[^\n]*\n' +poundComment = r'#[^\n]*\n' +Comment = group(slashStarComment, slashSlashComment, poundComment) + +# Parenthesis +OpenParen = group('\(', '\[', '\{') +CloseParen = group('\)', '\]', '\}') + +# Qouting for strings +TriQuote = r'"""[^\"\"\"]*"""' +MonoQuote = r'"([^"]|(\\"))*"' + +# ; +Terminator = r'(;'+maybe(Whitespace)+')+' + + + +rexs = [Identifiers, Whitespace, Number, Hexnumber, Terminator, OpenParen, CloseParen, MonoQuote, TriQuote, Comment] +names = ["Identifier", "Whitespace", "Number", "HexNumber", "Terminator", + "OpenParen", "CloseParen", "MonoQuote", "TriQuote", "Comment"] +ignores = ['Whitespace', 'Comment'] + + +def get_lexer(): + return IoLexer() + +class IoLexer(object): + def __init__(self): + super(IoLexer, self).__init__() + self.lexer = Lexer([parse_regex(r) for r in rexs], names, ignores) + + def tokenize(self, input): + tokens = self.lexer.tokenize(input) + self._magic_tokenize(tokens) + return tokens + + def _magic_tokenize(self, tokens): + i = -1 + while(i < len(tokens)-1): + i += 1 + if tokens[i].name != 'OpenParen': + continue + if tokens[i].source == '(' and (i == 0 or tokens[i-1].name != 'Identifier'): + token_to_add = '' + elif tokens[i].source == '[': + token_to_add = 'squareBrackets' + elif tokens[i].source == '{': + token_to_add = 'curlyBrackets' + else: + continue + + tokens.insert(i, Token('Identifier', token_to_add, + SourcePos(tokens[i].source_pos.i, + tokens[i].source_pos.lineno, + tokens[i].source_pos.columnno))) + i += 1 + + return tokens \ No newline at end of file Added: pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Thu Nov 12 13:51:20 2009 @@ -0,0 +1,178 @@ +import py +from pypy.rlib.parsing.lexer import Lexer, Token, SourcePos + +from pypy.lang.io.parser import get_lexer + +iolexer = get_lexer() +def test_lex_identifier(): + inp = "Compiler" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "Compiler", SourcePos(0, 0, 0)) + +def test_lex_identifier2(): + inp = "@Compiler" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "@", SourcePos(0, 0, 0)) + assert tokens[1] == Token("Identifier", "Compiler", SourcePos(1, 0, 1)) + +def test_lex_identifier3(): + inp = "_" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "_", SourcePos(0, 0, 0)) + + +def test_lex_identifier4(): + inp = "@@@@" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "@@@@", SourcePos(0, 0, 0)) + +def test_lex_identifier5(): + inp = ":=" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", ":=", SourcePos(0, 0, 0)) + +def test_lex_identifier6(): + inp = "::=::" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "::=::", SourcePos(0, 0, 0)) + +def test_lex_ignores_whitespace(): + inp = "Foo bar" + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token("Identifier", "Foo", SourcePos(0, 0, 0)) + assert tokens[1] == Token("Identifier", "bar", SourcePos(10, 0, 10)) + +def test_lex_numbers(): + inp = '12345678901234567890' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Number', '12345678901234567890', SourcePos(0, 0, 0)) + +def test_lex_numbers2(): + inp = '.239' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Number', '.239', SourcePos(0, 0, 0)) + +def test_lex_numbers3(): + inp = '1.239' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Number', '1.239', SourcePos(0, 0, 0)) + +def test_lex_numbers4(): + inp = '1.239e+123' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Number', '1.239e+123', SourcePos(0, 0, 0)) + +def test_lex_numbers5(): + inp = '1239e+123' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Number', '1239e+123', SourcePos(0, 0, 0)) + +def test_lex_hexnumber(): + inp = '0xCAFFEE' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('HexNumber', '0xCAFFEE', SourcePos(0, 0, 0)) + +def test_lex_hexnumber(): + inp = '0XCAFFEE' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('HexNumber', '0XCAFFEE', SourcePos(0, 0, 0)) + +def test_lex_terminator(): + inp = ';' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0)) + +def test_lex_terminator1(): + inp = ';;' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Terminator', ';;', SourcePos(0, 0, 0)) + assert len(tokens) == 1 + +def test_lex_terminator2(): + inp = '; ;' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Terminator', '; ;', SourcePos(0, 0, 0)) + assert len(tokens) == 1 + +def test_lex_open_paren(): + inp = '()' + tokens = iolexer.tokenize(inp) + assert Token('OpenParen', '(', SourcePos(0, 0, 0)) in tokens + +def test_lex_close_paren(): + inp = '()' + tokens = iolexer.tokenize(inp) + assert Token('CloseParen', ')', SourcePos(1, 0, 1)) in tokens + +def test_lex_monoquote(): + inp = '"MonoQuote" "qertz"' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('MonoQuote', '"MonoQuote"', SourcePos(0, 0, 0)) + +def test_lex_monoquote_escapes(): + inp = '"a\\"a"' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('MonoQuote', '"a\\"a"', SourcePos(0, 0, 0)) + +def test_lex_triquote(): + inp = '"""TriQuote"""' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('TriQuote', '"""TriQuote"""', SourcePos(0, 0, 0)) + +def test_lex_pound_comment_is_ignored(): + inp = """a # foo bar + """ + tokens = iolexer.tokenize(inp) + assert tokens[0].name != 'Comment' + assert len(tokens) == 1 + +def test_lex_pound_comment_is_ignored(): + inp = """a // foo bar + """ + tokens = iolexer.tokenize(inp) + assert tokens[0].name != 'Comment' + assert len(tokens) == 1 + +def test_lex_slash_star_comment_is_ignored(): + inp = """a /* foo bar */ q""" + tokens = iolexer.tokenize(inp) + assert tokens[0].name != 'Comment' + assert tokens[1].name != 'Comment' + assert len(tokens) == 2 + +def test_lex_paren_adds_anon_message(): + inp = '()' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Identifier', '', SourcePos(0, 0, 0)) + assert tokens[1] == Token('OpenParen', '(', SourcePos(0, 0, 0)) + +def test_lex_paren_adds_only_when_no_receiver(): + inp = 'foo ()' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Identifier', 'foo', SourcePos(0, 0, 0)) + assert tokens[1] == Token('OpenParen', '(', SourcePos(4, 0, 4)) + +def test_lex_open_paren(): + inp = '()' + tokens = iolexer.tokenize(inp) + assert Token('OpenParen', '(', SourcePos(0, 0, 0)) in tokens + +def test_lex_close_paren(): + inp = '()' + tokens = iolexer.tokenize(inp) + assert Token('CloseParen', ')', SourcePos(1, 0, 1)) in tokens + + +def test_lex_squareBrackets(): + inp = '[]' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Identifier', 'squareBrackets', SourcePos(0, 0, 0)) + assert tokens[1] == Token('OpenParen', '[', SourcePos(0, 0, 0)) + assert tokens[2] == Token('CloseParen', ']', SourcePos(1, 0, 1)) + +def test_lex_curlyBrackets(): + inp = '{}' + tokens = iolexer.tokenize(inp) + assert tokens[0] == Token('Identifier', 'curlyBrackets', SourcePos(0, 0, 0)) + assert tokens[1] == Token('OpenParen', '{', SourcePos(0, 0, 0)) + assert tokens[2] == Token('CloseParen', '}', SourcePos(1, 0, 1)) \ No newline at end of file From arigo at codespeak.net Thu Nov 12 13:57:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 13:57:21 +0100 (CET) Subject: [pypy-svn] r69223 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112125721.4674016802C@codespeak.net> Author: arigo Date: Thu Nov 12 13:57:20 2009 New Revision: 69223 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: Progress... Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Thu Nov 12 13:57:20 2009 @@ -200,12 +200,13 @@ class W_UnicodeTranslateError(W_UnicodeError): """Unicode translation error.""" - def __init__(self, space, w_obj, w_start, w_end, w_reason): - self.object = space.unicode_w(w_obj) + def __init__(self, space, w_object, w_start, w_end, w_reason): + self.object = space.unicode_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_obj, w_start, w_end, w_reason]) + W_BaseException.__init__(self, space, [w_object, w_start, + w_end, w_reason]) def descr_str(self, space): return space.appexec([space.wrap(self)], """(self): @@ -220,10 +221,10 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_translate_error(space, w_subtype, w_obj, w_start, w_end, - w_reason): +def descr_new_unicode_translate_error(space, w_subtype, w_object, + w_start, w_end, w_reason): exc = space.allocate_instance(W_UnicodeTranslateError, w_subtype) - W_UnicodeTranslateError.__init__(exc, space, w_obj, w_start, + W_UnicodeTranslateError.__init__(exc, space, w_object, w_start, w_end, w_reason) return space.wrap(exc) @@ -401,4 +402,111 @@ code = readwrite_attrproperty_w('w_code', W_SystemExit) ) +W_EOFError = _new_exception('EOFError', W_StandardError, + """Read beyond end of file.""") +W_IndentationError = _new_exception('IndentationError', W_SyntaxError, + """Improper indentation.""") + +W_TabError = _new_exception('TabError', W_IndentationError, + """Improper mixture of spaces and tabs.""") + +W_ZeroDivisionError = _new_exception('ZeroDivisionError', W_ArithmeticError, + """Second argument to a division or modulo operation was zero.""") + +W_SystemError = _new_exception('SystemError', W_StandardError, + """Internal error in the Python interpreter. + +Please report this to the Python maintainer, along with the traceback, +the Python version, and the hardware/OS platform and version.""") + +W_AssertionError = _new_exception('AssertionError', W_StandardError, + """Assertion failed.""") + +class W_UnicodeDecodeError(W_UnicodeError): + """Unicode decoding error.""" + + def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + self.encoding = space.str_w(w_encoding) + self.object = space.str_w(w_object) + self.start = space.int_w(w_start) + self.end = space.int_w(w_end) + self.reason = space.str_w(w_reason) + W_BaseException.__init__(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + + def descr_str(self, space): + return space.appexec([self], """(self): + if self.end == self.start + 1: + return "%r codec can't decode byte 0x%02x in position %d: %s"%( + self.encoding, + ord(self.object[self.start]), self.start, self.reason) + return "%r codec can't decode bytes in position %d-%d: %s" % ( + self.encoding, self.start, self.end - 1, self.reason) + """) + + descr_str.unwrap_spec = ['self', ObjSpace] + +def descr_new_unicode_decode_error(space, w_subtype, w_encoding, w_object, + w_start, w_end, w_reason): + exc = space.allocate_instance(W_UnicodeDecodeError, w_subtype) + W_UnicodeDecodeError.__init__(exc, space, w_encoding, w_object, w_start, + w_end, w_reason) + return space.wrap(exc) + +W_UnicodeDecodeError.typedef = TypeDef( + 'UnicodeDecodeError', + W_UnicodeError.typedef, + __doc__ = W_UnicodeTranslateError.__doc__, + __new__ = interp2app(descr_new_unicode_decode_error), + __str__ = interp2app(W_UnicodeDecodeError.descr_str), + encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), + object = readwrite_attrproperty('object', W_UnicodeDecodeError, 'str_w'), + start = readwrite_attrproperty('start', W_UnicodeDecodeError, 'int_w'), + end = readwrite_attrproperty('end', W_UnicodeDecodeError, 'int_w'), + reason = readwrite_attrproperty('reason', W_UnicodeDecodeError, 'str_w'), +) + +W_TypeError = _new_exception('TypeError', W_StandardError, + """Inappropriate argument type.""") + +W_IndexError = _new_exception('IndexError', W_LookupError, + """Sequence index out of range.""") + +W_RuntimeWarning = _new_exception('RuntimeWarning', W_Warning, + """Base class for warnings about dubious runtime behavior.""") + +W_KeyboardInterrupt = _new_exception('KeyboardInterrupt', W_BaseException, + """Program interrupted by user.""") + +W_UserWarning = _new_exception('UserWarning', W_Warning, + """Base class for warnings generated by user code.""") + +W_SyntaxWarning = _new_exception('SyntaxWarning', W_Warning, + """Base class for warnings about dubious syntax.""") + +W_UnicodeWarning = _new_exception('UnicodeWarning', W_Warning, + """Base class for warnings about Unicode related problems, mostly +related to conversion problems.""") + +W_ImportWarning = _new_exception('ImportWarning', W_Warning, + """Base class for warnings about probable mistakes in module imports""") + +W_MemoryError = _new_exception('MemoryError', W_StandardError, + """Out of memory.""") + +W_UnboundLocalError = _new_exception('UnboundLocalError', W_NameError, + """Local name referenced but not bound to a value.""") + +W_NotImplementedError = _new_exception('NotImplementedError', W_RuntimeError, + """Method or function hasn't been implemented yet.""") + +W_AttributeError = _new_exception('AttributeError', W_StandardError, + """Attribute not found.""") + +W_OverflowError = _new_exception('OverflowError', W_ArithmeticError, + """Result too large to be represented.""") + + +##class UnicodeEncodeError(UnicodeError): +## """Unicode encoding error.""".......... Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Thu Nov 12 13:57:20 2009 @@ -57,6 +57,9 @@ ut.object = u'y' assert ut.object == u'y' assert str(ut) == "can't translate characters in position 1-4: bah" + ut.start = 4 + ut.object = u'012345' + assert str(ut) == "can't translate character u'\\x34' in position 4: bah" def test_key_error(self): from tempexceptions import KeyError @@ -85,3 +88,19 @@ assert SystemExit().code is None assert SystemExit("x").code == "x" assert SystemExit(1, 2).code == (1, 2) + + def test_unicode_decode_error(self): + from tempexceptions import UnicodeDecodeError + ud = UnicodeDecodeError("x", "y", 1, 5, "bah") + assert ud.encoding == 'x' + assert ud.object == 'y' + assert ud.start == 1 + assert ud.end == 5 + assert ud.reason == 'bah' + assert ud.args == ('x', 'y', 1, 5, 'bah') + assert ud.message == '' + ud.object = 'z9' + assert ud.object == 'z9' + assert str(ud) == "'x' codec can't decode bytes in position 1-4: bah" + ud.end = 2 + assert str(ud) == "'x' codec can't decode byte 0x39 in position 1: bah" From afa at codespeak.net Thu Nov 12 13:59:16 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 12 Nov 2009 13:59:16 +0100 (CET) Subject: [pypy-svn] r69224 - pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc Message-ID: <20091112125916.13E2216802C@codespeak.net> Author: afa Date: Thu Nov 12 13:59:15 2009 New Revision: 69224 Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Log: be more careful when extracting the symbol from a "constant indirect jump" Modified: pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/trackgcroot.py Thu Nov 12 13:59:15 2009 @@ -648,7 +648,7 @@ for s in insn.all_sources_of(loc): sources.append(s) for source in sources: - m = re.match("DWORD PTR (.+)", source) + m = re.match("DWORD PTR " + self.LABEL, source) if m: reg.append(m.group(1)) if reg: From arigo at codespeak.net Thu Nov 12 14:11:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 14:11:57 +0100 (CET) Subject: [pypy-svn] r69225 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112131157.2C5D216802F@codespeak.net> Author: arigo Date: Thu Nov 12 14:11:56 2009 New Revision: 69225 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: Finish this. Missing: WindowsError. Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Thu Nov 12 14:11:56 2009 @@ -209,14 +209,14 @@ w_end, w_reason]) def descr_str(self, space): - return space.appexec([space.wrap(self)], """(self): + return space.appexec([space.wrap(self)], r"""(self): if self.end == self.start + 1: badchar = ord(self.object[self.start]) if badchar <= 0xff: - return "can't translate character u'\\\\x%02x' in position %d: %s" % (badchar, self.start, self.reason) + return "can't translate character u'\\x%02x' in position %d: %s" % (badchar, self.start, self.reason) if badchar <= 0xffff: - return "can't translate character u'\\\\u%04x' in position %d: %s"%(badchar, self.start, self.reason) - return "can't translate character u'\\\\U%08x' in position %d: %s"%(badchar, self.start, self.reason) + return "can't translate character u'\\u%04x' in position %d: %s"%(badchar, self.start, self.reason) + return "can't translate character u'\\U%08x' in position %d: %s"%(badchar, self.start, self.reason) return "can't translate characters in position %d-%d: %s" % (self.start, self.end - 1, self.reason) """) descr_str.unwrap_spec = ['self', ObjSpace] @@ -444,7 +444,6 @@ return "%r codec can't decode bytes in position %d-%d: %s" % ( self.encoding, self.start, self.end - 1, self.reason) """) - descr_str.unwrap_spec = ['self', ObjSpace] def descr_new_unicode_decode_error(space, w_subtype, w_encoding, w_object, @@ -457,7 +456,7 @@ W_UnicodeDecodeError.typedef = TypeDef( 'UnicodeDecodeError', W_UnicodeError.typedef, - __doc__ = W_UnicodeTranslateError.__doc__, + __doc__ = W_UnicodeDecodeError.__doc__, __new__ = interp2app(descr_new_unicode_decode_error), __str__ = interp2app(W_UnicodeDecodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), @@ -507,6 +506,51 @@ W_OverflowError = _new_exception('OverflowError', W_ArithmeticError, """Result too large to be represented.""") +class W_UnicodeEncodeError(W_UnicodeError): + """Unicode encoding error.""" -##class UnicodeEncodeError(UnicodeError): -## """Unicode encoding error.""".......... + def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + self.encoding = space.str_w(w_encoding) + self.object = space.unicode_w(w_object) + self.start = space.int_w(w_start) + self.end = space.int_w(w_end) + self.reason = space.str_w(w_reason) + W_BaseException.__init__(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + + def descr_str(self, space): + return space.appexec([self], r"""(self): + if self.end == self.start + 1: + badchar = ord(self.object[self.start]) + if badchar <= 0xff: + return "%r codec can't encode character u'\\x%02x' in position %d: %s"%( + self.encoding, badchar, self.start, self.reason) + if badchar <= 0xffff: + return "%r codec can't encode character u'\\u%04x' in position %d: %s"%( + self.encoding, badchar, self.start, self.reason) + return "%r codec can't encode character u'\\U%08x' in position %d: %s"%( + self.encoding, badchar, self.start, self.reason) + return "%r codec can't encode characters in position %d-%d: %s" % ( + self.encoding, self.start, self.end - 1, self.reason) + """) + descr_str.unwrap_spec = ['self', ObjSpace] + +def descr_new_unicode_encode_error(space, w_subtype, w_encoding, w_object, + w_start, w_end, w_reason): + exc = space.allocate_instance(W_UnicodeEncodeError, w_subtype) + W_UnicodeEncodeError.__init__(exc, space, w_encoding, w_object, w_start, + w_end, w_reason) + return space.wrap(exc) + +W_UnicodeEncodeError.typedef = TypeDef( + 'UnicodeEncodeError', + W_UnicodeError.typedef, + __doc__ = W_UnicodeEncodeError.__doc__, + __new__ = interp2app(descr_new_unicode_encode_error), + __str__ = interp2app(W_UnicodeEncodeError.descr_str), + encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), + object = readwrite_attrproperty('object', W_UnicodeEncodeError, 'unicode_w'), + start = readwrite_attrproperty('start', W_UnicodeEncodeError, 'int_w'), + end = readwrite_attrproperty('end', W_UnicodeEncodeError, 'int_w'), + reason = readwrite_attrproperty('reason', W_UnicodeEncodeError, 'str_w'), +) Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Thu Nov 12 14:11:56 2009 @@ -104,3 +104,19 @@ assert str(ud) == "'x' codec can't decode bytes in position 1-4: bah" ud.end = 2 assert str(ud) == "'x' codec can't decode byte 0x39 in position 1: bah" + + def test_unicode_encode_error(self): + from tempexceptions import UnicodeEncodeError + ue = UnicodeEncodeError("x", u"y", 1, 5, "bah") + assert ue.encoding == 'x' + assert ue.object == u'y' + assert ue.start == 1 + assert ue.end == 5 + assert ue.reason == 'bah' + assert ue.args == ('x', u'y', 1, 5, 'bah') + assert ue.message == '' + ue.object = u'z9' + assert ue.object == u'z9' + assert str(ue) == "'x' codec can't encode characters in position 1-4: bah" + ue.end = 2 + assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" From arigo at codespeak.net Thu Nov 12 14:12:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 14:12:19 +0100 (CET) Subject: [pypy-svn] r69226 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112131219.BA8F816802F@codespeak.net> Author: arigo Date: Thu Nov 12 14:12:19 2009 New Revision: 69226 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/ (props changed) pypy/branch/faster-raise/pypy/module/tempexceptions/test/ (props changed) Log: fixeol From arigo at codespeak.net Thu Nov 12 14:30:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 14:30:39 +0100 (CET) Subject: [pypy-svn] r69227 - in pypy/branch/faster-raise/pypy/module/tempexceptions: . test Message-ID: <20091112133039.7CB2116802A@codespeak.net> Author: arigo Date: Thu Nov 12 14:30:39 2009 New Revision: 69227 Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Log: Fix bugs, improve tests. Add W_WindowsError, untested for now. Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py Thu Nov 12 14:30:39 2009 @@ -1,4 +1,4 @@ - +import sys from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): @@ -53,5 +53,7 @@ 'ValueError' : 'interp_exceptions.W_ValueError', 'Warning' : 'interp_exceptions.W_Warning', 'ZeroDivisionError' : 'interp_exceptions.W_ZeroDivisionError', - } + + if sys.platform.startswith("win"): + interpleveldefs['WindowsError'] = 'interp_exceptions.W_WindowsError' Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/interp_exceptions.py Thu Nov 12 14:30:39 2009 @@ -74,6 +74,7 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError def readwrite_attrproperty(name, cls, unwrapname): def fget(space, obj): @@ -282,27 +283,77 @@ self.args_w = [args_w[0], args_w[1]] def descr_str(self, space): - if not space.is_w(self.w_filename, space.w_None): - return space.wrap("[Errno %d] %s: %s" % (space.int_w(self.w_errno), - space.str_w(self.w_strerror), - space.str_w(self.w_filename))) - if (not space.is_w(self.w_errno, space.w_None) and - not space.is_w(self.w_errno, space.w_None)): - return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), - space.str_w(self.w_strerror))) - return W_BaseException.__str__(self, space) + return self.str(space, self.w_errno) descr_str.unwrap_spec = ['self', ObjSpace] + def str(self, space, w_errno): # for WindowsError + if (not space.is_w(w_errno, space.w_None) and + not space.is_w(self.w_strerror, space.w_None)): + if not space.is_w(self.w_filename, space.w_None): + return space.wrap("[Errno %d] %s: %s" % ( + space.int_w(w_errno), + space.str_w(self.w_strerror), + space.str_w(self.w_filename))) + return space.wrap("[Errno %d] %s" % (space.int_w(w_errno), + space.str_w(self.w_strerror))) + return W_BaseException.descr_str(self, space) + W_EnvironmentError.typedef = TypeDef( 'EnvironmentError', W_StandardError.typedef, + __doc__ = W_EnvironmentError.__doc__, __new__ = _new(W_EnvironmentError), - __str__ = interp2app(W_EnvironmentError.descr_str) + __str__ = interp2app(W_EnvironmentError.descr_str), + errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), + strerror = readwrite_attrproperty_w('w_strerror', W_EnvironmentError), + filename = readwrite_attrproperty_w('w_filename', W_EnvironmentError), ) W_OSError = _new_exception('OSError', W_EnvironmentError, """OS system call failed.""") +class W_WindowsError(W_OSError): + """MS-Windows OS system call failed.""" + + def __init__(self, space, args_w): + W_OSError.__init__(self, space, args_w) + # Set errno to the POSIX errno, and winerror to the Win32 + # error code. + try: + errno = space.int_w(self.w_errno) + except OperationError: + errno = 22 # EINVAL + else: + errno = self._winerror_to_errno.get(errno, 22) # EINVAL + self.w_winerror = self.w_errno + self.w_errno = space.wrap(errno) + + def descr_str(self, space): + return self.str(space, self.w_winerror) + descr_str.unwrap_spec = ['self', ObjSpace] + + # copied from CPython: PC/errmap.h + _winerror_to_errno = { + 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, + 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, + 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, 29: 13, 30: 13, + 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, 36: 13, 53: 2, 65: 13, + 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, 108: 13, 109: 32, 112: 28, + 114: 9, 128: 10, 129: 10, 130: 9, 132: 13, 145: 41, 158: 13, + 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, + 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, + 200: 8, 201: 8, 202: 8, 206: 2, 215: 11, 1816: 12, + } + +W_WindowsError.typedef = TypeDef( + "WindowsError", + W_OSError.typedef, + __doc__ = W_WindowsError.__doc__, + __new__ = _new(W_WindowsError), + __str__ = interp2app(W_WindowsError.descr_str), + winerror = readwrite_attrproperty_w('w_winerror', W_WindowsError), + ) + W_DeprecationWarning = _new_exception('DeprecationWarning', W_Warning, """Base class for warnings about deprecated features.""") Modified: pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py Thu Nov 12 14:30:39 2009 @@ -71,6 +71,10 @@ ee = EnvironmentError(3, "x", "y") assert str(ee) == "[Errno 3] x: y" assert str(EnvironmentError(3, "x")) == "[Errno 3] x" + assert ee.errno == 3 + assert ee.strerror == "x" + assert ee.filename == "y" + assert EnvironmentError(3, "x").filename is None def test_syntax_error(self): from tempexceptions import SyntaxError From arigo at codespeak.net Thu Nov 12 15:04:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 15:04:51 +0100 (CET) Subject: [pypy-svn] r69229 - in pypy/branch/faster-raise/pypy: interpreter lib module/exceptions module/exceptions/test module/tempexceptions objspace/std Message-ID: <20091112140451.F1A6516800B@codespeak.net> Author: arigo Date: Thu Nov 12 15:04:51 2009 New Revision: 69229 Added: pypy/branch/faster-raise/pypy/module/exceptions/ - copied from r69227, pypy/branch/faster-raise/pypy/module/tempexceptions/ Removed: pypy/branch/faster-raise/pypy/lib/_exceptions.py pypy/branch/faster-raise/pypy/module/tempexceptions/ Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/module/exceptions/__init__.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py pypy/branch/faster-raise/pypy/objspace/std/objspace.py Log: Progress. Now we are entering the land of real obscureness. Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Thu Nov 12 15:04:51 2009 @@ -389,12 +389,19 @@ def make_builtins(self): "NOT_RPYTHON: only for initializing the space." + from pypy.module.exceptions import Module + w_name_exceptions = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name_exceptions) + from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.setitem(w_modules, w_name_exceptions, + self.wrap(self.exceptions_module)) + from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) @@ -405,6 +412,8 @@ bootstrap_modules = ['sys', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] + self.export_builtin_exceptions() + # initialize with "bootstrap types" from objspace (e.g. w_None) for name, value in self.__dict__.items(): if name.startswith('w_') and not name.endswith('Type'): @@ -430,6 +439,18 @@ self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'), w_builtin_module_names) + def export_builtin_exceptions(self): + """NOT_RPYTHON""" + w_dic = self.exceptions_module.getdict() + names_w = self.unpackiterable(self.call_function(self.getattr(w_dic, self.wrap("keys")))) + + for w_name in names_w: + name = self.str_w(w_name) + if not name.startswith('__'): + excname = name + w_exc = self.getitem(w_dic, w_name) + setattr(self, "w_"+excname, w_exc) + def install_mixedmodule(self, mixedname, installed_builtin_modules): """NOT_RPYTHON""" modname = self.setbuiltinmodule(mixedname) Modified: pypy/branch/faster-raise/pypy/module/exceptions/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/__init__.py Thu Nov 12 15:04:51 2009 @@ -2,8 +2,6 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - applevel_name = 'tempexceptions' - appleveldefs = {} interpleveldefs = { Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/tempexceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Thu Nov 12 15:04:51 2009 @@ -3,10 +3,10 @@ class AppTestExc(object): def setup_class(cls): - cls.space = gettestobjspace(usemodules=('tempexceptions',)) + cls.space = gettestobjspace(usemodules=('exceptions',)) def test_baseexc(self): - from tempexceptions import BaseException + from exceptions import BaseException assert str(BaseException()) == '' assert repr(BaseException()) == 'BaseException()' @@ -25,7 +25,7 @@ assert x.xyz == 3 def test_exc(self): - from tempexceptions import Exception, BaseException + from exceptions import Exception, BaseException assert issubclass(Exception, BaseException) assert isinstance(Exception(), Exception) @@ -33,7 +33,7 @@ assert repr(Exception(3, "x")) == "Exception(3, 'x')" def test_custom_class(self): - from tempexceptions import Exception + from exceptions import Exception class MyException(Exception): def __init__(self, x): @@ -46,7 +46,7 @@ assert str(MyException("x")) == "x" def test_unicode_translate_error(self): - from tempexceptions import UnicodeTranslateError + from exceptions import UnicodeTranslateError ut = UnicodeTranslateError(u"x", 1, 5, "bah") assert ut.object == u'x' assert ut.start == 1 @@ -62,12 +62,12 @@ assert str(ut) == "can't translate character u'\\x34' in position 4: bah" def test_key_error(self): - from tempexceptions import KeyError + from exceptions import KeyError assert str(KeyError('s')) == "'s'" def test_environment_error(self): - from tempexceptions import EnvironmentError + from exceptions import EnvironmentError ee = EnvironmentError(3, "x", "y") assert str(ee) == "[Errno 3] x: y" assert str(EnvironmentError(3, "x")) == "[Errno 3] x" @@ -77,7 +77,7 @@ assert EnvironmentError(3, "x").filename is None def test_syntax_error(self): - from tempexceptions import SyntaxError + from exceptions import SyntaxError s = SyntaxError(3) assert str(s) == '3' assert str(SyntaxError("a", "b", 123)) == "a" @@ -88,13 +88,13 @@ assert str(SyntaxError("msg", ("file.py", 2, 3, 4))) == "msg (file.py, line 2)" def test_system_exit(self): - from tempexceptions import SystemExit + from exceptions import SystemExit assert SystemExit().code is None assert SystemExit("x").code == "x" assert SystemExit(1, 2).code == (1, 2) def test_unicode_decode_error(self): - from tempexceptions import UnicodeDecodeError + from exceptions import UnicodeDecodeError ud = UnicodeDecodeError("x", "y", 1, 5, "bah") assert ud.encoding == 'x' assert ud.object == 'y' @@ -110,7 +110,7 @@ assert str(ud) == "'x' codec can't decode byte 0x39 in position 1: bah" def test_unicode_encode_error(self): - from tempexceptions import UnicodeEncodeError + from exceptions import UnicodeEncodeError ue = UnicodeEncodeError("x", u"y", 1, 5, "bah") assert ue.encoding == 'x' assert ue.object == u'y' Modified: pypy/branch/faster-raise/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/objspace.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/objspace.py Thu Nov 12 15:04:51 2009 @@ -322,9 +322,7 @@ setattr(self, 'w_' + typedef.name, w_type) # exceptions & builtins - w_mod = self.setup_exceptions() self.make_builtins() - self.sys.setmodule(w_mod) # the type of old-style classes self.w_classobj = self.builtin.get('__metaclass__') @@ -358,61 +356,6 @@ self.setattr(w___pypy__, self.wrap('get_tproxy_controller'), self.wrap(app_proxy_controller)) - def create_builtin_module(self, pyname, publicname): - """NOT_RPYTHON - helper function which returns the wrapped module and its dict. - """ - # generate on-the-fly - class Fake: pass - fake = Fake() - from pypy import lib - fname = os.path.join(os.path.split(lib.__file__)[0], pyname) - fake.filename = fname - fake.code = compile(file(fname).read(), fname, "exec") - fake.modname = publicname - w_dic = PyPyCacheDir.build_applevelinterp_dict(fake, self) - from pypy.interpreter.module import Module - mod = Module(self, self.wrap(publicname), w_dic) - w_mod = self.wrap(mod) - return w_mod, w_dic - - def setup_exceptions(self): - """NOT_RPYTHON""" - ## hacking things in - def call(w_type, w_args): - space = self - # too early for unpackiterable as well :-( - name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) - dic = space.unwrap(space.getitem(w_args, space.wrap(2))) - dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) - bases = list(bases) - if not bases: - bases = [space.w_object] - res = W_TypeObject(space, name, bases, dic) - res.ready() - return res - try: - # note that we hide the real call method by an instance variable! - self.call = call - mod, w_dic = self.create_builtin_module('_exceptions.py', 'exceptions') - - self.w_IndexError = self.getitem(w_dic, self.wrap("IndexError")) - self.w_StopIteration = self.getitem(w_dic, self.wrap("StopIteration")) - finally: - del self.call # revert - - names_w = self.unpackiterable(self.call_function(self.getattr(w_dic, self.wrap("keys")))) - - for w_name in names_w: - name = self.str_w(w_name) - if not name.startswith('__'): - excname = name - w_exc = self.getitem(w_dic, w_name) - setattr(self, "w_"+excname, w_exc) - - return mod - def createexecutioncontext(self): # add space specific fields to execution context # note that this method must not call space methods that might need an From afa at codespeak.net Thu Nov 12 15:08:58 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 12 Nov 2009 15:08:58 +0100 (CET) Subject: [pypy-svn] r69230 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091112140858.196EE16802F@codespeak.net> Author: afa Date: Thu Nov 12 15:08:57 2009 New Revision: 69230 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Test and fix for WindowsError Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Thu Nov 12 15:08:57 2009 @@ -283,20 +283,17 @@ self.args_w = [args_w[0], args_w[1]] def descr_str(self, space): - return self.str(space, self.w_errno) - descr_str.unwrap_spec = ['self', ObjSpace] - - def str(self, space, w_errno): # for WindowsError - if (not space.is_w(w_errno, space.w_None) and + if (not space.is_w(self.w_errno, space.w_None) and not space.is_w(self.w_strerror, space.w_None)): if not space.is_w(self.w_filename, space.w_None): return space.wrap("[Errno %d] %s: %s" % ( - space.int_w(w_errno), + space.int_w(self.w_errno), space.str_w(self.w_strerror), space.str_w(self.w_filename))) - return space.wrap("[Errno %d] %s" % (space.int_w(w_errno), + return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), space.str_w(self.w_strerror))) return W_BaseException.descr_str(self, space) + descr_str.unwrap_spec = ['self', ObjSpace] W_EnvironmentError.typedef = TypeDef( 'EnvironmentError', @@ -329,7 +326,16 @@ self.w_errno = space.wrap(errno) def descr_str(self, space): - return self.str(space, self.w_winerror) + if (not space.is_w(self.w_winerror, space.w_None) and + not space.is_w(self.w_strerror, space.w_None)): + if not space.is_w(self.w_filename, space.w_None): + return space.wrap("[Error %d] %s: %s" % ( + space.int_w(self.w_winerror), + space.str_w(self.w_strerror), + space.str_w(self.w_filename))) + return space.wrap("[Error %d] %s" % (space.int_w(self.w_winerror), + space.str_w(self.w_strerror))) + return W_BaseException.descr_str(self, space) descr_str.unwrap_spec = ['self', ObjSpace] # copied from CPython: PC/errmap.h Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Thu Nov 12 15:08:57 2009 @@ -76,6 +76,18 @@ assert ee.filename == "y" assert EnvironmentError(3, "x").filename is None + def test_windows_error(self): + try: + from exceptions import WindowsError + except ImportError: + skip('WindowsError not present') + ee = WindowsError(3, "x", "y") + assert str(ee) == "[Error 3] x: y" + # winerror=3 (ERROR_PATH_NOT_FOUND) maps to errno=2 (ENOENT) + assert ee.winerror == 3 + assert ee.errno == 2 + assert str(WindowsError(3, "x")) == "[Error 3] x" + def test_syntax_error(self): from exceptions import SyntaxError s = SyntaxError(3) From pedronis at codespeak.net Thu Nov 12 16:42:04 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 12 Nov 2009 16:42:04 +0100 (CET) Subject: [pypy-svn] r69231 - pypy/build/bot2/pypybuildbot Message-ID: <20091112154204.AAF0116802C@codespeak.net> Author: pedronis Date: Thu Nov 12 16:42:03 2009 New Revision: 69231 Modified: pypy/build/bot2/pypybuildbot/master.py Log: run own tests on mac nightly Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Nov 12 16:42:03 2009 @@ -37,7 +37,7 @@ status.putChild('summary', summary.Summary(['own', 'applevel', 'lib-python', 'jit', 'stackless', - 'windows', 'maemo', 'other'])) + 'windows', 'mac', 'maemo', 'other'])) pypybuilds = load('pypybuildbot.builds') @@ -76,7 +76,8 @@ 'change_source': [], 'schedulers': [ Nightly("nightly", [LINUX32, CPYLINUX32, APPLVLLINUX32, CPYWIN32, - STACKLESSAPPLVLLINUX32, JITCPYLINUX32], + STACKLESSAPPLVLLINUX32, JITCPYLINUX32, + MACOSX32], hour=4, minute=45), Nightly("nightly-benchmark", [JITBENCH], hour=2, minute=25), @@ -98,7 +99,7 @@ "slavenames": ["minime"], "builddir": MACOSX32, "factory": pypyOwnTestFactory, - "category": 'own' + "category": 'mac' }, {"name": CPYLINUX32, "slavenames": ["wyvern", "cobra"], From cfbolz at codespeak.net Thu Nov 12 17:45:30 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 12 Nov 2009 17:45:30 +0100 (CET) Subject: [pypy-svn] r69232 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091112164530.0E5BD16802C@codespeak.net> Author: cfbolz Date: Thu Nov 12 17:45:30 2009 New Revision: 69232 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (cfbolz, hpk): this passes nowaday. xfail would have shown this :-) Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Thu Nov 12 17:45:30 2009 @@ -122,7 +122,6 @@ ([20], 2432902008176640000L)) def test_factorialrec(self): - skip("does not make sense yet") self.run_source(''' def main(n): if n > 1: From afa at codespeak.net Thu Nov 12 18:06:03 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 12 Nov 2009 18:06:03 +0100 (CET) Subject: [pypy-svn] r69233 - in pypy/branch/faster-raise/pypy: module/exceptions rlib Message-ID: <20091112170603.5BF6B16802A@codespeak.net> Author: afa Date: Thu Nov 12 18:06:02 2009 New Revision: 69233 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/rlib/rwin32.py Log: Build at translation time the winerror->errno conversion dict. Note that it would be a bad idea to call _dosmaperr() in the WindowsError constructor, because this changes the global errno. Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Thu Nov 12 18:06:02 2009 @@ -75,6 +75,7 @@ GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError +from pypy.rlib import rwin32 def readwrite_attrproperty(name, cls, unwrapname): def fget(space, obj): @@ -319,9 +320,9 @@ try: errno = space.int_w(self.w_errno) except OperationError: - errno = 22 # EINVAL + errno = self._default_errno else: - errno = self._winerror_to_errno.get(errno, 22) # EINVAL + errno = self._winerror_to_errno.get(errno, self._default_errno) self.w_winerror = self.w_errno self.w_errno = space.wrap(errno) @@ -338,18 +339,10 @@ return W_BaseException.descr_str(self, space) descr_str.unwrap_spec = ['self', ObjSpace] - # copied from CPython: PC/errmap.h - _winerror_to_errno = { - 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, - 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, - 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, 29: 13, 30: 13, - 31: 13, 32: 13, 33: 13, 34: 13, 35: 13, 36: 13, 53: 2, 65: 13, - 67: 2, 80: 17, 82: 13, 83: 13, 89: 11, 108: 13, 109: 32, 112: 28, - 114: 9, 128: 10, 129: 10, 130: 9, 132: 13, 145: 41, 158: 13, - 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, - 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, - 200: 8, 201: 8, 202: 8, 206: 2, 215: 11, 1816: 12, - } + if hasattr(rwin32, 'build_winerror_to_errno'): + _winerror_to_errno, _default_errno = rwin32.build_winerror_to_errno() + else: + _winerror_to_errno, _default_errno = {}, 22 # EINVAL W_WindowsError.typedef = TypeDef( "WindowsError", Modified: pypy/branch/faster-raise/pypy/rlib/rwin32.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/rwin32.py (original) +++ pypy/branch/faster-raise/pypy/rlib/rwin32.py Thu Nov 12 18:06:02 2009 @@ -6,7 +6,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import intmask -import os, sys +import os, sys, errno # This module can be imported on any platform, # but most symbols are not usable... @@ -14,8 +14,13 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h'], + includes = ['windows.h', 'errno.h'], libraries = ['kernel32'], + separate_module_sources = [""" + long WINAPI pypy_dosmaperr(long winerror) + { _dosmaperr(winerror); return errno; } + """], + export_symbols = ["pypy_dosmaperr"], ) else: eci = ExternalCompilationInfo() @@ -60,7 +65,6 @@ """.split(): locals()[name] = rffi_platform.ConstantInteger(name) - for k, v in rffi_platform.configure(CConfig).items(): globals()[k] = v @@ -93,6 +97,20 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + dosmaperr = winexternal('pypy_dosmaperr', [rffi.LONG], rffi.LONG) + + + def build_winerror_to_errno(): + """Build a dictionary mapping windows error numbers to POSIX errno. + The function returns the dict, and the default value for codes not + in the dict.""" + default = errno.EINVAL + errors = {} + for i in range(1, 65000): + error = dosmaperr(i) + if error != default: + errors[i] = error + return errors, default # A bit like strerror... def FormatError(code): From arigo at codespeak.net Thu Nov 12 18:13:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 18:13:14 +0100 (CET) Subject: [pypy-svn] r69234 - in pypy/branch/faster-raise/pypy: interpreter module/exceptions module/exceptions/test objspace/std Message-ID: <20091112171314.D49E516802A@codespeak.net> Author: arigo Date: Thu Nov 12 18:13:12 2009 New Revision: 69234 Modified: pypy/branch/faster-raise/pypy/interpreter/typedef.py pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py pypy/branch/faster-raise/pypy/objspace/std/objspace.py pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py pypy/branch/faster-raise/pypy/objspace/std/typeobject.py Log: (fijal, pedronis around, arigo) Delicate. Allows multiple inheritance of exception classes, as much as CPython (probably). See comment in interp_exceptions._new_exception(). Modified: pypy/branch/faster-raise/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/typedef.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/typedef.py Thu Nov 12 18:13:12 2009 @@ -24,6 +24,7 @@ self.weakrefable |= __base.weakrefable self.rawdict = {} self.acceptable_as_base_class = True + self.applevel_subclasses_base = None # xxx used by faking self.fakedcpytype = None self.add_entries(**rawdict) Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Thu Nov 12 18:13:12 2009 @@ -140,10 +140,12 @@ self.w_dict = w_dict -def _new(cls): +def _new(cls, basecls=None): + if basecls is None: + basecls = cls def descr_new_base_exception(space, w_subtype, args_w): exc = space.allocate_instance(cls, w_subtype) - cls.__init__(exc, space, args_w) + basecls.__init__(exc, space, args_w) return space.wrap(exc) descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] descr_new_base_exception.func_name = 'descr_new_' + cls.__name__ @@ -162,21 +164,30 @@ ) def _new_exception(name, base, docstring, **kwargs): - class W_Exception(base): + # Create a subclass W_Exc of the class 'base'. Note that there is + # hackery going on on the typedef of W_Exc: when we make further + # app-level subclasses, they inherit at interp-level from 'realbase' + # instead of W_Exc. This allows multiple inheritance to work (see + # test_multiple_inheritance in test_exc.py). + + class W_Exc(base): __doc__ = docstring - W_Exception.__name__ = 'W_' + name + W_Exc.__name__ = 'W_' + name + + realbase = base.typedef.applevel_subclasses_base or base for k, v in kwargs.items(): - kwargs[k] = interp2app(v.__get__(None, W_Exception)) - W_Exception.typedef = TypeDef( + kwargs[k] = interp2app(v.__get__(None, realbase)) + W_Exc.typedef = TypeDef( name, base.typedef, - __doc__ = W_Exception.__doc__, - __new__ = _new(W_Exception), + __doc__ = W_Exc.__doc__, + __new__ = _new(W_Exc, realbase), **kwargs ) - return W_Exception + W_Exc.typedef.applevel_subclasses_base = realbase + return W_Exc W_Exception = _new_exception('Exception', W_BaseException, """Common base class for all non-exit exceptions.""") Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Thu Nov 12 18:13:12 2009 @@ -33,7 +33,7 @@ assert repr(Exception(3, "x")) == "Exception(3, 'x')" def test_custom_class(self): - from exceptions import Exception + from exceptions import Exception, BaseException, LookupError class MyException(Exception): def __init__(self, x): @@ -43,6 +43,8 @@ return self.x assert issubclass(MyException, Exception) + assert issubclass(MyException, BaseException) + assert not issubclass(MyException, LookupError) assert str(MyException("x")) == "x" def test_unicode_translate_error(self): @@ -136,3 +138,26 @@ assert str(ue) == "'x' codec can't encode characters in position 1-4: bah" ue.end = 2 assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" + + def test_multiple_inheritance(self): + class A(LookupError, ValueError): + pass + assert issubclass(A, A) + assert issubclass(A, Exception) + assert issubclass(A, LookupError) + assert issubclass(A, ValueError) + assert not issubclass(A, KeyError) + a = A() + assert isinstance(a, A) + assert isinstance(a, Exception) + assert isinstance(a, LookupError) + assert isinstance(a, ValueError) + assert not isinstance(a, KeyError) + + try: + class B(UnicodeTranslateError, UnicodeEncodeError): + pass + except TypeError: + pass + else: + fail("bah") Modified: pypy/branch/faster-raise/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/objspace.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/objspace.py Thu Nov 12 18:13:12 2009 @@ -558,16 +558,18 @@ # the purpose of the above check is to avoid the code below # to be annotated at all for 'cls' if it is not necessary w_subtype = w_type.check_user_subclass(w_subtype) + if cls.typedef.applevel_subclasses_base is not None: + cls = cls.typedef.applevel_subclasses_base subcls = get_unique_interplevel_subclass( self.config, cls, w_subtype.hasdict, w_subtype.nslots != 0, w_subtype.needsdel, w_subtype.weakrefable) instance = instantiate(subcls) + assert isinstance(instance, cls) instance.user_setup(self, w_subtype) else: raise OperationError(self.w_TypeError, self.wrap("%s.__new__(%s): only for the type %s" % ( w_type.name, w_subtype.getname(self, '?'), w_type.name))) - assert isinstance(instance, cls) return instance allocate_instance._annspecialcase_ = "specialize:arg(1)" Modified: pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py Thu Nov 12 18:13:12 2009 @@ -91,8 +91,12 @@ for descrname, descrvalue in rawdict.items(): dict_w[descrname] = w(descrvalue) + if typedef.applevel_subclasses_base is not None: + overridetypedef = typedef.applevel_subclasses_base.typedef + else: + overridetypedef = typedef w_type = W_TypeObject(space, typedef.name, bases_w, dict_w, - overridetypedef=typedef) + overridetypedef=overridetypedef) w_type.lazyloaders = lazyloaders return w_type Modified: pypy/branch/faster-raise/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/typeobject.py Thu Nov 12 18:13:12 2009 @@ -76,7 +76,6 @@ w_self.hasdict = False w_self.needsdel = False w_self.weakrefable = False - w_self.w_same_layout_as = None w_self.weak_subclasses = [] w_self.__flags__ = 0 # or _HEAPTYPE w_self.instancetypedef = overridetypedef @@ -87,6 +86,7 @@ else: setup_user_defined_type(w_self) custom_metaclass = not space.is_w(space.type(w_self), space.w_type) + w_self.w_same_layout_as = get_parent_layout(w_self) if space.config.objspace.std.withtypeversion: if w_self.instancetypedef.hasdict or custom_metaclass: @@ -518,7 +518,6 @@ hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase) create_all_slots(w_self, hasoldstylebase) - w_self.w_same_layout_as = get_parent_layout(w_self) ensure_common_attributes(w_self) def setup_builtin_type(w_self): From arigo at codespeak.net Thu Nov 12 18:24:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 18:24:38 +0100 (CET) Subject: [pypy-svn] r69235 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091112172438.74A8D16802A@codespeak.net> Author: arigo Date: Thu Nov 12 18:24:36 2009 New Revision: 69235 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: (fijal, arigo) - assignment to Exception.args - add __module__ Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Thu Nov 12 18:24:36 2009 @@ -129,6 +129,9 @@ def descr_getargs(space, self): return space.newtuple(self.args_w) + def descr_setargs(space, self, w_newargs): + self.args_w = space.unpackiterable(w_newargs) + def getdict(self): if self.w_dict is None: self.w_dict = self.space.newdict() @@ -154,13 +157,15 @@ W_BaseException.typedef = TypeDef( 'BaseException', __doc__ = W_BaseException.__doc__, + __module__ = 'exceptions', __new__ = _new(W_BaseException), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_BaseException), message = interp_attrproperty_w('w_message', W_BaseException), - args = GetSetProperty(W_BaseException.descr_getargs), + args = GetSetProperty(W_BaseException.descr_getargs, + W_BaseException.descr_setargs), ) def _new_exception(name, base, docstring, **kwargs): @@ -183,6 +188,7 @@ name, base.typedef, __doc__ = W_Exc.__doc__, + __module__ = 'exceptions', __new__ = _new(W_Exc, realbase), **kwargs ) @@ -245,6 +251,7 @@ 'UnicodeTranslateError', W_UnicodeError.typedef, __doc__ = W_UnicodeTranslateError.__doc__, + __module__ = 'exceptions', __new__ = interp2app(descr_new_unicode_translate_error), __str__ = interp2app(W_UnicodeTranslateError.descr_str), object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), @@ -311,6 +318,7 @@ 'EnvironmentError', W_StandardError.typedef, __doc__ = W_EnvironmentError.__doc__, + __module__ = 'exceptions', __new__ = _new(W_EnvironmentError), __str__ = interp2app(W_EnvironmentError.descr_str), errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), @@ -359,6 +367,7 @@ "WindowsError", W_OSError.typedef, __doc__ = W_WindowsError.__doc__, + __module__ = 'exceptions', __new__ = _new(W_WindowsError), __str__ = interp2app(W_WindowsError.descr_str), winerror = readwrite_attrproperty_w('w_winerror', W_WindowsError), @@ -433,6 +442,7 @@ __new__ = _new(W_SyntaxError), __str__ = interp2app(W_SyntaxError.descr_str), __doc__ = W_SyntaxError.__doc__, + __module__ = 'exceptions', msg = readwrite_attrproperty_w('w_msg', W_SyntaxError), filename = readwrite_attrproperty_w('w_filename', W_SyntaxError), lineno = readwrite_attrproperty_w('w_lineno', W_SyntaxError), @@ -460,6 +470,7 @@ W_BaseException.typedef, __new__ = _new(W_SystemExit), __doc__ = W_SystemExit.__doc__, + __module__ = 'exceptions', code = readwrite_attrproperty_w('w_code', W_SystemExit) ) @@ -518,6 +529,7 @@ 'UnicodeDecodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeDecodeError.__doc__, + __module__ = 'exceptions', __new__ = interp2app(descr_new_unicode_decode_error), __str__ = interp2app(W_UnicodeDecodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), @@ -607,6 +619,7 @@ 'UnicodeEncodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeEncodeError.__doc__, + __module__ = 'exceptions', __new__ = interp2app(descr_new_unicode_encode_error), __str__ = interp2app(W_UnicodeEncodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Thu Nov 12 18:24:36 2009 @@ -23,6 +23,8 @@ x = BaseException() x.xyz = 3 assert x.xyz == 3 + x.args = [42] + assert x.args == (42,) def test_exc(self): from exceptions import Exception, BaseException @@ -140,6 +142,7 @@ assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" def test_multiple_inheritance(self): + from exceptions import LookupError, ValueError, Exception class A(LookupError, ValueError): pass assert issubclass(A, A) @@ -154,6 +157,7 @@ assert isinstance(a, ValueError) assert not isinstance(a, KeyError) + from exceptions import UnicodeDecodeError, UnicodeEncodeError try: class B(UnicodeTranslateError, UnicodeEncodeError): pass @@ -161,3 +165,10 @@ pass else: fail("bah") + + def test_doc_and_module(self): + import exceptions + for name, e in exceptions.__dict__.items(): + if isinstance(e, type) and issubclass(e, exceptions.BaseException): + assert e.__doc__, e + assert e.__module__ == 'exceptions', e From fijal at codespeak.net Thu Nov 12 18:52:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 18:52:28 +0100 (CET) Subject: [pypy-svn] r69236 - in pypy/branch/faster-raise/pypy/objspace/std: . test Message-ID: <20091112175228.07FA016802A@codespeak.net> Author: fijal Date: Thu Nov 12 18:52:28 2009 New Revision: 69236 Modified: pypy/branch/faster-raise/pypy/objspace/std/test/test_proxy_internals.py pypy/branch/faster-raise/pypy/objspace/std/transparent.py Log: (arigo, pedronis, fijal) Make sure we don't get None out of __pypy__.tproxy, test it Modified: pypy/branch/faster-raise/pypy/objspace/std/test/test_proxy_internals.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/test/test_proxy_internals.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/test/test_proxy_internals.py Thu Nov 12 18:52:28 2009 @@ -1,7 +1,8 @@ """ test proxy internals like code, traceback, frame """ -from pypy.conftest import gettestobjspace +import py +from pypy.conftest import gettestobjspace, option class AppProxy(object): def setup_class(cls): @@ -21,6 +22,28 @@ return get_proxy """) +class AppTestProxyInterpOnly(AppProxy): + def setup_class(cls): + if option.runappdirect: + py.test.skip("interp only test") + from pypy.interpreter.typedef import TypeDef, interp2app + from pypy.interpreter.baseobjspace import Wrappable + + class W_Stuff(Wrappable): + pass + + def descr_new(space, w_subtype): + return W_Stuff() + + W_Stuff.typedef = TypeDef( + 'Stuff', + __new__ = interp2app(descr_new), + ) + cls.w_Stuff = cls.space.gettypefor(W_Stuff) + + def test_unproxyable(self): + raises(TypeError, self.get_proxy, self.Stuff()) + class AppTestProxyInternals(AppProxy): def test_traceback_basic(self): try: @@ -31,7 +54,7 @@ tb = self.get_proxy(e[2]) assert tb.tb_frame is e[2].tb_frame - + def test_traceback_catch(self): try: try: Modified: pypy/branch/faster-raise/pypy/objspace/std/transparent.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/transparent.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/transparent.py Thu Nov 12 18:52:28 2009 @@ -26,7 +26,6 @@ completely controlled by the controller.""" from pypy.interpreter.typedef import Function, PyTraceback, PyFrame, \ PyCode, GeneratorIterator - if not space.is_true(space.callable(w_controller)): raise OperationError(space.w_TypeError, space.wrap("controller should be function")) @@ -49,14 +48,12 @@ return W_Transparent(space, w_type, w_controller) else: raise OperationError(space.w_TypeError, space.wrap("type expected as first argument")) - try: - w_lookup = w_type or w_type.w_besttype - for k, v in type_cache.cache: - if w_lookup == k: - return v(space, w_type, w_controller) - except KeyError: - raise OperationError(space.w_TypeError, space.wrap("Object type %s could not "\ - "be wrapped (YET)" % w_type.getname(space, "?"))) + w_lookup = w_type + for k, v in type_cache.cache: + if w_lookup == k: + return v(space, w_type, w_controller) + raise OperationError(space.w_TypeError, space.wrap("Object type %s could not "\ + "be wrapped (YET)" % w_type.getname(space, "?"))) def register_proxyable(space, cls): tpdef = cls.typedef From fijal at codespeak.net Thu Nov 12 18:53:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 18:53:07 +0100 (CET) Subject: [pypy-svn] r69237 - pypy/branch/faster-raise/pypy/module/exceptions Message-ID: <20091112175307.8E67C16802A@codespeak.net> Author: fijal Date: Thu Nov 12 18:53:06 2009 New Revision: 69237 Modified: pypy/branch/faster-raise/pypy/module/exceptions/__init__.py Log: (arigo, fijal, pedronis) Make Exceptions proxyable (again) Modified: pypy/branch/faster-raise/pypy/module/exceptions/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/__init__.py Thu Nov 12 18:53:06 2009 @@ -37,8 +37,8 @@ 'StopIteration' : 'interp_exceptions.W_StopIteration', 'SyntaxError' : 'interp_exceptions.W_SyntaxError', 'SyntaxWarning' : 'interp_exceptions.W_SyntaxWarning', - 'SystemError' : 'interp_exceptions.W_SystemError', 'SystemExit' : 'interp_exceptions.W_SystemExit', + 'SystemError' : 'interp_exceptions.W_SystemError', 'TabError' : 'interp_exceptions.W_TabError', 'TypeError' : 'interp_exceptions.W_TypeError', 'UnboundLocalError' : 'interp_exceptions.W_UnboundLocalError', @@ -55,3 +55,11 @@ if sys.platform.startswith("win"): interpleveldefs['WindowsError'] = 'interp_exceptions.W_WindowsError' + + def setup_after_space_initialization(self): + from pypy.objspace.std.transparent import register_proxyable + from pypy.module.exceptions import interp_exceptions + + for name, exc in interp_exceptions.__dict__.items(): + if isinstance(exc, type) and issubclass(exc, interp_exceptions.W_BaseException): + register_proxyable(self.space, exc) From fijal at codespeak.net Thu Nov 12 18:57:09 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 18:57:09 +0100 (CET) Subject: [pypy-svn] r69238 - pypy/branch/faster-raise/pypy/interpreter/test Message-ID: <20091112175709.5FDBD16802F@codespeak.net> Author: fijal Date: Thu Nov 12 18:57:08 2009 New Revision: 69238 Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_module.py Log: fix obscure test Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_module.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_module.py Thu Nov 12 18:57:08 2009 @@ -54,13 +54,13 @@ r = repr(sys) assert r == "" - import _exceptions # known to be in pypy/lib - r = repr(_exceptions) - assert r.startswith("') - nofile = type(_exceptions)('nofile', 'foo') + nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" - m = type(_exceptions).__new__(type(_exceptions)) + m = type(_pypy_interact).__new__(type(_pypy_interact)) assert repr(m).startswith(" Author: fijal Date: Thu Nov 12 19:07:26 2009 New Revision: 69239 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Log: (arigo, pedronis, fijal) Fix translation issue Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Thu Nov 12 19:07:26 2009 @@ -130,7 +130,7 @@ return space.newtuple(self.args_w) def descr_setargs(space, self, w_newargs): - self.args_w = space.unpackiterable(w_newargs) + self.args_w = space.viewiterable(w_newargs) def getdict(self): if self.w_dict is None: From antocuni at codespeak.net Thu Nov 12 19:21:07 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 12 Nov 2009 19:21:07 +0100 (CET) Subject: [pypy-svn] r69240 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091112182107.01421168030@codespeak.net> Author: antocuni Date: Thu Nov 12 19:21:07 2009 New Revision: 69240 Modified: pypy/trunk/pypy/jit/backend/cli/method.py Log: two new subcategories for logging the time spent in the cli jit backend Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Thu Nov 12 19:21:07 2009 @@ -1,5 +1,6 @@ import py import os +from pypy.rlib.debug import debug_start, debug_stop from pypy.tool.pairtype import extendabletype from pypy.rpython.ootypesystem import ootype from pypy.translator.cli import dotnet @@ -164,15 +165,20 @@ def compile(self): # ---- + debug_start('jit-backend-emit_ops') if self.nocast: self.compute_types() self.emit_load_inputargs() - self.emit_preamble() + self.emit_preamble() self.emit_operations(self.cliloop.operations) self.emit_branches() self.emit_end() + debug_stop('jit-backend-emit_ops') # ---- - return self.finish_code() + debug_start('jit-backend-finish_code') + res = self.finish_code() + debug_stop('jit-backend-finish_code') + return res def _parseopt(self, text): text = text.lower() From fijal at codespeak.net Thu Nov 12 19:24:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 12 Nov 2009 19:24:21 +0100 (CET) Subject: [pypy-svn] r69241 - in pypy/branch/faster-raise/pypy/module/pypyjit: . test Message-ID: <20091112182421.84697168039@codespeak.net> Author: fijal Date: Thu Nov 12 19:24:20 2009 New Revision: 69241 Modified: pypy/branch/faster-raise/pypy/module/pypyjit/policy.py pypy/branch/faster-raise/pypy/module/pypyjit/test/test_policy.py Log: Look into our new shiny module Modified: pypy/branch/faster-raise/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/policy.py Thu Nov 12 19:24:20 2009 @@ -8,7 +8,7 @@ if '.' in modname: modname, _ = modname.split('.', 1) - if modname in ['pypyjit', 'signal', 'micronumpy', 'math']: + if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions']: return True return False Modified: pypy/branch/faster-raise/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/test/test_policy.py Thu Nov 12 19:24:20 2009 @@ -30,6 +30,7 @@ assert not pypypolicy.look_inside_pypy_module('posix.interp_expat') assert pypypolicy.look_inside_pypy_module('__builtin__.operation') assert pypypolicy.look_inside_pypy_module('__builtin__.abstractinst') + assert pypypolicy.look_inside_pypy_module('exceptions.interp_exceptions') for modname in 'pypyjit', 'signal', 'micronumpy', 'math': assert pypypolicy.look_inside_pypy_module(modname) assert pypypolicy.look_inside_pypy_module(modname + '.foo') From arigo at codespeak.net Thu Nov 12 19:32:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 12 Nov 2009 19:32:57 +0100 (CET) Subject: [pypy-svn] r69242 - pypy/branch/faster-raise/pypy/tool Message-ID: <20091112183257.822BF168032@codespeak.net> Author: arigo Date: Thu Nov 12 19:32:57 2009 New Revision: 69242 Modified: pypy/branch/faster-raise/pypy/tool/sourcetools.py Log: Don't generate names that are incrediiiiibly long. Modified: pypy/branch/faster-raise/pypy/tool/sourcetools.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/sourcetools.py (original) +++ pypy/branch/faster-raise/pypy/tool/sourcetools.py Thu Nov 12 19:32:57 2009 @@ -236,12 +236,13 @@ 'a' <= chr(i) <= 'z' or 'A' <= chr(i) <= 'Z') and chr(i) or '_') for i in range(256)]) +PY_IDENTIFIER_MAX = 120 def valid_identifier(stuff): stuff = str(stuff).translate(PY_IDENTIFIER) if not stuff or ('0' <= stuff[0] <= '9'): stuff = '_' + stuff - return stuff + return stuff[:PY_IDENTIFIER_MAX] CO_VARARGS = 0x0004 CO_VARKEYWORDS = 0x0008 From benjamin at codespeak.net Fri Nov 13 02:03:45 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 13 Nov 2009 02:03:45 +0100 (CET) Subject: [pypy-svn] r69243 - pypy/trunk/pypy/doc Message-ID: <20091113010345.B62E2168027@codespeak.net> Author: benjamin Date: Fri Nov 13 02:03:44 2009 New Revision: 69243 Modified: pypy/trunk/pypy/doc/rlib.txt Log: fix module name Modified: pypy/trunk/pypy/doc/rlib.txt ============================================================================== --- pypy/trunk/pypy/doc/rlib.txt (original) +++ pypy/trunk/pypy/doc/rlib.txt Fri Nov 13 02:03:44 2009 @@ -285,7 +285,7 @@ anything except a. To parse a regular expression and to get a matcher for it, you can use the -function ``make_runner(s)`` in the ``pypy.rlib.parsing.parseregex`` module. It +function ``make_runner(s)`` in the ``pypy.rlib.parsing.regexparse`` module. It returns a object with a ``recognize(input)`` method that returns True or False depending on whether ``input`` matches the string or not. From benjamin at codespeak.net Fri Nov 13 02:23:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 13 Nov 2009 02:23:59 +0100 (CET) Subject: [pypy-svn] r69244 - pypy/trunk/pypy/rlib/parsing Message-ID: <20091113012359.4FB1A168029@codespeak.net> Author: benjamin Date: Fri Nov 13 02:23:58 2009 New Revision: 69244 Modified: pypy/trunk/pypy/rlib/parsing/lexer.py Log: remove unused import Modified: pypy/trunk/pypy/rlib/parsing/lexer.py ============================================================================== --- pypy/trunk/pypy/rlib/parsing/lexer.py (original) +++ pypy/trunk/pypy/rlib/parsing/lexer.py Fri Nov 13 02:23:58 2009 @@ -1,5 +1,4 @@ import py -from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.parsing import deterministic, regex class Token(object): From cfbolz at codespeak.net Fri Nov 13 10:27:21 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 13 Nov 2009 10:27:21 +0100 (CET) Subject: [pypy-svn] r69246 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091113092721.118AE16802D@codespeak.net> Author: cfbolz Date: Fri Nov 13 10:27:20 2009 New Revision: 69246 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: update Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Fri Nov 13 10:27:20 2009 @@ -13,18 +13,16 @@ - set up the buildbot on the Mac Mini DONE - directly call assembler for residual portal calls - making the CLI backend working with logging DONE - - set up nightly run on Mac (Samuele, Carl Friedrich) + - set up nightly run on Mac DONE - debug CLI crashes (Anto) - compress the virtuals part of resume data more (Samuele, Carl Friedrich) - try to do something non-insane about Python-level exceptions IN-PROGRESS - improve operations logging DONE? - - rewrite exceptions module to be a mixed module (Armin, Maciek) + - rewrite exceptions module to be a mixed module DONE - make the assembler produced by generate_failure smaller - - merge guard_nonnull(x, ...), guard_class(x, ...) DONE - put the class into the structure to get only one promote when using an instance - look into failing pypy-c-jit apptests - - fix buildbot on py.test 1.1 branch BRANCH MERGED From fijal at codespeak.net Fri Nov 13 10:43:06 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 13 Nov 2009 10:43:06 +0100 (CET) Subject: [pypy-svn] r69247 - pypy/extradoc/planning Message-ID: <20091113094306.AB91D168027@codespeak.net> Author: fijal Date: Fri Nov 13 10:43:06 2009 New Revision: 69247 Modified: pypy/extradoc/planning/jit.txt Log: Add a task we know we won't handle on a sprint Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Fri Nov 13 10:43:06 2009 @@ -19,6 +19,8 @@ - think about code memory management +- forcing virtualizables should only force fields affected, not everything + Python interpreter: - goal: on average <=5 guards per original bytecode @@ -26,7 +28,6 @@ - raising an exception tends to escape frames, due to the traceback capturing - prevent jitting really general */** calls - - improve test running, compile only once - module/__builtin__/app_inspect.py forces frames for globals() and locals(). From pedronis at codespeak.net Fri Nov 13 10:43:09 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 13 Nov 2009 10:43:09 +0100 (CET) Subject: [pypy-svn] r69248 - pypy/extradoc/sprintinfo/ddorf2009 Message-ID: <20091113094309.2955F16802F@codespeak.net> Author: pedronis Date: Fri Nov 13 10:43:08 2009 New Revision: 69248 Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: (all) planning for today Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Fri Nov 13 10:43:08 2009 @@ -4,29 +4,21 @@ - Samuele - Carl Friedrich - Maciek - - Holger - TASKS ====== - - set up the buildbot on the Mac Mini DONE - - directly call assembler for residual portal calls - - making the CLI backend working with logging DONE - - set up nightly run on Mac DONE - - debug CLI crashes (Anto) - - compress the virtuals part of resume data more (Samuele, Carl Friedrich) - - try to do something non-insane about Python-level exceptions IN-PROGRESS - - improve operations logging DONE? - - rewrite exceptions module to be a mixed module DONE - - make the assembler produced by generate_failure smaller + - directly call assembler for residual portal calls POSTPONED + - compress the virtuals part of resume data more (Samuele, Carl Friedrich) POSTPONED + - try to do something non-insane about Python-level exceptions IN-PROGRESS (Maciek, Armin) + - make the assembler produced by generate_failure smaller POSTPONED - put the class into the structure to get only one promote when using an instance - - look into failing pypy-c-jit apptests - + - look into failing pypy-c-jit apptests, pypy-c-jit translate.py POSTPONED + - write blog post (Samuele, Carl Friedrich) - - at the end of the sprint, migrate tasks to jit.txt + - at the end of the sprint, migrate tasks to jit.txt (Carl Friedrich) Talks ----- From fijal at codespeak.net Fri Nov 13 11:35:49 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 13 Nov 2009 11:35:49 +0100 (CET) Subject: [pypy-svn] r69249 - pypy/branch/faster-raise/pypy/annotation Message-ID: <20091113103549.55C27168025@codespeak.net> Author: fijal Date: Fri Nov 13 11:35:47 2009 New Revision: 69249 Modified: pypy/branch/faster-raise/pypy/annotation/description.py Log: (fijal, arigo) Fix for running on top of pypy-c, where OSError.errno is of a different type than regular slots. Modified: pypy/branch/faster-raise/pypy/annotation/description.py ============================================================================== --- pypy/branch/faster-raise/pypy/annotation/description.py (original) +++ pypy/branch/faster-raise/pypy/annotation/description.py Fri Nov 13 11:35:47 2009 @@ -450,7 +450,7 @@ # is of type FunctionType. But bookkeeper.immutablevalue() # will do the right thing in s_get_value(). - if type(value) is MemberDescriptorType: + if type(value) in MemberDescriptorTypes: # skip __slots__, showing up in the class as 'member' objects return if name == '__init__' and self.is_builtin_exception_class(): @@ -884,5 +884,9 @@ class Sample(object): __slots__ = 'x' -MemberDescriptorType = type(Sample.x) +MemberDescriptorTypes = [type(Sample.x)] del Sample +try: + MemberDescriptorTypes.append(type(OSError.errno)) +except AttributeError: # on CPython <= 2.4 + pass From fijal at codespeak.net Fri Nov 13 13:19:13 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 13 Nov 2009 13:19:13 +0100 (CET) Subject: [pypy-svn] r69253 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091113121913.EF1E8168025@codespeak.net> Author: fijal Date: Fri Nov 13 13:19:13 2009 New Revision: 69253 Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/pytraceback.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Log: (Experimental, arigo & fijal) Remove force_f_back in record_application_traceback and force f_back on leave only if exception is raised (and only one frame at a time) Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Fri Nov 13 13:19:13 2009 @@ -138,6 +138,10 @@ def _unchain(self, frame): #assert frame is self.gettopframe() --- slowish + if frame.last_exception is not None: + f_back = frame.f_back() + frame.f_back_some = f_back + frame.f_back_forced = True if self.some_frame is frame: self.some_frame = frame.f_back_some else: Modified: pypy/branch/faster-raise/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pytraceback.py Fri Nov 13 13:19:13 2009 @@ -49,7 +49,6 @@ self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): - frame.force_f_back() if frame.pycode.hidden_applevel: return tb = operror.application_traceback Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Fri Nov 13 13:19:13 2009 @@ -248,6 +248,7 @@ class Frame(object): _f_back_some = None _f_forward = None + last_exception = None def __init__(self, ec, virtual_with_base_frame=None): self.ec = ec @@ -757,3 +758,12 @@ assert ec.gettopframe() is frame ec._unchain(frame) assert ec.gettopframe() is None + + def test_unchain_with_exception(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + frame3 = self.Frame(ec, frame2) + ec._chain(frame3) + frame3.last_exception = 3 + ec._unchain(frame3) + assert frame3.f_back_some is frame2 + assert frame3.f_back_forced From afa at codespeak.net Fri Nov 13 13:36:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 13:36:22 +0100 (CET) Subject: [pypy-svn] r69254 - pypy/branch/msvc-asmgcroot-2 Message-ID: <20091113123622.B91E8168026@codespeak.net> Author: afa Date: Fri Nov 13 13:36:20 2009 New Revision: 69254 Added: pypy/branch/msvc-asmgcroot-2/ - copied from r69253, pypy/trunk/ Log: a branch to merge the changes made in branch/msvc-asmgcroot From fijal at codespeak.net Fri Nov 13 13:57:58 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 13 Nov 2009 13:57:58 +0100 (CET) Subject: [pypy-svn] r69256 - pypy/branch/faster-raise/pypy/translator/platform Message-ID: <20091113125758.8AF1F168026@codespeak.net> Author: fijal Date: Fri Nov 13 13:57:56 2009 New Revision: 69256 Modified: pypy/branch/faster-raise/pypy/translator/platform/__init__.py Log: (everybody) Encode error messages using utf-8 Modified: pypy/branch/faster-raise/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/platform/__init__.py (original) +++ pypy/branch/faster-raise/pypy/translator/platform/__init__.py Fri Nov 13 13:57:56 2009 @@ -108,7 +108,7 @@ def _handle_error(self, returncode, stderr, stdout, outname): if returncode != 0: errorfile = outname.new(ext='errors') - errorfile.write(stderr) + errorfile.write(stderr.encode("utf-8")) stderrlines = stderr.splitlines() for line in stderrlines[:50]: log.ERROR(line) From pedronis at codespeak.net Fri Nov 13 14:09:45 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 13 Nov 2009 14:09:45 +0100 (CET) Subject: [pypy-svn] r69257 - pypy/trunk/pypy/translator/platform Message-ID: <20091113130945.4F84D168027@codespeak.net> Author: pedronis Date: Fri Nov 13 14:09:44 2009 New Revision: 69257 Modified: pypy/trunk/pypy/translator/platform/__init__.py Log: fix -Ojit translation, obscure Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Fri Nov 13 14:09:44 2009 @@ -108,7 +108,7 @@ def _handle_error(self, returncode, stderr, stdout, outname): if returncode != 0: errorfile = outname.new(ext='errors') - errorfile.write(stderr) + errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines[:50]: log.ERROR(line) From afa at codespeak.net Fri Nov 13 14:49:14 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 14:49:14 +0100 (CET) Subject: [pypy-svn] r69259 - in pypy/branch/msvc-asmgcroot-2/pypy/translator: c c/gcc c/gcc/test c/gcc/test/msvc c/src c/test platform Message-ID: <20091113134914.CD05C168026@codespeak.net> Author: afa Date: Fri Nov 13 14:49:13 2009 New Revision: 69259 Added: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/autopath.py - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/autopath.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/instruction.py - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/instruction.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/ - copied from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track0.s - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track0.s pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track1.s - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track1.s pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track2.s - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track2.s pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track8.s - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track8.s pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track_switch0.s - copied unchanged from r69257, pypy/branch/msvc-asmgcroot/pypy/translator/c/gcc/test/msvc/track_switch0.s Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/genc.py pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/char.h pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/debug.h pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/mem.h pypy/branch/msvc-asmgcroot-2/pypy/translator/c/test/test_newgc.py pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/windows.py Log: Merge forward Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_asmgcroot.py Fri Nov 13 14:49:13 2009 @@ -6,18 +6,20 @@ from pypy.annotation.listdef import s_list_of_strings from pypy import conftest -def setup_module(module): - if sys.platform == 'win32': - if not ('mingw' in os.popen('gcc --version').read() and - 'GNU' in os.popen('make --version').read()): - py.test.skip("mingw32 and MSYS are required for asmgcc on Windows") - class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved # instructions: should_be_moving = False @classmethod + def make_config(cls): + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.gc = cls.gcpolicy + config.translation.gcrootfinder = "asmgcc" + return config + + @classmethod def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] @@ -29,12 +31,7 @@ else: print 'Result: "%s"' % (res,) return 0 - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) - config.translation.gc = cls.gcpolicy - config.translation.gcrootfinder = "asmgcc" - if sys.platform == 'win32': - config.translation.cc = 'mingw32' + config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() a.build_types(main, [s_list_of_strings]) @@ -51,7 +48,7 @@ def run(arg0, arg1): lines = [] - print >> sys.stderr, 'RUN: starting', exe_name + print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: @@ -162,6 +159,33 @@ assert res == 4900 +class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): + # for the individual tests see + # ====> ../../test/test_newgc.py + + @classmethod + def setup_class(cls): + if sys.platform != 'win32': + py.test.skip("mingw32 specific test") + if not ('mingw' in os.popen('gcc --version').read() and + 'GNU' in os.popen('make --version').read()): + py.test.skip("mingw32 and MSYS are required for this test") + + test_newgc.TestSemiSpaceGC.setup_class.im_func(cls) + + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.cc = 'mingw32' + return config + + + def test_callback_with_collect(self): + py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/test_trackgcroot.py Fri Nov 13 14:49:13 2009 @@ -4,11 +4,11 @@ from pypy.translator.c.gcc.trackgcroot import format_callshape from pypy.translator.c.gcc.trackgcroot import LOC_NOWHERE, LOC_REG from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED -from pypy.translator.c.gcc.trackgcroot import GcRootTracker -from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker +from pypy.translator.c.gcc.trackgcroot import ElfAssemblerParser +from pypy.translator.c.gcc.trackgcroot import DarwinAssemblerParser from pypy.translator.c.gcc.trackgcroot import compress_callshape from pypy.translator.c.gcc.trackgcroot import decompress_callshape -from pypy.translator.c.gcc.trackgcroot import OFFSET_LABELS +from pypy.translator.c.gcc.trackgcroot import PARSERS from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -63,7 +63,7 @@ \tMORE STUFF """ lines = source.splitlines(True) - parts = list(GcRootTracker().find_functions(iter(lines))) + parts = list(ElfAssemblerParser().find_functions(iter(lines))) assert len(parts) == 5 assert parts[0] == (False, lines[:2]) assert parts[1] == (True, lines[2:5]) @@ -96,7 +96,7 @@ \t.section stuff """ lines = source.splitlines(True) - parts = list(GcRootTracker(format='darwin').find_functions(iter(lines))) + parts = list(DarwinAssemblerParser().find_functions(iter(lines))) assert len(parts) == 7 assert parts[0] == (False, lines[:3]) assert parts[1] == (True, lines[3:7]) @@ -108,7 +108,7 @@ def test_computegcmaptable(): tests = [] - for format in ('elf', 'darwin'): + for format in ('elf', 'darwin', 'msvc'): for path in this_dir.join(format).listdir("track*.s"): n = path.purebasename[5:] try: @@ -120,15 +120,20 @@ for format, _, path in tests: yield check_computegcmaptable, format, path -r_globallabel = re.compile(r"([\w]+)=[.]+") -r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") + +r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") +r_gcroot_constant = re.compile(r";\tmov\t.+, .+_constant_always_one_") def check_computegcmaptable(format, path): + if format == 'msvc': + r_globallabel = re.compile(r"([\w]+)::") + else: + r_globallabel = re.compile(r"([\w]+)=[.]+") print - print path.basename + print path.dirpath().basename + '/' + path.basename lines = path.readlines() expectedlines = lines[:] - tracker = FunctionGcRootTracker(lines, format=format) + tracker = PARSERS[format].FunctionGcRootTracker(lines) table = tracker.computegcmaptable(verbose=sys.maxint) tabledict = {} seen = {} @@ -148,10 +153,22 @@ got = tabledict[label] assert format_callshape(got) == expected seen[label] = True - expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) - expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + if format == 'msvc': + expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s::\n' % (label,)) + else: + expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s=.+%d\n' % (label, + tracker.OFFSET_LABELS)) + if format == 'msvc' and r_gcroot_constant.match(line): + expectedlines[i] = ';' + expectedlines[i] + expectedlines[i+1] = (expectedlines[i+1] + .replace('\timul\t', '\tmov\t') + + '\t; GCROOT\n') prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % [key for key in tabledict if key not in seen]) + print lines + print expectedlines assert lines == expectedlines Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py Fri Nov 13 14:49:13 2009 @@ -1,302 +1,40 @@ #! /usr/bin/env python - +import autopath import re, sys, os, random -LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' -r_functionstart_elf = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") -r_functionend_elf = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") - -# darwin -r_textstart = re.compile(r"\t.text\s*$") -# see -# http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html -OTHERSECTIONS = ['section', 'zerofill', - 'const', 'static_const', 'cstring', - 'literal4', 'literal8', 'literal16', - 'constructor', 'desctructor', - 'symbol_stub', - 'data', 'static_data', - 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', - 'dyld', 'mod_init_func', 'mod_term_func', - 'const_data' - ] -r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") -r_functionstart_darwin = re.compile(r"_(\w+):\s*$") - -if sys.platform != 'darwin': - OFFSET_LABELS = 2**30 -else: - OFFSET_LABELS = 0 - -# inside functions -r_label = re.compile(LABEL+"[:]\s*$") -r_rel_label = re.compile(r"(\d+):\s*$") -r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") -r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) -r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") -r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") -OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' -r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") -r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") -r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") -r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") -LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" -LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" -r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") -r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") -r_localvarnofp = re.compile(LOCALVAR) -r_localvarfp = re.compile(LOCALVARFP) -r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") -r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") - - -class GcRootTracker(object): - - def __init__(self, verbose=0, shuffle=False, format='elf'): - self.verbose = verbose - self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py - self.format = format - self.clear() - - def clear(self): - self.gcmaptable = [] - self.seen_main = False - - def dump_raw_table(self, output): - print >> output, "seen_main = %d" % (self.seen_main,) - for entry in self.gcmaptable: - print >> output, entry - - def reload_raw_table(self, input): - firstline = input.readline() - assert firstline.startswith("seen_main = ") - self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) - for line in input: - entry = eval(line) - assert type(entry) is tuple - self.gcmaptable.append(entry) - - def dump(self, output): - assert self.seen_main - shapes = {} - shapelines = [] - shapeofs = 0 - def _globalname(name): - if self.format in ('darwin', 'mingw32'): - return '_' + name - return name - def _globl(name): - print >> output, "\t.globl %s" % _globalname(name) - def _label(name): - print >> output, "%s:" % _globalname(name) - def _variant(**kwargs): - txt = kwargs[self.format] - print >> output, "\t%s" % txt - - print >> output, "\t.text" - _globl('pypy_asm_stackwalk') - _variant(elf='.type pypy_asm_stackwalk, @function', - darwin='', - mingw32='') - _label('pypy_asm_stackwalk') - print >> output, """\ - /* See description in asmgcroot.py */ - movl 4(%esp), %edx /* my argument, which is the callback */ - movl %esp, %eax /* my frame top address */ - pushl %eax /* ASM_FRAMEDATA[6] */ - pushl %ebp /* ASM_FRAMEDATA[5] */ - pushl %edi /* ASM_FRAMEDATA[4] */ - pushl %esi /* ASM_FRAMEDATA[3] */ - pushl %ebx /* ASM_FRAMEDATA[2] */ - - /* Add this ASM_FRAMEDATA to the front of the circular linked */ - /* list. Let's call it 'self'. */ - movl __gcrootanchor+4, %eax /* next = gcrootanchor->next */ - pushl %eax /* self->next = next */ - pushl $__gcrootanchor /* self->prev = gcrootanchor */ - movl %esp, __gcrootanchor+4 /* gcrootanchor->next = self */ - movl %esp, (%eax) /* next->prev = self */ - - /* note: the Mac OS X 16 bytes aligment must be respected. */ - call *%edx /* invoke the callback */ - - /* Detach this ASM_FRAMEDATA from the circular linked list */ - popl %esi /* prev = self->prev */ - popl %edi /* next = self->next */ - movl %edi, 4(%esi) /* prev->next = next */ - movl %esi, (%edi) /* next->prev = prev */ - - popl %ebx /* restore from ASM_FRAMEDATA[2] */ - popl %esi /* restore from ASM_FRAMEDATA[3] */ - popl %edi /* restore from ASM_FRAMEDATA[4] */ - popl %ebp /* restore from ASM_FRAMEDATA[5] */ - popl %ecx /* ignored ASM_FRAMEDATA[6] */ - /* the return value is the one of the 'call' above, */ - /* because %eax (and possibly %edx) are unmodified */ - ret -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', - darwin='', - mingw32='') - print >> output, '\t.data' - print >> output, '\t.align\t4' - _globl('__gcrootanchor') - _label('__gcrootanchor') - print >> output, """\ - /* A circular doubly-linked list of all */ - /* the ASM_FRAMEDATAs currently alive */ - .long\t__gcrootanchor /* prev */ - .long\t__gcrootanchor /* next */ -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _globl('__gcmapstart') - _label('__gcmapstart') - for label, state, is_range in self.gcmaptable: - try: - n = shapes[state] - except KeyError: - n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] - shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( - shapeofs, - ', '.join(bytes))) - shapeofs += len(bytes) - if is_range: - n = ~ n - print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) - print >> output, '\t.long\t%d' % (n,) - _globl('__gcmapend') - _label('__gcmapend') - _variant(elf='.section\t.rodata', - darwin='.const', - mingw32='') - _globl('__gccallshapes') - _label('__gccallshapes') - output.writelines(shapelines) - - def find_functions(self, iterlines): - _find_functions = getattr(self, '_find_functions_' + self.format) - return _find_functions(iterlines) - - def _find_functions_elf(self, iterlines): - functionlines = [] - in_function = False - for line in iterlines: - if r_functionstart_elf.match(line): - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - in_function = True - functionlines = [] - functionlines.append(line) - if r_functionend_elf.match(line): - assert in_function, ( - "missed the start of the current function") - yield True, functionlines - in_function = False - functionlines = [] - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - - def _find_functions_darwin(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - assert not in_text, "unexpected repeated .text start: %d" % n - in_text = True - elif r_sectionstart.match(line): - if in_function: - yield in_function, functionlines - functionlines = [] - in_text = False - in_function = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - - if functionlines: - yield in_function, functionlines - - def _find_functions_mingw32(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - in_text = True - elif r_sectionstart.match(line): - in_text = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - if functionlines: - yield in_function, functionlines - - def process(self, iterlines, newfile, entrypoint='main', filename='?'): - if self.format in ('darwin', 'mingw32'): - entrypoint = '_' + entrypoint - for in_function, lines in self.find_functions(iterlines): - if in_function: - lines = self.process_function(lines, entrypoint, filename) - newfile.writelines(lines) - if self.verbose == 1: - sys.stderr.write('\n') - - def process_function(self, lines, entrypoint, filename): - tracker = FunctionGcRootTracker(lines, filetag=getidentifier(filename), - format=self.format) - is_main = tracker.funcname == entrypoint - tracker.is_stack_bottom = is_main - if self.verbose == 1: - sys.stderr.write('.') - elif self.verbose > 1: - print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, - tracker.funcname) - table = tracker.computegcmaptable(self.verbose) - if self.verbose > 1: - for label, state in table: - print >> sys.stderr, label, '\t', format_callshape(state) - table = compress_gcmaptable(table) - if self.shuffle and random.random() < 0.5: - self.gcmaptable[:0] = table - else: - self.gcmaptable.extend(table) - self.seen_main |= is_main - return tracker.lines - +from pypy.translator.c.gcc.instruction import Insn, Label, InsnCall, InsnRet +from pypy.translator.c.gcc.instruction import InsnFunctionStart, InsnStop +from pypy.translator.c.gcc.instruction import InsnSetLocal, InsnCopyLocal +from pypy.translator.c.gcc.instruction import InsnPrologue, InsnEpilogue +from pypy.translator.c.gcc.instruction import InsnGCROOT +from pypy.translator.c.gcc.instruction import InsnStackAdjust +from pypy.translator.c.gcc.instruction import InsnCannotFollowEsp +from pypy.translator.c.gcc.instruction import LocalVar, somenewvalue, frameloc +from pypy.translator.c.gcc.instruction import LOC_REG, LOC_NOWHERE, LOC_MASK +from pypy.translator.c.gcc.instruction import LOC_EBP_BASED, LOC_ESP_BASED class FunctionGcRootTracker(object): + skip = 0 - def __init__(self, lines, filetag=0, format='elf'): - if format == 'elf': - match = r_functionstart_elf.match(lines[0]) - funcname = match.group(1) - match = r_functionend_elf.match(lines[-1]) - assert funcname == match.group(1) - assert funcname == match.group(2) - elif format == 'darwin': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - elif format == 'mingw32': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - else: - assert False, "unknown format: %s" % format + @classmethod + def init_regexp(cls): + cls.r_label = re.compile(cls.LABEL+"[:]\s*$") + cls.r_globl = re.compile(r"\t[.]globl\t"+cls.LABEL+"\s*$") + cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) + + cls.r_insn = re.compile(r"\t([a-z]\w*)\s") + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") + + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") + cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") + def __init__(self, funcname, lines, filetag=0): self.funcname = funcname self.lines = lines self.uses_frame_pointer = False - self.r_localvar = r_localvarnofp + self.r_localvar = self.r_localvarnofp self.filetag = filetag # a "stack bottom" function is either main() or a callback from C code self.is_stack_bottom = False @@ -316,6 +54,9 @@ self.dump() return self.gettable() + def replace_symbols(self, operand): + return operand + def gettable(self): """Returns a list [(label_after_call, callshape_tuple)] See format_callshape() for more details about callshape_tuple. @@ -333,7 +74,7 @@ shape = [retaddr] # the first gcroots are always the ones corresponding to # the callee-saved registers - for reg in CALLEE_SAVE_REGISTERS: + for reg in self.CALLEE_SAVE_REGISTERS: shape.append(LOC_NOWHERE) gcroots = [] for localvar, tag in insn.gcroots.items(): @@ -341,17 +82,17 @@ loc = localvar.getlocation(insn.framesize, self.uses_frame_pointer) else: - assert localvar in REG2LOC, "%s: %s" % (self.funcname, - localvar) - loc = REG2LOC[localvar] + assert localvar in self.REG2LOC, "%s: %s" % (self.funcname, + localvar) + loc = self.REG2LOC[localvar] assert isinstance(loc, int) if tag is None: gcroots.append(loc) else: - regindex = CALLEE_SAVE_REGISTERS.index(tag) + regindex = self.CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc if LOC_NOWHERE in shape and not self.is_stack_bottom: - reg = CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] + reg = self.CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) gcroots.sort() @@ -362,13 +103,13 @@ def findlabels(self): self.labels = {} # {name: Label()} for lineno, line in enumerate(self.lines): - match = r_label.match(line) + match = self.r_label.match(line) label = None if match: label = match.group(1) else: # labels used by: j* NNNf - match = r_rel_label.match(line) + match = self.r_rel_label.match(line) if match: label = "rel %d" % lineno if label: @@ -388,30 +129,35 @@ lst.append(previnsn) def parse_instructions(self): - self.insns = [InsnFunctionStart()] + self.insns = [InsnFunctionStart(self.CALLEE_SAVE_REGISTERS)] ignore_insns = False for lineno, line in enumerate(self.lines): + if lineno < self.skip: + continue self.currentlineno = lineno insn = [] - match = r_insn.match(line) - if match: + match = self.r_insn.match(line) + + if self.r_bottom_marker.match(line): + self.is_stack_bottom = True + elif match: if not ignore_insns: opname = match.group(1) try: meth = getattr(self, 'visit_' + opname) except AttributeError: - meth = self.find_missing_visit_method(opname) + self.find_missing_visit_method(opname) + meth = getattr(self, 'visit_' + opname) + line = line.rsplit(';', 1)[0] insn = meth(line) - elif r_gcroot_marker.match(line): + elif self.r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) - elif r_bottom_marker.match(line): - self.is_stack_bottom = True elif line == '\t/* ignore_in_trackgcroot */\n': ignore_insns = True elif line == '\t/* end_ignore_in_trackgcroot */\n': ignore_insns = False else: - match = r_label.match(line) + match = self.r_label.match(line) if match: insn = self.labels[match.group(1)] @@ -421,18 +167,17 @@ else: self.append_instruction(insn) - del self.currentlineno + del self.currentlineno - def find_missing_visit_method(self, opname): + @classmethod + def find_missing_visit_method(cls, opname): # only for operations that are no-ops as far as we are concerned prefix = opname - while prefix not in self.IGNORE_OPS_WITH_PREFIXES: + while prefix not in cls.IGNORE_OPS_WITH_PREFIXES: prefix = prefix[:-1] if not prefix: raise UnrecognizedOperation(opname) - visit_nop = FunctionGcRootTracker.__dict__['visit_nop'] - setattr(FunctionGcRootTracker, 'visit_' + opname, visit_nop) - return self.visit_nop + setattr(cls, 'visit_' + opname, cls.visit_nop) def list_call_insns(self): return [insn for insn in self.insns if isinstance(insn, InsnCall)] @@ -461,8 +206,8 @@ for insn1, delta1 in deltas.items(): if hasattr(insn1, 'framesize'): size_at_insn.append(insn1.framesize + delta1) - assert len(size_at_insn) > 0, ( - "cannot reach the start of the function??") + if not size_at_insn: + continue size_at_insn = size_at_insn[0] for insn1, delta1 in deltas.items(): size_at_insn1 = size_at_insn - delta1 @@ -480,20 +225,24 @@ elif isinstance(localvar, (list, tuple)): return [fixvar(var) for var in localvar] - match = r_localvar_esp.match(localvar) + match = self.r_localvar_esp.match(localvar) if match: - if localvar == '0(%esp)': # for pushl and popl, by - hint = None # default ebp addressing is - else: # a bit nicer + if localvar == self.TOP_OF_STACK: # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer hint = 'esp' ofs_from_esp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_esp += int(match.group(2) or '0') localvar = ofs_from_esp - insn.framesize assert localvar != 0 # that's the return address return LocalVar(localvar, hint=hint) elif self.uses_frame_pointer: - match = r_localvar_ebp.match(localvar) + match = self.r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_ebp += int(match.group(2) or '0') localvar = ofs_from_ebp - 4 assert localvar != 0 # that's the return address return LocalVar(localvar, hint='ebp') @@ -553,10 +302,10 @@ # script); otherwise invent a name and add the label to tracker.lines. label = None # this checks for a ".globl NAME" followed by "NAME:" - match = r_globl.match(self.lines[call.lineno+1]) + match = self.r_globl.match(self.lines[call.lineno+1]) if match: label1 = match.group(1) - match = r_globllabel.match(self.lines[call.lineno+2]) + match = self.r_globllabel.match(self.lines[call.lineno+2]) if match: label2 = match.group(1) if label1 == label2: @@ -569,22 +318,26 @@ break k += 1 self.labels[label] = None - # These global symbols are not directly labels pointing to the - # code location because such global labels in the middle of - # functions confuse gdb. Instead, we add to the global symbol's - # value a big constant, which is subtracted again when we need - # the original value for gcmaptable.s. That's a hack. - self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, - OFFSET_LABELS)) - self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) + if self.format == 'msvc': + self.lines.insert(call.lineno+1, '%s::\n' % (label,)) + self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) + else: + # These global symbols are not directly labels pointing to the + # code location because such global labels in the middle of + # functions confuse gdb. Instead, we add to the global symbol's + # value a big constant, which is subtracted again when we need + # the original value for gcmaptable.s. That's a hack. + self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, + self.OFFSET_LABELS)) + self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label # ____________________________________________________________ def _visit_gcroot_marker(self, line): - match = r_gcroot_marker.match(line) + match = self.r_gcroot_marker.match(line) loc = match.group(1) - return InsnGCROOT(loc) + return InsnGCROOT(self.replace_symbols(loc)) def visit_nop(self, line): return [] @@ -615,15 +368,15 @@ visit_xorw = visit_nop def visit_addl(self, line, sign=+1): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if target == '%esp': - count = match.group(1) - if not count.startswith('$'): + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if target == self.ESP: + count = self.extract_immediate(source) + if count is None: # strange instruction - I've seen 'subl %eax, %esp' return InsnCannotFollowEsp() - return InsnStackAdjust(sign * int(count[1:])) + return InsnStackAdjust(sign * count) elif self.r_localvar.match(target): return InsnSetLocal(target, [source, target]) else: @@ -633,7 +386,7 @@ return self.visit_addl(line, sign=-1) def unary_insn(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) if self.r_localvar.match(target): return InsnSetLocal(target) @@ -641,14 +394,14 @@ return [] def binary_insn(self, line): - match = r_binaryinsn.match(line) + match = self.r_binaryinsn.match(line) if not match: raise UnrecognizedOperation(line) - source = match.group(1) - target = match.group(2) + source = match.group("source") + target = match.group("target") if self.r_localvar.match(target): return InsnSetLocal(target, [source]) - elif target == '%esp': + elif target == self.ESP: raise UnrecognizedOperation(line) else: return [] @@ -673,9 +426,9 @@ visit_cmovno = binary_insn def visit_andl(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for andl $-16, %esp used to align the stack in main(). # The exact amount of adjutment is not known yet, so we use # an odd-valued estimate to make sure the real value is not used @@ -685,12 +438,12 @@ return self.binary_insn(line) def visit_leal(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for leal -12(%ebp), %esp in function epilogues - source = match.group(1) - match = r_localvar_ebp.match(source) + source = match.group("source") + match = self.r_localvar_ebp.match(source) if match: if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') @@ -698,7 +451,7 @@ assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp else: - match = r_localvar_esp.match(source) + match = self.r_localvar_esp.match(source) # leal 12(%esp), %esp if match: return InsnStackAdjust(int(match.group(1))) @@ -709,7 +462,9 @@ return self.binary_insn(line) def insns_for_copy(self, source, target): - if source == '%esp' or target == '%esp': + source = self.replace_symbols(source) + target = self.replace_symbols(target) + if source == self.ESP or target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) elif self.r_localvar.match(target): if self.r_localvar.match(source): @@ -720,14 +475,14 @@ return [] def visit_movl(self, line): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if source == '%esp' and target == '%ebp': + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if source == self.ESP and target == self.EBP: return self._visit_prologue() - elif source == '%ebp' and target == '%esp': + elif source == self.EBP and target == self.ESP: return self._visit_epilogue() - if source == '%esp' and self.funcname.startswith('VALGRIND_'): + if source == self.ESP and self.funcname.startswith('VALGRIND_'): return [] # in VALGRIND_XXX functions, there is a dummy-looking # mov %esp, %eax. Shows up only when compiling with # gcc -fno-unit-at-a-time. @@ -736,25 +491,25 @@ visit_mov = visit_movl def visit_pushl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') + return [InsnStackAdjust(-4)] + self.insns_for_copy(source, self.TOP_OF_STACK) def visit_pushw(self, line): return [InsnStackAdjust(-2)] # rare but not impossible def _visit_pop(self, target): - return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] + return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+4)] def visit_popl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) return self._visit_pop(target) def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer self.uses_frame_pointer = True - self.r_localvar = r_localvarfp + self.r_localvar = self.r_localvarfp return [InsnPrologue()] def _visit_epilogue(self): @@ -763,38 +518,38 @@ return [InsnEpilogue(4)] def visit_leave(self, line): - return self._visit_epilogue() + self._visit_pop('%ebp') + return self._visit_epilogue() + self._visit_pop(self.EBP) def visit_ret(self, line): - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) def visit_jmp(self, line): tablelabels = [] - match = r_jmp_switch.match(line) + match = self.r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. tablelabels.append(match.group(1)) - elif r_unaryinsn_star.match(line): + elif self.r_unaryinsn_star.match(line): # maybe a jmp similar to the above, but stored in a # registry: # movl L9341(%eax), %eax # jmp *%eax - operand = r_unaryinsn_star.match(line).group(1)[1:] + operand = self.r_unaryinsn_star.match(line).group(1) def walker(insn, locs): sources = [] for loc in locs: for s in insn.all_sources_of(loc): # if the source looks like 8(%eax,%edx,4) # %eax is the real source, %edx is an offset. - match = r_jmp_source.match(s) - if match and not r_localvar_esp.match(s): + match = self.r_jmp_source.match(s) + if match and not self.r_localvar_esp.match(s): sources.append(match.group(1)) else: sources.append(s) for source in sources: - label_match = re.compile(LABEL).match(source) + label_match = re.compile(self.LABEL).match(source) if label_match: tablelabels.append(label_match.group(0)) return @@ -809,37 +564,43 @@ assert len(tablelabels) <= 1 if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 - while not r_jmptable_end.match(self.lines[tablelin]): - match = r_jmptable_item.match(self.lines[tablelin]) + while not self.r_jmptable_end.match(self.lines[tablelin]): + # skip empty lines + if (not self.lines[tablelin].strip() + or self.lines[tablelin].startswith(';')): + tablelin += 1 + continue + match = self.r_jmptable_item.match(self.lines[tablelin]) if not match: - raise NoPatternMatch(self.lines[tablelin]) + raise NoPatternMatch(repr(self.lines[tablelin])) label = match.group(1) if label != '0': self.register_jump_to(label) tablelin += 1 return InsnStop() - if r_unaryinsn_star.match(line): + if self.r_unaryinsn_star.match(line): # that looks like an indirect tail-call. # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) try: self.conditional_jump(line) except KeyError: # label not found: check if it's a tail-call turned into a jump - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) assert not target.startswith('.') # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() def register_jump_to(self, label): - self.labels[label].previous_insns.append(self.insns[-1]) + if not isinstance(self.insns[-1], InsnStop): + self.labels[label].previous_insns.append(self.insns[-1]) def conditional_jump(self, line): - match = r_jump.match(line) + match = self.r_jump.match(line) if not match: - match = r_jump_rel_label.match(line) + match = self.r_jump_rel_label.match(line) if not match: raise UnrecognizedOperation(line) # j* NNNf @@ -878,260 +639,116 @@ def visit_xchgl(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS # which is to use a marker no-op "xchgl %ebx, %ebx" - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if source == target: return [] raise UnrecognizedOperation(line) def visit_call(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) + if match is None: - assert r_unaryinsn_star.match(line) # indirect call - else: - target = match.group(1) - if target in FUNCTIONS_NOT_RETURNING: - return InsnStop() - if sys.platform == 'win32' and target == '__alloca': - # in functions with large stack requirements, windows - # needs a call to _alloca(), to turn reserved pages - # into committed memory. - # With mingw32 gcc at least, %esp is not used before - # this call. So we don't bother to compute the exact - # stack effect. - return [InsnCannotFollowEsp()] - if target in self.labels: - lineoffset = self.labels[target].lineno - self.currentlineno - if lineoffset >= 0: - assert lineoffset in (1,2) - return [InsnStackAdjust(-4)] + assert self.r_unaryinsn_star.match(line) # indirect call + return [InsnCall(self.currentlineno), + InsnSetLocal(self.EAX)] # the result is there + + target = match.group(1) + + if self.format in ('msvc'): + # On win32, the address of a foreign function must be + # computed, the optimizer may store it in a register. We + # could ignore this, except when the function need special + # processing (not returning, __stdcall...) + def find_register(target): + reg = [] + def walker(insn, locs): + sources = [] + for loc in locs: + for s in insn.all_sources_of(loc): + sources.append(s) + for source in sources: + m = re.match("DWORD PTR " + self.LABEL, source) + if m: + reg.append(m.group(1)) + if reg: + return + yield tuple(sources) + insn = InsnStop() + insn.previous_insns = [self.insns[-1]] + self.walk_instructions_backwards(walker, insn, (target,)) + return reg + + if match and self.r_localvarfp.match(target): + sources = find_register(target) + if sources: + target, = sources + + if target in self.FUNCTIONS_NOT_RETURNING: + return [InsnStop(), InsnCannotFollowEsp()] + if self.format == 'mingw32' and target == '__alloca': + # in functions with large stack requirements, windows + # needs a call to _alloca(), to turn reserved pages + # into committed memory. + # With mingw32 gcc at least, %esp is not used before + # this call. So we don't bother to compute the exact + # stack effect. + return [InsnCannotFollowEsp()] + if target in self.labels: + lineoffset = self.labels[target].lineno - self.currentlineno + if lineoffset >= 0: + assert lineoffset in (1,2) + return [InsnStackAdjust(-4)] + insns = [InsnCall(self.currentlineno), - InsnSetLocal('%eax')] # the result is there - if sys.platform == 'win32': + InsnSetLocal(self.EAX)] # the result is there + if self.format in ('mingw32', 'msvc'): # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size - if match and '@' in target: - insns.append(InsnStackAdjust(int(target.split('@')[1]))) + if '@' in target and not target.startswith('@'): + insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) + # Some (intrinsic?) functions use the "fastcall" calling convention + # XXX without any declaration, how can we guess the stack effect? + if target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: + insns.append(InsnStackAdjust(16)) return insns -class UnrecognizedOperation(Exception): - pass - -class NoPatternMatch(Exception): - pass - -class SomeNewValue(object): - pass -somenewvalue = SomeNewValue() +class ElfFunctionGcRootTracker(FunctionGcRootTracker): + format = 'elf' -class LocalVar(object): - # A local variable location at position 'ofs_from_frame_end', - # which is counted from the end of the stack frame (so it is always - # negative, unless it refers to arguments of the current function). - def __init__(self, ofs_from_frame_end, hint=None): - self.ofs_from_frame_end = ofs_from_frame_end - self.hint = hint - - def __repr__(self): - return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') - - def __hash__(self): - return hash(self.ofs_from_frame_end) - - def __cmp__(self, other): - if isinstance(other, LocalVar): - return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) - else: - return 1 - - def getlocation(self, framesize, uses_frame_pointer): - if (self.hint == 'esp' or not uses_frame_pointer - or self.ofs_from_frame_end % 2 != 0): - # try to use esp-relative addressing - ofs_from_esp = framesize + self.ofs_from_frame_end - if ofs_from_esp % 2 == 0: - return frameloc(LOC_ESP_BASED, ofs_from_esp) - # we can get an odd value if the framesize is marked as bogus - # by visit_andl() - assert uses_frame_pointer - ofs_from_ebp = self.ofs_from_frame_end + 4 - return frameloc(LOC_EBP_BASED, ofs_from_ebp) - - -class Insn(object): - _args_ = [] - _locals_ = [] - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join([str(getattr(self, name)) - for name in self._args_])) - def requestgcroots(self, tracker): - return {} - - def source_of(self, localvar, tag): - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class Label(Insn): - _args_ = ['label', 'lineno'] - def __init__(self, label, lineno): - self.label = label - self.lineno = lineno - self.previous_insns = [] # all insns that jump (or fallthrough) here - -class InsnFunctionStart(Insn): - framesize = 0 - previous_insns = () - def __init__(self): - self.arguments = {} - for reg in CALLEE_SAVE_REGISTERS: - self.arguments[reg] = somenewvalue - - def source_of(self, localvar, tag): - if localvar not in self.arguments: - if localvar in ('%eax', '%edx', '%ecx'): - # xxx this might show a bug in trackgcroot.py failing to - # figure out which instruction stored a value in these - # registers. However, this case also occurs when the - # the function's calling convention was optimized by gcc: - # the 3 registers above are then used to pass arguments - pass - else: - assert (isinstance(localvar, LocalVar) and - localvar.ofs_from_frame_end > 0), ( - "must come from an argument to the function, got %r" % - (localvar,)) - self.arguments[localvar] = somenewvalue - return self.arguments[localvar] - - def all_sources_of(self, localvar): - return [] - -class InsnSetLocal(Insn): - _args_ = ['target', 'sources'] - _locals_ = ['target', 'sources'] - - def __init__(self, target, sources=()): - self.target = target - self.sources = sources - - def source_of(self, localvar, tag): - if localvar == self.target: - return somenewvalue - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return self.sources - return [localvar] - -class InsnCopyLocal(Insn): - _args_ = ['source', 'target'] - _locals_ = ['source', 'target'] - - def __init__(self, source, target): - self.source = source - self.target = target - - def source_of(self, localvar, tag): - if localvar == self.target: - return self.source - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return [self.source] - return [localvar] - -class InsnStackAdjust(Insn): - _args_ = ['delta'] - def __init__(self, delta): - assert delta % 2 == 0 # should be "% 4", but there is the special - self.delta = delta # case of 'pushw' to handle - -class InsnCannotFollowEsp(InsnStackAdjust): - def __init__(self): - self.delta = -7 # use an odd value as marker - -class InsnStop(Insn): - pass + ESP = '%esp' + EBP = '%ebp' + EAX = '%eax' + CALLEE_SAVE_REGISTERS = ['%ebx', '%esi', '%edi', '%ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 2**30 + TOP_OF_STACK = '0(%esp)' + + r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") + r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") + LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" + LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") + r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") + + r_rel_label = re.compile(r"(\d+):\s*$") + r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -class InsnRet(InsnStop): - framesize = 0 - def requestgcroots(self, tracker): - # no need to track the value of these registers in the caller - # function if we are the main(), or if we are flagged as a - # "bottom" function (a callback from C code) - if tracker.is_stack_bottom: - return {} - else: - return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) - -class InsnCall(Insn): - _args_ = ['lineno', 'gcroots'] - def __init__(self, lineno): - # 'gcroots' is a dict built by side-effect during the call to - # FunctionGcRootTracker.trackgcroots(). Its meaning is as - # follows: the keys are the locations that contain gc roots - # (register names or LocalVar instances). The value - # corresponding to a key is the "tag", which is None for a - # normal gc root, or else the name of a callee-saved register. - # In the latter case it means that this is only a gc root if the - # corresponding register in the caller was really containing a - # gc pointer. A typical example: - # - # InsnCall({LocalVar(-8)': None, - # '%esi': '%esi', - # LocalVar(-12)': '%ebx'}) - # - # means that the value at -8 from the frame end is a gc root - # across this call; that %esi is a gc root if it was in the - # caller (typically because %esi is not modified at all in the - # current function); and that the value at -12 from the frame - # end is a gc root if %ebx was a gc root in the caller - # (typically because the current function saves and restores - # %ebx from there in the prologue and epilogue). - self.gcroots = {} - self.lineno = lineno - - def source_of(self, localvar, tag): - tag1 = self.gcroots.setdefault(localvar, tag) - assert tag1 == tag, ( - "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( - localvar, tag1, tag)) - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class InsnGCROOT(Insn): - _args_ = ['loc'] - _locals_ = ['loc'] - def __init__(self, loc): - self.loc = loc - def requestgcroots(self, tracker): - return {self.loc: None} - -class InsnPrologue(Insn): - def __setattr__(self, attr, value): - if attr == 'framesize': - assert value == 4, ("unrecognized function prologue - " - "only supports push %ebp; movl %esp, %ebp") - Insn.__setattr__(self, attr, value) - -class InsnEpilogue(Insn): - def __init__(self, framesize=None): - if framesize is not None: - self.framesize = framesize + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") - -if sys.platform != 'win32': FUNCTIONS_NOT_RETURNING = { 'abort': None, '_exit': None, @@ -1139,31 +756,684 @@ '___assert_rtn': None, 'L___assert_rtn$stub': None } -else: + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = match.group(1) + match = self.r_functionend.match(lines[-1]) + assert funcname == match.group(1) + assert funcname == match.group(2) + super(ElfFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def extract_immediate(self, value): + if not value.startswith('$'): + return None + return int(value[1:]) + +ElfFunctionGcRootTracker.init_regexp() + +class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): + format = 'darwin' + + r_functionstart = re.compile(r"_(\w+):\s*$") + OFFSET_LABELS = 0 + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = '_' + match.group(1) + FunctionGcRootTracker.__init__(self, funcname, lines, filetag) + +class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): + format = 'mingw32' + +class MsvcFunctionGcRootTracker(FunctionGcRootTracker): + format = 'msvc' + ESP = 'esp' + EBP = 'ebp' + EAX = 'eax' + CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + TOP_OF_STACK = 'DWORD PTR [esp]' + + OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' + LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 0 + + r_functionstart = re.compile(r"; Function compile flags: ") + r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") + r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") + r_symboldefine = re.compile(r"([_A-Za-z0-9$]+) = ([-0-9]+)\s*;.+\n") + + LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]?\d*\]" + LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") + r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") + + r_rel_label = re.compile(r"$1") # never matches + r_jump_rel_label = re.compile(r"$1") # never matches + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"[^\t\n;]") + + r_gcroot_marker = re.compile(r"$1") # never matches + r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot") + r_bottom_marker = re.compile(r"; .+\tpypy_asm_stack_bottom\(\);") + FUNCTIONS_NOT_RETURNING = { '_abort': None, '__exit': None, '__assert': None, '__wassert': None, + '__imp__abort': None, + '__imp___wassert': None, + 'DWORD PTR __imp__abort': None, + 'DWORD PTR __imp___wassert': None, } -CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] -CALLEE_SAVE_REGISTERS = CALLEE_SAVE_REGISTERS_NOEBP + ['%ebp'] + @classmethod + def init_regexp(cls): + super(MsvcFunctionGcRootTracker, cls).init_regexp() + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+r"),\s*(?P"+cls.OPERAND+r")\s*(?:;.+)?$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+cls.LABEL+"\s*$") + + def __init__(self, lines, filetag=0): + self.defines = {} + for i, line in enumerate(lines): + if self.r_symboldefine.match(line): + match = self.r_symboldefine.match(line) + name = match.group(1) + value = int(match.group(2)) + self.defines[name] = value + continue + + match = self.r_codestart.match(line) + if match: + self.skip = i + break + + funcname = match.group(1) + super(MsvcFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def replace_symbols(self, operand): + for name, value in self.defines.items(): + operand = operand.replace(name, str(value)) + return operand + + for name in ''' + push pop mov lea + xor sub add + '''.split(): + locals()['visit_' + name] = getattr(FunctionGcRootTracker, + 'visit_' + name + 'l') + + visit_int = FunctionGcRootTracker.visit_nop + visit_npad = FunctionGcRootTracker.visit_nop + # probably not GC pointers + visit_cdq = FunctionGcRootTracker.visit_nop + + def extract_immediate(self, value): + try: + return int(value) + except ValueError: + return None + + def _visit_gcroot_marker(self, line=None): + # two possible patterns: + # 1. mov reg, DWORD PTR _always_one_ + # imul target, reg + # 2. mov reg, DWORD PTR _always_one_ + # imul reg, target + assert self.lines[self.currentlineno].startswith("\tmov\t") + mov = self.r_binaryinsn.match(self.lines[self.currentlineno]) + assert re.match("DWORD PTR .+_always_one_", mov.group("source")) + reg = mov.group("target") + + self.lines[self.currentlineno] = ";" + self.lines[self.currentlineno] + + # the 'imul' must appear in the same block; the 'reg' must not + # appear in the instructions between + imul = None + lineno = self.currentlineno + 1 + stop = False + while not stop: + line = self.lines[lineno] + if line == '\n': + stop = True + elif line.startswith("\tjmp\t"): + stop = True + elif self.r_gcroot_marker_var.search(line): + stop = True + elif line.startswith("\tmov\t%s," % (reg,)): + # mov reg, + stop = True + elif line.startswith("\txor\t%s, %s" % (reg, reg)): + # xor reg, reg + stop = True + elif line.startswith("\timul\t"): + imul = self.r_binaryinsn.match(line) + imul_arg1 = imul.group("target") + imul_arg2 = imul.group("source") + break + # the register may not appear in other instructions + elif reg in line: + assert False, (line, lineno) + + lineno += 1 + else: + # No imul, the returned value is not used in this function + return [] + + if reg == imul_arg2: + self.lines[lineno] = ";" + self.lines[lineno] + return InsnGCROOT(self.replace_symbols(imul_arg1)) + else: + assert reg == imul_arg1 + self.lines[lineno] = "\tmov\t%s, %s\n" % (imul_arg1, imul_arg2) + if imul_arg2.startswith('OFFSET '): + # ignore static global variables + pass + else: + self.lines[lineno] += "\t; GCROOT\n" + + return [] + + def insns_for_copy(self, source, target): + if self.r_gcroot_marker_var.match(source): + return self._visit_gcroot_marker() + if self.lines[self.currentlineno].endswith("\t; GCROOT\n"): + insns = [InsnGCROOT(self.replace_symbols(source))] + else: + insns = [] + return insns + super(MsvcFunctionGcRootTracker, self).insns_for_copy(source, target) + + +MsvcFunctionGcRootTracker.init_regexp() + +class AssemblerParser(object): + def __init__(self, verbose=0, shuffle=False): + self.verbose = verbose + self.shuffle = shuffle + self.gcmaptable = [] + self.seen_main = False + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + for in_function, lines in self.find_functions(iterlines): + if in_function: + lines = self.process_function(lines, entrypoint, filename) + self.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + + def write_newfile(self, newfile, lines, grist): + newfile.writelines(lines) + + def process_function(self, lines, entrypoint, filename): + tracker = self.FunctionGcRootTracker( + lines, filetag=getidentifier(filename)) + is_main = tracker.funcname == entrypoint + tracker.is_stack_bottom = is_main + if self.verbose == 1: + sys.stderr.write('.') + elif self.verbose > 1: + print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, + tracker.funcname) + table = tracker.computegcmaptable(self.verbose) + if self.verbose > 1: + for label, state in table: + print >> sys.stderr, label, '\t', format_callshape(state) + table = compress_gcmaptable(table) + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = table + else: + self.gcmaptable.extend(table) + self.seen_main |= is_main + return tracker.lines + +class ElfAssemblerParser(AssemblerParser): + format = "elf" + FunctionGcRootTracker = ElfFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + +class DarwinAssemblerParser(AssemblerParser): + format = "darwin" + FunctionGcRootTracker = DarwinFunctionGcRootTracker + + r_textstart = re.compile(r"\t.text\s*$") + + # see + # http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html + OTHERSECTIONS = ['section', 'zerofill', + 'const', 'static_const', 'cstring', + 'literal4', 'literal8', 'literal16', + 'constructor', 'desctructor', + 'symbol_stub', + 'data', 'static_data', + 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', + 'dyld', 'mod_init_func', 'mod_term_func', + 'const_data' + ] + r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + assert not in_text, "unexpected repeated .text start: %d" % n + in_text = True + elif cls.r_sectionstart.match(line): + if in_function: + yield in_function, functionlines + functionlines = [] + in_text = False + in_function = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + + if functionlines: + yield in_function, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(DarwinAssemblerParser, self).process_function( + lines, entrypoint, filename) + +class Mingw32AssemblerParser(DarwinAssemblerParser): + format = "mingw32" + FunctionGcRootTracker = Mingw32FunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + in_text = True + elif cls.r_sectionstart.match(line): + in_text = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + if functionlines: + yield in_function, functionlines + +class MsvcAssemblerParser(AssemblerParser): + format = "msvc" + FunctionGcRootTracker = MsvcFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(MsvcAssemblerParser, self).process_function( + lines, entrypoint, filename) + + def write_newfile(self, newfile, lines, grist): + newlines = [] + for line in lines: + # truncate long comments + if line.startswith(";"): + line = line[:-1][:500] + '\n' + + # Workaround a bug in the .s files generated by msvc + # compiler: every string or float constant is exported + # with a name built after its value, and will conflict + # with other modules. + if line.startswith("PUBLIC\t__real@"): + line = '; ' + line + elif line.startswith("PUBLIC\t??_C@"): + line = '; ' + line + elif line == "PUBLIC\t__$ArrayPad$\n": + line = '; ' + line + + # Because we insert labels in the code, some "SHORT" jumps + # are now longer than 127 bytes. We turn them all into + # "NEAR" jumps. Note that the assembler allocates space + # for a near jump, but still generates a short jump when + # it can. + line = line.replace('\tjmp\tSHORT ', '\tjmp\t') + line = line.replace('\tjne\tSHORT ', '\tjne\t') + line = line.replace('\tje\tSHORT ', '\tje\t') + + newlines.append(line) + + if line == "\t.model\tflat\n": + newlines.append("\tassume fs:nothing\n") + + newfile.writelines(newlines) + +PARSERS = { + 'elf': ElfAssemblerParser, + 'darwin': DarwinAssemblerParser, + 'mingw32': Mingw32AssemblerParser, + 'msvc': MsvcAssemblerParser, + } + +class GcRootTracker(object): + + def __init__(self, verbose=0, shuffle=False, format='elf'): + self.verbose = verbose + self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py + self.format = format + self.gcmaptable = [] + self.seen_main = False + + def dump_raw_table(self, output): + print >> output, "seen_main = %d" % (self.seen_main,) + for entry in self.gcmaptable: + print >> output, entry + + def reload_raw_table(self, input): + firstline = input.readline() + assert firstline.startswith("seen_main = ") + self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) + for line in input: + entry = eval(line) + assert type(entry) is tuple + self.gcmaptable.append(entry) + + def dump(self, output): + assert self.seen_main + shapes = {} + shapelines = [] + shapeofs = 0 + def _globalname(name, disp=""): + if self.format in ('darwin', 'mingw32', 'msvc'): + name = '_' + name + + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s + %s" % (name, disp) + else: + return name + + def _globl(name): + if self.format == 'msvc': + print >> output, "PUBLIC %s" % _globalname(name) + else: + print >> output, "\t.globl %s" % _globalname(name) + def _label(name): + print >> output, "%s:" % _globalname(name) + def _variant(**kwargs): + txt = kwargs[self.format] + print >> output, "\t%s" % txt + + def _comment(comment): + if self.format == 'msvc': + print >> output, "; %s" % comment + else: + print >> output, "/* %s */" % comment + + def _movl(source, target, comment): + if self.format == 'msvc': + print >> output, "\tmov\t%s, %s\t\t; %s" % (target, source, comment) + else: + print >> output, "\tmovl\t%s, %s\t\t/* %s */ " % (source, target, comment) + + def _pushl(source, comment): + if self.format == 'msvc': + print >> output, "\tpush\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpushl\t%s\t\t/* %s */ " % (source, comment) + + def _popl(source, comment): + if self.format == 'msvc': + print >> output, "\tpop\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpopl\t%s\t\t/* %s */ " % (source, comment) + + + def _register(name, disp=""): + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s(%%%s)" % (disp, name) + else: + return '%' + name + + def _offset(name): + if self.format == 'msvc': + return "OFFSET %s" % _globalname(name) + else: + return "$%s" % _globalname(name) + + def _call(arg, comment): + if self.format == 'msvc': + print >> output, "\tcall\t%s\t\t;%s" % (arg, comment) + else: + print >> output, "\tcall\t%s\t\t/* %s */" % (arg, comment) + + def _indirectjmp(arg): + if self.format == 'msvc': + return "DWORD PTR " + arg + else: + return "*%" + arg + + if self.format == 'msvc': + print >> output, """\ + TITLE gcmaptable.s + .686P + .XMM + .model flat + """ + + _variant(elf='\t.text', + darwin='\t.text', + mingw32='\t.text', + msvc='_TEXT\tSEGMENT') + + _globl('pypy_asm_stackwalk') + _variant(elf='.type pypy_asm_stackwalk, @function', + darwin='', + mingw32='', + msvc='') + _label('pypy_asm_stackwalk') + _comment("See description in asmgcroot.py") + _movl(_register("esp", disp="4"), _register("edx"), "my argument, which is the callback") + _movl(_register("esp"), _register("eax"), "my frame top address") + _pushl(_register("eax"), "ASM_FRAMEDATA[6]") + _pushl(_register("ebp"), "ASM_FRAMEDATA[5]") + _pushl(_register("edi"), "ASM_FRAMEDATA[4]") + _pushl(_register("esi"), "ASM_FRAMEDATA[3]") + _pushl(_register("ebx"), "ASM_FRAMEDATA[2]") + + print >> output + _comment("Add this ASM_FRAMEDATA to the front of the circular linked") + _comment("list. Let's call it 'self'.") + print >> output + _movl(_globalname("__gcrootanchor", disp=4), _register("eax"), "next = gcrootanchor->next") + _pushl(_register("eax"), "self->next = next") + _pushl(_offset("__gcrootanchor"), "self->prev = gcrootanchor") + _movl(_register("esp"), _globalname("__gcrootanchor", disp=4), "gcrootanchor->next = self") + _movl(_register("esp"), _register("eax", "0"), "next->prev = self") + print >> output + + _comment("note: the Mac OS X 16 bytes aligment must be respected.") + _call(_indirectjmp("edx"), "invoke the callback") + print >> output + + _comment("Detach this ASM_FRAMEDATA from the circular linked list") + _popl(_register("esi"), "prev = self->prev") + _popl(_register("edi"), "next = self->next") + _movl(_register("edi"), _register("esi", disp="4"), "prev->next = next") + _movl(_register("esi"), _register("edi", disp="0"), "next->prev = prev") + print >> output + + _popl(_register("ebx"), "restore from ASM_FRAMEDATA[2]") + _popl(_register("esi"), "restore from ASM_FRAMEDATA[3]") + _popl(_register("edi"), "restore from ASM_FRAMEDATA[4]") + _popl(_register("ebp"), "restore from ASM_FRAMEDATA[5]") + _popl(_register("ecx"), "ignored ASM_FRAMEDATA[6]") + _comment("the return value is the one of the 'call' above,") + _comment("because %eax (and possibly %edx) are unmodified") + + print >> output, "\tret" + + _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', + darwin='', + mingw32='', + msvc='') + + if self.format == 'msvc': + for label, state, is_range in self.gcmaptable: + print >> output, "EXTERN %s:NEAR" % label + + if self.format == 'msvc': + print >> output, '_DATA SEGMENT' + + _comment("A circular doubly-linked list of all") + _comment("the ASM_FRAMEDATAs currently alive") + if self.format == 'msvc': + _globl('__gcrootanchor') + print >> output, '%s\tDD FLAT:___gcrootanchor ; prev' % _globalname("__gcrootanchor") + print >> output, '\tDD FLAT:___gcrootanchor ; next' + else: + print >> output, '\t.data' + print >> output, '\t.align\t4' + _globl('__gcrootanchor') + _label('__gcrootanchor') + print >> output, """\ + .long\t__gcrootanchor /* prev */ + .long\t__gcrootanchor /* next */ +""".replace("__gcrootanchor", _globalname("__gcrootanchor")) + + _globl('__gcmapstart') + if self.format == 'msvc': + print >> output, '%s' % _globalname('__gcmapstart'), + else: + _label('__gcmapstart') + for label, state, is_range in self.gcmaptable: + try: + n = shapes[state] + except KeyError: + n = shapes[state] = shapeofs + bytes = [str(b) for b in compress_callshape(state)] + if self.format == 'msvc': + shapelines.append('\tDB\t%s\t;%s\n' % ( + ', '.join(bytes), + shapeofs)) + else: + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) + shapeofs += len(bytes) + if is_range: + n = ~ n + if self.format == 'msvc': + print >> output, '\tDD\t%s' % (label,) + print >> output, '\tDD\t%d' % (n,) + else: + print >> output, '\t.long\t%s-%d' % ( + label, + PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) + print >> output, '\t.long\t%d' % (n,) + + _globl('__gcmapend') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gcmapend') + else: + _label('__gcmapend') + _variant(elf='.section\t.rodata', + darwin='.const', + mingw32='', + msvc='') + + _globl('__gccallshapes') + if self.format == 'msvc': + print >> output, _globalname('__gccallshapes'), + else: + _label('__gccallshapes') + output.writelines(shapelines) + + if self.format == 'msvc': + print >> output, "_DATA ENDS" + print >> output, "END" + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) + for in_function, lines in parser.find_functions(iterlines): + if in_function: + lines = parser.process_function(lines, entrypoint, filename) + parser.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = parser.gcmaptable + else: + self.gcmaptable.extend(parser.gcmaptable) + self.seen_main |= parser.seen_main + + +class UnrecognizedOperation(Exception): + pass + +class NoPatternMatch(Exception): + pass -LOC_NOWHERE = 0 -LOC_REG = 1 -LOC_EBP_BASED = 2 -LOC_ESP_BASED = 3 -LOC_MASK = 0x03 - -REG2LOC = {} -for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): - REG2LOC[_reg] = LOC_REG | (_i<<2) - -def frameloc(base, offset): - assert base in (LOC_EBP_BASED, LOC_ESP_BASED) - assert offset % 4 == 0 - return base | offset # __________ debugging output __________ @@ -1179,7 +1449,7 @@ elif kind == LOC_REG: reg = loc >> 2 assert 0 <= reg <= 3 - return CALLEE_SAVE_REGISTERS[reg] + return ElfFunctionGcRootTracker.CALLEE_SAVE_REGISTERS[reg] else: if kind == LOC_EBP_BASED: result = '(%ebp)' @@ -1304,6 +1574,12 @@ verbose = 1 shuffle = False output_raw_table = False + if sys.platform == 'darwin': + format = 'darwin' + elif sys.platform == 'win32': + format = 'mingw32' + else: + format = 'elf' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1314,14 +1590,11 @@ elif sys.argv[1] == '-t': del sys.argv[1] output_raw_table = True + elif sys.argv[1].startswith('-f'): + format = sys.argv[1][2:] + del sys.argv[1] else: break - if sys.platform == 'darwin': - format = 'darwin' - elif sys.platform == 'win32': - format = 'mingw32' - else: - format = 'elf' tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) for fn in sys.argv[1:]: f = open(fn, 'r') @@ -1332,7 +1605,7 @@ tracker.reload_raw_table(f) f.close() else: - assert fn.endswith('.s') + assert fn.endswith('.s'), fn lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: @@ -1345,6 +1618,5 @@ f.close() if output_raw_table: tracker.dump_raw_table(sys.stdout) - tracker.clear() if not output_raw_table: tracker.dump(sys.stdout) Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/genc.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/genc.py Fri Nov 13 14:49:13 2009 @@ -488,23 +488,56 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - sfiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] - lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] - gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] + trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] + if self.translator.platform.name == 'msvc': + trackgcfiles = [f for f in trackgcfiles + if f.startswith(('implement', 'testing', + '../module_cache/module'))] + sfiles = ['%s.s' % (c,) for c in trackgcfiles] + lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] + gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] mk.definition('ASMFILES', sfiles) mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') - mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') - mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: - python = "" - mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') - mk.rule('gcmaptable.s', '$(GCMAPFILES)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + python = '' + + if self.translator.platform.name == 'msvc': + lblofiles = [] + for cfile in mk.cfiles: + f = cfile[:-2] + if f in trackgcfiles: + ofile = '%s.lbl.obj' % (f,) + else: + ofile = '%s.obj' % (f,) + + lblofiles.append(ofile) + mk.definition('ASMLBLOBJFILES', lblofiles) + mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') + # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory + # even in debug builds + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Oi /Ob1') + mk.rule('.SUFFIXES', '.s', []) + mk.rule('.s.obj', '', + 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + mk.rule('.c.gcmap', '', + ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + ) + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + + else: + mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') + mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + mk.rule('%.lbl.s %.gcmap', '%.s', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') else: mk.definition('DEBUGFLAGS', '-O1 -g') Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/char.h ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/char.h (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/char.h Fri Nov 13 14:49:13 2009 @@ -6,8 +6,8 @@ /*** binary operations ***/ -#define OP_CHAR_EQ(x,y,r) r = ((x) == (y)) -#define OP_CHAR_NE(x,y,r) r = ((x) != (y)) +#define OP_CHAR_EQ(x,y,r) r = ((unsigned char)(x) == (unsigned char)(y)) +#define OP_CHAR_NE(x,y,r) r = ((unsigned char)(x) != (unsigned char)(y)) #define OP_CHAR_LE(x,y,r) r = ((unsigned char)(x) <= (unsigned char)(y)) #define OP_CHAR_GT(x,y,r) r = ((unsigned char)(x) > (unsigned char)(y)) #define OP_CHAR_LT(x,y,r) r = ((unsigned char)(x) < (unsigned char)(y)) Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/debug.h (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/debug.h Fri Nov 13 14:49:13 2009 @@ -139,12 +139,18 @@ return 1; } +#if defined(_MSC_VER) || defined(__MINGW32__) +#define PYPY_LONG_LONG_PRINTF_FORMAT "I64" +#else +#define PYPY_LONG_LONG_PRINTF_FORMAT "ll" +#endif + static void display_startstop(const char *prefix, const char *postfix, const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); - fprintf(pypy_debug_file, "%s[%llx] %s%s%s\n%s", + fprintf(pypy_debug_file, "%s[%"PYPY_LONG_LONG_PRINTF_FORMAT"x] %s%s%s\n%s", colors, timestamp, prefix, category, postfix, debug_stop_colors); Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/mem.h (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/src/mem.h Fri Nov 13 14:49:13 2009 @@ -17,14 +17,18 @@ could make two copies of the local variable (e.g. one in the stack and one in a register), pass one to GCROOT, and later use the other one. In practice the pypy_asm_gcroot() is often a no-op in the final - machine code and doesn't prevent most optimizations. Getting the - asm() right was tricky, though. The asm() is not volatile so that - gcc is free to delete it if the output variable is not used at all. - We need to prevent gcc from moving the asm() *before* the call that - could cause a collection; this is the purpose of the (unused) - __gcnoreorderhack input argument. Any memory input argument would - have this effect: as far as gcc knows the call instruction can modify - arbitrary memory, thus creating the order dependency that we want. */ + machine code and doesn't prevent most optimizations. */ +#ifndef _MSC_VER + +/* With gcc, getting the asm() right was tricky, though. The asm() is + not volatile so that gcc is free to delete it if the output variable + is not used at all. We need to prevent gcc from moving the asm() + *before* the call that could cause a collection; this is the purpose + of the (unused) __gcnoreorderhack input argument. Any memory input + argument would have this effect: as far as gcc knows the call + instruction can modify arbitrary memory, thus creating the order + dependency that we want. */ + #define pypy_asm_gcroot(p) ({void*_r; \ asm ("/* GCROOT %0 */" : "=g" (_r) : \ "0" (p), "m" (__gcnoreorderhack)); \ @@ -36,6 +40,31 @@ /* marker for trackgcroot.py */ #define pypy_asm_stack_bottom() asm volatile ("/* GC_STACK_BOTTOM */" : : ) +#else + +/* With the msvc Microsoft Compiler, the optimizer seems free to move + any code (even asm) that involves local memory (registers and stack). + The _ReadWriteBarrier function has an effect only where the content + of a global variable is *really* used. trackgcroot.py will remove + the extra instructions: the access to _constant_always_one_ is + removed, and the multiplication is replaced with a simple move. */ + +static __forceinline void* +pypy_asm_gcroot(void* _r1) +{ + static volatile int _constant_always_one_ = 1; + (long)_r1 *= _constant_always_one_; + _ReadWriteBarrier(); + return _r1; +} + +#define pypy_asm_keepalive(v) __asm { } +static __declspec(noinline) void pypy_asm_stack_bottom() { } + +#endif + + + #define OP_GC_ASMGCROOT_STATIC(i, r) r = \ i == 0 ? (void*)&__gcmapstart : \ i == 1 ? (void*)&__gcmapend : \ Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/test/test_newgc.py Fri Nov 13 14:49:13 2009 @@ -622,7 +622,7 @@ import gc slong = cast_type_to_ffitype(rffi.LONG) - from pypy.rpython.lltypesystem.ll2ctypes import libc_name + from pypy.rlib.libffi import get_libc_name def callback(ll_args, ll_res, stuff): gc.collect() @@ -637,7 +637,7 @@ res[0] = -1 def f(): - libc = CDLL(libc_name) + libc = CDLL(get_libc_name()) qsort = libc.getpointer('qsort', [ffi_type_pointer, slong, slong, ffi_type_pointer], ffi_type_void) Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/windows.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/windows.py Fri Nov 13 14:49:13 2009 @@ -96,6 +96,15 @@ # Probably not a msvc compiler... self.version = 0 + # Try to find a masm assembler + returncode, stdout, stderr = _run_subprocess('ml.exe', '', + env=self.c_environ) + r = re.search('Macro Assembler', stderr) + if r is None and os.path.exists('c:/masm32/bin/ml.exe'): + self.masm = 'c:/masm32/bin/ml.exe' + else: + self.masm = 'ml.exe' + # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] @@ -133,6 +142,12 @@ def _args_for_shared(self, args): return ['/dll'] + args + def check___thread(self): + # __declspec(thread) does not seem to work when using assembler. + # Returning False will cause the program to use TlsAlloc functions. + # see src/thread_nt.h + return False + def _link_args_from_eci(self, eci, standalone): # Windows needs to resolve all symbols even for DLLs args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) @@ -177,10 +192,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr) stderrlines = stderr.splitlines() - for line in stderrlines[:5]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 5: - log.ERROR('...') raise CompilationError(stdout, stderr) @@ -225,20 +238,30 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags + list(eci.compile_extra)), ('LDFLAGS', self.link_flags + list(eci.link_extra)), - ('CC', self.cc) + ('CC', self.cc), + ('CC_LINK', self.link), + ('MASM', self.masm), ] for args in definitions: m.definition(*args) rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS)'), - ('%.obj', '%.c', '$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS)'), + ('.c.obj', '', '$(CC) /nologo $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: m.rule(*rule) + if self.version < 80: + m.rule('$(TARGET)', '$(OBJECTS)', + '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + else: + m.rule('$(TARGET)', '$(OBJECTS)', + ['$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', + ]) + return m def execute_makefile(self, path_to_makefile): @@ -251,14 +274,29 @@ try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))]) finally: oldcwd.chdir() self._handle_error(returncode, stdout, stderr, path.join('make')) class NMakefile(posix.GnuMakefile): - pass # for the moment + def write(self, out=None): + # nmake expands macros when it parses rules. + # Write all macros before the rules. + if out is None: + f = self.makefile_dir.join('Makefile').open('w') + else: + f = out + for line in self.lines: + if not isinstance(line, posix.Rule): + line.write(f) + for line in self.lines: + if isinstance(line, posix.Rule): + line.write(f) + f.flush() + if out is None: + f.close() class MingwPlatform(posix.BasePosix): From david at codespeak.net Fri Nov 13 15:17:04 2009 From: david at codespeak.net (david at codespeak.net) Date: Fri, 13 Nov 2009 15:17:04 +0100 (CET) Subject: [pypy-svn] r69262 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20091113141704.17C49168026@codespeak.net> Author: david Date: Fri Nov 13 15:17:03 2009 New Revision: 69262 Modified: pypy/branch/io-lang/pypy/lang/io/parser.py Log: Add Comma token Modified: pypy/branch/io-lang/pypy/lang/io/parser.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parser.py (original) +++ pypy/branch/io-lang/pypy/lang/io/parser.py Fri Nov 13 15:17:03 2009 @@ -45,11 +45,13 @@ # ; Terminator = r'(;'+maybe(Whitespace)+')+' +# , +Comma = r'\,' -rexs = [Identifiers, Whitespace, Number, Hexnumber, Terminator, OpenParen, CloseParen, MonoQuote, TriQuote, Comment] +rexs = [Identifiers, Whitespace, Number, Hexnumber, Terminator, OpenParen, CloseParen, MonoQuote, TriQuote, Comment, Comma] names = ["Identifier", "Whitespace", "Number", "HexNumber", "Terminator", - "OpenParen", "CloseParen", "MonoQuote", "TriQuote", "Comment"] + "OpenParen", "CloseParen", "MonoQuote", "TriQuote", "Comment", "Comma"] ignores = ['Whitespace', 'Comment'] From david at codespeak.net Fri Nov 13 15:19:06 2009 From: david at codespeak.net (david at codespeak.net) Date: Fri, 13 Nov 2009 15:19:06 +0100 (CET) Subject: [pypy-svn] r69263 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20091113141906.BA166168026@codespeak.net> Author: david Date: Fri Nov 13 15:19:06 2009 New Revision: 69263 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Log: Lexer tests for the comma token Modified: pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Fri Nov 13 15:19:06 2009 @@ -175,4 +175,9 @@ tokens = iolexer.tokenize(inp) assert tokens[0] == Token('Identifier', 'curlyBrackets', SourcePos(0, 0, 0)) assert tokens[1] == Token('OpenParen', '{', SourcePos(0, 0, 0)) - assert tokens[2] == Token('CloseParen', '}', SourcePos(1, 0, 1)) \ No newline at end of file + assert tokens[2] == Token('CloseParen', '}', SourcePos(1, 0, 1)) + +def test_lex_comma_token(): + inp = '(1, 2)' + tokens = iolexer.tokenize(inp) + assert tokens[3] == Token('Comma', ',', SourcePos(2, 0, 2)) \ No newline at end of file From david at codespeak.net Fri Nov 13 15:20:05 2009 From: david at codespeak.net (david at codespeak.net) Date: Fri, 13 Nov 2009 15:20:05 +0100 (CET) Subject: [pypy-svn] r69264 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20091113142005.601F6168026@codespeak.net> Author: david Date: Fri Nov 13 15:20:04 2009 New Revision: 69264 Added: pypy/branch/io-lang/pypy/lang/io/compiler.py pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py Log: Add Compiler object and implement tokensForString method to provide access to the lexer from io level Added: pypy/branch/io-lang/pypy/lang/io/compiler.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/compiler.py Fri Nov 13 15:20:04 2009 @@ -0,0 +1,19 @@ +from pypy.lang.io.register import register_method +from pypy.lang.io.model import W_Object, W_Number +from pypy.lang.io.parser import get_lexer + at register_method('Compiler', 'tokensForString') +def compiler_tokens_for_string(space, w_target, w_message, w_context): + input = w_message.arguments[0].eval(space, w_context, w_target).value + io_tokens = [] + for token in get_lexer().tokenize(input): + t = W_Object(space) + if token.source in ['[', '{'] and len(io_tokens) > 0: + io_tokens[-1].slots['character'].value = token.source_pos.columnno + + t.slots['character'] = W_Number(space, len(token.source) + token.source_pos.columnno) + t.slots['line'] = W_Number(space, token.source_pos.lineno + 1) + t.slots['type'] = space.w_sequence.clone_and_init(token.name) + t.slots['name'] = space.w_sequence.clone_and_init(token.source) + io_tokens.append(t) + + return space.w_list.clone_and_init(space, io_tokens) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/objspace.py (original) +++ pypy/branch/io-lang/pypy/lang/io/objspace.py Fri Nov 13 15:20:04 2009 @@ -12,6 +12,7 @@ import pypy.lang.io.map import pypy.lang.io.coroutine import pypy.lang.io.sequence +import pypy.lang.io.compiler class ObjSpace(object): """docstring for ObjSpace""" @@ -29,6 +30,7 @@ self.w_call = W_Object(self, [self.w_object]) self.w_map = W_Map(self, [self.w_object]) self.w_coroutine = W_Coroutine.w_getmain(self) + self.w_compiler = W_Object(self, [self.w_object]) # flow control objects self.w_normal = W_Object(self, [self.w_object]) self.w_break = W_Object(self, [self.w_object]) @@ -42,6 +44,7 @@ # default stop state self.stop_status = self.w_normal self.w_return_value = self.w_nil + self.init_w_object() self.init_w_protos() @@ -57,7 +60,6 @@ self.init_w_number() - self.init_w_core() self.init_w_call() @@ -72,6 +74,7 @@ self.init_stored_messages() + self.init_w_compiler() def init_w_map(self): for key, function in cfunction_definitions['Map'].items(): @@ -109,6 +112,7 @@ self.w_core.slots['Number'] = self.w_number self.w_core.slots['Sequence'] = self.w_sequence self.w_core.slots['ImmutableSequence'] = self.w_immutable_sequence + self.w_core.slots['Compiler'] = self.w_compiler def init_w_number(self): self.w_number = instantiate(W_Number) @@ -157,7 +161,11 @@ self.w_core.slots['Eol'] = self.w_eol self.w_core.slots['Eol'].slots['type'] = W_ImmutableSequence(self, 'Eol') - + def init_w_compiler(self): + self.w_compiler.slots['type'] = W_ImmutableSequence(self, 'Compiler') + for key, function in cfunction_definitions['Compiler'].items(): + self.w_compiler.slots[key] = W_CFunction(self, function) + def init_w_coroutine(self): for key, function in cfunction_definitions['Coroutine'].items(): self.w_coroutine.slots[key] = W_CFunction(self, function) Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py Fri Nov 13 15:20:04 2009 @@ -0,0 +1,170 @@ +from pypy.lang.io.parserhack import interpret +from pypy.lang.io.model import W_Object, W_Number +import py + +def test_compiler_is_present(): + inp = "Compiler" + res, space = interpret(inp) + assert isinstance(res, W_Object) + assert res.slots['type'].value == 'Compiler' + assert res.protos == [space.w_object] + +def test_compiler_token_for_number_is_number(): + inp = 'Compiler tokensForString("1")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Number' + assert res.items[0].slots['name'].value == "1" + assert isinstance(res.items[0], W_Object) + +def test_compiler_token_HexNumber(): + inp = 'Compiler tokensForString("0x1")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 3 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'HexNumber' + assert res.items[0].slots['name'].value == "0x1" + assert isinstance(res.items[0], W_Object) + +def test_compiler_token_open_paren(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "(" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_close_paren(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == ")" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_anon_message(): + inp = 'Compiler tokensForString(" ()")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "" + assert isinstance(res.items[0], W_Object) + +def test_compiler_parse_paren_produces_anon_message(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert isinstance(res.items[0], W_Object) + +# curlyBraces +def test_compiler_token_open_squareBrackets(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "[" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_squareBrackets(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == "]" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_squareBrackets_message(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "squareBrackets" + assert isinstance(res.items[0], W_Object) + +# curlyBrackets +def test_compiler_token_open_curlyBrackets(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "{" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_curlyBrackets(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == "}" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_curlyBrackets_message(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "curlyBrackets" + assert isinstance(res.items[0], W_Object) + +def test_compiler_identifier_token(): + inp = 'Compiler tokensForString("foo")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 3 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == 'foo' + assert isinstance(res.items[0], W_Object) + + +def test_compiler_terminator_token(): + inp = 'Compiler tokensForString(";")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Terminator' + assert res.items[0].slots['name'].value == ";" + assert isinstance(res.items[0], W_Object) + +def test_compiler_comma_token(): + inp = 'Compiler tokensForString("(1,2)")' + res, space = interpret(inp) + assert res.items[3].slots['character'].value == 3 + assert res.items[3].slots['line'].value == 1 + assert res.items[3].slots['type'].value == 'Comma' + assert res.items[3].slots['name'].value == "," + assert isinstance(res.items[3], W_Object) + +def test_compiler_triquote_token(): + py.test.skip('Problem in the parserhack') + inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")' + res, space = interpret(inp) + assert res.items[0].slots['type'].value == 'TriQuote' + assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\"" + assert isinstance(res.items[0], W_Object) + +def test_compiler_monoquote_token(): + py.test.skip('Problem in the parserhack') + inp = 'Compiler tokensForString("\\\"lorem\\\"")' + res, space = interpret(inp) + assert res.items[0].slots['name'].value == "\"lorem\"" + assert res.items[0].slots['type'].value == 'MonoQuote' + assert isinstance(res.items[0], W_Object) + +def test_compiler_comment_token(): + py.test.skip("These Tokens are ignored by the lexer") + inp = 'Compiler tokensForString("xxx")' + res, space = interpret(inp) + assert res.items[0].slots['type'].value == 'Comment' + assert res.items[0].slots['name'].value == "??" + assert isinstance(res.items[0], space.x) \ No newline at end of file From afa at codespeak.net Fri Nov 13 15:46:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 15:46:17 +0100 (CET) Subject: [pypy-svn] r69265 - pypy/branch/msvc-asmgcroot-2/pypy/translator/platform Message-ID: <20091113144617.8E199168026@codespeak.net> Author: afa Date: Fri Nov 13 15:46:17 2009 New Revision: 69265 Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/__init__.py Log: Merge r69257 from trunk Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/__init__.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/platform/__init__.py Fri Nov 13 15:46:17 2009 @@ -108,7 +108,7 @@ def _handle_error(self, returncode, stderr, stdout, outname): if returncode != 0: errorfile = outname.new(ext='errors') - errorfile.write(stderr) + errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines[:50]: log.ERROR(line) From afa at codespeak.net Fri Nov 13 18:18:04 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 18:18:04 +0100 (CET) Subject: [pypy-svn] r69267 - pypy/branch/msvc-asmgcroot-2/pypy/rpython/memory/gctransform Message-ID: <20091113171804.BA425168026@codespeak.net> Author: afa Date: Fri Nov 13 18:18:03 2009 New Revision: 69267 Modified: pypy/branch/msvc-asmgcroot-2/pypy/rpython/memory/gctransform/asmgcroot.py Log: This was somehow forgotten during the merge Modified: pypy/branch/msvc-asmgcroot-2/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/rpython/memory/gctransform/asmgcroot.py Fri Nov 13 18:18:03 2009 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation from pypy.translator.unsimplify import copyvar from pypy.rlib.debug import ll_assert +import sys # @@ -296,6 +297,7 @@ return # the item may have been not found because the main array was # not sorted. Sort it and try again. + win32_follow_gcmap_jmp(gcmapstart, gcmapend) sort_gcmap(gcmapstart, gcmapend) item = search_in_gcmap(gcmapstart, gcmapend, retaddr) if item: @@ -384,6 +386,22 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +if sys.platform == 'win32': + def win32_follow_gcmap_jmp(start, end): + # The initial gcmap table contains addresses to a JMP + # instruction that jumps indirectly to the real code. + # Replace them with the target addresses. + while start < end: + code = rffi.cast(rffi.CCHARP, start.address[0])[0] + if code == '\xe9': # jmp + rel32 = rffi.cast(rffi.LONGP, start.address[0]+1)[0] + target = start.address[0] + (rel32 + 5) + start.address[0] = target + start += arrayitemsize +else: + def win32_follow_gcmap_jmp(start, end): + pass + def _compare_gcmap_entries(addr1, addr2): key1 = addr1.address[0] key2 = addr2.address[0] From afa at codespeak.net Fri Nov 13 19:53:23 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 19:53:23 +0100 (CET) Subject: [pypy-svn] r69270 - pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc Message-ID: <20091113185323.89C09168026@codespeak.net> Author: afa Date: Fri Nov 13 19:53:21 2009 New Revision: 69270 Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py Log: Replace tabs in literals. Now that the buildbot failures are the same than the ones on trunk ({jit}, {own}) the branch is ready to merge! Modified: pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/trackgcroot.py Fri Nov 13 19:53:21 2009 @@ -1275,10 +1275,10 @@ if self.format == 'msvc': print >> output, """\ - TITLE gcmaptable.s + TITLE\tgcmaptable.s .686P .XMM - .model flat + .model\tflat """ _variant(elf='\t.text', @@ -1410,7 +1410,7 @@ output.writelines(shapelines) if self.format == 'msvc': - print >> output, "_DATA ENDS" + print >> output, "_DATA\tENDS" print >> output, "END" def process(self, iterlines, newfile, entrypoint='main', filename='?'): From afa at codespeak.net Fri Nov 13 20:24:31 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 20:24:31 +0100 (CET) Subject: [pypy-svn] r69272 - in pypy/trunk/pypy: rpython/memory/gctransform translator/c translator/c/gcc translator/c/gcc/test translator/c/gcc/test/msvc translator/c/src translator/c/test translator/platform Message-ID: <20091113192431.7204616802A@codespeak.net> Author: afa Date: Fri Nov 13 20:24:30 2009 New Revision: 69272 Added: pypy/trunk/pypy/translator/c/gcc/autopath.py - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/autopath.py pypy/trunk/pypy/translator/c/gcc/instruction.py - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/instruction.py pypy/trunk/pypy/translator/c/gcc/test/msvc/ - copied from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/ pypy/trunk/pypy/translator/c/gcc/test/msvc/track0.s - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track0.s pypy/trunk/pypy/translator/c/gcc/test/msvc/track1.s - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track1.s pypy/trunk/pypy/translator/c/gcc/test/msvc/track2.s - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track2.s pypy/trunk/pypy/translator/c/gcc/test/msvc/track8.s - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track8.s pypy/trunk/pypy/translator/c/gcc/test/msvc/track_switch0.s - copied unchanged from r69270, pypy/branch/msvc-asmgcroot-2/pypy/translator/c/gcc/test/msvc/track_switch0.s Modified: pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/trunk/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/trunk/pypy/translator/c/gcc/trackgcroot.py pypy/trunk/pypy/translator/c/genc.py pypy/trunk/pypy/translator/c/src/char.h pypy/trunk/pypy/translator/c/src/debug.h pypy/trunk/pypy/translator/c/src/mem.h pypy/trunk/pypy/translator/c/test/test_newgc.py pypy/trunk/pypy/translator/platform/windows.py Log: Merge branch msvc-asmgcroot: support asmgcroot for the Microsoft compiler This is necessary for a framework gc to work with the jit. (the alternatives being --gc=boehm, or --cc=mingw32) Now "translate.py -Ojit" should work out of the box. The MASM assembler (ml.exe) is used. It may come with your installation of Visual Studio or with the Platform SDK; otherwise it can be installed from http://www.masm32.com/ The script trackgcroot.py was refactored, hopefully in a more object-oriented way, mostly because the assembler generated by the MSVC is very different from gcc. Also, the markers inserted in the code to indicater gc roots are different, because the compiler seems to optimize even inline __asm blocks. Moreover an "extern" function pointer often does not point to the function code, but inside a relocation map which contains a jmp to the real code. Modified: pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py Fri Nov 13 20:24:30 2009 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation from pypy.translator.unsimplify import copyvar from pypy.rlib.debug import ll_assert +import sys # @@ -296,6 +297,7 @@ return # the item may have been not found because the main array was # not sorted. Sort it and try again. + win32_follow_gcmap_jmp(gcmapstart, gcmapend) sort_gcmap(gcmapstart, gcmapend) item = search_in_gcmap(gcmapstart, gcmapend, retaddr) if item: @@ -384,6 +386,22 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +if sys.platform == 'win32': + def win32_follow_gcmap_jmp(start, end): + # The initial gcmap table contains addresses to a JMP + # instruction that jumps indirectly to the real code. + # Replace them with the target addresses. + while start < end: + code = rffi.cast(rffi.CCHARP, start.address[0])[0] + if code == '\xe9': # jmp + rel32 = rffi.cast(rffi.LONGP, start.address[0]+1)[0] + target = start.address[0] + (rel32 + 5) + start.address[0] = target + start += arrayitemsize +else: + def win32_follow_gcmap_jmp(start, end): + pass + def _compare_gcmap_entries(addr1, addr2): key1 = addr1.address[0] key2 = addr2.address[0] Modified: pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_asmgcroot.py Fri Nov 13 20:24:30 2009 @@ -6,18 +6,20 @@ from pypy.annotation.listdef import s_list_of_strings from pypy import conftest -def setup_module(module): - if sys.platform == 'win32': - if not ('mingw' in os.popen('gcc --version').read() and - 'GNU' in os.popen('make --version').read()): - py.test.skip("mingw32 and MSYS are required for asmgcc on Windows") - class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved # instructions: should_be_moving = False @classmethod + def make_config(cls): + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.gc = cls.gcpolicy + config.translation.gcrootfinder = "asmgcc" + return config + + @classmethod def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] @@ -29,12 +31,7 @@ else: print 'Result: "%s"' % (res,) return 0 - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) - config.translation.gc = cls.gcpolicy - config.translation.gcrootfinder = "asmgcc" - if sys.platform == 'win32': - config.translation.cc = 'mingw32' + config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() a.build_types(main, [s_list_of_strings]) @@ -51,7 +48,7 @@ def run(arg0, arg1): lines = [] - print >> sys.stderr, 'RUN: starting', exe_name + print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: @@ -162,6 +159,33 @@ assert res == 4900 +class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): + # for the individual tests see + # ====> ../../test/test_newgc.py + + @classmethod + def setup_class(cls): + if sys.platform != 'win32': + py.test.skip("mingw32 specific test") + if not ('mingw' in os.popen('gcc --version').read() and + 'GNU' in os.popen('make --version').read()): + py.test.skip("mingw32 and MSYS are required for this test") + + test_newgc.TestSemiSpaceGC.setup_class.im_func(cls) + + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.cc = 'mingw32' + return config + + + def test_callback_with_collect(self): + py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/trunk/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/test/test_trackgcroot.py Fri Nov 13 20:24:30 2009 @@ -4,11 +4,11 @@ from pypy.translator.c.gcc.trackgcroot import format_callshape from pypy.translator.c.gcc.trackgcroot import LOC_NOWHERE, LOC_REG from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED -from pypy.translator.c.gcc.trackgcroot import GcRootTracker -from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker +from pypy.translator.c.gcc.trackgcroot import ElfAssemblerParser +from pypy.translator.c.gcc.trackgcroot import DarwinAssemblerParser from pypy.translator.c.gcc.trackgcroot import compress_callshape from pypy.translator.c.gcc.trackgcroot import decompress_callshape -from pypy.translator.c.gcc.trackgcroot import OFFSET_LABELS +from pypy.translator.c.gcc.trackgcroot import PARSERS from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -63,7 +63,7 @@ \tMORE STUFF """ lines = source.splitlines(True) - parts = list(GcRootTracker().find_functions(iter(lines))) + parts = list(ElfAssemblerParser().find_functions(iter(lines))) assert len(parts) == 5 assert parts[0] == (False, lines[:2]) assert parts[1] == (True, lines[2:5]) @@ -96,7 +96,7 @@ \t.section stuff """ lines = source.splitlines(True) - parts = list(GcRootTracker(format='darwin').find_functions(iter(lines))) + parts = list(DarwinAssemblerParser().find_functions(iter(lines))) assert len(parts) == 7 assert parts[0] == (False, lines[:3]) assert parts[1] == (True, lines[3:7]) @@ -108,7 +108,7 @@ def test_computegcmaptable(): tests = [] - for format in ('elf', 'darwin'): + for format in ('elf', 'darwin', 'msvc'): for path in this_dir.join(format).listdir("track*.s"): n = path.purebasename[5:] try: @@ -120,15 +120,20 @@ for format, _, path in tests: yield check_computegcmaptable, format, path -r_globallabel = re.compile(r"([\w]+)=[.]+") -r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") + +r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") +r_gcroot_constant = re.compile(r";\tmov\t.+, .+_constant_always_one_") def check_computegcmaptable(format, path): + if format == 'msvc': + r_globallabel = re.compile(r"([\w]+)::") + else: + r_globallabel = re.compile(r"([\w]+)=[.]+") print - print path.basename + print path.dirpath().basename + '/' + path.basename lines = path.readlines() expectedlines = lines[:] - tracker = FunctionGcRootTracker(lines, format=format) + tracker = PARSERS[format].FunctionGcRootTracker(lines) table = tracker.computegcmaptable(verbose=sys.maxint) tabledict = {} seen = {} @@ -148,10 +153,22 @@ got = tabledict[label] assert format_callshape(got) == expected seen[label] = True - expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) - expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + if format == 'msvc': + expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s::\n' % (label,)) + else: + expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s=.+%d\n' % (label, + tracker.OFFSET_LABELS)) + if format == 'msvc' and r_gcroot_constant.match(line): + expectedlines[i] = ';' + expectedlines[i] + expectedlines[i+1] = (expectedlines[i+1] + .replace('\timul\t', '\tmov\t') + + '\t; GCROOT\n') prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % [key for key in tabledict if key not in seen]) + print lines + print expectedlines assert lines == expectedlines Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Fri Nov 13 20:24:30 2009 @@ -1,302 +1,40 @@ #! /usr/bin/env python - +import autopath import re, sys, os, random -LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' -r_functionstart_elf = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") -r_functionend_elf = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") - -# darwin -r_textstart = re.compile(r"\t.text\s*$") -# see -# http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html -OTHERSECTIONS = ['section', 'zerofill', - 'const', 'static_const', 'cstring', - 'literal4', 'literal8', 'literal16', - 'constructor', 'desctructor', - 'symbol_stub', - 'data', 'static_data', - 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', - 'dyld', 'mod_init_func', 'mod_term_func', - 'const_data' - ] -r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") -r_functionstart_darwin = re.compile(r"_(\w+):\s*$") - -if sys.platform != 'darwin': - OFFSET_LABELS = 2**30 -else: - OFFSET_LABELS = 0 - -# inside functions -r_label = re.compile(LABEL+"[:]\s*$") -r_rel_label = re.compile(r"(\d+):\s*$") -r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") -r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) -r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") -r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") -OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' -r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") -r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") -r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") -r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") -LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" -LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" -r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") -r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") -r_localvarnofp = re.compile(LOCALVAR) -r_localvarfp = re.compile(LOCALVARFP) -r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") -r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") - - -class GcRootTracker(object): - - def __init__(self, verbose=0, shuffle=False, format='elf'): - self.verbose = verbose - self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py - self.format = format - self.clear() - - def clear(self): - self.gcmaptable = [] - self.seen_main = False - - def dump_raw_table(self, output): - print >> output, "seen_main = %d" % (self.seen_main,) - for entry in self.gcmaptable: - print >> output, entry - - def reload_raw_table(self, input): - firstline = input.readline() - assert firstline.startswith("seen_main = ") - self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) - for line in input: - entry = eval(line) - assert type(entry) is tuple - self.gcmaptable.append(entry) - - def dump(self, output): - assert self.seen_main - shapes = {} - shapelines = [] - shapeofs = 0 - def _globalname(name): - if self.format in ('darwin', 'mingw32'): - return '_' + name - return name - def _globl(name): - print >> output, "\t.globl %s" % _globalname(name) - def _label(name): - print >> output, "%s:" % _globalname(name) - def _variant(**kwargs): - txt = kwargs[self.format] - print >> output, "\t%s" % txt - - print >> output, "\t.text" - _globl('pypy_asm_stackwalk') - _variant(elf='.type pypy_asm_stackwalk, @function', - darwin='', - mingw32='') - _label('pypy_asm_stackwalk') - print >> output, """\ - /* See description in asmgcroot.py */ - movl 4(%esp), %edx /* my argument, which is the callback */ - movl %esp, %eax /* my frame top address */ - pushl %eax /* ASM_FRAMEDATA[6] */ - pushl %ebp /* ASM_FRAMEDATA[5] */ - pushl %edi /* ASM_FRAMEDATA[4] */ - pushl %esi /* ASM_FRAMEDATA[3] */ - pushl %ebx /* ASM_FRAMEDATA[2] */ - - /* Add this ASM_FRAMEDATA to the front of the circular linked */ - /* list. Let's call it 'self'. */ - movl __gcrootanchor+4, %eax /* next = gcrootanchor->next */ - pushl %eax /* self->next = next */ - pushl $__gcrootanchor /* self->prev = gcrootanchor */ - movl %esp, __gcrootanchor+4 /* gcrootanchor->next = self */ - movl %esp, (%eax) /* next->prev = self */ - - /* note: the Mac OS X 16 bytes aligment must be respected. */ - call *%edx /* invoke the callback */ - - /* Detach this ASM_FRAMEDATA from the circular linked list */ - popl %esi /* prev = self->prev */ - popl %edi /* next = self->next */ - movl %edi, 4(%esi) /* prev->next = next */ - movl %esi, (%edi) /* next->prev = prev */ - - popl %ebx /* restore from ASM_FRAMEDATA[2] */ - popl %esi /* restore from ASM_FRAMEDATA[3] */ - popl %edi /* restore from ASM_FRAMEDATA[4] */ - popl %ebp /* restore from ASM_FRAMEDATA[5] */ - popl %ecx /* ignored ASM_FRAMEDATA[6] */ - /* the return value is the one of the 'call' above, */ - /* because %eax (and possibly %edx) are unmodified */ - ret -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', - darwin='', - mingw32='') - print >> output, '\t.data' - print >> output, '\t.align\t4' - _globl('__gcrootanchor') - _label('__gcrootanchor') - print >> output, """\ - /* A circular doubly-linked list of all */ - /* the ASM_FRAMEDATAs currently alive */ - .long\t__gcrootanchor /* prev */ - .long\t__gcrootanchor /* next */ -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _globl('__gcmapstart') - _label('__gcmapstart') - for label, state, is_range in self.gcmaptable: - try: - n = shapes[state] - except KeyError: - n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] - shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( - shapeofs, - ', '.join(bytes))) - shapeofs += len(bytes) - if is_range: - n = ~ n - print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) - print >> output, '\t.long\t%d' % (n,) - _globl('__gcmapend') - _label('__gcmapend') - _variant(elf='.section\t.rodata', - darwin='.const', - mingw32='') - _globl('__gccallshapes') - _label('__gccallshapes') - output.writelines(shapelines) - - def find_functions(self, iterlines): - _find_functions = getattr(self, '_find_functions_' + self.format) - return _find_functions(iterlines) - - def _find_functions_elf(self, iterlines): - functionlines = [] - in_function = False - for line in iterlines: - if r_functionstart_elf.match(line): - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - in_function = True - functionlines = [] - functionlines.append(line) - if r_functionend_elf.match(line): - assert in_function, ( - "missed the start of the current function") - yield True, functionlines - in_function = False - functionlines = [] - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - - def _find_functions_darwin(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - assert not in_text, "unexpected repeated .text start: %d" % n - in_text = True - elif r_sectionstart.match(line): - if in_function: - yield in_function, functionlines - functionlines = [] - in_text = False - in_function = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - - if functionlines: - yield in_function, functionlines - - def _find_functions_mingw32(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - in_text = True - elif r_sectionstart.match(line): - in_text = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - if functionlines: - yield in_function, functionlines - - def process(self, iterlines, newfile, entrypoint='main', filename='?'): - if self.format in ('darwin', 'mingw32'): - entrypoint = '_' + entrypoint - for in_function, lines in self.find_functions(iterlines): - if in_function: - lines = self.process_function(lines, entrypoint, filename) - newfile.writelines(lines) - if self.verbose == 1: - sys.stderr.write('\n') - - def process_function(self, lines, entrypoint, filename): - tracker = FunctionGcRootTracker(lines, filetag=getidentifier(filename), - format=self.format) - is_main = tracker.funcname == entrypoint - tracker.is_stack_bottom = is_main - if self.verbose == 1: - sys.stderr.write('.') - elif self.verbose > 1: - print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, - tracker.funcname) - table = tracker.computegcmaptable(self.verbose) - if self.verbose > 1: - for label, state in table: - print >> sys.stderr, label, '\t', format_callshape(state) - table = compress_gcmaptable(table) - if self.shuffle and random.random() < 0.5: - self.gcmaptable[:0] = table - else: - self.gcmaptable.extend(table) - self.seen_main |= is_main - return tracker.lines - +from pypy.translator.c.gcc.instruction import Insn, Label, InsnCall, InsnRet +from pypy.translator.c.gcc.instruction import InsnFunctionStart, InsnStop +from pypy.translator.c.gcc.instruction import InsnSetLocal, InsnCopyLocal +from pypy.translator.c.gcc.instruction import InsnPrologue, InsnEpilogue +from pypy.translator.c.gcc.instruction import InsnGCROOT +from pypy.translator.c.gcc.instruction import InsnStackAdjust +from pypy.translator.c.gcc.instruction import InsnCannotFollowEsp +from pypy.translator.c.gcc.instruction import LocalVar, somenewvalue, frameloc +from pypy.translator.c.gcc.instruction import LOC_REG, LOC_NOWHERE, LOC_MASK +from pypy.translator.c.gcc.instruction import LOC_EBP_BASED, LOC_ESP_BASED class FunctionGcRootTracker(object): + skip = 0 - def __init__(self, lines, filetag=0, format='elf'): - if format == 'elf': - match = r_functionstart_elf.match(lines[0]) - funcname = match.group(1) - match = r_functionend_elf.match(lines[-1]) - assert funcname == match.group(1) - assert funcname == match.group(2) - elif format == 'darwin': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - elif format == 'mingw32': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - else: - assert False, "unknown format: %s" % format + @classmethod + def init_regexp(cls): + cls.r_label = re.compile(cls.LABEL+"[:]\s*$") + cls.r_globl = re.compile(r"\t[.]globl\t"+cls.LABEL+"\s*$") + cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) + + cls.r_insn = re.compile(r"\t([a-z]\w*)\s") + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") + + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") + cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") + def __init__(self, funcname, lines, filetag=0): self.funcname = funcname self.lines = lines self.uses_frame_pointer = False - self.r_localvar = r_localvarnofp + self.r_localvar = self.r_localvarnofp self.filetag = filetag # a "stack bottom" function is either main() or a callback from C code self.is_stack_bottom = False @@ -316,6 +54,9 @@ self.dump() return self.gettable() + def replace_symbols(self, operand): + return operand + def gettable(self): """Returns a list [(label_after_call, callshape_tuple)] See format_callshape() for more details about callshape_tuple. @@ -333,7 +74,7 @@ shape = [retaddr] # the first gcroots are always the ones corresponding to # the callee-saved registers - for reg in CALLEE_SAVE_REGISTERS: + for reg in self.CALLEE_SAVE_REGISTERS: shape.append(LOC_NOWHERE) gcroots = [] for localvar, tag in insn.gcroots.items(): @@ -341,17 +82,17 @@ loc = localvar.getlocation(insn.framesize, self.uses_frame_pointer) else: - assert localvar in REG2LOC, "%s: %s" % (self.funcname, - localvar) - loc = REG2LOC[localvar] + assert localvar in self.REG2LOC, "%s: %s" % (self.funcname, + localvar) + loc = self.REG2LOC[localvar] assert isinstance(loc, int) if tag is None: gcroots.append(loc) else: - regindex = CALLEE_SAVE_REGISTERS.index(tag) + regindex = self.CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc if LOC_NOWHERE in shape and not self.is_stack_bottom: - reg = CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] + reg = self.CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) gcroots.sort() @@ -362,13 +103,13 @@ def findlabels(self): self.labels = {} # {name: Label()} for lineno, line in enumerate(self.lines): - match = r_label.match(line) + match = self.r_label.match(line) label = None if match: label = match.group(1) else: # labels used by: j* NNNf - match = r_rel_label.match(line) + match = self.r_rel_label.match(line) if match: label = "rel %d" % lineno if label: @@ -388,30 +129,35 @@ lst.append(previnsn) def parse_instructions(self): - self.insns = [InsnFunctionStart()] + self.insns = [InsnFunctionStart(self.CALLEE_SAVE_REGISTERS)] ignore_insns = False for lineno, line in enumerate(self.lines): + if lineno < self.skip: + continue self.currentlineno = lineno insn = [] - match = r_insn.match(line) - if match: + match = self.r_insn.match(line) + + if self.r_bottom_marker.match(line): + self.is_stack_bottom = True + elif match: if not ignore_insns: opname = match.group(1) try: meth = getattr(self, 'visit_' + opname) except AttributeError: - meth = self.find_missing_visit_method(opname) + self.find_missing_visit_method(opname) + meth = getattr(self, 'visit_' + opname) + line = line.rsplit(';', 1)[0] insn = meth(line) - elif r_gcroot_marker.match(line): + elif self.r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) - elif r_bottom_marker.match(line): - self.is_stack_bottom = True elif line == '\t/* ignore_in_trackgcroot */\n': ignore_insns = True elif line == '\t/* end_ignore_in_trackgcroot */\n': ignore_insns = False else: - match = r_label.match(line) + match = self.r_label.match(line) if match: insn = self.labels[match.group(1)] @@ -421,18 +167,17 @@ else: self.append_instruction(insn) - del self.currentlineno + del self.currentlineno - def find_missing_visit_method(self, opname): + @classmethod + def find_missing_visit_method(cls, opname): # only for operations that are no-ops as far as we are concerned prefix = opname - while prefix not in self.IGNORE_OPS_WITH_PREFIXES: + while prefix not in cls.IGNORE_OPS_WITH_PREFIXES: prefix = prefix[:-1] if not prefix: raise UnrecognizedOperation(opname) - visit_nop = FunctionGcRootTracker.__dict__['visit_nop'] - setattr(FunctionGcRootTracker, 'visit_' + opname, visit_nop) - return self.visit_nop + setattr(cls, 'visit_' + opname, cls.visit_nop) def list_call_insns(self): return [insn for insn in self.insns if isinstance(insn, InsnCall)] @@ -461,8 +206,8 @@ for insn1, delta1 in deltas.items(): if hasattr(insn1, 'framesize'): size_at_insn.append(insn1.framesize + delta1) - assert len(size_at_insn) > 0, ( - "cannot reach the start of the function??") + if not size_at_insn: + continue size_at_insn = size_at_insn[0] for insn1, delta1 in deltas.items(): size_at_insn1 = size_at_insn - delta1 @@ -480,20 +225,24 @@ elif isinstance(localvar, (list, tuple)): return [fixvar(var) for var in localvar] - match = r_localvar_esp.match(localvar) + match = self.r_localvar_esp.match(localvar) if match: - if localvar == '0(%esp)': # for pushl and popl, by - hint = None # default ebp addressing is - else: # a bit nicer + if localvar == self.TOP_OF_STACK: # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer hint = 'esp' ofs_from_esp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_esp += int(match.group(2) or '0') localvar = ofs_from_esp - insn.framesize assert localvar != 0 # that's the return address return LocalVar(localvar, hint=hint) elif self.uses_frame_pointer: - match = r_localvar_ebp.match(localvar) + match = self.r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_ebp += int(match.group(2) or '0') localvar = ofs_from_ebp - 4 assert localvar != 0 # that's the return address return LocalVar(localvar, hint='ebp') @@ -553,10 +302,10 @@ # script); otherwise invent a name and add the label to tracker.lines. label = None # this checks for a ".globl NAME" followed by "NAME:" - match = r_globl.match(self.lines[call.lineno+1]) + match = self.r_globl.match(self.lines[call.lineno+1]) if match: label1 = match.group(1) - match = r_globllabel.match(self.lines[call.lineno+2]) + match = self.r_globllabel.match(self.lines[call.lineno+2]) if match: label2 = match.group(1) if label1 == label2: @@ -569,22 +318,26 @@ break k += 1 self.labels[label] = None - # These global symbols are not directly labels pointing to the - # code location because such global labels in the middle of - # functions confuse gdb. Instead, we add to the global symbol's - # value a big constant, which is subtracted again when we need - # the original value for gcmaptable.s. That's a hack. - self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, - OFFSET_LABELS)) - self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) + if self.format == 'msvc': + self.lines.insert(call.lineno+1, '%s::\n' % (label,)) + self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) + else: + # These global symbols are not directly labels pointing to the + # code location because such global labels in the middle of + # functions confuse gdb. Instead, we add to the global symbol's + # value a big constant, which is subtracted again when we need + # the original value for gcmaptable.s. That's a hack. + self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, + self.OFFSET_LABELS)) + self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label # ____________________________________________________________ def _visit_gcroot_marker(self, line): - match = r_gcroot_marker.match(line) + match = self.r_gcroot_marker.match(line) loc = match.group(1) - return InsnGCROOT(loc) + return InsnGCROOT(self.replace_symbols(loc)) def visit_nop(self, line): return [] @@ -615,15 +368,15 @@ visit_xorw = visit_nop def visit_addl(self, line, sign=+1): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if target == '%esp': - count = match.group(1) - if not count.startswith('$'): + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if target == self.ESP: + count = self.extract_immediate(source) + if count is None: # strange instruction - I've seen 'subl %eax, %esp' return InsnCannotFollowEsp() - return InsnStackAdjust(sign * int(count[1:])) + return InsnStackAdjust(sign * count) elif self.r_localvar.match(target): return InsnSetLocal(target, [source, target]) else: @@ -633,7 +386,7 @@ return self.visit_addl(line, sign=-1) def unary_insn(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) if self.r_localvar.match(target): return InsnSetLocal(target) @@ -641,14 +394,14 @@ return [] def binary_insn(self, line): - match = r_binaryinsn.match(line) + match = self.r_binaryinsn.match(line) if not match: raise UnrecognizedOperation(line) - source = match.group(1) - target = match.group(2) + source = match.group("source") + target = match.group("target") if self.r_localvar.match(target): return InsnSetLocal(target, [source]) - elif target == '%esp': + elif target == self.ESP: raise UnrecognizedOperation(line) else: return [] @@ -673,9 +426,9 @@ visit_cmovno = binary_insn def visit_andl(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for andl $-16, %esp used to align the stack in main(). # The exact amount of adjutment is not known yet, so we use # an odd-valued estimate to make sure the real value is not used @@ -685,12 +438,12 @@ return self.binary_insn(line) def visit_leal(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for leal -12(%ebp), %esp in function epilogues - source = match.group(1) - match = r_localvar_ebp.match(source) + source = match.group("source") + match = self.r_localvar_ebp.match(source) if match: if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') @@ -698,7 +451,7 @@ assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp else: - match = r_localvar_esp.match(source) + match = self.r_localvar_esp.match(source) # leal 12(%esp), %esp if match: return InsnStackAdjust(int(match.group(1))) @@ -709,7 +462,9 @@ return self.binary_insn(line) def insns_for_copy(self, source, target): - if source == '%esp' or target == '%esp': + source = self.replace_symbols(source) + target = self.replace_symbols(target) + if source == self.ESP or target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) elif self.r_localvar.match(target): if self.r_localvar.match(source): @@ -720,14 +475,14 @@ return [] def visit_movl(self, line): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if source == '%esp' and target == '%ebp': + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if source == self.ESP and target == self.EBP: return self._visit_prologue() - elif source == '%ebp' and target == '%esp': + elif source == self.EBP and target == self.ESP: return self._visit_epilogue() - if source == '%esp' and self.funcname.startswith('VALGRIND_'): + if source == self.ESP and self.funcname.startswith('VALGRIND_'): return [] # in VALGRIND_XXX functions, there is a dummy-looking # mov %esp, %eax. Shows up only when compiling with # gcc -fno-unit-at-a-time. @@ -736,25 +491,25 @@ visit_mov = visit_movl def visit_pushl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') + return [InsnStackAdjust(-4)] + self.insns_for_copy(source, self.TOP_OF_STACK) def visit_pushw(self, line): return [InsnStackAdjust(-2)] # rare but not impossible def _visit_pop(self, target): - return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] + return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+4)] def visit_popl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) return self._visit_pop(target) def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer self.uses_frame_pointer = True - self.r_localvar = r_localvarfp + self.r_localvar = self.r_localvarfp return [InsnPrologue()] def _visit_epilogue(self): @@ -763,38 +518,38 @@ return [InsnEpilogue(4)] def visit_leave(self, line): - return self._visit_epilogue() + self._visit_pop('%ebp') + return self._visit_epilogue() + self._visit_pop(self.EBP) def visit_ret(self, line): - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) def visit_jmp(self, line): tablelabels = [] - match = r_jmp_switch.match(line) + match = self.r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. tablelabels.append(match.group(1)) - elif r_unaryinsn_star.match(line): + elif self.r_unaryinsn_star.match(line): # maybe a jmp similar to the above, but stored in a # registry: # movl L9341(%eax), %eax # jmp *%eax - operand = r_unaryinsn_star.match(line).group(1)[1:] + operand = self.r_unaryinsn_star.match(line).group(1) def walker(insn, locs): sources = [] for loc in locs: for s in insn.all_sources_of(loc): # if the source looks like 8(%eax,%edx,4) # %eax is the real source, %edx is an offset. - match = r_jmp_source.match(s) - if match and not r_localvar_esp.match(s): + match = self.r_jmp_source.match(s) + if match and not self.r_localvar_esp.match(s): sources.append(match.group(1)) else: sources.append(s) for source in sources: - label_match = re.compile(LABEL).match(source) + label_match = re.compile(self.LABEL).match(source) if label_match: tablelabels.append(label_match.group(0)) return @@ -809,37 +564,43 @@ assert len(tablelabels) <= 1 if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 - while not r_jmptable_end.match(self.lines[tablelin]): - match = r_jmptable_item.match(self.lines[tablelin]) + while not self.r_jmptable_end.match(self.lines[tablelin]): + # skip empty lines + if (not self.lines[tablelin].strip() + or self.lines[tablelin].startswith(';')): + tablelin += 1 + continue + match = self.r_jmptable_item.match(self.lines[tablelin]) if not match: - raise NoPatternMatch(self.lines[tablelin]) + raise NoPatternMatch(repr(self.lines[tablelin])) label = match.group(1) if label != '0': self.register_jump_to(label) tablelin += 1 return InsnStop() - if r_unaryinsn_star.match(line): + if self.r_unaryinsn_star.match(line): # that looks like an indirect tail-call. # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) try: self.conditional_jump(line) except KeyError: # label not found: check if it's a tail-call turned into a jump - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) assert not target.startswith('.') # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() def register_jump_to(self, label): - self.labels[label].previous_insns.append(self.insns[-1]) + if not isinstance(self.insns[-1], InsnStop): + self.labels[label].previous_insns.append(self.insns[-1]) def conditional_jump(self, line): - match = r_jump.match(line) + match = self.r_jump.match(line) if not match: - match = r_jump_rel_label.match(line) + match = self.r_jump_rel_label.match(line) if not match: raise UnrecognizedOperation(line) # j* NNNf @@ -878,260 +639,116 @@ def visit_xchgl(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS # which is to use a marker no-op "xchgl %ebx, %ebx" - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if source == target: return [] raise UnrecognizedOperation(line) def visit_call(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) + if match is None: - assert r_unaryinsn_star.match(line) # indirect call - else: - target = match.group(1) - if target in FUNCTIONS_NOT_RETURNING: - return InsnStop() - if sys.platform == 'win32' and target == '__alloca': - # in functions with large stack requirements, windows - # needs a call to _alloca(), to turn reserved pages - # into committed memory. - # With mingw32 gcc at least, %esp is not used before - # this call. So we don't bother to compute the exact - # stack effect. - return [InsnCannotFollowEsp()] - if target in self.labels: - lineoffset = self.labels[target].lineno - self.currentlineno - if lineoffset >= 0: - assert lineoffset in (1,2) - return [InsnStackAdjust(-4)] + assert self.r_unaryinsn_star.match(line) # indirect call + return [InsnCall(self.currentlineno), + InsnSetLocal(self.EAX)] # the result is there + + target = match.group(1) + + if self.format in ('msvc'): + # On win32, the address of a foreign function must be + # computed, the optimizer may store it in a register. We + # could ignore this, except when the function need special + # processing (not returning, __stdcall...) + def find_register(target): + reg = [] + def walker(insn, locs): + sources = [] + for loc in locs: + for s in insn.all_sources_of(loc): + sources.append(s) + for source in sources: + m = re.match("DWORD PTR " + self.LABEL, source) + if m: + reg.append(m.group(1)) + if reg: + return + yield tuple(sources) + insn = InsnStop() + insn.previous_insns = [self.insns[-1]] + self.walk_instructions_backwards(walker, insn, (target,)) + return reg + + if match and self.r_localvarfp.match(target): + sources = find_register(target) + if sources: + target, = sources + + if target in self.FUNCTIONS_NOT_RETURNING: + return [InsnStop(), InsnCannotFollowEsp()] + if self.format == 'mingw32' and target == '__alloca': + # in functions with large stack requirements, windows + # needs a call to _alloca(), to turn reserved pages + # into committed memory. + # With mingw32 gcc at least, %esp is not used before + # this call. So we don't bother to compute the exact + # stack effect. + return [InsnCannotFollowEsp()] + if target in self.labels: + lineoffset = self.labels[target].lineno - self.currentlineno + if lineoffset >= 0: + assert lineoffset in (1,2) + return [InsnStackAdjust(-4)] + insns = [InsnCall(self.currentlineno), - InsnSetLocal('%eax')] # the result is there - if sys.platform == 'win32': + InsnSetLocal(self.EAX)] # the result is there + if self.format in ('mingw32', 'msvc'): # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size - if match and '@' in target: - insns.append(InsnStackAdjust(int(target.split('@')[1]))) + if '@' in target and not target.startswith('@'): + insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) + # Some (intrinsic?) functions use the "fastcall" calling convention + # XXX without any declaration, how can we guess the stack effect? + if target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: + insns.append(InsnStackAdjust(16)) return insns -class UnrecognizedOperation(Exception): - pass - -class NoPatternMatch(Exception): - pass - -class SomeNewValue(object): - pass -somenewvalue = SomeNewValue() +class ElfFunctionGcRootTracker(FunctionGcRootTracker): + format = 'elf' -class LocalVar(object): - # A local variable location at position 'ofs_from_frame_end', - # which is counted from the end of the stack frame (so it is always - # negative, unless it refers to arguments of the current function). - def __init__(self, ofs_from_frame_end, hint=None): - self.ofs_from_frame_end = ofs_from_frame_end - self.hint = hint - - def __repr__(self): - return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') - - def __hash__(self): - return hash(self.ofs_from_frame_end) - - def __cmp__(self, other): - if isinstance(other, LocalVar): - return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) - else: - return 1 - - def getlocation(self, framesize, uses_frame_pointer): - if (self.hint == 'esp' or not uses_frame_pointer - or self.ofs_from_frame_end % 2 != 0): - # try to use esp-relative addressing - ofs_from_esp = framesize + self.ofs_from_frame_end - if ofs_from_esp % 2 == 0: - return frameloc(LOC_ESP_BASED, ofs_from_esp) - # we can get an odd value if the framesize is marked as bogus - # by visit_andl() - assert uses_frame_pointer - ofs_from_ebp = self.ofs_from_frame_end + 4 - return frameloc(LOC_EBP_BASED, ofs_from_ebp) - - -class Insn(object): - _args_ = [] - _locals_ = [] - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join([str(getattr(self, name)) - for name in self._args_])) - def requestgcroots(self, tracker): - return {} - - def source_of(self, localvar, tag): - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class Label(Insn): - _args_ = ['label', 'lineno'] - def __init__(self, label, lineno): - self.label = label - self.lineno = lineno - self.previous_insns = [] # all insns that jump (or fallthrough) here - -class InsnFunctionStart(Insn): - framesize = 0 - previous_insns = () - def __init__(self): - self.arguments = {} - for reg in CALLEE_SAVE_REGISTERS: - self.arguments[reg] = somenewvalue - - def source_of(self, localvar, tag): - if localvar not in self.arguments: - if localvar in ('%eax', '%edx', '%ecx'): - # xxx this might show a bug in trackgcroot.py failing to - # figure out which instruction stored a value in these - # registers. However, this case also occurs when the - # the function's calling convention was optimized by gcc: - # the 3 registers above are then used to pass arguments - pass - else: - assert (isinstance(localvar, LocalVar) and - localvar.ofs_from_frame_end > 0), ( - "must come from an argument to the function, got %r" % - (localvar,)) - self.arguments[localvar] = somenewvalue - return self.arguments[localvar] - - def all_sources_of(self, localvar): - return [] - -class InsnSetLocal(Insn): - _args_ = ['target', 'sources'] - _locals_ = ['target', 'sources'] - - def __init__(self, target, sources=()): - self.target = target - self.sources = sources - - def source_of(self, localvar, tag): - if localvar == self.target: - return somenewvalue - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return self.sources - return [localvar] - -class InsnCopyLocal(Insn): - _args_ = ['source', 'target'] - _locals_ = ['source', 'target'] - - def __init__(self, source, target): - self.source = source - self.target = target - - def source_of(self, localvar, tag): - if localvar == self.target: - return self.source - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return [self.source] - return [localvar] - -class InsnStackAdjust(Insn): - _args_ = ['delta'] - def __init__(self, delta): - assert delta % 2 == 0 # should be "% 4", but there is the special - self.delta = delta # case of 'pushw' to handle - -class InsnCannotFollowEsp(InsnStackAdjust): - def __init__(self): - self.delta = -7 # use an odd value as marker - -class InsnStop(Insn): - pass + ESP = '%esp' + EBP = '%ebp' + EAX = '%eax' + CALLEE_SAVE_REGISTERS = ['%ebx', '%esi', '%edi', '%ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 2**30 + TOP_OF_STACK = '0(%esp)' + + r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") + r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") + LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" + LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") + r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") + + r_rel_label = re.compile(r"(\d+):\s*$") + r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -class InsnRet(InsnStop): - framesize = 0 - def requestgcroots(self, tracker): - # no need to track the value of these registers in the caller - # function if we are the main(), or if we are flagged as a - # "bottom" function (a callback from C code) - if tracker.is_stack_bottom: - return {} - else: - return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) - -class InsnCall(Insn): - _args_ = ['lineno', 'gcroots'] - def __init__(self, lineno): - # 'gcroots' is a dict built by side-effect during the call to - # FunctionGcRootTracker.trackgcroots(). Its meaning is as - # follows: the keys are the locations that contain gc roots - # (register names or LocalVar instances). The value - # corresponding to a key is the "tag", which is None for a - # normal gc root, or else the name of a callee-saved register. - # In the latter case it means that this is only a gc root if the - # corresponding register in the caller was really containing a - # gc pointer. A typical example: - # - # InsnCall({LocalVar(-8)': None, - # '%esi': '%esi', - # LocalVar(-12)': '%ebx'}) - # - # means that the value at -8 from the frame end is a gc root - # across this call; that %esi is a gc root if it was in the - # caller (typically because %esi is not modified at all in the - # current function); and that the value at -12 from the frame - # end is a gc root if %ebx was a gc root in the caller - # (typically because the current function saves and restores - # %ebx from there in the prologue and epilogue). - self.gcroots = {} - self.lineno = lineno - - def source_of(self, localvar, tag): - tag1 = self.gcroots.setdefault(localvar, tag) - assert tag1 == tag, ( - "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( - localvar, tag1, tag)) - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class InsnGCROOT(Insn): - _args_ = ['loc'] - _locals_ = ['loc'] - def __init__(self, loc): - self.loc = loc - def requestgcroots(self, tracker): - return {self.loc: None} - -class InsnPrologue(Insn): - def __setattr__(self, attr, value): - if attr == 'framesize': - assert value == 4, ("unrecognized function prologue - " - "only supports push %ebp; movl %esp, %ebp") - Insn.__setattr__(self, attr, value) - -class InsnEpilogue(Insn): - def __init__(self, framesize=None): - if framesize is not None: - self.framesize = framesize + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") - -if sys.platform != 'win32': FUNCTIONS_NOT_RETURNING = { 'abort': None, '_exit': None, @@ -1139,31 +756,684 @@ '___assert_rtn': None, 'L___assert_rtn$stub': None } -else: + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = match.group(1) + match = self.r_functionend.match(lines[-1]) + assert funcname == match.group(1) + assert funcname == match.group(2) + super(ElfFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def extract_immediate(self, value): + if not value.startswith('$'): + return None + return int(value[1:]) + +ElfFunctionGcRootTracker.init_regexp() + +class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): + format = 'darwin' + + r_functionstart = re.compile(r"_(\w+):\s*$") + OFFSET_LABELS = 0 + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = '_' + match.group(1) + FunctionGcRootTracker.__init__(self, funcname, lines, filetag) + +class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): + format = 'mingw32' + +class MsvcFunctionGcRootTracker(FunctionGcRootTracker): + format = 'msvc' + ESP = 'esp' + EBP = 'ebp' + EAX = 'eax' + CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + TOP_OF_STACK = 'DWORD PTR [esp]' + + OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' + LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 0 + + r_functionstart = re.compile(r"; Function compile flags: ") + r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") + r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") + r_symboldefine = re.compile(r"([_A-Za-z0-9$]+) = ([-0-9]+)\s*;.+\n") + + LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]?\d*\]" + LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") + r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") + + r_rel_label = re.compile(r"$1") # never matches + r_jump_rel_label = re.compile(r"$1") # never matches + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"[^\t\n;]") + + r_gcroot_marker = re.compile(r"$1") # never matches + r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot") + r_bottom_marker = re.compile(r"; .+\tpypy_asm_stack_bottom\(\);") + FUNCTIONS_NOT_RETURNING = { '_abort': None, '__exit': None, '__assert': None, '__wassert': None, + '__imp__abort': None, + '__imp___wassert': None, + 'DWORD PTR __imp__abort': None, + 'DWORD PTR __imp___wassert': None, } -CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] -CALLEE_SAVE_REGISTERS = CALLEE_SAVE_REGISTERS_NOEBP + ['%ebp'] + @classmethod + def init_regexp(cls): + super(MsvcFunctionGcRootTracker, cls).init_regexp() + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+r"),\s*(?P"+cls.OPERAND+r")\s*(?:;.+)?$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+cls.LABEL+"\s*$") + + def __init__(self, lines, filetag=0): + self.defines = {} + for i, line in enumerate(lines): + if self.r_symboldefine.match(line): + match = self.r_symboldefine.match(line) + name = match.group(1) + value = int(match.group(2)) + self.defines[name] = value + continue + + match = self.r_codestart.match(line) + if match: + self.skip = i + break + + funcname = match.group(1) + super(MsvcFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def replace_symbols(self, operand): + for name, value in self.defines.items(): + operand = operand.replace(name, str(value)) + return operand + + for name in ''' + push pop mov lea + xor sub add + '''.split(): + locals()['visit_' + name] = getattr(FunctionGcRootTracker, + 'visit_' + name + 'l') + + visit_int = FunctionGcRootTracker.visit_nop + visit_npad = FunctionGcRootTracker.visit_nop + # probably not GC pointers + visit_cdq = FunctionGcRootTracker.visit_nop + + def extract_immediate(self, value): + try: + return int(value) + except ValueError: + return None + + def _visit_gcroot_marker(self, line=None): + # two possible patterns: + # 1. mov reg, DWORD PTR _always_one_ + # imul target, reg + # 2. mov reg, DWORD PTR _always_one_ + # imul reg, target + assert self.lines[self.currentlineno].startswith("\tmov\t") + mov = self.r_binaryinsn.match(self.lines[self.currentlineno]) + assert re.match("DWORD PTR .+_always_one_", mov.group("source")) + reg = mov.group("target") + + self.lines[self.currentlineno] = ";" + self.lines[self.currentlineno] + + # the 'imul' must appear in the same block; the 'reg' must not + # appear in the instructions between + imul = None + lineno = self.currentlineno + 1 + stop = False + while not stop: + line = self.lines[lineno] + if line == '\n': + stop = True + elif line.startswith("\tjmp\t"): + stop = True + elif self.r_gcroot_marker_var.search(line): + stop = True + elif line.startswith("\tmov\t%s," % (reg,)): + # mov reg, + stop = True + elif line.startswith("\txor\t%s, %s" % (reg, reg)): + # xor reg, reg + stop = True + elif line.startswith("\timul\t"): + imul = self.r_binaryinsn.match(line) + imul_arg1 = imul.group("target") + imul_arg2 = imul.group("source") + break + # the register may not appear in other instructions + elif reg in line: + assert False, (line, lineno) + + lineno += 1 + else: + # No imul, the returned value is not used in this function + return [] + + if reg == imul_arg2: + self.lines[lineno] = ";" + self.lines[lineno] + return InsnGCROOT(self.replace_symbols(imul_arg1)) + else: + assert reg == imul_arg1 + self.lines[lineno] = "\tmov\t%s, %s\n" % (imul_arg1, imul_arg2) + if imul_arg2.startswith('OFFSET '): + # ignore static global variables + pass + else: + self.lines[lineno] += "\t; GCROOT\n" + + return [] + + def insns_for_copy(self, source, target): + if self.r_gcroot_marker_var.match(source): + return self._visit_gcroot_marker() + if self.lines[self.currentlineno].endswith("\t; GCROOT\n"): + insns = [InsnGCROOT(self.replace_symbols(source))] + else: + insns = [] + return insns + super(MsvcFunctionGcRootTracker, self).insns_for_copy(source, target) + + +MsvcFunctionGcRootTracker.init_regexp() + +class AssemblerParser(object): + def __init__(self, verbose=0, shuffle=False): + self.verbose = verbose + self.shuffle = shuffle + self.gcmaptable = [] + self.seen_main = False + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + for in_function, lines in self.find_functions(iterlines): + if in_function: + lines = self.process_function(lines, entrypoint, filename) + self.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + + def write_newfile(self, newfile, lines, grist): + newfile.writelines(lines) + + def process_function(self, lines, entrypoint, filename): + tracker = self.FunctionGcRootTracker( + lines, filetag=getidentifier(filename)) + is_main = tracker.funcname == entrypoint + tracker.is_stack_bottom = is_main + if self.verbose == 1: + sys.stderr.write('.') + elif self.verbose > 1: + print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, + tracker.funcname) + table = tracker.computegcmaptable(self.verbose) + if self.verbose > 1: + for label, state in table: + print >> sys.stderr, label, '\t', format_callshape(state) + table = compress_gcmaptable(table) + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = table + else: + self.gcmaptable.extend(table) + self.seen_main |= is_main + return tracker.lines + +class ElfAssemblerParser(AssemblerParser): + format = "elf" + FunctionGcRootTracker = ElfFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + +class DarwinAssemblerParser(AssemblerParser): + format = "darwin" + FunctionGcRootTracker = DarwinFunctionGcRootTracker + + r_textstart = re.compile(r"\t.text\s*$") + + # see + # http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html + OTHERSECTIONS = ['section', 'zerofill', + 'const', 'static_const', 'cstring', + 'literal4', 'literal8', 'literal16', + 'constructor', 'desctructor', + 'symbol_stub', + 'data', 'static_data', + 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', + 'dyld', 'mod_init_func', 'mod_term_func', + 'const_data' + ] + r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + assert not in_text, "unexpected repeated .text start: %d" % n + in_text = True + elif cls.r_sectionstart.match(line): + if in_function: + yield in_function, functionlines + functionlines = [] + in_text = False + in_function = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + + if functionlines: + yield in_function, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(DarwinAssemblerParser, self).process_function( + lines, entrypoint, filename) + +class Mingw32AssemblerParser(DarwinAssemblerParser): + format = "mingw32" + FunctionGcRootTracker = Mingw32FunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + in_text = True + elif cls.r_sectionstart.match(line): + in_text = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + if functionlines: + yield in_function, functionlines + +class MsvcAssemblerParser(AssemblerParser): + format = "msvc" + FunctionGcRootTracker = MsvcFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(MsvcAssemblerParser, self).process_function( + lines, entrypoint, filename) + + def write_newfile(self, newfile, lines, grist): + newlines = [] + for line in lines: + # truncate long comments + if line.startswith(";"): + line = line[:-1][:500] + '\n' + + # Workaround a bug in the .s files generated by msvc + # compiler: every string or float constant is exported + # with a name built after its value, and will conflict + # with other modules. + if line.startswith("PUBLIC\t__real@"): + line = '; ' + line + elif line.startswith("PUBLIC\t??_C@"): + line = '; ' + line + elif line == "PUBLIC\t__$ArrayPad$\n": + line = '; ' + line + + # Because we insert labels in the code, some "SHORT" jumps + # are now longer than 127 bytes. We turn them all into + # "NEAR" jumps. Note that the assembler allocates space + # for a near jump, but still generates a short jump when + # it can. + line = line.replace('\tjmp\tSHORT ', '\tjmp\t') + line = line.replace('\tjne\tSHORT ', '\tjne\t') + line = line.replace('\tje\tSHORT ', '\tje\t') + + newlines.append(line) + + if line == "\t.model\tflat\n": + newlines.append("\tassume fs:nothing\n") + + newfile.writelines(newlines) + +PARSERS = { + 'elf': ElfAssemblerParser, + 'darwin': DarwinAssemblerParser, + 'mingw32': Mingw32AssemblerParser, + 'msvc': MsvcAssemblerParser, + } + +class GcRootTracker(object): + + def __init__(self, verbose=0, shuffle=False, format='elf'): + self.verbose = verbose + self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py + self.format = format + self.gcmaptable = [] + self.seen_main = False + + def dump_raw_table(self, output): + print >> output, "seen_main = %d" % (self.seen_main,) + for entry in self.gcmaptable: + print >> output, entry + + def reload_raw_table(self, input): + firstline = input.readline() + assert firstline.startswith("seen_main = ") + self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) + for line in input: + entry = eval(line) + assert type(entry) is tuple + self.gcmaptable.append(entry) + + def dump(self, output): + assert self.seen_main + shapes = {} + shapelines = [] + shapeofs = 0 + def _globalname(name, disp=""): + if self.format in ('darwin', 'mingw32', 'msvc'): + name = '_' + name + + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s + %s" % (name, disp) + else: + return name + + def _globl(name): + if self.format == 'msvc': + print >> output, "PUBLIC %s" % _globalname(name) + else: + print >> output, "\t.globl %s" % _globalname(name) + def _label(name): + print >> output, "%s:" % _globalname(name) + def _variant(**kwargs): + txt = kwargs[self.format] + print >> output, "\t%s" % txt + + def _comment(comment): + if self.format == 'msvc': + print >> output, "; %s" % comment + else: + print >> output, "/* %s */" % comment + + def _movl(source, target, comment): + if self.format == 'msvc': + print >> output, "\tmov\t%s, %s\t\t; %s" % (target, source, comment) + else: + print >> output, "\tmovl\t%s, %s\t\t/* %s */ " % (source, target, comment) + + def _pushl(source, comment): + if self.format == 'msvc': + print >> output, "\tpush\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpushl\t%s\t\t/* %s */ " % (source, comment) + + def _popl(source, comment): + if self.format == 'msvc': + print >> output, "\tpop\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpopl\t%s\t\t/* %s */ " % (source, comment) + + + def _register(name, disp=""): + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s(%%%s)" % (disp, name) + else: + return '%' + name + + def _offset(name): + if self.format == 'msvc': + return "OFFSET %s" % _globalname(name) + else: + return "$%s" % _globalname(name) + + def _call(arg, comment): + if self.format == 'msvc': + print >> output, "\tcall\t%s\t\t;%s" % (arg, comment) + else: + print >> output, "\tcall\t%s\t\t/* %s */" % (arg, comment) + + def _indirectjmp(arg): + if self.format == 'msvc': + return "DWORD PTR " + arg + else: + return "*%" + arg + + if self.format == 'msvc': + print >> output, """\ + TITLE\tgcmaptable.s + .686P + .XMM + .model\tflat + """ + + _variant(elf='\t.text', + darwin='\t.text', + mingw32='\t.text', + msvc='_TEXT\tSEGMENT') + + _globl('pypy_asm_stackwalk') + _variant(elf='.type pypy_asm_stackwalk, @function', + darwin='', + mingw32='', + msvc='') + _label('pypy_asm_stackwalk') + _comment("See description in asmgcroot.py") + _movl(_register("esp", disp="4"), _register("edx"), "my argument, which is the callback") + _movl(_register("esp"), _register("eax"), "my frame top address") + _pushl(_register("eax"), "ASM_FRAMEDATA[6]") + _pushl(_register("ebp"), "ASM_FRAMEDATA[5]") + _pushl(_register("edi"), "ASM_FRAMEDATA[4]") + _pushl(_register("esi"), "ASM_FRAMEDATA[3]") + _pushl(_register("ebx"), "ASM_FRAMEDATA[2]") + + print >> output + _comment("Add this ASM_FRAMEDATA to the front of the circular linked") + _comment("list. Let's call it 'self'.") + print >> output + _movl(_globalname("__gcrootanchor", disp=4), _register("eax"), "next = gcrootanchor->next") + _pushl(_register("eax"), "self->next = next") + _pushl(_offset("__gcrootanchor"), "self->prev = gcrootanchor") + _movl(_register("esp"), _globalname("__gcrootanchor", disp=4), "gcrootanchor->next = self") + _movl(_register("esp"), _register("eax", "0"), "next->prev = self") + print >> output + + _comment("note: the Mac OS X 16 bytes aligment must be respected.") + _call(_indirectjmp("edx"), "invoke the callback") + print >> output + + _comment("Detach this ASM_FRAMEDATA from the circular linked list") + _popl(_register("esi"), "prev = self->prev") + _popl(_register("edi"), "next = self->next") + _movl(_register("edi"), _register("esi", disp="4"), "prev->next = next") + _movl(_register("esi"), _register("edi", disp="0"), "next->prev = prev") + print >> output + + _popl(_register("ebx"), "restore from ASM_FRAMEDATA[2]") + _popl(_register("esi"), "restore from ASM_FRAMEDATA[3]") + _popl(_register("edi"), "restore from ASM_FRAMEDATA[4]") + _popl(_register("ebp"), "restore from ASM_FRAMEDATA[5]") + _popl(_register("ecx"), "ignored ASM_FRAMEDATA[6]") + _comment("the return value is the one of the 'call' above,") + _comment("because %eax (and possibly %edx) are unmodified") + + print >> output, "\tret" + + _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', + darwin='', + mingw32='', + msvc='') + + if self.format == 'msvc': + for label, state, is_range in self.gcmaptable: + print >> output, "EXTERN %s:NEAR" % label + + if self.format == 'msvc': + print >> output, '_DATA SEGMENT' + + _comment("A circular doubly-linked list of all") + _comment("the ASM_FRAMEDATAs currently alive") + if self.format == 'msvc': + _globl('__gcrootanchor') + print >> output, '%s\tDD FLAT:___gcrootanchor ; prev' % _globalname("__gcrootanchor") + print >> output, '\tDD FLAT:___gcrootanchor ; next' + else: + print >> output, '\t.data' + print >> output, '\t.align\t4' + _globl('__gcrootanchor') + _label('__gcrootanchor') + print >> output, """\ + .long\t__gcrootanchor /* prev */ + .long\t__gcrootanchor /* next */ +""".replace("__gcrootanchor", _globalname("__gcrootanchor")) + + _globl('__gcmapstart') + if self.format == 'msvc': + print >> output, '%s' % _globalname('__gcmapstart'), + else: + _label('__gcmapstart') + for label, state, is_range in self.gcmaptable: + try: + n = shapes[state] + except KeyError: + n = shapes[state] = shapeofs + bytes = [str(b) for b in compress_callshape(state)] + if self.format == 'msvc': + shapelines.append('\tDB\t%s\t;%s\n' % ( + ', '.join(bytes), + shapeofs)) + else: + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) + shapeofs += len(bytes) + if is_range: + n = ~ n + if self.format == 'msvc': + print >> output, '\tDD\t%s' % (label,) + print >> output, '\tDD\t%d' % (n,) + else: + print >> output, '\t.long\t%s-%d' % ( + label, + PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) + print >> output, '\t.long\t%d' % (n,) + + _globl('__gcmapend') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gcmapend') + else: + _label('__gcmapend') + _variant(elf='.section\t.rodata', + darwin='.const', + mingw32='', + msvc='') + + _globl('__gccallshapes') + if self.format == 'msvc': + print >> output, _globalname('__gccallshapes'), + else: + _label('__gccallshapes') + output.writelines(shapelines) + + if self.format == 'msvc': + print >> output, "_DATA\tENDS" + print >> output, "END" + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) + for in_function, lines in parser.find_functions(iterlines): + if in_function: + lines = parser.process_function(lines, entrypoint, filename) + parser.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = parser.gcmaptable + else: + self.gcmaptable.extend(parser.gcmaptable) + self.seen_main |= parser.seen_main + + +class UnrecognizedOperation(Exception): + pass + +class NoPatternMatch(Exception): + pass -LOC_NOWHERE = 0 -LOC_REG = 1 -LOC_EBP_BASED = 2 -LOC_ESP_BASED = 3 -LOC_MASK = 0x03 - -REG2LOC = {} -for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): - REG2LOC[_reg] = LOC_REG | (_i<<2) - -def frameloc(base, offset): - assert base in (LOC_EBP_BASED, LOC_ESP_BASED) - assert offset % 4 == 0 - return base | offset # __________ debugging output __________ @@ -1179,7 +1449,7 @@ elif kind == LOC_REG: reg = loc >> 2 assert 0 <= reg <= 3 - return CALLEE_SAVE_REGISTERS[reg] + return ElfFunctionGcRootTracker.CALLEE_SAVE_REGISTERS[reg] else: if kind == LOC_EBP_BASED: result = '(%ebp)' @@ -1304,6 +1574,12 @@ verbose = 1 shuffle = False output_raw_table = False + if sys.platform == 'darwin': + format = 'darwin' + elif sys.platform == 'win32': + format = 'mingw32' + else: + format = 'elf' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1314,14 +1590,11 @@ elif sys.argv[1] == '-t': del sys.argv[1] output_raw_table = True + elif sys.argv[1].startswith('-f'): + format = sys.argv[1][2:] + del sys.argv[1] else: break - if sys.platform == 'darwin': - format = 'darwin' - elif sys.platform == 'win32': - format = 'mingw32' - else: - format = 'elf' tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) for fn in sys.argv[1:]: f = open(fn, 'r') @@ -1332,7 +1605,7 @@ tracker.reload_raw_table(f) f.close() else: - assert fn.endswith('.s') + assert fn.endswith('.s'), fn lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: @@ -1345,6 +1618,5 @@ f.close() if output_raw_table: tracker.dump_raw_table(sys.stdout) - tracker.clear() if not output_raw_table: tracker.dump(sys.stdout) Modified: pypy/trunk/pypy/translator/c/genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/genc.py (original) +++ pypy/trunk/pypy/translator/c/genc.py Fri Nov 13 20:24:30 2009 @@ -488,23 +488,56 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - sfiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] - lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] - gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] + trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] + if self.translator.platform.name == 'msvc': + trackgcfiles = [f for f in trackgcfiles + if f.startswith(('implement', 'testing', + '../module_cache/module'))] + sfiles = ['%s.s' % (c,) for c in trackgcfiles] + lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] + gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] mk.definition('ASMFILES', sfiles) mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') - mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') - mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: - python = "" - mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') - mk.rule('gcmaptable.s', '$(GCMAPFILES)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + python = '' + + if self.translator.platform.name == 'msvc': + lblofiles = [] + for cfile in mk.cfiles: + f = cfile[:-2] + if f in trackgcfiles: + ofile = '%s.lbl.obj' % (f,) + else: + ofile = '%s.obj' % (f,) + + lblofiles.append(ofile) + mk.definition('ASMLBLOBJFILES', lblofiles) + mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') + # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory + # even in debug builds + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Oi /Ob1') + mk.rule('.SUFFIXES', '.s', []) + mk.rule('.s.obj', '', + 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + mk.rule('.c.gcmap', '', + ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + ) + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + + else: + mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') + mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + mk.rule('%.lbl.s %.gcmap', '%.s', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') else: mk.definition('DEBUGFLAGS', '-O1 -g') Modified: pypy/trunk/pypy/translator/c/src/char.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/char.h (original) +++ pypy/trunk/pypy/translator/c/src/char.h Fri Nov 13 20:24:30 2009 @@ -6,8 +6,8 @@ /*** binary operations ***/ -#define OP_CHAR_EQ(x,y,r) r = ((x) == (y)) -#define OP_CHAR_NE(x,y,r) r = ((x) != (y)) +#define OP_CHAR_EQ(x,y,r) r = ((unsigned char)(x) == (unsigned char)(y)) +#define OP_CHAR_NE(x,y,r) r = ((unsigned char)(x) != (unsigned char)(y)) #define OP_CHAR_LE(x,y,r) r = ((unsigned char)(x) <= (unsigned char)(y)) #define OP_CHAR_GT(x,y,r) r = ((unsigned char)(x) > (unsigned char)(y)) #define OP_CHAR_LT(x,y,r) r = ((unsigned char)(x) < (unsigned char)(y)) Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Fri Nov 13 20:24:30 2009 @@ -139,12 +139,18 @@ return 1; } +#if defined(_MSC_VER) || defined(__MINGW32__) +#define PYPY_LONG_LONG_PRINTF_FORMAT "I64" +#else +#define PYPY_LONG_LONG_PRINTF_FORMAT "ll" +#endif + static void display_startstop(const char *prefix, const char *postfix, const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); - fprintf(pypy_debug_file, "%s[%llx] %s%s%s\n%s", + fprintf(pypy_debug_file, "%s[%"PYPY_LONG_LONG_PRINTF_FORMAT"x] %s%s%s\n%s", colors, timestamp, prefix, category, postfix, debug_stop_colors); Modified: pypy/trunk/pypy/translator/c/src/mem.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/mem.h (original) +++ pypy/trunk/pypy/translator/c/src/mem.h Fri Nov 13 20:24:30 2009 @@ -17,14 +17,18 @@ could make two copies of the local variable (e.g. one in the stack and one in a register), pass one to GCROOT, and later use the other one. In practice the pypy_asm_gcroot() is often a no-op in the final - machine code and doesn't prevent most optimizations. Getting the - asm() right was tricky, though. The asm() is not volatile so that - gcc is free to delete it if the output variable is not used at all. - We need to prevent gcc from moving the asm() *before* the call that - could cause a collection; this is the purpose of the (unused) - __gcnoreorderhack input argument. Any memory input argument would - have this effect: as far as gcc knows the call instruction can modify - arbitrary memory, thus creating the order dependency that we want. */ + machine code and doesn't prevent most optimizations. */ +#ifndef _MSC_VER + +/* With gcc, getting the asm() right was tricky, though. The asm() is + not volatile so that gcc is free to delete it if the output variable + is not used at all. We need to prevent gcc from moving the asm() + *before* the call that could cause a collection; this is the purpose + of the (unused) __gcnoreorderhack input argument. Any memory input + argument would have this effect: as far as gcc knows the call + instruction can modify arbitrary memory, thus creating the order + dependency that we want. */ + #define pypy_asm_gcroot(p) ({void*_r; \ asm ("/* GCROOT %0 */" : "=g" (_r) : \ "0" (p), "m" (__gcnoreorderhack)); \ @@ -36,6 +40,31 @@ /* marker for trackgcroot.py */ #define pypy_asm_stack_bottom() asm volatile ("/* GC_STACK_BOTTOM */" : : ) +#else + +/* With the msvc Microsoft Compiler, the optimizer seems free to move + any code (even asm) that involves local memory (registers and stack). + The _ReadWriteBarrier function has an effect only where the content + of a global variable is *really* used. trackgcroot.py will remove + the extra instructions: the access to _constant_always_one_ is + removed, and the multiplication is replaced with a simple move. */ + +static __forceinline void* +pypy_asm_gcroot(void* _r1) +{ + static volatile int _constant_always_one_ = 1; + (long)_r1 *= _constant_always_one_; + _ReadWriteBarrier(); + return _r1; +} + +#define pypy_asm_keepalive(v) __asm { } +static __declspec(noinline) void pypy_asm_stack_bottom() { } + +#endif + + + #define OP_GC_ASMGCROOT_STATIC(i, r) r = \ i == 0 ? (void*)&__gcmapstart : \ i == 1 ? (void*)&__gcmapend : \ Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_newgc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_newgc.py Fri Nov 13 20:24:30 2009 @@ -622,7 +622,7 @@ import gc slong = cast_type_to_ffitype(rffi.LONG) - from pypy.rpython.lltypesystem.ll2ctypes import libc_name + from pypy.rlib.libffi import get_libc_name def callback(ll_args, ll_res, stuff): gc.collect() @@ -637,7 +637,7 @@ res[0] = -1 def f(): - libc = CDLL(libc_name) + libc = CDLL(get_libc_name()) qsort = libc.getpointer('qsort', [ffi_type_pointer, slong, slong, ffi_type_pointer], ffi_type_void) Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Fri Nov 13 20:24:30 2009 @@ -96,6 +96,15 @@ # Probably not a msvc compiler... self.version = 0 + # Try to find a masm assembler + returncode, stdout, stderr = _run_subprocess('ml.exe', '', + env=self.c_environ) + r = re.search('Macro Assembler', stderr) + if r is None and os.path.exists('c:/masm32/bin/ml.exe'): + self.masm = 'c:/masm32/bin/ml.exe' + else: + self.masm = 'ml.exe' + # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] @@ -133,6 +142,12 @@ def _args_for_shared(self, args): return ['/dll'] + args + def check___thread(self): + # __declspec(thread) does not seem to work when using assembler. + # Returning False will cause the program to use TlsAlloc functions. + # see src/thread_nt.h + return False + def _link_args_from_eci(self, eci, standalone): # Windows needs to resolve all symbols even for DLLs args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) @@ -177,10 +192,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr) stderrlines = stderr.splitlines() - for line in stderrlines[:5]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 5: - log.ERROR('...') raise CompilationError(stdout, stderr) @@ -225,20 +238,30 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags + list(eci.compile_extra)), ('LDFLAGS', self.link_flags + list(eci.link_extra)), - ('CC', self.cc) + ('CC', self.cc), + ('CC_LINK', self.link), + ('MASM', self.masm), ] for args in definitions: m.definition(*args) rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS)'), - ('%.obj', '%.c', '$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS)'), + ('.c.obj', '', '$(CC) /nologo $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: m.rule(*rule) + if self.version < 80: + m.rule('$(TARGET)', '$(OBJECTS)', + '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + else: + m.rule('$(TARGET)', '$(OBJECTS)', + ['$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', + ]) + return m def execute_makefile(self, path_to_makefile): @@ -251,14 +274,29 @@ try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))]) finally: oldcwd.chdir() self._handle_error(returncode, stdout, stderr, path.join('make')) class NMakefile(posix.GnuMakefile): - pass # for the moment + def write(self, out=None): + # nmake expands macros when it parses rules. + # Write all macros before the rules. + if out is None: + f = self.makefile_dir.join('Makefile').open('w') + else: + f = out + for line in self.lines: + if not isinstance(line, posix.Rule): + line.write(f) + for line in self.lines: + if isinstance(line, posix.Rule): + line.write(f) + f.flush() + if out is None: + f.close() class MingwPlatform(posix.BasePosix): From afa at codespeak.net Fri Nov 13 20:26:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 20:26:11 +0100 (CET) Subject: [pypy-svn] r69273 - pypy/branch/msvc-asmgcroot-2 Message-ID: <20091113192611.0414316802A@codespeak.net> Author: afa Date: Fri Nov 13 20:26:11 2009 New Revision: 69273 Removed: pypy/branch/msvc-asmgcroot-2/ Log: kill merged branch From afa at codespeak.net Fri Nov 13 20:26:16 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 13 Nov 2009 20:26:16 +0100 (CET) Subject: [pypy-svn] r69274 - pypy/branch/msvc-asmgcroot Message-ID: <20091113192616.D74C516802D@codespeak.net> Author: afa Date: Fri Nov 13 20:26:16 2009 New Revision: 69274 Removed: pypy/branch/msvc-asmgcroot/ Log: kill merged branch From fijal at codespeak.net Sat Nov 14 10:12:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 10:12:42 +0100 (CET) Subject: [pypy-svn] r69275 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091114091242.A890E168027@codespeak.net> Author: fijal Date: Sat Nov 14 10:12:39 2009 New Revision: 69275 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: (arigo, fijal, IC train) Fix pickling of exceptions Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 10:12:39 2009 @@ -142,6 +142,17 @@ raise OperationError( space.w_TypeError, space.wrap("setting exceptions's dictionary to a non-dict") ) self.w_dict = w_dict + def descr_reduce(self, space): + lst = [self.getclass(space), space.newtuple(self.args_w)] + if self.w_dict is not None and space.is_true(self.w_dict): + lst = lst + [self.w_dict] + return space.newtuple(lst) + descr_reduce.unwrap_spec = ['self', ObjSpace] + + def descr_setstate(self, space, w_dict): + w_olddict = self.getdict() + space.call_method(w_olddict, 'update', w_dict) + descr_setstate.unwrap_spec = ['self', ObjSpace, W_Root] def _new(cls, basecls=None): if basecls is None: @@ -163,6 +174,8 @@ __repr__ = interp2app(W_BaseException.descr_repr), __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_BaseException), + __reduce__ = interp2app(W_BaseException.descr_reduce), + __setstate__ = interp2app(W_BaseException.descr_setstate), message = interp_attrproperty_w('w_message', W_BaseException), args = GetSetProperty(W_BaseException.descr_getargs, W_BaseException.descr_setargs), @@ -308,7 +321,7 @@ return space.wrap("[Errno %d] %s: %s" % ( space.int_w(self.w_errno), space.str_w(self.w_strerror), - space.str_w(self.w_filename))) + space.str_w(space.repr(self.w_filename)))) return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), space.str_w(self.w_strerror))) return W_BaseException.descr_str(self, space) Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 14 10:12:39 2009 @@ -73,7 +73,7 @@ def test_environment_error(self): from exceptions import EnvironmentError ee = EnvironmentError(3, "x", "y") - assert str(ee) == "[Errno 3] x: y" + assert str(ee) == "[Errno 3] x: 'y'" assert str(EnvironmentError(3, "x")) == "[Errno 3] x" assert ee.errno == 3 assert ee.strerror == "x" @@ -172,3 +172,21 @@ if isinstance(e, type) and issubclass(e, exceptions.BaseException): assert e.__doc__, e assert e.__module__ == 'exceptions', e + + def test_reduce(self): + from exceptions import LookupError + + le = LookupError(1, 2, "a") + assert le.__reduce__() == (LookupError, (1, 2, "a")) + le.xyz = (1, 2) + assert le.__reduce__() == (LookupError, (1, 2, "a"), {"xyz": (1, 2)}) + + def test_setstate(self): + from exceptions import FutureWarning + + fw = FutureWarning() + fw.__setstate__({"xyz": (1, 2)}) + assert fw.xyz == (1, 2) + fw.__setstate__({'z': 1}) + assert fw.z == 1 + assert fw.xyz == (1, 2) From fijal at codespeak.net Sat Nov 14 13:15:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 13:15:55 +0100 (CET) Subject: [pypy-svn] r69276 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091114121555.7035C168027@codespeak.net> Author: fijal Date: Sat Nov 14 13:15:54 2009 New Revision: 69276 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: (arigo, fijal) Support getitem Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 13:15:54 2009 @@ -132,6 +132,10 @@ def descr_setargs(space, self, w_newargs): self.args_w = space.viewiterable(w_newargs) + def descr_getitem(self, space, w_index): + return space.getitem(space.newtuple(self.args_w), w_index) + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + def getdict(self): if self.w_dict is None: self.w_dict = self.space.newdict() @@ -174,6 +178,7 @@ __repr__ = interp2app(W_BaseException.descr_repr), __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, cls=W_BaseException), + __getitem__ = interp2app(W_BaseException.descr_getitem), __reduce__ = interp2app(W_BaseException.descr_reduce), __setstate__ = interp2app(W_BaseException.descr_setstate), message = interp_attrproperty_w('w_message', W_BaseException), Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 14 13:15:54 2009 @@ -25,6 +25,9 @@ assert x.xyz == 3 x.args = [42] assert x.args == (42,) + assert x[0] == 42 + x.args = (1, 2, 3) + assert x[1:2] == (2,) def test_exc(self): from exceptions import Exception, BaseException From fijal at codespeak.net Sat Nov 14 13:17:13 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 13:17:13 +0100 (CET) Subject: [pypy-svn] r69277 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091114121713.36DD1168027@codespeak.net> Author: fijal Date: Sat Nov 14 13:17:12 2009 New Revision: 69277 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: (arigo, fijal) Apparently stdlib modify message attribute Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 13:17:12 2009 @@ -181,7 +181,7 @@ __getitem__ = interp2app(W_BaseException.descr_getitem), __reduce__ = interp2app(W_BaseException.descr_reduce), __setstate__ = interp2app(W_BaseException.descr_setstate), - message = interp_attrproperty_w('w_message', W_BaseException), + message = readwrite_attrproperty_w('w_message', W_BaseException), args = GetSetProperty(W_BaseException.descr_getargs, W_BaseException.descr_setargs), ) Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 14 13:17:12 2009 @@ -28,6 +28,8 @@ assert x[0] == 42 x.args = (1, 2, 3) assert x[1:2] == (2,) + x.message = "xyz" + assert x.message == "xyz" def test_exc(self): from exceptions import Exception, BaseException From fijal at codespeak.net Sat Nov 14 13:53:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 13:53:07 +0100 (CET) Subject: [pypy-svn] r69278 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091114125307.195C7168024@codespeak.net> Author: fijal Date: Sat Nov 14 13:53:06 2009 New Revision: 69278 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: (arigo, fijal) Split __init__ and __new__ in Exceptions Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 13:53:06 2009 @@ -73,7 +73,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict -from pypy.interpreter.gateway import interp2app +from pypy.interpreter.gateway import interp2app, Arguments from pypy.interpreter.error import OperationError from pypy.rlib import rwin32 @@ -98,14 +98,19 @@ and will be deprecated at some point. """ w_dict = None + args_w = [] - def __init__(self, space, args_w): - self.args_w = args_w + def __init__(self, space): self.space = space + self.w_message = space.w_None + + def descr_init(self, space, args_w): + self.args_w = args_w if len(args_w) == 1: self.w_message = args_w[0] else: self.w_message = space.wrap("") + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): lgt = len(self.args_w) @@ -161,11 +166,11 @@ def _new(cls, basecls=None): if basecls is None: basecls = cls - def descr_new_base_exception(space, w_subtype, args_w): + def descr_new_base_exception(space, w_subtype, __args__): exc = space.allocate_instance(cls, w_subtype) - basecls.__init__(exc, space, args_w) + basecls.__init__(exc, space) return space.wrap(exc) - descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] + descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, Arguments] descr_new_base_exception.func_name = 'descr_new_' + cls.__name__ return interp2app(descr_new_base_exception) @@ -174,6 +179,7 @@ __doc__ = W_BaseException.__doc__, __module__ = 'exceptions', __new__ = _new(W_BaseException), + __init__ = interp2app(W_BaseException.descr_init), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, @@ -237,13 +243,20 @@ class W_UnicodeTranslateError(W_UnicodeError): """Unicode translation error.""" - def __init__(self, space, w_object, w_start, w_end, w_reason): + object = u'' + start = 0 + end = 0 + reason = '' + + def descr_init(self, space, w_object, w_start, w_end, w_reason): self.object = space.unicode_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_object, w_start, - w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_object, w_start, + w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([space.wrap(self)], r"""(self): @@ -258,19 +271,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_translate_error(space, w_subtype, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeTranslateError, w_subtype) - W_UnicodeTranslateError.__init__(exc, space, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeTranslateError.typedef = TypeDef( 'UnicodeTranslateError', W_UnicodeError.typedef, __doc__ = W_UnicodeTranslateError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_translate_error), + __new__ = _new(W_UnicodeTranslateError), + __init__ = interp2app(W_UnicodeTranslateError.descr_init), __str__ = interp2app(W_UnicodeTranslateError.descr_str), object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), start = readwrite_attrproperty('start', W_UnicodeTranslateError, 'int_w'), @@ -307,17 +314,21 @@ class W_EnvironmentError(W_StandardError): """Base class for I/O related errors.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) + def __init__(self, space): self.w_errno = space.w_None self.w_strerror = space.w_None - self.w_filename = space.w_None + self.w_filename = space.w_None + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): + W_BaseException.descr_init(self, space, args_w) if 2 <= len(args_w) <= 3: self.w_errno = args_w[0] self.w_strerror = args_w[1] if len(args_w) == 3: self.w_filename = args_w[2] self.args_w = [args_w[0], args_w[1]] + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): if (not space.is_w(self.w_errno, space.w_None) and @@ -338,6 +349,7 @@ __doc__ = W_EnvironmentError.__doc__, __module__ = 'exceptions', __new__ = _new(W_EnvironmentError), + __init__ = interp2app(W_EnvironmentError.descr_init), __str__ = interp2app(W_EnvironmentError.descr_str), errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), strerror = readwrite_attrproperty_w('w_strerror', W_EnvironmentError), @@ -350,10 +362,14 @@ class W_WindowsError(W_OSError): """MS-Windows OS system call failed.""" - def __init__(self, space, args_w): - W_OSError.__init__(self, space, args_w) + def __init__(self, space): + self.w_winerror = space.w_None + W_OSError.__init__(self, space) + + def descr_init(self, space, args_w): # Set errno to the POSIX errno, and winerror to the Win32 # error code. + W_OSError.descr_init(self, space, args_w) try: errno = space.int_w(self.w_errno) except OperationError: @@ -362,6 +378,7 @@ errno = self._winerror_to_errno.get(errno, self._default_errno) self.w_winerror = self.w_errno self.w_errno = space.wrap(errno) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): if (not space.is_w(self.w_winerror, space.w_None) and @@ -387,6 +404,7 @@ __doc__ = W_WindowsError.__doc__, __module__ = 'exceptions', __new__ = _new(W_WindowsError), + __init__ = interp2app(W_WindowsError.descr_init), __str__ = interp2app(W_WindowsError.descr_str), winerror = readwrite_attrproperty_w('w_winerror', W_WindowsError), ) @@ -413,8 +431,15 @@ class W_SyntaxError(W_StandardError): """Invalid syntax.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) + def __init__(self, space): + self.w_filename = space.w_None + self.w_lineno = space.w_None + self.w_offset = space.w_None + self.w_text = space.w_None + self.w_msg = space.w_None + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): # that's not a self.w_message!!! if len(args_w) > 0: self.w_msg = args_w[0] @@ -426,11 +451,8 @@ self.w_lineno = values_w[1] self.w_offset = values_w[2] self.w_text = values_w[3] - else: - self.w_filename = space.w_None - self.w_lineno = space.w_None - self.w_offset = space.w_None - self.w_text = space.w_None + W_BaseException.descr_init(self, space, args_w) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): return space.appexec([self], """(self): @@ -458,6 +480,7 @@ 'SyntaxError', W_StandardError.typedef, __new__ = _new(W_SyntaxError), + __init__ = interp2app(W_SyntaxError.descr_init), __str__ = interp2app(W_SyntaxError.descr_str), __doc__ = W_SyntaxError.__doc__, __module__ = 'exceptions', @@ -474,19 +497,23 @@ class W_SystemExit(W_BaseException): """Request to exit from the interpreter.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) - if len(args_w) == 0: - self.w_code = space.w_None - elif len(args_w) == 1: + def __init__(self, space): + self.w_code = space.w_None + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): + if len(args_w) == 1: self.w_code = args_w[0] - else: + elif len(args_w) > 1: self.w_code = space.newtuple(args_w) + W_BaseException.descr_init(self, space, args_w) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] W_SystemExit.typedef = TypeDef( 'SystemExit', W_BaseException.typedef, __new__ = _new(W_SystemExit), + __init__ = interp2app(W_SystemExit.descr_init), __doc__ = W_SystemExit.__doc__, __module__ = 'exceptions', code = readwrite_attrproperty_w('w_code', W_SystemExit) @@ -515,15 +542,22 @@ class W_UnicodeDecodeError(W_UnicodeError): """Unicode decoding error.""" + encoding = '' + object = '' + start = 0 + end = 0 + reason = '' - def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): self.encoding = space.str_w(w_encoding) self.object = space.str_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_encoding, w_object, - w_start, w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([self], """(self): @@ -536,19 +570,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_decode_error(space, w_subtype, w_encoding, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeDecodeError, w_subtype) - W_UnicodeDecodeError.__init__(exc, space, w_encoding, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeDecodeError.typedef = TypeDef( 'UnicodeDecodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeDecodeError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_decode_error), + __new__ = _new(W_UnicodeDecodeError), + __init__ = interp2app(W_UnicodeDecodeError.descr_init), __str__ = interp2app(W_UnicodeDecodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), object = readwrite_attrproperty('object', W_UnicodeDecodeError, 'str_w'), @@ -600,14 +628,22 @@ class W_UnicodeEncodeError(W_UnicodeError): """Unicode encoding error.""" - def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + encoding = '' + object = u'' + start = 0 + end = 0 + reason = '' + + def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): self.encoding = space.str_w(w_encoding) self.object = space.unicode_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_encoding, w_object, - w_start, w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([self], r"""(self): @@ -626,19 +662,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_encode_error(space, w_subtype, w_encoding, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeEncodeError, w_subtype) - W_UnicodeEncodeError.__init__(exc, space, w_encoding, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeEncodeError.typedef = TypeDef( 'UnicodeEncodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeEncodeError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_encode_error), + __new__ = _new(W_UnicodeEncodeError), + __init__ = interp2app(W_UnicodeEncodeError.descr_init), __str__ = interp2app(W_UnicodeEncodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), object = readwrite_attrproperty('object', W_UnicodeEncodeError, 'unicode_w'), Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 14 13:53:06 2009 @@ -31,6 +31,15 @@ x.message = "xyz" assert x.message == "xyz" + def test_kwargs(self): + from exceptions import Exception + class X(Exception): + def __init__(self, x=3): + self.x = x + + x = X(x=8) + assert x.x == 8 + def test_exc(self): from exceptions import Exception, BaseException From fijal at codespeak.net Sat Nov 14 14:08:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:08:05 +0100 (CET) Subject: [pypy-svn] r69279 - pypy/branch/faster-raise/pypy/module/exceptions Message-ID: <20091114130805.E75E5168025@codespeak.net> Author: fijal Date: Sat Nov 14 14:08:05 2009 New Revision: 69279 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Log: Add strange attribute, nobody ever sets it Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 14:08:05 2009 @@ -436,15 +436,14 @@ self.w_lineno = space.w_None self.w_offset = space.w_None self.w_text = space.w_None - self.w_msg = space.w_None + self.w_msg = space.wrap('') + self.w_print_file_and_line = space.w_None # what's that? W_BaseException.__init__(self, space) def descr_init(self, space, args_w): # that's not a self.w_message!!! if len(args_w) > 0: self.w_msg = args_w[0] - else: - self.w_msg = space.wrap('') if len(args_w) == 2: values_w = space.viewiterable(args_w[1], 4) self.w_filename = values_w[0] @@ -489,6 +488,8 @@ lineno = readwrite_attrproperty_w('w_lineno', W_SyntaxError), offset = readwrite_attrproperty_w('w_offset', W_SyntaxError), text = readwrite_attrproperty_w('w_text', W_SyntaxError), + print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', + W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, From fijal at codespeak.net Sat Nov 14 14:39:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:39:45 +0100 (CET) Subject: [pypy-svn] r69280 - pypy/trunk/pypy/rpython Message-ID: <20091114133945.2F2A016802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:39:44 2009 New Revision: 69280 Modified: pypy/trunk/pypy/rpython/normalizecalls.py pypy/trunk/pypy/rpython/rpbc.py Log: (arigo, fijal) Improve speec of jitting by not looping over all callfamilies (1/2) Modified: pypy/trunk/pypy/rpython/normalizecalls.py ============================================================================== --- pypy/trunk/pypy/rpython/normalizecalls.py (original) +++ pypy/trunk/pypy/rpython/normalizecalls.py Sat Nov 14 14:39:44 2009 @@ -14,8 +14,12 @@ def normalize_call_familes(annotator): for callfamily in annotator.bookkeeper.pbc_maximal_call_families.infos(): + if not callfamily.modified: + assert callfamily.normalized + continue normalize_calltable(annotator, callfamily) callfamily.normalized = True + callfamily.modified = False def normalize_calltable(annotator, callfamily): """Try to normalize all rows of a table.""" Modified: pypy/trunk/pypy/rpython/rpbc.py ============================================================================== --- pypy/trunk/pypy/rpython/rpbc.py (original) +++ pypy/trunk/pypy/rpython/rpbc.py Sat Nov 14 14:39:44 2009 @@ -484,7 +484,8 @@ try: thisattrvalue = frozendesc.attrcache[attr] except KeyError: - warning("Desc %r has no attribute %r" % (frozendesc, attr)) + if not frozendesc.has_attribute(attr): + warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) setattr(result, mangled_name, llvalue) From fijal at codespeak.net Sat Nov 14 14:40:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:40:21 +0100 (CET) Subject: [pypy-svn] r69281 - pypy/trunk/pypy/annotation Message-ID: <20091114134021.7DB5216802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:40:21 2009 New Revision: 69281 Modified: pypy/trunk/pypy/annotation/description.py Log: (arigo, fijal) Improve speec of jitting by not looping over all callfamilies (2/2) Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sat Nov 14 14:40:21 2009 @@ -14,6 +14,7 @@ """ overridden = False normalized = False + modified = True def __init__(self, desc): self.descs = { desc: True } @@ -21,6 +22,7 @@ self.total_calltable_size = 0 def update(self, other): + self.modified = True self.normalized = self.normalized or other.normalized self.descs.update(other.descs) for shape, table in other.calltables.items(): @@ -33,7 +35,7 @@ # call at which call site. Each call site gets a row of graphs, # sharable with other call sites. Each column is a FunctionDesc. # There is one such table per "call shape". - table = self.calltables.setdefault(callshape, []) + table = self.calltables.get(callshape, []) for i, existing_row in enumerate(table): if existing_row == row: # XXX maybe use a dict again here? return i @@ -43,6 +45,7 @@ try: self.calltable_lookup_row(callshape, row) except LookupError: + self.modified = True table = self.calltables.setdefault(callshape, []) table.append(row) self.total_calltable_size += 1 @@ -802,6 +805,15 @@ self.knowntype = new_or_old_class(pyobj) assert bool(pyobj), "__nonzero__ unsupported on frozen PBC %r" %(pyobj,) + def has_attribute(self, attr): + if attr in self.attrcache: + return True + try: + self._read_attribute(attr) + return True + except AttributeError: + return False + def read_attribute(self, attr): try: return self.attrcache[attr] From fijal at codespeak.net Sat Nov 14 14:41:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:41:33 +0100 (CET) Subject: [pypy-svn] r69282 - pypy/branch/direct-assembler-call Message-ID: <20091114134133.BA80B16802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:41:33 2009 New Revision: 69282 Added: pypy/branch/direct-assembler-call/ - copied from r69281, pypy/trunk/ Log: A branch to implement direct calls from assembler to more assembler From fijal at codespeak.net Sat Nov 14 14:42:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:42:34 +0100 (CET) Subject: [pypy-svn] r69283 - in pypy/branch/direct-assembler-call/pypy/jit/metainterp: . test Message-ID: <20091114134234.5D6E116802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:42:34 2009 New Revision: 69283 Modified: pypy/branch/direct-assembler-call/pypy/jit/metainterp/pyjitpl.py pypy/branch/direct-assembler-call/pypy/jit/metainterp/test/test_recursive.py pypy/branch/direct-assembler-call/pypy/jit/metainterp/warmstate.py Log: (arigo, fijal) IN-PROGRESS start thinking about direct calls Modified: pypy/branch/direct-assembler-call/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/direct-assembler-call/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/direct-assembler-call/pypy/jit/metainterp/pyjitpl.py Sat Nov 14 14:42:34 2009 @@ -42,16 +42,10 @@ argtypes = unrolling_iterable(self.argtypes) def wrapped(self, orgpc): args = (self, ) - #if DEBUG >= DEBUG_DETAILED: - # s = '%s:%d\t%s' % (self.jitcode.name, orgpc, name) - #else: - s = '' for argspec in argtypes: if argspec == "box": box = self.load_arg() args += (box, ) - #if DEBUG >= DEBUG_DETAILED: - # s += '\t' + box.repr_rpython() elif argspec == "constbox": args += (self.load_const_arg(), ) elif argspec == "int": @@ -82,12 +76,7 @@ args += (methdescr, ) else: assert 0, "unknown argtype declaration: %r" % (argspec,) - #if DEBUG >= DEBUG_DETAILED: - # debug_print(s) val = func(*args) - #if DEBUG >= DEBUG_DETAILED: - # reprboxes = ' '.join([box.repr_rpython() for box in self.env]) - # debug_print(' \x1b[34menv=[%s]\x1b[0m' % (reprboxes,)) if val is None: val = False return val @@ -675,6 +664,9 @@ greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): return self.perform_call(portal_code, varargs[1:], greenkey) + token = warmrunnerstate.get_assembler_token(greenkey) + if token is not None: + return self.metainterp.direct_assembler_call(varargs, token) return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") @@ -1924,6 +1916,15 @@ max_key = key return max_key + def direct_assembler_call(self, varargs, token): + """ Generate a direct call to assembler for portal entry point. + """ + num_green_args = self.staticdata.num_green_args + assert self.staticdata.virtualizable_info is None # XXX + args = varargs[num_green_args + 1:] + xxx + self.framestack[-1].execute_varargs(rop.CALL_ASSEMBLER, args, + descr=token, exc=False) class GenerateMergePoint(Exception): def __init__(self, args, target_loop_token): Modified: pypy/branch/direct-assembler-call/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/direct-assembler-call/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/direct-assembler-call/pypy/jit/metainterp/test/test_recursive.py Sat Nov 14 14:42:34 2009 @@ -647,7 +647,22 @@ self.check_aborted_count(1) self.check_history(call=1) self.check_tree_loop_count(3) - + + def test_directly_call_assembler(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno : str(codeno), + can_inline = lambda codeno : False) + + def portal(codeno): + i = 0 + while i < 10: + driver.can_enter_jit(codeno = codeno, i = i) + driver.jit_merge_point(codeno = codeno, i = i) + if codeno == 2: + portal(1) + i += 1 + + self.meta_interp(portal, [2], inline=True) class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/direct-assembler-call/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/direct-assembler-call/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/direct-assembler-call/pypy/jit/metainterp/warmstate.py Sat Nov 14 14:42:34 2009 @@ -445,6 +445,7 @@ unwrap_greenkey = self.make_unwrap_greenkey() if can_inline_ptr is None: def can_inline_callable(*greenargs): + # XXX shouldn't it be False by default? return True else: rtyper = self.warmrunnerdesc.rtyper @@ -462,7 +463,16 @@ greenargs = unwrap_greenkey(greenkey) return can_inline(*greenargs) self.can_inline_callable = can_inline_greenkey - + + get_jitcell = self.make_jitcell_getter() + def get_assembler_token(greenkey): + greenargs = unwrap_greenkey(greenkey) + cell = get_jitcell(*greenargs) + if cell.counter >= 0: + return None + return cell.entry_loop_token + self.get_assembler_token = get_assembler_token + # get_location_ptr = self.warmrunnerdesc.get_printable_location_ptr if get_location_ptr is None: From fijal at codespeak.net Sat Nov 14 14:45:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:45:32 +0100 (CET) Subject: [pypy-svn] r69284 - pypy/branch/faster-raise/pypy/interpreter/pyparser Message-ID: <20091114134532.3A11416802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:45:31 2009 New Revision: 69284 Modified: pypy/branch/faster-raise/pypy/interpreter/pyparser/error.py Log: (fijal, arigo) Kill unused line. Modified: pypy/branch/faster-raise/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyparser/error.py Sat Nov 14 14:45:31 2009 @@ -8,7 +8,6 @@ self.offset = offset self.text = text self.filename = filename - self.print_file_and_line = False def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), From fijal at codespeak.net Sat Nov 14 14:46:38 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 14:46:38 +0100 (CET) Subject: [pypy-svn] r69285 - in pypy/branch/faster-raise/pypy: interpreter module/exceptions module/exceptions/test objspace/std Message-ID: <20091114134638.689E816802D@codespeak.net> Author: fijal Date: Sat Nov 14 14:46:37 2009 New Revision: 69285 Modified: pypy/branch/faster-raise/pypy/interpreter/typedef.py pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py Log: (fijal, arigo) Move descr_del_dict to interpreter/typedef.py and use it from module/exceptions. Modified: pypy/branch/faster-raise/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/typedef.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/typedef.py Sat Nov 14 14:46:37 2009 @@ -562,6 +562,9 @@ def descr_set_dict(space, w_obj, w_dict): w_obj.setdict(space, w_dict) +def descr_del_dict(space, w_obj): # blame CPython for the existence of this one + w_obj.setdict(space, space.newdict()) + def descr_get_weakref(space, w_obj): lifeline = w_obj.getweakref() if lifeline is None: Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 14 14:46:37 2009 @@ -72,7 +72,8 @@ from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ - GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict + GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict,\ + descr_del_dict from pypy.interpreter.gateway import interp2app, Arguments from pypy.interpreter.error import OperationError from pypy.rlib import rwin32 @@ -182,7 +183,7 @@ __init__ = interp2app(W_BaseException.descr_init), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), - __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, + __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict, cls=W_BaseException), __getitem__ = interp2app(W_BaseException.descr_getitem), __reduce__ = interp2app(W_BaseException.descr_reduce), Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 14 14:46:37 2009 @@ -40,6 +40,15 @@ x = X(x=8) assert x.x == 8 + def test_catch_with_unpack(self): + from exceptions import LookupError + + try: + raise LookupError(1, 2) + except LookupError, (one, two): + assert one == 1 + assert two == 2 + def test_exc(self): from exceptions import Exception, BaseException Modified: pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/stdtypedef.py Sat Nov 14 14:46:37 2009 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty, Member from pypy.interpreter.typedef import descr_get_dict, descr_set_dict -from pypy.interpreter.typedef import no_hash_descr +from pypy.interpreter.typedef import no_hash_descr, descr_del_dict from pypy.interpreter.baseobjspace import SpaceCache from pypy.objspace.std.model import StdObjSpaceMultiMethod from pypy.objspace.std.multimethod import FailedToImplement @@ -39,9 +39,6 @@ a = a.base return True -def descr_del_dict(space, w_obj): # blame CPython for the existence of this one - w_obj.setdict(space, space.newdict()) - std_dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict) std_dict_descr.name = '__dict__' From fijal at codespeak.net Sat Nov 14 15:20:06 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 15:20:06 +0100 (CET) Subject: [pypy-svn] r69286 - pypy/branch/faster-raise Message-ID: <20091114142006.D69E316802D@codespeak.net> Author: fijal Date: Sat Nov 14 15:20:06 2009 New Revision: 69286 Removed: pypy/branch/faster-raise/pytest_resultlog.py Modified: pypy/branch/faster-raise/ (props changed) Log: Remote pylib from svn:externals From fijal at codespeak.net Sat Nov 14 15:24:21 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 14 Nov 2009 15:24:21 +0100 (CET) Subject: [pypy-svn] r69287 - in pypy/branch/faster-raise: dotviewer lib-python py pypy pypy/annotation pypy/annotation/test pypy/bin pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/doc/tool pypy/jit pypy/jit/backend pypy/jit/backend/cli pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/metainterp pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tl/test pypy/jit/tl/tla pypy/jit/tool pypy/lang/gameboy pypy/lang/gameboy/debug pypy/lang/gameboy/profiling pypy/lang/gameboy/profiling/evaluation pypy/lang/gameboy/test pypy/lang/gameboy/tool pypy/lang/js pypy/lang/js/test pypy/lang/js/test/ecma pypy/lang/prolog/interpreter pypy/lang/scheme pypy/lang/scheme/test pypy/lang/smalltalk/test pypy/lang/smalltalk/tool pypy/lib pypy/lib/app_test/ctypes_tests pypy/lib/distributed pypy/lib/test2 pypy/module/__builtin__/test pypy/module/_codecs/test pypy/module/_file/test pypy/module/_minimal_curses pypy/module/_sre/test pypy/module/bz2/test pypy/module/gc pypy/module/pypyjit/test pypy/module/sys pypy/module/sys/test pypy/objspace/std/test pypy/rlib pypy/rlib/parsing pypy/rlib/parsing/test pypy/rlib/rsdl pypy/rlib/rsdl/test pypy/rlib/rstruct pypy/rlib/test pypy/rpython pypy/rpython/memory/gctransform pypy/rpython/microbench pypy/rpython/module/test pypy/rpython/test pypy/tool pypy/tool/algo/test pypy/tool/bench pypy/tool/pytest pypy/tool/pytest/test pypy/tool/rest pypy/tool/test pypy/translator pypy/translator/backendopt/test pypy/translator/benchmark pypy/translator/c pypy/translator/c/gcc pypy/translator/c/gcc/test pypy/translator/c/gcc/test/msvc pypy/translator/c/src pypy/translator/c/test pypy/translator/cli pypy/translator/cli/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/jvm pypy/translator/microbench/pybench pypy/translator/platform pypy/translator/platform/test pypy/translator/sandbox pypy/translator/sandbox/test pypy/translator/test pypy/translator/tool Message-ID: <20091114142421.034ED16802D@codespeak.net> Author: fijal Date: Sat Nov 14 15:24:15 2009 New Revision: 69287 Added: pypy/branch/faster-raise/py/ - copied from r69286, pypy/trunk/py/ pypy/branch/faster-raise/pypy/tool/difftime.py - copied unchanged from r69286, pypy/trunk/pypy/tool/difftime.py pypy/branch/faster-raise/pypy/tool/rest/ (props changed) - copied from r69286, pypy/trunk/pypy/tool/rest/ pypy/branch/faster-raise/pypy/translator/c/gcc/autopath.py - copied unchanged from r69286, pypy/trunk/pypy/translator/c/gcc/autopath.py pypy/branch/faster-raise/pypy/translator/c/gcc/instruction.py - copied unchanged from r69286, pypy/trunk/pypy/translator/c/gcc/instruction.py pypy/branch/faster-raise/pypy/translator/c/gcc/test/msvc/ - copied from r69286, pypy/trunk/pypy/translator/c/gcc/test/msvc/ Modified: pypy/branch/faster-raise/dotviewer/conftest.py pypy/branch/faster-raise/lib-python/conftest.py pypy/branch/faster-raise/pypy/annotation/description.py pypy/branch/faster-raise/pypy/annotation/test/autopath.py pypy/branch/faster-raise/pypy/bin/autopath.py pypy/branch/faster-raise/pypy/bin/py.py pypy/branch/faster-raise/pypy/config/autopath.py pypy/branch/faster-raise/pypy/config/config.py pypy/branch/faster-raise/pypy/config/makerestdoc.py pypy/branch/faster-raise/pypy/config/pypyoption.py pypy/branch/faster-raise/pypy/config/test/test_makerestdoc.py pypy/branch/faster-raise/pypy/config/test/test_pypyoption.py pypy/branch/faster-raise/pypy/conftest.py pypy/branch/faster-raise/pypy/doc/config/autopath.py pypy/branch/faster-raise/pypy/doc/config/generate.py pypy/branch/faster-raise/pypy/doc/config/makemodules.py pypy/branch/faster-raise/pypy/doc/config/objspace.std.immutable_builtintypes.txt (contents, props changed) pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules._locale.txt (props changed) pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules._winreg.txt (contents, props changed) pypy/branch/faster-raise/pypy/doc/confrest.py pypy/branch/faster-raise/pypy/doc/confrest_oldpy.py pypy/branch/faster-raise/pypy/doc/conftest.py pypy/branch/faster-raise/pypy/doc/index.txt pypy/branch/faster-raise/pypy/doc/rlib.txt pypy/branch/faster-raise/pypy/doc/statistic/confrest.py pypy/branch/faster-raise/pypy/doc/test_redirections.py pypy/branch/faster-raise/pypy/doc/tool/makeref.py pypy/branch/faster-raise/pypy/doc/tool/mydot.py pypy/branch/faster-raise/pypy/jit/backend/autopath.py pypy/branch/faster-raise/pypy/jit/backend/cli/method.py pypy/branch/faster-raise/pypy/jit/backend/test/conftest.py pypy/branch/faster-raise/pypy/jit/backend/x86/autopath.py pypy/branch/faster-raise/pypy/jit/backend/x86/detect_sse2.py (props changed) pypy/branch/faster-raise/pypy/jit/backend/x86/valgrind.py (props changed) pypy/branch/faster-raise/pypy/jit/conftest.py pypy/branch/faster-raise/pypy/jit/metainterp/codewriter.py pypy/branch/faster-raise/pypy/jit/metainterp/logger.py pypy/branch/faster-raise/pypy/jit/metainterp/optimizeopt.py pypy/branch/faster-raise/pypy/jit/metainterp/optimizeutil.py (props changed) pypy/branch/faster-raise/pypy/jit/metainterp/pyjitpl.py pypy/branch/faster-raise/pypy/jit/metainterp/test/test_codewriter.py (contents, props changed) pypy/branch/faster-raise/pypy/jit/metainterp/test/test_list.py pypy/branch/faster-raise/pypy/jit/metainterp/test/test_logger.py pypy/branch/faster-raise/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/faster-raise/pypy/jit/tl/autopath.py pypy/branch/faster-raise/pypy/jit/tl/conftest.py pypy/branch/faster-raise/pypy/jit/tl/spli/autopath.py pypy/branch/faster-raise/pypy/jit/tl/targettlc.py pypy/branch/faster-raise/pypy/jit/tl/targettlr.py pypy/branch/faster-raise/pypy/jit/tl/test/test_pypyjit.py pypy/branch/faster-raise/pypy/jit/tl/tla/targettla.py pypy/branch/faster-raise/pypy/jit/tl/tla/tla_assembler.py pypy/branch/faster-raise/pypy/jit/tool/autopath.py pypy/branch/faster-raise/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py pypy/branch/faster-raise/pypy/lang/gameboy/debug/gameboy_debug_parts.py (props changed) pypy/branch/faster-raise/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py pypy/branch/faster-raise/pypy/lang/gameboy/profiling/gameboyTest.py pypy/branch/faster-raise/pypy/lang/gameboy/test/test_cartridge.py pypy/branch/faster-raise/pypy/lang/gameboy/test/test_rom.py pypy/branch/faster-raise/pypy/lang/gameboy/tool/autopath.py pypy/branch/faster-raise/pypy/lang/gameboy/video_meta.py (props changed) pypy/branch/faster-raise/pypy/lang/js/autopath.py pypy/branch/faster-raise/pypy/lang/js/jsparser.py pypy/branch/faster-raise/pypy/lang/js/test/ecma/conftest.py pypy/branch/faster-raise/pypy/lang/js/test/test_interactive.py pypy/branch/faster-raise/pypy/lang/js/test/test_parser.py pypy/branch/faster-raise/pypy/lang/prolog/interpreter/autopath.py pypy/branch/faster-raise/pypy/lang/prolog/interpreter/conftest.py pypy/branch/faster-raise/pypy/lang/prolog/interpreter/interactive.py pypy/branch/faster-raise/pypy/lang/prolog/interpreter/parsing.py pypy/branch/faster-raise/pypy/lang/scheme/autopath.py pypy/branch/faster-raise/pypy/lang/scheme/execution.py pypy/branch/faster-raise/pypy/lang/scheme/test/test_interactive.py pypy/branch/faster-raise/pypy/lang/smalltalk/test/test_miniimage.py pypy/branch/faster-raise/pypy/lang/smalltalk/tool/analyseimage.py pypy/branch/faster-raise/pypy/lang/smalltalk/tool/autopath.py pypy/branch/faster-raise/pypy/lib/_csv.py (props changed) pypy/branch/faster-raise/pypy/lib/_sha256.py (props changed) pypy/branch/faster-raise/pypy/lib/_sha512.py (props changed) pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/conftest.py pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/test_win32.py (contents, props changed) pypy/branch/faster-raise/pypy/lib/distributed/socklayer.py pypy/branch/faster-raise/pypy/lib/test2/autopath.py pypy/branch/faster-raise/pypy/module/__builtin__/test/autopath.py pypy/branch/faster-raise/pypy/module/__builtin__/test/test_import.py pypy/branch/faster-raise/pypy/module/_codecs/test/autopath.py pypy/branch/faster-raise/pypy/module/_file/test/test_file_extra.py pypy/branch/faster-raise/pypy/module/_minimal_curses/fficurses.py pypy/branch/faster-raise/pypy/module/_sre/test/autopath.py pypy/branch/faster-raise/pypy/module/_sre/test/test_app_sre.py pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_compdecomp.py pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_file.py pypy/branch/faster-raise/pypy/module/gc/__init__.py pypy/branch/faster-raise/pypy/module/pypyjit/test/conftest.py pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/faster-raise/pypy/module/sys/test/autopath.py pypy/branch/faster-raise/pypy/module/sys/version.py pypy/branch/faster-raise/pypy/objspace/std/test/test_complexobject.py pypy/branch/faster-raise/pypy/rlib/parsing/ebnfparse.py pypy/branch/faster-raise/pypy/rlib/parsing/lexer.py pypy/branch/faster-raise/pypy/rlib/parsing/makepackrat.py pypy/branch/faster-raise/pypy/rlib/parsing/regexparse.py pypy/branch/faster-raise/pypy/rlib/parsing/test/autopath.py pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonlexer.py pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonparse.py pypy/branch/faster-raise/pypy/rlib/rsdl/RMix_helper.py (props changed) pypy/branch/faster-raise/pypy/rlib/rsdl/eci.py pypy/branch/faster-raise/pypy/rlib/rsdl/test/autopath.py pypy/branch/faster-raise/pypy/rlib/rstruct/__init__.py (props changed) pypy/branch/faster-raise/pypy/rlib/rwinreg.py (contents, props changed) pypy/branch/faster-raise/pypy/rlib/test/test_listsort.py pypy/branch/faster-raise/pypy/rlib/test/test_timer.py (props changed) pypy/branch/faster-raise/pypy/rlib/timer.py (props changed) pypy/branch/faster-raise/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/faster-raise/pypy/rpython/microbench/autopath.py pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_path.py pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_stat.py (contents, props changed) pypy/branch/faster-raise/pypy/rpython/normalizecalls.py pypy/branch/faster-raise/pypy/rpython/rpbc.py pypy/branch/faster-raise/pypy/rpython/test/test_rbuiltin.py pypy/branch/faster-raise/pypy/test_all.py pypy/branch/faster-raise/pypy/tool/algo/test/autopath.py pypy/branch/faster-raise/pypy/tool/ansi_mandelbrot.py pypy/branch/faster-raise/pypy/tool/ansi_print.py pypy/branch/faster-raise/pypy/tool/autopath.py pypy/branch/faster-raise/pypy/tool/bench/pypyresult.py pypy/branch/faster-raise/pypy/tool/genstatistic.py pypy/branch/faster-raise/pypy/tool/option.py pypy/branch/faster-raise/pypy/tool/pytest/appsupport.py pypy/branch/faster-raise/pypy/tool/pytest/autopath.py pypy/branch/faster-raise/pypy/tool/pytest/genreportdata.py pypy/branch/faster-raise/pypy/tool/pytest/htmlreport.py pypy/branch/faster-raise/pypy/tool/pytest/test/test_appsupport.py (props changed) pypy/branch/faster-raise/pypy/tool/pytest/test/test_new_count.py pypy/branch/faster-raise/pypy/tool/statistic_over_time.py pypy/branch/faster-raise/pypy/tool/test/autopath.py pypy/branch/faster-raise/pypy/tool/test/test_conftest1.py pypy/branch/faster-raise/pypy/tool/test/test_pytestsupport.py pypy/branch/faster-raise/pypy/tool/udir.py pypy/branch/faster-raise/pypy/tool/watchdog_nt.py (props changed) pypy/branch/faster-raise/pypy/translator/autopath.py pypy/branch/faster-raise/pypy/translator/backendopt/test/test_writeanalyze.py pypy/branch/faster-raise/pypy/translator/benchmark/autopath.py pypy/branch/faster-raise/pypy/translator/benchmark/benchmarks.py pypy/branch/faster-raise/pypy/translator/benchmark/jitbench.py pypy/branch/faster-raise/pypy/translator/c/autopath.py pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_asmgcroot.py pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/faster-raise/pypy/translator/c/gcc/trackgcroot.py pypy/branch/faster-raise/pypy/translator/c/genc.py pypy/branch/faster-raise/pypy/translator/c/src/char.h pypy/branch/faster-raise/pypy/translator/c/src/debug.h pypy/branch/faster-raise/pypy/translator/c/src/mem.h pypy/branch/faster-raise/pypy/translator/c/test/autopath.py pypy/branch/faster-raise/pypy/translator/c/test/test_extfunc.py pypy/branch/faster-raise/pypy/translator/c/test/test_newgc.py pypy/branch/faster-raise/pypy/translator/cli/conftest.py pypy/branch/faster-raise/pypy/translator/cli/test/autopath.py pypy/branch/faster-raise/pypy/translator/driver.py pypy/branch/faster-raise/pypy/translator/goal/autopath.py pypy/branch/faster-raise/pypy/translator/goal/targetgbfullprofiling.py (contents, props changed) pypy/branch/faster-raise/pypy/translator/goal/targetgbimplementation.py pypy/branch/faster-raise/pypy/translator/goal/targetgbprofiling.py (props changed) pypy/branch/faster-raise/pypy/translator/goal/targetgbrom4.py pypy/branch/faster-raise/pypy/translator/goal/targetpreimportedpypy.py (contents, props changed) pypy/branch/faster-raise/pypy/translator/goal/targetpypystandalone.py pypy/branch/faster-raise/pypy/translator/goal/test2/autopath.py pypy/branch/faster-raise/pypy/translator/goal/translate.py pypy/branch/faster-raise/pypy/translator/interactive.py pypy/branch/faster-raise/pypy/translator/jvm/conftest.py pypy/branch/faster-raise/pypy/translator/jvm/genjvm.py pypy/branch/faster-raise/pypy/translator/microbench/pybench/autopath.py pypy/branch/faster-raise/pypy/translator/platform/__init__.py pypy/branch/faster-raise/pypy/translator/platform/test/test_darwin.py pypy/branch/faster-raise/pypy/translator/platform/test/test_maemo.py pypy/branch/faster-raise/pypy/translator/platform/windows.py pypy/branch/faster-raise/pypy/translator/sandbox/autopath.py pypy/branch/faster-raise/pypy/translator/sandbox/test/autopath.py pypy/branch/faster-raise/pypy/translator/test/autopath.py pypy/branch/faster-raise/pypy/translator/test/test_driver.py pypy/branch/faster-raise/pypy/translator/tool/autopath.py Log: Merge forward svn merge -r 69169:HEAD svn+ssh://codespeak.net/svn/pypy/trunk/ . Modified: pypy/branch/faster-raise/dotviewer/conftest.py ============================================================================== --- pypy/branch/faster-raise/dotviewer/conftest.py (original) +++ pypy/branch/faster-raise/dotviewer/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ import py def pytest_addoption(parser): - group = parser.addgroup("dotviever") + group = parser.getgroup("dotviever") group.addoption('--pygame', action="store_true", dest="pygame", default=False, help="allow interactive tests using Pygame") Modified: pypy/branch/faster-raise/lib-python/conftest.py ============================================================================== --- pypy/branch/faster-raise/lib-python/conftest.py (original) +++ pypy/branch/faster-raise/lib-python/conftest.py Sat Nov 14 15:24:15 2009 @@ -28,7 +28,7 @@ # def pytest_addoption(parser): - group = parser.addgroup("complicance testing options") + group = parser.getgroup("complicance testing options") group.addoption('-T', '--timeout', action="store", type="string", default="1000", dest="timeout", help="fail a test module after the given timeout. " Modified: pypy/branch/faster-raise/pypy/annotation/description.py ============================================================================== --- pypy/branch/faster-raise/pypy/annotation/description.py (original) +++ pypy/branch/faster-raise/pypy/annotation/description.py Sat Nov 14 15:24:15 2009 @@ -14,6 +14,7 @@ """ overridden = False normalized = False + modified = True def __init__(self, desc): self.descs = { desc: True } @@ -21,6 +22,7 @@ self.total_calltable_size = 0 def update(self, other): + self.modified = True self.normalized = self.normalized or other.normalized self.descs.update(other.descs) for shape, table in other.calltables.items(): @@ -33,7 +35,7 @@ # call at which call site. Each call site gets a row of graphs, # sharable with other call sites. Each column is a FunctionDesc. # There is one such table per "call shape". - table = self.calltables.setdefault(callshape, []) + table = self.calltables.get(callshape, []) for i, existing_row in enumerate(table): if existing_row == row: # XXX maybe use a dict again here? return i @@ -43,6 +45,7 @@ try: self.calltable_lookup_row(callshape, row) except LookupError: + self.modified = True table = self.calltables.setdefault(callshape, []) table.append(row) self.total_calltable_size += 1 @@ -802,6 +805,15 @@ self.knowntype = new_or_old_class(pyobj) assert bool(pyobj), "__nonzero__ unsupported on frozen PBC %r" %(pyobj,) + def has_attribute(self, attr): + if attr in self.attrcache: + return True + try: + self._read_attribute(attr) + return True + except AttributeError: + return False + def read_attribute(self, attr): try: return self.attrcache[attr] Modified: pypy/branch/faster-raise/pypy/annotation/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/annotation/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/annotation/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/bin/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/bin/autopath.py (original) +++ pypy/branch/faster-raise/pypy/bin/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/bin/py.py ============================================================================== --- pypy/branch/faster-raise/pypy/bin/py.py (original) +++ pypy/branch/faster-raise/pypy/bin/py.py Sat Nov 14 15:24:15 2009 @@ -12,7 +12,7 @@ pass from pypy.tool import option -from py.compat.optparse import make_option +from optparse import make_option from pypy.interpreter import main, interactive, error, gateway from pypy.config.config import OptionDescription, BoolOption, StrOption from pypy.config.config import Config, to_optparse Modified: pypy/branch/faster-raise/pypy/config/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/autopath.py (original) +++ pypy/branch/faster-raise/pypy/config/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/config/config.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/config.py (original) +++ pypy/branch/faster-raise/pypy/config/config.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py -from py.compat import optparse +import optparse from pypy.tool.pairtype import extendabletype SUPPRESS_USAGE = optparse.SUPPRESS_USAGE Modified: pypy/branch/faster-raise/pypy/config/makerestdoc.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/makerestdoc.py (original) +++ pypy/branch/faster-raise/pypy/config/makerestdoc.py Sat Nov 14 15:24:15 2009 @@ -1,13 +1,13 @@ import py -from py.__.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link -from py.__.rest.rst import Directive, Em, Quote, Text +from pypy.tool.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link +from pypy.tool.rest.rst import Directive, Em, Quote, Text from pypy.config.config import ChoiceOption, BoolOption, StrOption, IntOption from pypy.config.config import FloatOption, OptionDescription, Option, Config from pypy.config.config import ArbitraryOption, DEFAULT_OPTION_NAME from pypy.config.config import _getnegation -configdocdir = py.magic.autopath().dirpath().dirpath().join("doc", "config") +configdocdir = py.path.local(__file__).dirpath().dirpath().join("doc", "config") def get_fullpath(opt, path): if path: @@ -212,7 +212,7 @@ """ register a :config: ReST link role for use in documentation. """ try: from docutils.parsers.rst import directives, states, roles - from py.__.rest.directive import register_linkrole + from pypy.tool.rest.directive import register_linkrole except ImportError: return # enable :config: link role Modified: pypy/branch/faster-raise/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/pypyoption.py (original) +++ pypy/branch/faster-raise/pypy/config/pypyoption.py Sat Nov 14 15:24:15 2009 @@ -5,7 +5,7 @@ from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConflictConfigError -modulepath = py.magic.autopath().dirpath().dirpath().join("module") +modulepath = py.path.local(__file__).dirpath().dirpath().join("module") all_modules = [p.basename for p in modulepath.listdir() if p.check(dir=True, dotfile=False) and p.join('__init__.py').check()] @@ -77,6 +77,7 @@ "bz2" : ["pypy.module.bz2.interp_bz2"], "pyexpat" : ["pypy.module.pyexpat.interp_pyexpat"], "_ssl" : ["pypy.module._ssl.interp_ssl"], + "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], } def get_module_validator(modname): Modified: pypy/branch/faster-raise/pypy/config/test/test_makerestdoc.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/test/test_makerestdoc.py (original) +++ pypy/branch/faster-raise/pypy/config/test/test_makerestdoc.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ from pypy.config.config import * from pypy.config.makerestdoc import make_cmdline_overview -from py.__.misc.rest import process as restcheck +from pypy.tool.rest.rest import process as restcheck tempdir = py.test.ensuretemp('config') Modified: pypy/branch/faster-raise/pypy/config/test/test_pypyoption.py ============================================================================== --- pypy/branch/faster-raise/pypy/config/test/test_pypyoption.py (original) +++ pypy/branch/faster-raise/pypy/config/test/test_pypyoption.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ from pypy.config.config import Config, ConfigError from pypy.config.translationoption import set_opt_level -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() def test_required(): conf = get_pypy_config() Modified: pypy/branch/faster-raise/pypy/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/conftest.py (original) +++ pypy/branch/faster-raise/pypy/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py, sys, os -from py.__.test.outcome import Failed +from py.impl.test.outcome import Failed from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.tool.pytest import appsupport @@ -9,18 +9,14 @@ from pypy.tool.udir import udir from pypy.tool.autopath import pypydir -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() # pytest settings pytest_plugins = "resultlog", rsyncdirs = ['.', '../lib-python', '../demo'] rsyncignore = ['_cache'] -# XXX workaround for a py.test bug clashing with lib/py symlink -# do we really need the latter? -empty_conftest = type(sys)('conftest') -empty_conftest.__file__ = "?" -sys.modules['pypy.lib.py.conftest'] = empty_conftest +collect_ignore = ['./lib/py'] # PyPy's command line extra options (these are added # to py.test's standard options) @@ -36,7 +32,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("pypy options") + group = parser.getgroup("pypy options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") group.addoption('-A', '--runappdirect', action="store_true", Modified: pypy/branch/faster-raise/pypy/doc/config/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/config/autopath.py (original) +++ pypy/branch/faster-raise/pypy/doc/config/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/doc/config/generate.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/config/generate.py (original) +++ pypy/branch/faster-raise/pypy/doc/config/generate.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ from pypy.config import pypyoption, translationoption, config from pypy.doc.config.confrest import all_optiondescrs -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() for descr in all_optiondescrs: prefix = descr._name Modified: pypy/branch/faster-raise/pypy/doc/config/makemodules.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/config/makemodules.py (original) +++ pypy/branch/faster-raise/pypy/doc/config/makemodules.py Sat Nov 14 15:24:15 2009 @@ -2,7 +2,7 @@ import py from pypy.config import pypyoption, translationoption, config -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() if __name__ == '__main__': c = config.Config(pypyoption.pypy_optiondescription).usemodules Modified: pypy/branch/faster-raise/pypy/doc/config/objspace.std.immutable_builtintypes.txt ============================================================================== --- pypy/branch/faster-raise/pypy/doc/config/objspace.std.immutable_builtintypes.txt (original) +++ pypy/branch/faster-raise/pypy/doc/config/objspace.std.immutable_builtintypes.txt Sat Nov 14 15:24:15 2009 @@ -1 +1 @@ -Disallow modification of builtin types. Enabled by default. +Disallow modification of builtin types. Enabled by default. Modified: pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules._winreg.txt ============================================================================== --- pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules._winreg.txt (original) +++ pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules._winreg.txt Sat Nov 14 15:24:15 2009 @@ -1,2 +1,2 @@ -Use the built-in '_winreg' module, provides access to the Windows registry. -This module is expected to be working and is included by default on Windows. +Use the built-in '_winreg' module, provides access to the Windows registry. +This module is expected to be working and is included by default on Windows. Modified: pypy/branch/faster-raise/pypy/doc/confrest.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/confrest.py (original) +++ pypy/branch/faster-raise/pypy/doc/confrest.py Sat Nov 14 15:24:15 2009 @@ -34,7 +34,7 @@ class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' Modified: pypy/branch/faster-raise/pypy/doc/confrest_oldpy.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/confrest_oldpy.py (original) +++ pypy/branch/faster-raise/pypy/doc/confrest_oldpy.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import py -from py.__.misc.rest import convert_rest_html, strip_html_header -from py.__.misc.difftime import worded_time +from pypy.tool.rest.rest import convert_rest_html, strip_html_header +from pypy.tool.difftime import worded_time html = py.xml.html @@ -104,7 +104,7 @@ class Project: - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "py lib" prefix_title = "" # we have a logo already containing "py lib" encoding = 'latin1' @@ -186,7 +186,9 @@ id = 'docinfoline')) page.contentspace.append(py.xml.raw(content)) - outputpath.ensure().write(page.unicode().encode(encoding)) + f = outputpath.open('w') + f.write(page.unicode().encode(encoding)) + f.close() # XXX this function comes from apigen/linker.py, put it # somewhere in py lib Modified: pypy/branch/faster-raise/pypy/doc/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/conftest.py (original) +++ pypy/branch/faster-raise/pypy/doc/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,12 +1,12 @@ import py from pypy.config.makerestdoc import register_config_role -docdir = py.magic.autopath().dirpath() +docdir = py.path.local(__file__).dirpath() pytest_plugins = "pytest_restdoc" def pytest_addoption(parser): - group = parser.addgroup("pypy-doc options") + group = parser.getgroup("pypy-doc options") group.addoption('--pypy-doctests', action="store_true", dest="pypy_doctests", default=False, help="enable doctests in .txt files") Modified: pypy/branch/faster-raise/pypy/doc/index.txt ============================================================================== --- pypy/branch/faster-raise/pypy/doc/index.txt (original) +++ pypy/branch/faster-raise/pypy/doc/index.txt Sat Nov 14 15:24:15 2009 @@ -1,59 +1,331 @@ +================================================= +PyPy - a Python_ implementation written in Python +================================================= -The PyPy project aims at producing a flexible and fast Python_ -implementation. The guiding idea is to translate a Python-level -description of the Python language itself to lower level languages. -Rumors have it that the secret goal is being faster-than-C which is -nonsense, isn't it? `more...`_ +.. _Python: http://www.python.org/dev/doc/maint24/ref/ref.html -Getting into PyPy ... -============================================= +.. sectnum:: +.. contents:: :depth: 1 -* `Release 1.1`_: the latest official release -* `PyPy Blog`_: news and status info about PyPy +PyPy User Documentation +=============================================== -* `Documentation`_: extensive documentation and papers_ about PyPy. +`getting started`_ provides hands-on instructions +including a two-liner to run the PyPy Python interpreter +on your system, examples on advanced features and +entry points for using PyPy's translation tool chain. -* `Getting Started`_: Getting started and playing with PyPy. +`FAQ`_ contains some frequently asked questions. -Mailing lists, bug tracker, IRC channel -============================================= +New features of PyPy's Python Interpreter and +Translation Framework: -* `Development mailing list`_: development and conceptual - discussions. + * `What PyPy can do for your objects`_ + * `Stackless and coroutines`_ + * `JIT Generation in PyPy`_ + * `Sandboxing Python code`_ -* `Subversion commit mailing list`_: updates to code and - documentation. +`PyPy Prolog Interpreter`_ describes an implementation of +Prolog that makes use of our Translation Tool chain. -* `Development bug/feature tracker`_: filing bugs and feature requests. +Status_ of the project. -* `Sprint mailing list`_: mailing list for organising upcoming sprints. -* **IRC channel #pypy on freenode**: Many of the core developers are hanging out - at #pypy on irc.freenode.net. You are welcome to join and ask questions - (if they are not already developed in the FAQ_). - You can find logs of the channel here_. +Project Documentation +===================================== -.. XXX play1? +PyPy was funded by the EU for several years. See the `web site of the EU +project`_ for more details. + +.. _`web site of the EU project`: http://pypy.org + +architecture_ gives a complete view of PyPy's basic design. + +`coding guide`_ helps you to write code for PyPy (especially also describes +coding in RPython a bit). + +`sprint reports`_ lists reports written at most of our sprints, from +2003 to the present. + +`talks and related projects`_ lists presentations +and related projects. + +`ideas for PyPy related projects`_ which might be a good way to get +into PyPy. + +`PyPy video documentation`_ is a page linking to the videos (e.g. of talks and +introductions) that are available. + +`Technical reports`_ is a page that contains links to the +reports that we submitted to the European Union. + +`development methodology`_ describes our sprint-driven approach. + +`license`_ contains licensing details (basically a straight MIT-license). + +`Glossary`_ of PyPy words to help you align your inner self with +the PyPy universe. + + +Status +=================================== + +PyPy can be used to run Python programs on Linux, OS/X, +Windows, on top of .NET, and on top of Java. +It is recommended to try out the current Subversion HEAD, +which contains `major improvements`__ since the last release. + +.. __: http://codespeak.net/pipermail/pypy-dev/2007q4/004103.html + +PyPy is mainly developed on Linux and Mac OS X. Windows is supported, +but platform-specific bugs tend to take longer before we notice and fix +them. Linux 64-bit machines are supported (though it may also take some +time before we notice and fix bugs). + +PyPy's own tests `summary`_, daily updated, run through BuildBot infrastructure. +You can also find CPython's compliance tests run with compiled ``pypy-c`` +exeuctables there. + +information dating from early 2007: + +`PyPy LOC statistics`_ shows LOC statistics about PyPy. + +`PyPy statistics`_ is a page with various statistics about the PyPy project. + +`compatibility matrix`_ is a diagram that shows which of the various features +of the PyPy interpreter work together with which other features. + + +Source Code Documentation +=============================================== + +`object spaces`_ discusses the object space interface +and several implementations. + +`bytecode interpreter`_ explains the basic mechanisms +of the bytecode interpreter and virtual machine. + +`interpreter optimizations`_ describes our various strategies for +improving the performance of our interpreter, including alternative +object implementations (for strings, dictionaries and lists) in the +standard object space. + +`translation`_ is a detailed overview of our translation process. The +rtyper_ is the largest component of our translation process. + +`dynamic-language translation`_ is a paper that describes +the translation process, especially the flow object space +and the annotator in detail. This document is also part +of the `EU reports`_. + +`low-level encapsulation`_ describes how our approach hides +away a lot of low level details. This document is also part +of the `EU reports`_. + +`translation aspects`_ describes how we weave different +properties into our interpreter during the translation +process. This document is also part of the `EU reports`_. + +`garbage collector`_ strategies that can be used by the virtual +machines produced by the translation process. + +`parser`_ contains (outdated, unfinished) documentation about +the parser. + +`rlib`_ describes some modules that can be used when implementing programs in +RPython. + +`configuration documentation`_ describes the various configuration options that +allow you to customize PyPy. + +`CLI backend`_ describes the details of the .NET backend. + +`JIT Generation in PyPy`_ describes how we produce the Python Just-in-time Compiler +from our Python interpreter. -Meeting PyPy developers -======================= -The PyPy developers are organising sprints and presenting results at -conferences all year round. They will be happy to meet in person with -anyone interested in the project. Watch out for sprint announcements -on the `development mailing list`_. -.. _Python: http://docs.python.org/index.html -.. _`more...`: architecture.html#mission-statement -.. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ -.. _here: http://tismerysoft.de/pypy/irc-logs/pypy -.. _`sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint -.. _`subversion commit mailing list`: http://codespeak.net/mailman/listinfo/pypy-svn -.. _`development mailing list`: http://codespeak.net/mailman/listinfo/pypy-dev .. _`FAQ`: faq.html -.. _`Documentation`: docindex.html -.. _`Getting Started`: getting-started.html -.. _papers: extradoc.html -.. _`Release 1.1`: release-1.1.0.html +.. _Glossary: glossary.html +.. _`PyPy video documentation`: video-index.html +.. _parser: parser.html +.. _`development methodology`: dev_method.html +.. _`sprint reports`: sprint-reports.html +.. _`talks and related projects`: extradoc.html +.. _`license`: ../../LICENSE +.. _`PyPy LOC statistics`: http://codespeak.net/~hpk/pypy-stat/ +.. _`PyPy statistics`: http://codespeak.net/pypy/trunk/pypy/doc/statistic +.. _`object spaces`: objspace.html +.. _`interpreter optimizations`: interpreter-optimizations.html +.. _`translation`: translation.html +.. _`dynamic-language translation`: dynamic-language-translation.html +.. _`low-level encapsulation`: low-level-encapsulation.html +.. _`translation aspects`: translation-aspects.html +.. _`configuration documentation`: config/ +.. _`coding guide`: coding-guide.html +.. _`architecture`: architecture.html +.. _`getting started`: getting-started.html +.. _`theory`: theory.html +.. _`bytecode interpreter`: interpreter.html +.. _`EU reports`: index-report.html +.. _`Technical reports`: index-report.html +.. _`summary`: http://codespeak.net:8099/summary +.. _`ideas for PyPy related projects`: project-ideas.html +.. _`Nightly builds and benchmarks`: http://tuatara.cs.uni-duesseldorf.de/benchmark.html +.. _`directory reference`: +.. _`rlib`: rlib.html +.. _`Sandboxing Python code`: sandbox.html + +PyPy directory cross-reference +------------------------------ + +Here is a fully referenced alphabetical two-level deep +directory overview of PyPy: + +============================ =========================================== +Directory explanation/links +============================ =========================================== +`annotation/`_ `type inferencing code`_ for `RPython`_ programs + +`bin/`_ command-line scripts, mainly `py.py`_ and `translatorshell.py`_ + +`config/`_ handles the numerous options for building and running PyPy + +`doc/`_ text versions of PyPy developer documentation + +`doc/config/`_ documentation for the numerous translation options + +`doc/discussion/`_ drafts of ideas and documentation + +``doc/*/`` other specific documentation topics or tools + +`interpreter/`_ `bytecode interpreter`_ and related objects + (frames, functions, modules,...) + +`interpreter/pyparser/`_ interpreter-level Python source parser + +`interpreter/astcompiler/`_ interpreter-level bytecode compiler, via an AST + representation + +`lang/`_ interpreters for non-Python languages, written in RPython_ + +`lang/js/`_ a JavaScript interpreter (in-progress) + +`lang/prolog/`_ a `Prolog interpreter`_ + +`lib/`_ PyPy's wholesale reimplementations of CPython modules_ + and experimental new application-level modules + +`lib/app_test/`_ tests for the reimplementations, running on top of CPython + +`lib/distributed/`_ distributed execution prototype, based on `transparent proxies`_ + +`module/`_ contains `mixed modules`_ implementing core modules with + both application and interpreter level code. + Not all are finished and working. Use the ``--withmod-xxx`` + or ``--allworkingmodules`` translation options. + +`objspace/`_ `object space`_ implementations + +`objspace/trace.py`_ the `trace object space`_ monitoring bytecode and space operations + +`objspace/dump.py`_ the dump object space saves a large, searchable log file + with all operations + +`objspace/taint.py`_ the `taint object space`_, providing object tainting + +`objspace/thunk.py`_ the `thunk object space`_, providing unique object features + +`objspace/flow/`_ the FlowObjSpace_ implementing `abstract interpretation` + +`objspace/std/`_ the StdObjSpace_ implementing CPython's objects and types + +`rlib/`_ a `"standard library"`_ for RPython_ programs + +`rpython/`_ the `RPython Typer`_ + +`rpython/lltypesystem/`_ the `low-level type system`_ for C-like backends + +`rpython/ootypesystem/`_ the `object-oriented type system`_ for OO backends + +`rpython/memory/`_ the `garbage collector`_ construction framework + +`tool/`_ various utilities and hacks used from various places + +`tool/algo/`_ general-purpose algorithmic and mathematic + tools + +`tool/pytest/`_ support code for our `testing methods`_ + +`translator/`_ translation_ backends and support code + +`translator/backendopt/`_ general optimizations that run before a backend generates code + +`translator/c/`_ the `GenC backend`_, producing C code from an + RPython program (generally via the rtyper_) + +`translator/cli/`_ the `CLI backend`_ for `.NET`_ (Microsoft CLR or Mono_) + +`translator/goal/`_ our `main PyPy-translation scripts`_ live here + +`translator/jvm/`_ the Java backend + +`translator/stackless/`_ the `Stackless Transform`_ + +`translator/tool/`_ helper tools for translation, including the Pygame + `graph viewer`_ + +``*/test/`` many directories have a test subdirectory containing test + modules (see `Testing in PyPy`_) + +``_cache/`` holds cache files from internally `translating application + level to interpreterlevel`_ code. +============================ =========================================== + +.. _`bytecode interpreter`: interpreter.html +.. _`translating application level to interpreterlevel`: geninterp.html +.. _documentation: index.html +.. _`Testing in PyPy`: coding-guide.html#testing-in-pypy +.. _`mixed modules`: coding-guide.html#mixed-modules +.. _`modules`: coding-guide.html#modules +.. _`basil`: http://people.cs.uchicago.edu/~jriehl/BasilTalk.pdf +.. _`object space`: objspace.html +.. _FlowObjSpace: objspace.html#the-flow-object-space +.. _`trace object space`: objspace.html#the-trace-object-space +.. _`taint object space`: objspace-proxies.html#taint +.. _`thunk object space`: objspace-proxies.html#thunk +.. _`transparent proxies`: objspace-proxies.html#tproxy +.. _`What PyPy can do for your objects`: objspace-proxies.html +.. _`Stackless and coroutines`: stackless.html +.. _StdObjSpace: objspace.html#the-standard-object-space +.. _`abstract interpretation`: theory.html#abstract-interpretation +.. _`rpython`: coding-guide.html#rpython +.. _`type inferencing code`: translation.html#the-annotation-pass +.. _`RPython Typer`: translation.html#rpython-typer +.. _`testing methods`: coding-guide.html#testing-in-pypy +.. _`translation`: translation.html +.. _`GenC backend`: translation.html#genc +.. _`CLI backend`: cli-backend.html +.. _`py.py`: getting-started.html#main-entry-point +.. _`translatorshell.py`: getting-started.html#try-out-the-translator +.. _JIT: jit/index.html +.. _`JIT Generation in PyPy`: jit/index.html +.. _`just-in-time compiler generator`: jit/index.html +.. _rtyper: rtyper.html +.. _`low-level type system`: rtyper.html#low-level-type +.. _`object-oriented type system`: rtyper.html#oo-type +.. _`garbage collector`: garbage_collection.html +.. _`Stackless Transform`: translation.html#the-stackless-transform +.. _`PyPy Prolog Interpreter`: prolog-interpreter.html +.. _`Prolog Interpreter`: prolog-interpreter.html +.. _`main PyPy-translation scripts`: getting-started.html#translating-the-pypy-python-interpreter +.. _`translate.py`: getting-started.html#translating-the-pypy-python-interpreter +.. _`.NET`: http://www.microsoft.com/net/ +.. _Mono: http://www.mono-project.com/ +.. _`"standard library"`: rlib.html +.. _`graph viewer`: getting-started.html#try-out-the-translator +.. _`compatibility matrix`: image/compat-matrix.png + +.. include:: _ref.txt + Modified: pypy/branch/faster-raise/pypy/doc/rlib.txt ============================================================================== --- pypy/branch/faster-raise/pypy/doc/rlib.txt (original) +++ pypy/branch/faster-raise/pypy/doc/rlib.txt Sat Nov 14 15:24:15 2009 @@ -285,7 +285,7 @@ anything except a. To parse a regular expression and to get a matcher for it, you can use the -function ``make_runner(s)`` in the ``pypy.rlib.parsing.parseregex`` module. It +function ``make_runner(s)`` in the ``pypy.rlib.parsing.regexparse`` module. It returns a object with a ``recognize(input)`` method that returns True or False depending on whether ``input`` matches the string or not. Modified: pypy/branch/faster-raise/pypy/doc/statistic/confrest.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/statistic/confrest.py (original) +++ pypy/branch/faster-raise/pypy/doc/statistic/confrest.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py -from py.__.doc.confrest import * +from pypy.doc.confrest import * class PyPyPage(Page): def fill_menubar(self): @@ -17,7 +17,7 @@ " ", id="menubar") class Project(Project): - mydir = py.magic.autopath().dirpath() + mydir = py.path.local(__file__).dirpath() title = "PyPy" stylesheet = 'style.css' encoding = 'latin1' Modified: pypy/branch/faster-raise/pypy/doc/test_redirections.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/test_redirections.py (original) +++ pypy/branch/faster-raise/pypy/doc/test_redirections.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import py -redir = py.magic.autopath().dirpath('redirections') +redir = py.path.local(__file__).dirpath('redirections') def checkexist(path): print "checking", path Modified: pypy/branch/faster-raise/pypy/doc/tool/makeref.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/tool/makeref.py (original) +++ pypy/branch/faster-raise/pypy/doc/tool/makeref.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import py -py.magic.autopath() +py.path.local(__file__) import pypy pypydir = py.path.local(pypy.__file__).dirpath() distdir = pypydir.dirpath() Modified: pypy/branch/faster-raise/pypy/doc/tool/mydot.py ============================================================================== --- pypy/branch/faster-raise/pypy/doc/tool/mydot.py (original) +++ pypy/branch/faster-raise/pypy/doc/tool/mydot.py Sat Nov 14 15:24:15 2009 @@ -62,7 +62,7 @@ if __name__ == '__main__': - from py.compat import optparse + import optparse parser = optparse.OptionParser() parser.add_option("-T", dest="format", help="output format") Modified: pypy/branch/faster-raise/pypy/jit/backend/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/backend/autopath.py (original) +++ pypy/branch/faster-raise/pypy/jit/backend/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/faster-raise/pypy/jit/backend/cli/method.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,6 @@ import py import os +from pypy.rlib.debug import debug_start, debug_stop from pypy.tool.pairtype import extendabletype from pypy.rpython.ootypesystem import ootype from pypy.translator.cli import dotnet @@ -164,15 +165,20 @@ def compile(self): # ---- + debug_start('jit-backend-emit_ops') if self.nocast: self.compute_types() self.emit_load_inputargs() - self.emit_preamble() + self.emit_preamble() self.emit_operations(self.cliloop.operations) self.emit_branches() self.emit_end() + debug_stop('jit-backend-emit_ops') # ---- - return self.finish_code() + debug_start('jit-backend-finish_code') + res = self.finish_code() + debug_stop('jit-backend-finish_code') + return res def _parseopt(self, text): text = text.lower() Modified: pypy/branch/faster-raise/pypy/jit/backend/test/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/backend/test/conftest.py (original) +++ pypy/branch/faster-raise/pypy/jit/backend/test/conftest.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup('random test options') + group = parser.getgroup('random test options') group.addoption('--random-seed', action="store", type="int", default=random.randrange(0, 10000), dest="randomseed", Modified: pypy/branch/faster-raise/pypy/jit/backend/x86/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/backend/x86/autopath.py (original) +++ pypy/branch/faster-raise/pypy/jit/backend/x86/autopath.py Sat Nov 14 15:24:15 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/branch/faster-raise/pypy/jit/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/conftest.py (original) +++ pypy/branch/faster-raise/pypy/jit/conftest.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ option = py.test.config.option def pytest_addoption(parser): - group = parser.addgroup("JIT options") + group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", default=False, dest="run_slow_tests", help="run all the compiled tests (instead of just a few)") Modified: pypy/branch/faster-raise/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/codewriter.py Sat Nov 14 15:24:15 2009 @@ -65,6 +65,8 @@ self.counter = 0 self.class_sizes = [] self._class_sizes_seen = {} + self.list_of_addr2name = [] + self._functions_addr_seen = {} # set later with .start() self.metainterp_sd = None @@ -172,7 +174,9 @@ portal_code = self.make_portal_bytecode(portal_graph) self.metainterp_sd.info_from_codewriter(portal_code, leave_code, - self.class_sizes) + self.class_sizes, + self.list_of_addr2name, + portal_runner_ptr) def _start(self, metainterp_sd, portal_runner_ptr): self.metainterp_sd = metainterp_sd @@ -306,6 +310,8 @@ methdescr.setup(jitcodes) def getcalldescr(self, v_func, args, result, consider_effects_of=None): + if isinstance(v_func, Constant): + self.register_known_function(v_func.value) non_void_args = [x for x in args if x.concretetype is not lltype.Void] NON_VOID_ARGS = [x.concretetype for x in non_void_args] RESULT = result.concretetype @@ -330,6 +336,8 @@ self._class_sizes_seen[key] = True sizedescr = self.cpu.sizeof(STRUCT) self.class_sizes.append((vtable, sizedescr)) + vtable_addr = llmemory.cast_ptr_to_adr(vtable) + self.list_of_addr2name.append((vtable_addr, STRUCT.__name__)) def register_known_ooclass(self, cls, CLASS): # ootype only @@ -338,6 +346,16 @@ typedescr = self.cpu.typedescrof(CLASS) self.class_sizes.append((cls, typedescr)) + def register_known_function(self, func): + if self.rtyper.type_system.name == 'lltypesystem': + try: + obj = func._obj + except lltype.DelayedPointer: # probably ll_portal_runner + return + if obj not in self._functions_addr_seen: + self._functions_addr_seen[obj] = True + func_addr = llmemory.cast_ptr_to_adr(func) + self.list_of_addr2name.append((func_addr, obj._name)) class BytecodeMaker(object): Modified: pypy/branch/faster-raise/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/logger.py Sat Nov 14 15:24:15 2009 @@ -7,9 +7,10 @@ class Logger(object): - def __init__(self, ts, guard_number=False): - self.ts = ts - self.guard_number=guard_number + def __init__(self, metainterp_sd, guard_number=False): + self.metainterp_sd = metainterp_sd + self.ts = metainterp_sd.cpu.ts + self.guard_number = guard_number def log_loop(self, inputargs, operations, number=0, type=None): if type is None: @@ -57,7 +58,11 @@ elif isinstance(arg, BoxFloat): return 'f' + str(mv) elif isinstance(arg, self.ts.ConstAddr): - return 'ConstClass(cls' + str(mv) + ')' + addr = arg.getaddr(self.metainterp_sd.cpu) + name = self.metainterp_sd.get_name_from_address(addr) + if not name: + name = 'cls' + str(mv) + return 'ConstClass(' + name + ')' else: return '?' Modified: pypy/branch/faster-raise/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/optimizeopt.py Sat Nov 14 15:24:15 2009 @@ -688,6 +688,14 @@ elif value0.is_null(): self._optimize_nullness(op, op.args[1], expect_isnot) else: + cls0 = value0.get_constant_class(self.cpu) + if cls0 is not None: + cls1 = value1.get_constant_class(self.cpu) + if cls1 is not None and not cls0.same_constant(cls1): + # cannot be the same object, as we know that their + # class is different + self.make_constant_int(op.result, expect_isnot) + return self.optimize_default(op) def optimize_OOISNOT(self, op): Modified: pypy/branch/faster-raise/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/pyjitpl.py Sat Nov 14 15:24:15 2009 @@ -996,8 +996,8 @@ self.cpu = cpu self.stats = stats self.options = options - self.logger_noopt = Logger(cpu.ts) - self.logger_ops = Logger(cpu.ts, guard_number=True) + self.logger_noopt = Logger(self) + self.logger_ops = Logger(self, guard_number=True) RESULT = portal_graph.getreturnvar().concretetype self.result_type = history.getkind(RESULT) @@ -1022,15 +1022,21 @@ self.portal_code = None self.leave_code = None - self._class_sizes = None + self._class_sizes = None + self._addr2name_keys = [] + self._addr2name_values = [] def _freeze_(self): return True - def info_from_codewriter(self, portal_code, leave_code, class_sizes): + def info_from_codewriter(self, portal_code, leave_code, class_sizes, + list_of_addr2name, portal_runner_ptr): self.portal_code = portal_code self.leave_code = leave_code self._class_sizes = class_sizes + self._addr2name_keys = [key for key, value in list_of_addr2name] + self._addr2name_values = [value for key, value in list_of_addr2name] + self._portal_runner_ptr = portal_runner_ptr def finish_setup(self, optimizer=None): warmrunnerdesc = self.warmrunnerdesc @@ -1063,6 +1069,28 @@ class_sizes[vtable] = sizedescr self.cpu.set_class_sizes(class_sizes) + def get_name_from_address(self, addr): + # for debugging only + if we_are_translated(): + d = self.globaldata.addr2name + if d is None: + # Build the dictionary at run-time. This is needed + # because the keys are function/class addresses, so they + # can change from run to run. + k = llmemory.cast_ptr_to_adr(self._portal_runner_ptr) + d = {k: 'recursive call'} + keys = self._addr2name_keys + values = self._addr2name_values + for i in range(len(keys)): + d[keys[i]] = values[i] + self.globaldata.addr2name = d + return d.get(addr, '') + else: + for i in range(len(self._addr2name_keys)): + if addr == self._addr2name_keys[i]: + return self._addr2name_values[i] + return '' + def bytecode_for_address(self, fnaddress): if we_are_translated(): d = self.globaldata.indirectcall_dict @@ -1115,6 +1143,7 @@ def __init__(self, staticdata, prebuilt_fail_descr_list): self.initialized = False self.indirectcall_dict = None + self.addr2name = None self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_codewriter.py Sat Nov 14 15:24:15 2009 @@ -121,12 +121,14 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT): + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): return FakeMethDescr(CLASS, methname) + def sizeof(self, STRUCT): + return ('sizeof', STRUCT) if type_system == 'lltype': FakeCPU.ts = typesystem.llhelper @@ -366,6 +368,24 @@ assert jitcode._source.count('oononnull') == 2 assert jitcode._source.count('ooisnull') == 2 + def test_list_of_addr2name(self): + class A1: + def g(self): + self.x = 123 + return 5 + def f(): + a = A1() + a.y = a.g() + return a + graphs = self.make_graphs(f, []) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + assert len(cw.list_of_addr2name) == 2 + assert cw.list_of_addr2name[0][1].endswith('.A1') + assert cw.list_of_addr2name[1][1] == 'A1.g' + class ImmutableFieldsTests: def test_fields(self): Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_list.py Sat Nov 14 15:24:15 2009 @@ -399,6 +399,9 @@ n += a.x n = lst.pop() lst.append(n - 10 + a.x) + if a.x in lst: + pass + a.x = a.x + 1 - 1 a = lst.pop() b = lst.pop() return a * b Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_logger.py Sat Nov 14 15:24:15 2009 @@ -34,6 +34,14 @@ class TestLogger(object): ts = llhelper + def make_metainterp_sd(self): + class FakeMetaInterpSd: + class cpu: + ts = self.ts + def get_name_from_address(self, addr): + return 'Name' + return FakeMetaInterpSd() + def reparse(self, inp, namespace=None, check_equal=True): """ parse loop once, then log it and parse again. Checks that we get the same thing. @@ -41,7 +49,7 @@ if namespace is None: namespace = {} loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop, namespace) oloop = pure_parse(output, namespace=namespace) if check_equal: @@ -99,7 +107,7 @@ jump(i0, descr=target) ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts) + logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop) assert output.splitlines()[-1] == "jump(i0, descr=)" pure_parse(output) @@ -111,7 +119,7 @@ guard_true(i0, descr=fdescr) [i0] ''' loop = pure_parse(inp, namespace=namespace) - logger = Logger(self.ts, guard_number=True) + logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) @@ -119,18 +127,34 @@ def boom(): raise Exception namespace['fdescr'].get_index = boom - logger = Logger(self.ts, guard_number=False) + logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + def test_class_name(self): + from pypy.rpython.lltypesystem import lltype + AbcVTable = lltype.Struct('AbcVTable') + abcvtable = lltype.malloc(AbcVTable, immortal=True) + namespace = {'Name': abcvtable} + inp = ''' + [i0] + p = new_with_vtable(ConstClass(Name)) + ''' + loop = pure_parse(inp, namespace=namespace) + logger = Logger(self.make_metainterp_sd()) + output = logger.log_loop(loop) + assert output.splitlines()[-1].endswith( + " = new_with_vtable(ConstClass(Name))") + pure_parse(output, namespace=namespace) + def test_intro_loop(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_loop, [], [], 1, "foo") assert output.splitlines()[0] == "# Loop 1 : foo with 0 ops" pure_parse(output) def test_intro_bridge(self): - bare_logger = logger.Logger(self.ts) + bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_bridge, [], [], 3) assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" pure_parse(output) Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_optimizeopt.py Sat Nov 14 15:24:15 2009 @@ -1523,6 +1523,21 @@ """ self.optimize_loop(ops, "Not, Not, Not, Not, Not", expected) + def test_guard_class_oois(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable2)) [] + i = ooisnot(ConstPtr(myptr), p1) + guard_true(i) [] + jump(p1) + """ + expected = """ + [p1] + guard_class(p1, ConstClass(node_vtable2)) [] + jump(p1) + """ + self.optimize_loop(ops, "Not", expected) + # ---------- def make_fail_descr(self): Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py Sat Nov 14 15:24:15 2009 @@ -142,3 +142,14 @@ assert equaloplists(metainterp.history.operations, [ ResOperation(rop.SAME_AS, [b2], boxes[1]), ]) + +def test_get_name_from_address(): + class FakeMetaInterpSd(pyjitpl.MetaInterpStaticData): + def __init__(self): + pass + metainterp_sd = FakeMetaInterpSd() + metainterp_sd.info_from_codewriter(None, None, None, + [(123, "a"), (456, "b")]) + assert metainterp_sd.get_name_from_address(123) == 'a' + assert metainterp_sd.get_name_from_address(456) == 'b' + assert metainterp_sd.get_name_from_address(789) == '' Modified: pypy/branch/faster-raise/pypy/jit/tl/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/autopath.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/autopath.py Sat Nov 14 15:24:15 2009 @@ -21,7 +21,6 @@ """ - def __dirinfo(part): """ return (partdir, this_dir) and insert parent of partdir into sys.path. If the parent directories don't have the part @@ -33,13 +32,31 @@ except NameError: head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + error = None while head: partdir = head head, tail = os.path.split(head) if tail == part: + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + if not os.path.exists(checkfile): + error = "Cannot find %r" % (os.path.normpath(checkfile),) break else: - raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + error = "Cannot find the parent directory %r of the path %r" % ( + partdir, this_dir) + if not error: + # check for bogus end-of-line style (e.g. files checked out on + # Windows and moved to Unix) + f = open(__file__.replace('.pyc', '.py'), 'r') + data = f.read() + f.close() + if data.endswith('\r\n') or data.endswith('\r'): + error = ("Bad end-of-line style in the .py files. Typically " + "caused by a zip file or a checkout done on Windows and " + "moved to Unix or vice-versa.") + if error: + raise EnvironmentError("Invalid source tree - bogus checkout! " + + error) pypy_root = os.path.join(head, '') try: @@ -109,6 +126,9 @@ # set guaranteed attributes pypydir, this_dir = __dirinfo('pypy') +import py +libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) +libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) if __name__ == '__main__': __clone() Modified: pypy/branch/faster-raise/pypy/jit/tl/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/conftest.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit.py options") + group = parser.getgroup("pypyjit.py options") group.addoption('--ootype', action="store_true", dest="ootype", default=False, help="use ootype") Modified: pypy/branch/faster-raise/pypy/jit/tl/spli/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/spli/autopath.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/spli/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/jit/tl/targettlc.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/targettlc.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/targettlc.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import time import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlc import interp, interp_nonjit, ConstantPool from pypy.jit.metainterp.policy import JitPolicy from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/faster-raise/pypy/jit/tl/targettlr.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/targettlr.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/targettlr.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tlr import interpret from pypy.jit.backend.hlinfo import highleveljitinfo Modified: pypy/branch/faster-raise/pypy/jit/tl/test/test_pypyjit.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/test/test_pypyjit.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/test/test_pypyjit.py Sat Nov 14 15:24:15 2009 @@ -21,7 +21,7 @@ def check_crasher(func_name): try: JIT_EXECUTABLE.sysexec(CRASH_FILE, func_name) - except py.__.process.cmdexec.ExecutionFailed, e: + except py.impl.process.cmdexec.ExecutionFailed, e: print "stderr" print "------" print e.err Modified: pypy/branch/faster-raise/pypy/jit/tl/tla/targettla.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/tla/targettla.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/tla/targettla.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla import tla Modified: pypy/branch/faster-raise/pypy/jit/tl/tla/tla_assembler.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tl/tla/tla_assembler.py (original) +++ pypy/branch/faster-raise/pypy/jit/tl/tla/tla_assembler.py Sat Nov 14 15:24:15 2009 @@ -2,7 +2,7 @@ import sys import py -py.magic.autopath() +py.path.local(__file__) from pypy.jit.tl.tla.test_tla import assemble def usage(): Modified: pypy/branch/faster-raise/pypy/jit/tool/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/tool/autopath.py (original) +++ pypy/branch/faster-raise/pypy/jit/tool/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/debug/gameboy_debug_entry_point.py Sat Nov 14 15:24:15 2009 @@ -11,7 +11,7 @@ # ------------------------------------------------------------------------------ -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" # filename = ROM_PATH + "/rom9/rom9.gb" filename = "/home/tverwaes/roms/SuperMarioLand.gb" SOCKET_PORT = 55682 @@ -83,7 +83,7 @@ # ------------------------------------------------------------------------------ -MARIO_DIR = str(py.magic.autopath().dirpath().dirpath()\ +MARIO_DIR = str(py.path.local(__file__).dirpath().dirpath()\ .dirpath().dirpath()\ .dirpath().dirpath()) + "/mario" Modified: pypy/branch/faster-raise/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/profiling/evaluation/gameboy_evaluation_target.py Sat Nov 14 15:24:15 2009 @@ -8,7 +8,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/branch/faster-raise/pypy/lang/gameboy/profiling/gameboyTest.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/profiling/gameboyTest.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/profiling/gameboyTest.py Sat Nov 14 15:24:15 2009 @@ -7,7 +7,7 @@ debug.DEBUG_PRINT_LOGS = False -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" filename = "" if len(sys.argv) > 1: Modified: pypy/branch/faster-raise/pypy/lang/gameboy/test/test_cartridge.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/test/test_cartridge.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/test/test_cartridge.py Sat Nov 14 15:24:15 2009 @@ -8,7 +8,7 @@ def mapToByte(value): return ord(value) & 0xFF -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" CONTENT = "abcdefghijklmnopqrstuvwxyz1234567890" MAPPED_CONTENT = map_to_byte(CONTENT) Modified: pypy/branch/faster-raise/pypy/lang/gameboy/test/test_rom.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/test/test_rom.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/test/test_rom.py Sat Nov 14 15:24:15 2009 @@ -6,7 +6,7 @@ from pypy.lang.gameboy.gameboy import * -ROM_PATH = str(py.magic.autopath().dirpath().dirpath())+"/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath())+"/rom" EMULATION_CYCLES = 64 # ------------------------------------------------------------------------------ Modified: pypy/branch/faster-raise/pypy/lang/gameboy/tool/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/gameboy/tool/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lang/gameboy/tool/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lang/js/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/js/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lang/js/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lang/js/jsparser.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/js/jsparser.py (original) +++ pypy/branch/faster-raise/pypy/lang/js/jsparser.py Sat Nov 14 15:24:15 2009 @@ -2,7 +2,7 @@ from pypy.rlib.parsing.parsing import ParseError, Rule import py -GFILE = py.magic.autopath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/branch/faster-raise/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/branch/faster-raise/pypy/lang/js/test/ecma/conftest.py Sat Nov 14 15:24:15 2009 @@ -2,13 +2,13 @@ from pypy.lang.js.interpreter import * from pypy.lang.js.jsobj import W_Array, JsBaseExcept from pypy.rlib.parsing.parsing import ParseError -from py.__.test.outcome import Failed, ExceptionFailure +from py.impl.test.outcome import Failed, ExceptionFailure import pypy.lang.js as js from pypy.lang.js import interpreter interpreter.TEST = True -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() exclusionlist = ['shell.js', 'browser.js'] def pytest_addoption(parser): Modified: pypy/branch/faster-raise/pypy/lang/js/test/test_interactive.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/js/test/test_interactive.py (original) +++ pypy/branch/faster-raise/pypy/lang/js/test/test_interactive.py Sat Nov 14 15:24:15 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv): - return self._spawn(str(py.magic.autopath().dirpath().dirpath().join('js_interactive.py')), argv) + return self._spawn(str(py.path.local(__file__).dirpath().dirpath().join('js_interactive.py')), argv) def prompt_send(self, message): self.child.expect('js>') Modified: pypy/branch/faster-raise/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/js/test/test_parser.py (original) +++ pypy/branch/faster-raise/pypy/lang/js/test/test_parser.py Sat Nov 14 15:24:15 2009 @@ -9,7 +9,7 @@ from pypy import conftest import sys -GFILE = py.magic.autopath().dirpath().dirpath().join("jsgrammar.txt") +GFILE = py.path.local(__file__).dirpath().dirpath().join("jsgrammar.txt") try: t = GFILE.read(mode='U') Modified: pypy/branch/faster-raise/pypy/lang/prolog/interpreter/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/prolog/interpreter/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lang/prolog/interpreter/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lang/prolog/interpreter/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/prolog/interpreter/conftest.py (original) +++ pypy/branch/faster-raise/pypy/lang/prolog/interpreter/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import py, sys -rootdir = py.magic.autopath().dirpath() +rootdir = py.path.local(__file__).dirpath() Option = py.test.config.Option Modified: pypy/branch/faster-raise/pypy/lang/prolog/interpreter/interactive.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/prolog/interpreter/interactive.py (original) +++ pypy/branch/faster-raise/pypy/lang/prolog/interpreter/interactive.py Sat Nov 14 15:24:15 2009 @@ -7,7 +7,7 @@ import py import sys -#sys.path.append(str(py.magic.autopath().dirpath().dirpath())) +#sys.path.append(str(py.path.local(__file__).dirpath().dirpath())) from pypy.rlib.parsing.parsing import ParseError from pypy.rlib.parsing.deterministic import LexerError Modified: pypy/branch/faster-raise/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/branch/faster-raise/pypy/lang/prolog/interpreter/parsing.py Sat Nov 14 15:24:15 2009 @@ -3403,7 +3403,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/faster-raise/pypy/lang/scheme/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/scheme/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lang/scheme/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lang/scheme/execution.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/scheme/execution.py (original) +++ pypy/branch/faster-raise/pypy/lang/scheme/execution.py Sat Nov 14 15:24:15 2009 @@ -19,7 +19,7 @@ except (TypeError, AttributeError): pass -de_file = py.magic.autopath().dirpath().join("r5rs_derived_expr.ss") +de_file = py.path.local(__file__).dirpath().join("r5rs_derived_expr.ss") de_code = de_file.read() de_expr_lst = parse(de_code) Modified: pypy/branch/faster-raise/pypy/lang/scheme/test/test_interactive.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/scheme/test/test_interactive.py (original) +++ pypy/branch/faster-raise/pypy/lang/scheme/test/test_interactive.py Sat Nov 14 15:24:15 2009 @@ -20,7 +20,7 @@ return child def spawn(self, argv=[]): - path = py.magic.autopath()/".."/".."/"interactive.py" + path = py.path.local(__file__)/".."/".."/"interactive.py" return self._spawn(str(path), argv) def test_interactive(self): Modified: pypy/branch/faster-raise/pypy/lang/smalltalk/test/test_miniimage.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/smalltalk/test/test_miniimage.py (original) +++ pypy/branch/faster-raise/pypy/lang/smalltalk/test/test_miniimage.py Sat Nov 14 15:24:15 2009 @@ -12,7 +12,7 @@ def setup_module(module, filename='mini.image'): space = objspace.ObjSpace() - module.mini_image = py.magic.autopath().dirpath().dirpath().join(filename) + module.mini_image = py.path.local(__file__).dirpath().dirpath().join(filename) module.reader = open_miniimage(space) reader.initialize() module.image = squeakimage.SqueakImage() Modified: pypy/branch/faster-raise/pypy/lang/smalltalk/tool/analyseimage.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/smalltalk/tool/analyseimage.py (original) +++ pypy/branch/faster-raise/pypy/lang/smalltalk/tool/analyseimage.py Sat Nov 14 15:24:15 2009 @@ -6,7 +6,7 @@ from pypy.lang.smalltalk import interpreter import sys -mini_image = py.magic.autopath().dirpath().dirpath().join('mini.image') +mini_image = py.path.local(__file__).dirpath().dirpath().join('mini.image') def get_miniimage(space): return squeakimage.ImageReader(space, squeakimage.Stream(mini_image.open())) Modified: pypy/branch/faster-raise/pypy/lang/smalltalk/tool/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lang/smalltalk/tool/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lang/smalltalk/tool/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/conftest.py (original) +++ pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/conftest.py Sat Nov 14 15:24:15 2009 @@ -11,7 +11,7 @@ from pypy.translator.platform import platform from pypy.translator.tool.cbuild import ExternalCompilationInfo udir = py.test.ensuretemp('_ctypes_test') - cfile = py.magic.autopath().dirpath().join("_ctypes_test.c") + cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") if sys.platform == 'win32': libraries = ['oleaut32'] Modified: pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/test_win32.py ============================================================================== --- pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/test_win32.py (original) +++ pypy/branch/faster-raise/pypy/lib/app_test/ctypes_tests/test_win32.py Sat Nov 14 15:24:15 2009 @@ -1,80 +1,80 @@ -# Windows specific tests - -from ctypes import * -from ctypes.test import is_resource_enabled -from support import BaseCTypesTestChecker - -import py -import sys - -if sys.platform != "win32": - py.test.skip("win32-only tests") - -class TestWindows(BaseCTypesTestChecker): - def test_callconv_1(self): - # Testing stdcall function - - IsWindow = windll.user32.IsWindow - # ValueError: Procedure probably called with not enough arguments (4 bytes missing) - py.test.raises(ValueError, IsWindow) - - # This one should succeeed... - assert IsWindow(0) == 0 - - # ValueError: Procedure probably called with too many arguments (8 bytes in excess) - py.test.raises(ValueError, IsWindow, 0, 0, 0) - - def test_callconv_2(self): - # Calling stdcall function as cdecl - - IsWindow = cdll.user32.IsWindow - - # ValueError: Procedure called with not enough arguments (4 bytes missing) - # or wrong calling convention - py.test.raises(ValueError, IsWindow, None) - - if is_resource_enabled("SEH"): - def test_SEH(self): - # Call functions with invalid arguments, and make sure that access violations - # are trapped and raise an exception. - py.test.raises(WindowsError, windll.kernel32.GetModuleHandleA, 32) - -class TestWintypes(BaseCTypesTestChecker): - - def test_COMError(self): - import _ctypes - from _ctypes import COMError - assert COMError.__doc__ == "Raised when a COM method call failed." - - ex = COMError(-1, "text", ("details",)) - assert ex.hresult == -1 - assert ex.text == "text" - assert ex.details == ("details",) - assert (ex.hresult, ex.text, ex.details) == ex[:] - - def test_VARIANT(self): - from ctypes import wintypes - a = wintypes.VARIANT_BOOL() - assert a.value is False - b = wintypes.VARIANT_BOOL(3) - assert b.value is True - -class TestStructures(BaseCTypesTestChecker): - - def test_struct_by_value(self): - class POINT(Structure): - _fields_ = [("x", c_long), - ("y", c_long)] - - class RECT(Structure): - _fields_ = [("left", c_long), - ("top", c_long), - ("right", c_long), - ("bottom", c_long)] - - import conftest - dll = CDLL(str(conftest.sofile)) - - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - assert dll.PointInRect(byref(rect), pt) == 1 +# Windows specific tests + +from ctypes import * +from ctypes.test import is_resource_enabled +from support import BaseCTypesTestChecker + +import py +import sys + +if sys.platform != "win32": + py.test.skip("win32-only tests") + +class TestWindows(BaseCTypesTestChecker): + def test_callconv_1(self): + # Testing stdcall function + + IsWindow = windll.user32.IsWindow + # ValueError: Procedure probably called with not enough arguments (4 bytes missing) + py.test.raises(ValueError, IsWindow) + + # This one should succeeed... + assert IsWindow(0) == 0 + + # ValueError: Procedure probably called with too many arguments (8 bytes in excess) + py.test.raises(ValueError, IsWindow, 0, 0, 0) + + def test_callconv_2(self): + # Calling stdcall function as cdecl + + IsWindow = cdll.user32.IsWindow + + # ValueError: Procedure called with not enough arguments (4 bytes missing) + # or wrong calling convention + py.test.raises(ValueError, IsWindow, None) + + if is_resource_enabled("SEH"): + def test_SEH(self): + # Call functions with invalid arguments, and make sure that access violations + # are trapped and raise an exception. + py.test.raises(WindowsError, windll.kernel32.GetModuleHandleA, 32) + +class TestWintypes(BaseCTypesTestChecker): + + def test_COMError(self): + import _ctypes + from _ctypes import COMError + assert COMError.__doc__ == "Raised when a COM method call failed." + + ex = COMError(-1, "text", ("details",)) + assert ex.hresult == -1 + assert ex.text == "text" + assert ex.details == ("details",) + assert (ex.hresult, ex.text, ex.details) == ex[:] + + def test_VARIANT(self): + from ctypes import wintypes + a = wintypes.VARIANT_BOOL() + assert a.value is False + b = wintypes.VARIANT_BOOL(3) + assert b.value is True + +class TestStructures(BaseCTypesTestChecker): + + def test_struct_by_value(self): + class POINT(Structure): + _fields_ = [("x", c_long), + ("y", c_long)] + + class RECT(Structure): + _fields_ = [("left", c_long), + ("top", c_long), + ("right", c_long), + ("bottom", c_long)] + + import conftest + dll = CDLL(str(conftest.sofile)) + + pt = POINT(10, 10) + rect = RECT(0, 0, 20, 20) + assert dll.PointInRect(byref(rect), pt) == 1 Modified: pypy/branch/faster-raise/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/branch/faster-raise/pypy/lib/distributed/socklayer.py (original) +++ pypy/branch/faster-raise/pypy/lib/distributed/socklayer.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ import py from socket import socket -from py.__.green.msgstruct import decodemessage, message +from py.impl.green.msgstruct import decodemessage, message from socket import socket, AF_INET, SOCK_STREAM import marshal import sys Modified: pypy/branch/faster-raise/pypy/lib/test2/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/lib/test2/autopath.py (original) +++ pypy/branch/faster-raise/pypy/lib/test2/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/module/__builtin__/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/module/__builtin__/test/test_import.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/__builtin__/test/test_import.py (original) +++ pypy/branch/faster-raise/pypy/module/__builtin__/test/test_import.py Sat Nov 14 15:24:15 2009 @@ -76,7 +76,7 @@ code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) s2 = struct.pack("i", os.stat(str(p.join("x.py")))[stat.ST_MTIME]) - p.join("x.pyc").write(imp.get_magic() + s2 + s3) + p.join("x.pyc").write(imp.get_magic() + s2 + s3, mode='wb') else: w = space.wrap w_modname = w("compiled.x") @@ -92,7 +92,7 @@ stream.close() if space.config.objspace.usepycfiles: # also create a lone .pyc file - p.join('lone.pyc').write(p.join('x.pyc').read()) + p.join('lone.pyc').write(p.join('x.pyc').read(), mode='wb') return str(root) Modified: pypy/branch/faster-raise/pypy/module/_codecs/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_codecs/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/module/_codecs/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/module/_file/test/test_file_extra.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_file/test/test_file_extra.py (original) +++ pypy/branch/faster-raise/pypy/module/_file/test/test_file_extra.py Sat Nov 14 15:24:15 2009 @@ -19,7 +19,7 @@ def setup_module(mod): - udir.join('sample').write(SAMPLE) + udir.join('sample').write(SAMPLE, 'wb') class BaseROTests: Modified: pypy/branch/faster-raise/pypy/module/_minimal_curses/fficurses.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_minimal_curses/fficurses.py (original) +++ pypy/branch/faster-raise/pypy/module/_minimal_curses/fficurses.py Sat Nov 14 15:24:15 2009 @@ -5,6 +5,7 @@ import sys from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype +from pypy.rpython.tool import rffi_platform from pypy.rpython.extfunc import register_external from pypy.rpython.extregistry import ExtRegistryEntry from pypy.module._minimal_curses import interp_curses @@ -15,6 +16,7 @@ libraries = ['curses'], ) +rffi_platform.verify_eci(eci) INT = rffi.INT Modified: pypy/branch/faster-raise/pypy/module/_sre/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_sre/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/module/_sre/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/module/_sre/test/test_app_sre.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_sre/test/test_app_sre.py (original) +++ pypy/branch/faster-raise/pypy/module/_sre/test/test_app_sre.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp from pypy.conftest import gettestobjspace, option -from py.__.test.outcome import Skipped +from py.impl.test.outcome import Skipped def init_globals_hack(space): space.appexec([space.wrap(autopath.this_dir)], """(this_dir): Modified: pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_compdecomp.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_compdecomp.py (original) +++ pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_compdecomp.py Sat Nov 14 15:24:15 2009 @@ -25,7 +25,7 @@ mod.TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' mod.DATA = DATA - mod.BUGGY_DATA = py.magic.autopath().dirpath().join('data.bz2').read() + mod.BUGGY_DATA = py.path.local(__file__).dirpath().join('data.bz2').read() mod.decompress = decompress class AppTestBZ2Compressor(CheckAllocation): Modified: pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_file.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_file.py (original) +++ pypy/branch/faster-raise/pypy/module/bz2/test/test_bz2_file.py Sat Nov 14 15:24:15 2009 @@ -15,12 +15,12 @@ def create_temp_file(crlf=False): f = py.test.ensuretemp("bz2").join("foo") data = (DATA, DATA_CRLF)[crlf] - f.write(data) + f.write(data, 'wb') def create_broken_temp_file(): f = py.test.ensuretemp("bz2").join("foo") data = DATA[:100] - f.write(data) + f.write(data, 'wb') def decompress(data): import popen2 Modified: pypy/branch/faster-raise/pypy/module/gc/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/gc/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/gc/__init__.py Sat Nov 14 15:24:15 2009 @@ -14,3 +14,9 @@ 'garbage' : 'space.newlist([])', 'dump_heap_stats': 'interp_gc.dump_heap_stats', } + + def __init__(self, space, w_name): + ts = space.config.translation.type_system + if ts == 'ootype': + del self.interpleveldefs['dump_heap_stats'] + MixedModule.__init__(self, space, w_name) Modified: pypy/branch/faster-raise/pypy/module/pypyjit/test/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/test/conftest.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/test/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypyjit options") + group = parser.getgroup("pypyjit options") group.addoption("--pypy", action="store", default=None, dest="pypy_c", help="the location of the JIT enabled pypy-c") Modified: pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 14 15:24:15 2009 @@ -122,7 +122,6 @@ ([20], 2432902008176640000L)) def test_factorialrec(self): - skip("does not make sense yet") self.run_source(''' def main(n): if n > 1: Modified: pypy/branch/faster-raise/pypy/module/sys/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/sys/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/module/sys/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/module/sys/version.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/sys/version.py (original) +++ pypy/branch/faster-raise/pypy/module/sys/version.py Sat Nov 14 15:24:15 2009 @@ -87,10 +87,15 @@ # to depend on an external 'svn' executable in the path. rev = int(REV) try: - f = open(os.path.join(pypydir, '.svn', 'format'), 'r') - format = int(f.readline().strip()) - f.close() - if format <= 6: # Old XML-format + formatfile = os.path.join(pypydir, '.svn', 'format') + if os.path.exists(formatfile): + f = open(formatfile, 'r') + format = int(f.readline().strip()) + f.close() + oldformat = (format <= 6) # Old XML-format + else: + oldformat = False + if oldformat: f = open(os.path.join(pypydir, '.svn', 'entries'), 'r') for line in f: line = line.strip() Modified: pypy/branch/faster-raise/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/faster-raise/pypy/objspace/std/test/test_complexobject.py Sat Nov 14 15:24:15 2009 @@ -67,7 +67,7 @@ sys.path.append(%r) import helper return helper - """ % (str(py.magic.autopath().dirpath()))) + """ % (str(py.path.local(__file__).dirpath()))) def test_div(self): h = self.helper Modified: pypy/branch/faster-raise/pypy/rlib/parsing/ebnfparse.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/ebnfparse.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/ebnfparse.py Sat Nov 14 15:24:15 2009 @@ -2125,7 +2125,7 @@ # generated code between this line and its other occurence if __name__ == '__main__': - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/faster-raise/pypy/rlib/parsing/lexer.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/lexer.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/lexer.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,4 @@ import py -from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.parsing import deterministic, regex class Token(object): Modified: pypy/branch/faster-raise/pypy/rlib/parsing/makepackrat.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/makepackrat.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/makepackrat.py Sat Nov 14 15:24:15 2009 @@ -718,7 +718,7 @@ def test_generate(): - f = py.magic.autopath().dirpath().join("pypackrat.py") + f = py.path.local(__file__).dirpath().join("pypackrat.py") from pypackrat import PyPackratSyntaxParser p = PyPackratSyntaxParser(syntax) t = p.file() Modified: pypy/branch/faster-raise/pypy/rlib/parsing/regexparse.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/regexparse.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/regexparse.py Sat Nov 14 15:24:15 2009 @@ -1966,7 +1966,7 @@ def test_generate(): - f = py.magic.autopath() + f = py.path.local(__file__) oldcontent = f.read() s = "# GENERATED CODE BETWEEN THIS LINE AND ITS OTHER OCCURENCE\n".lower() pre, gen, after = oldcontent.split(s) Modified: pypy/branch/faster-raise/pypy/rlib/parsing/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonlexer.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonlexer.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonlexer.py Sat Nov 14 15:24:15 2009 @@ -230,7 +230,7 @@ assert tokens[i * 3].name == 'String' def test_self(): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlexer.tokenize(s) print tokens @@ -263,7 +263,7 @@ "op": "operator", } import tokenize, token - s = py.magic.autopath().read() + s = py.path.local(__file__).read() tokens = pythonlex(s) print [t.name for t in tokens][:20] tokens2 = list(tokenize.generate_tokens(iter(s.splitlines(True)).next)) Modified: pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonparse.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonparse.py (original) +++ pypy/branch/faster-raise/pypy/rlib/parsing/test/test_pythonparse.py Sat Nov 14 15:24:15 2009 @@ -7,7 +7,7 @@ from pypy.rlib.parsing.parsing import PackratParser, Symbol, ParseError, Rule from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function -grammar = py.magic.autopath().dirpath().join("pygrammar.txt").read(mode='rt') +grammar = py.path.local(__file__).dirpath().join("pygrammar.txt").read(mode='rt') def test_parse_grammar(): @@ -240,12 +240,12 @@ t = self.ToAST.transform(t) def test_parse_this(self): - s = py.magic.autopath().read() + s = py.path.local(__file__).read() t = self.parse(s) t = self.ToAST.transform(t) def test_parsing(self): - s = py.magic.autopath().dirpath().dirpath().join("parsing.py").read() + s = py.path.local(__file__).dirpath().dirpath().join("parsing.py").read() t = self.parse(s) t = self.ToAST.transform(t) Modified: pypy/branch/faster-raise/pypy/rlib/rsdl/eci.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/rsdl/eci.py (original) +++ pypy/branch/faster-raise/pypy/rlib/rsdl/eci.py Sat Nov 14 15:24:15 2009 @@ -9,7 +9,7 @@ includes = ['SDL.h'], include_dirs = ['/Library/Frameworks/SDL.framework/Headers'], link_files = [ - str(py.magic.autopath().dirpath().join('macosx-sdl-main/SDLMain.m')), + str(py.path.local(__file__).dirpath().join('macosx-sdl-main/SDLMain.m')), ], frameworks = ['SDL', 'Cocoa'] ) Modified: pypy/branch/faster-raise/pypy/rlib/rsdl/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/rsdl/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/rlib/rsdl/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/rlib/rwinreg.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/rwinreg.py (original) +++ pypy/branch/faster-raise/pypy/rlib/rwinreg.py Sat Nov 14 15:24:15 2009 @@ -1,134 +1,134 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.tool import rffi_platform as platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib import rwin32 - -eci = ExternalCompilationInfo( - includes = ['windows.h', - ], - libraries = ('Advapi32',) - ) -class CConfig: - _compilation_info_ = eci - - -constant_names = ''' -KEY_QUERY_VALUE KEY_SET_VALUE KEY_CREATE_SUB_KEY KEY_ENUMERATE_SUB_KEYS -KEY_NOTIFY KEY_CREATE_LINK KEY_READ KEY_WRITE KEY_EXECUTE KEY_ALL_ACCESS -KEY_WOW64_64KEY KEY_WOW64_32KEY REG_OPTION_RESERVED REG_OPTION_NON_VOLATILE -REG_OPTION_VOLATILE REG_OPTION_CREATE_LINK REG_OPTION_BACKUP_RESTORE -REG_OPTION_OPEN_LINK REG_LEGAL_OPTION REG_CREATED_NEW_KEY -REG_OPENED_EXISTING_KEY REG_WHOLE_HIVE_VOLATILE REG_REFRESH_HIVE -REG_NO_LAZY_FLUSH REG_NOTIFY_CHANGE_NAME REG_NOTIFY_CHANGE_ATTRIBUTES -REG_NOTIFY_CHANGE_LAST_SET REG_NOTIFY_CHANGE_SECURITY REG_LEGAL_CHANGE_FILTER -REG_NONE REG_SZ REG_EXPAND_SZ REG_BINARY REG_DWORD REG_DWORD_LITTLE_ENDIAN -REG_DWORD_BIG_ENDIAN REG_LINK REG_MULTI_SZ REG_RESOURCE_LIST -REG_FULL_RESOURCE_DESCRIPTOR REG_RESOURCE_REQUIREMENTS_LIST - -HKEY_LOCAL_MACHINE HKEY_CLASSES_ROOT HKEY_CURRENT_CONFIG HKEY_CURRENT_USER -HKEY_DYN_DATA HKEY_LOCAL_MACHINE HKEY_PERFORMANCE_DATATA HKEY_USERS -'''.split() -for name in constant_names: - setattr(CConfig, name, platform.DefinedConstantInteger(name)) - -constants = {} -cConfig = platform.configure(CConfig) -constants.update(cConfig) -globals().update(cConfig) - -def external(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv='win') - -HKEY = rwin32.HANDLE -PHKEY = rffi.CArrayPtr(HKEY) -REGSAM = rwin32.DWORD - -RegSetValue = external( - 'RegSetValueA', - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) - -RegSetValueEx = external( - 'RegSetValueExA', - [HKEY, rffi.CCHARP, rwin32.DWORD, - rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) - -RegQueryValue = external( - 'RegQueryValueA', - [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], - rffi.LONG) - -RegQueryValueEx = external( - 'RegQueryValueExA', - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) - -RegCreateKey = external( - 'RegCreateKeyA', - [HKEY, rffi.CCHARP, PHKEY], - rffi.LONG) - -RegDeleteValue = external( - 'RegDeleteValueA', - [HKEY, rffi.CCHARP], - rffi.LONG) - -RegDeleteKey = external( - 'RegDeleteKeyA', - [HKEY, rffi.CCHARP], - rffi.LONG) - -RegOpenKeyEx = external( - 'RegOpenKeyExA', - [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], - rffi.LONG) - -RegEnumValue = external( - 'RegEnumValueA', - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) - -RegEnumKeyEx = external( - 'RegEnumKeyExA', - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) - -RegQueryInfoKey = external( - 'RegQueryInfoKeyA', - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) - -RegCloseKey = external( - 'RegCloseKey', - [HKEY], - rffi.LONG) - -RegFlushKey = external( - 'RegFlushKey', - [HKEY], - rffi.LONG) - -RegLoadKey = external( - 'RegLoadKeyA', - [HKEY, rffi.CCHARP, rffi.CCHARP], - rffi.LONG) - -RegSaveKey = external( - 'RegSaveKeyA', - [HKEY, rffi.CCHARP, rffi.VOIDP], - rffi.LONG) - -RegConnectRegistry = external( - 'RegConnectRegistryA', - [rffi.CCHARP, HKEY, PHKEY], - rffi.LONG) +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.tool import rffi_platform as platform +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib import rwin32 + +eci = ExternalCompilationInfo( + includes = ['windows.h', + ], + libraries = ('Advapi32',) + ) +class CConfig: + _compilation_info_ = eci + + +constant_names = ''' +KEY_QUERY_VALUE KEY_SET_VALUE KEY_CREATE_SUB_KEY KEY_ENUMERATE_SUB_KEYS +KEY_NOTIFY KEY_CREATE_LINK KEY_READ KEY_WRITE KEY_EXECUTE KEY_ALL_ACCESS +KEY_WOW64_64KEY KEY_WOW64_32KEY REG_OPTION_RESERVED REG_OPTION_NON_VOLATILE +REG_OPTION_VOLATILE REG_OPTION_CREATE_LINK REG_OPTION_BACKUP_RESTORE +REG_OPTION_OPEN_LINK REG_LEGAL_OPTION REG_CREATED_NEW_KEY +REG_OPENED_EXISTING_KEY REG_WHOLE_HIVE_VOLATILE REG_REFRESH_HIVE +REG_NO_LAZY_FLUSH REG_NOTIFY_CHANGE_NAME REG_NOTIFY_CHANGE_ATTRIBUTES +REG_NOTIFY_CHANGE_LAST_SET REG_NOTIFY_CHANGE_SECURITY REG_LEGAL_CHANGE_FILTER +REG_NONE REG_SZ REG_EXPAND_SZ REG_BINARY REG_DWORD REG_DWORD_LITTLE_ENDIAN +REG_DWORD_BIG_ENDIAN REG_LINK REG_MULTI_SZ REG_RESOURCE_LIST +REG_FULL_RESOURCE_DESCRIPTOR REG_RESOURCE_REQUIREMENTS_LIST + +HKEY_LOCAL_MACHINE HKEY_CLASSES_ROOT HKEY_CURRENT_CONFIG HKEY_CURRENT_USER +HKEY_DYN_DATA HKEY_LOCAL_MACHINE HKEY_PERFORMANCE_DATATA HKEY_USERS +'''.split() +for name in constant_names: + setattr(CConfig, name, platform.DefinedConstantInteger(name)) + +constants = {} +cConfig = platform.configure(CConfig) +constants.update(cConfig) +globals().update(cConfig) + +def external(name, args, result): + return rffi.llexternal(name, args, result, compilation_info=eci, + calling_conv='win') + +HKEY = rwin32.HANDLE +PHKEY = rffi.CArrayPtr(HKEY) +REGSAM = rwin32.DWORD + +RegSetValue = external( + 'RegSetValueA', + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) + +RegSetValueEx = external( + 'RegSetValueExA', + [HKEY, rffi.CCHARP, rwin32.DWORD, + rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) + +RegQueryValue = external( + 'RegQueryValueA', + [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], + rffi.LONG) + +RegQueryValueEx = external( + 'RegQueryValueExA', + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) + +RegCreateKey = external( + 'RegCreateKeyA', + [HKEY, rffi.CCHARP, PHKEY], + rffi.LONG) + +RegDeleteValue = external( + 'RegDeleteValueA', + [HKEY, rffi.CCHARP], + rffi.LONG) + +RegDeleteKey = external( + 'RegDeleteKeyA', + [HKEY, rffi.CCHARP], + rffi.LONG) + +RegOpenKeyEx = external( + 'RegOpenKeyExA', + [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], + rffi.LONG) + +RegEnumValue = external( + 'RegEnumValueA', + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) + +RegEnumKeyEx = external( + 'RegEnumKeyExA', + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) + +RegQueryInfoKey = external( + 'RegQueryInfoKeyA', + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) + +RegCloseKey = external( + 'RegCloseKey', + [HKEY], + rffi.LONG) + +RegFlushKey = external( + 'RegFlushKey', + [HKEY], + rffi.LONG) + +RegLoadKey = external( + 'RegLoadKeyA', + [HKEY, rffi.CCHARP, rffi.CCHARP], + rffi.LONG) + +RegSaveKey = external( + 'RegSaveKeyA', + [HKEY, rffi.CCHARP, rffi.VOIDP], + rffi.LONG) + +RegConnectRegistry = external( + 'RegConnectRegistryA', + [rffi.CCHARP, HKEY, PHKEY], + rffi.LONG) Modified: pypy/branch/faster-raise/pypy/rlib/test/test_listsort.py ============================================================================== --- pypy/branch/faster-raise/pypy/rlib/test/test_listsort.py (original) +++ pypy/branch/faster-raise/pypy/rlib/test/test_listsort.py Sat Nov 14 15:24:15 2009 @@ -35,7 +35,7 @@ sorttest(lst1) def test_file(): - for fn in py.magic.autopath().dirpath().listdir(): + for fn in py.path.local(__file__).dirpath().listdir(): if fn.ext == '.py': lines1 = fn.readlines() sorttest(lines1) Modified: pypy/branch/faster-raise/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/faster-raise/pypy/rpython/memory/gctransform/asmgcroot.py Sat Nov 14 15:24:15 2009 @@ -8,6 +8,7 @@ from pypy.objspace.flow.model import SpaceOperation from pypy.translator.unsimplify import copyvar from pypy.rlib.debug import ll_assert +import sys # @@ -296,6 +297,7 @@ return # the item may have been not found because the main array was # not sorted. Sort it and try again. + win32_follow_gcmap_jmp(gcmapstart, gcmapend) sort_gcmap(gcmapstart, gcmapend) item = search_in_gcmap(gcmapstart, gcmapend, retaddr) if item: @@ -384,6 +386,22 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +if sys.platform == 'win32': + def win32_follow_gcmap_jmp(start, end): + # The initial gcmap table contains addresses to a JMP + # instruction that jumps indirectly to the real code. + # Replace them with the target addresses. + while start < end: + code = rffi.cast(rffi.CCHARP, start.address[0])[0] + if code == '\xe9': # jmp + rel32 = rffi.cast(rffi.LONGP, start.address[0]+1)[0] + target = start.address[0] + (rel32 + 5) + start.address[0] = target + start += arrayitemsize +else: + def win32_follow_gcmap_jmp(start, end): + pass + def _compare_gcmap_entries(addr1, addr2): key1 = addr1.address[0] key2 = addr2.address[0] Modified: pypy/branch/faster-raise/pypy/rpython/microbench/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/microbench/autopath.py (original) +++ pypy/branch/faster-raise/pypy/rpython/microbench/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_path.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_path.py (original) +++ pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_path.py Sat Nov 14 15:24:15 2009 @@ -11,7 +11,7 @@ from pypy.tool.udir import udir def test_exists(): - filename = impl.to_rstr(str(py.magic.autopath())) + filename = impl.to_rstr(str(py.path.local(__file__))) assert impl.ll_os_path_exists(filename) == True assert not impl.ll_os_path_exists(impl.to_rstr( "strange_filename_that_looks_improbable.sde")) Modified: pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_stat.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_stat.py (original) +++ pypy/branch/faster-raise/pypy/rpython/module/test/test_ll_os_stat.py Sat Nov 14 15:24:15 2009 @@ -1,17 +1,17 @@ -from pypy.rpython.module import ll_os_stat -import sys, os -import py - -class TestWin32Implementation: - def setup_class(cls): - if sys.platform != 'win32': - py.test.skip("win32 specific tests") - - def test_stat(self): - stat = ll_os_stat.win32_stat_llimpl - def check(f): - assert stat(f).st_mtime == os.stat(f).st_mtime - - check('c:/') - check('c:/temp') - check('c:/pagefile.sys') +from pypy.rpython.module import ll_os_stat +import sys, os +import py + +class TestWin32Implementation: + def setup_class(cls): + if sys.platform != 'win32': + py.test.skip("win32 specific tests") + + def test_stat(self): + stat = ll_os_stat.win32_stat_llimpl + def check(f): + assert stat(f).st_mtime == os.stat(f).st_mtime + + check('c:/') + check('c:/temp') + check('c:/pagefile.sys') Modified: pypy/branch/faster-raise/pypy/rpython/normalizecalls.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/normalizecalls.py (original) +++ pypy/branch/faster-raise/pypy/rpython/normalizecalls.py Sat Nov 14 15:24:15 2009 @@ -14,8 +14,12 @@ def normalize_call_familes(annotator): for callfamily in annotator.bookkeeper.pbc_maximal_call_families.infos(): + if not callfamily.modified: + assert callfamily.normalized + continue normalize_calltable(annotator, callfamily) callfamily.normalized = True + callfamily.modified = False def normalize_calltable(annotator, callfamily): """Try to normalize all rows of a table.""" Modified: pypy/branch/faster-raise/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/rpbc.py (original) +++ pypy/branch/faster-raise/pypy/rpython/rpbc.py Sat Nov 14 15:24:15 2009 @@ -484,7 +484,8 @@ try: thisattrvalue = frozendesc.attrcache[attr] except KeyError: - warning("Desc %r has no attribute %r" % (frozendesc, attr)) + if not frozendesc.has_attribute(attr): + warning("Desc %r has no attribute %r" % (frozendesc, attr)) continue llvalue = r_value.convert_const(thisattrvalue) setattr(result, mangled_name, llvalue) Modified: pypy/branch/faster-raise/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/branch/faster-raise/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/branch/faster-raise/pypy/rpython/test/test_rbuiltin.py Sat Nov 14 15:24:15 2009 @@ -294,7 +294,7 @@ def f(fn): fn = hlstr(fn) return os.path.exists(fn) - filename = self.string_to_ll(str(py.magic.autopath())) + filename = self.string_to_ll(str(py.path.local(__file__))) assert self.interpret(f, [filename]) == True #assert self.interpret(f, [ # self.string_to_ll("strange_filename_that_looks_improbable.sde")]) == False @@ -308,7 +308,7 @@ fn = hlstr(fn) return os.path.isdir(fn) assert self.interpret(f, [self.string_to_ll("/")]) == True - assert self.interpret(f, [self.string_to_ll(str(py.magic.autopath()))]) == False + assert self.interpret(f, [self.string_to_ll(str(py.path.local(__file__)))]) == False assert self.interpret(f, [self.string_to_ll("another/unlikely/directory/name")]) == False def test_pbc_isTrue(self): Modified: pypy/branch/faster-raise/pypy/test_all.py ============================================================================== --- pypy/branch/faster-raise/pypy/test_all.py (original) +++ pypy/branch/faster-raise/pypy/test_all.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ #! /usr/bin/env python -if __name__ == '__main__': +if __name__ == '__main__': import tool.autopath - import py - py.test.cmdline.main() + import py + py.cmdline.pytest() Modified: pypy/branch/faster-raise/pypy/tool/algo/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/algo/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/tool/algo/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/tool/ansi_mandelbrot.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/ansi_mandelbrot.py (original) +++ pypy/branch/faster-raise/pypy/tool/ansi_mandelbrot.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ import sys -from py.__.io.terminalwriter import ansi_print, get_terminal_width +from py.impl.io.terminalwriter import ansi_print, get_terminal_width """ Black 0;30 Dark Gray 1;30 Modified: pypy/branch/faster-raise/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/ansi_print.py (original) +++ pypy/branch/faster-raise/pypy/tool/ansi_print.py Sat Nov 14 15:24:15 2009 @@ -4,7 +4,7 @@ import sys -from py.__.io.terminalwriter import ansi_print +from py.impl.io.terminalwriter import ansi_print from pypy.tool.ansi_mandelbrot import Driver class AnsiLog: Modified: pypy/branch/faster-raise/pypy/tool/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/autopath.py (original) +++ pypy/branch/faster-raise/pypy/tool/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/tool/bench/pypyresult.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/bench/pypyresult.py (original) +++ pypy/branch/faster-raise/pypy/tool/bench/pypyresult.py Sat Nov 14 15:24:15 2009 @@ -52,7 +52,7 @@ if __name__ == "__main__": - x = py.magic.autopath().dirpath("bench-unix.benchmark_result") + x = py.path.local(__file__).dirpath("bench-unix.benchmark_result") db = ResultDB() db.parsepickle(x) Modified: pypy/branch/faster-raise/pypy/tool/genstatistic.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/genstatistic.py (original) +++ pypy/branch/faster-raise/pypy/tool/genstatistic.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ import autopath import py -from py.__.misc.cmdline import countloc +from py.impl.misc.cmdline import countloc from py.xml import raw pypydir = py.path.local(autopath.pypydir) Modified: pypy/branch/faster-raise/pypy/tool/option.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/option.py (original) +++ pypy/branch/faster-raise/pypy/tool/option.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ import os from pypy.config.pypyoption import get_pypy_config from pypy.config.config import Config, OptionDescription, to_optparse -from py.compat import optparse +import optparse extra_useage = """For detailed descriptions of all the options see http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" Modified: pypy/branch/faster-raise/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/pytest/appsupport.py (original) +++ pypy/branch/faster-raise/pypy/tool/pytest/appsupport.py Sat Nov 14 15:24:15 2009 @@ -1,9 +1,10 @@ import autopath import py -from py.__.magic import exprinfo +import py.impl.code.assertion +from py.impl.code import _assertionold as exprinfo from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from py.__.test.outcome import ExceptionFailure +from py.impl.test.outcome import ExceptionFailure # ____________________________________________________________ Modified: pypy/branch/faster-raise/pypy/tool/pytest/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/pytest/autopath.py (original) +++ pypy/branch/faster-raise/pypy/tool/pytest/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/tool/pytest/genreportdata.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/pytest/genreportdata.py (original) +++ pypy/branch/faster-raise/pypy/tool/pytest/genreportdata.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ import py import sys -mydir = py.magic.autopath().dirpath().realpath() +mydir = py.path.local(__file__).dirpath().realpath() from pypy.tool.pytest import htmlreport from pypy.tool.pytest import confpath Modified: pypy/branch/faster-raise/pypy/tool/pytest/htmlreport.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/pytest/htmlreport.py (original) +++ pypy/branch/faster-raise/pypy/tool/pytest/htmlreport.py Sat Nov 14 15:24:15 2009 @@ -232,7 +232,7 @@ t = self.rep.render_latest_table(self.rep.results) assert unicode(t) -mydir = py.magic.autopath().dirpath() +mydir = py.path.local(__file__).dirpath() def getpicklepath(): return mydir.join('.htmlreport.pickle') Modified: pypy/branch/faster-raise/pypy/tool/pytest/test/test_new_count.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/pytest/test/test_new_count.py (original) +++ pypy/branch/faster-raise/pypy/tool/pytest/test/test_new_count.py Sat Nov 14 15:24:15 2009 @@ -2,7 +2,7 @@ import py #from pypy.tool.pytest.confpath import testresultdir from pypy.tool.pytest.result import ResultFromMime -testpath = py.magic.autopath().dirpath('data') +testpath = py.path.local(__file__).dirpath('data') class TestResultCache: Modified: pypy/branch/faster-raise/pypy/tool/statistic_over_time.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/statistic_over_time.py (original) +++ pypy/branch/faster-raise/pypy/tool/statistic_over_time.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import py -from py.__.misc.cmdline.countloc import get_loccount +from py.impl.misc.cmdline.countloc import get_loccount import datetime import time Modified: pypy/branch/faster-raise/pypy/tool/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/tool/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/tool/test/test_conftest1.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/test/test_conftest1.py (original) +++ pypy/branch/faster-raise/pypy/tool/test/test_conftest1.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ import py -innertest = py.magic.autopath().dirpath('conftest1_innertest.py') +innertest = py.path.local(__file__).dirpath('conftest1_innertest.py') pytest_plugins = "pytest_pytester" class TestPyPyTests: Modified: pypy/branch/faster-raise/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/branch/faster-raise/pypy/tool/test/test_pytestsupport.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ import autopath -from py.__.magic import exprinfo +from py.impl.code import _assertionold as exprinfo from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.argument import Arguments Modified: pypy/branch/faster-raise/pypy/tool/udir.py ============================================================================== --- pypy/branch/faster-raise/pypy/tool/udir.py (original) +++ pypy/branch/faster-raise/pypy/tool/udir.py Sat Nov 14 15:24:15 2009 @@ -35,7 +35,7 @@ dir = local(dir) if basename is None: try: - p = py.magic.autopath().dirpath() + p = py.path.local(__file__).dirpath() basename = svn_info(py.path.svnwc(p).info().url) except: basename = '' Modified: pypy/branch/faster-raise/pypy/translator/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/faster-raise/pypy/translator/backendopt/test/test_writeanalyze.py Sat Nov 14 15:24:15 2009 @@ -130,6 +130,21 @@ result = wa.analyze(fgraph.startblock.operations[0]) assert not result + def test_contains(self): + def g(x, y, z): + l = [x] + return f(l, y, z) + def f(x, y, z): + return y in x + + + t, wa = self.translate(g, [int, int, int]) + ggraph = graphof(t, g) + assert ggraph.startblock.operations[-1].opname == 'direct_call' + + result = wa.analyze(ggraph.startblock.operations[-1]) + assert not result + class TestLLtype(BaseTestCanRaise): type_system = 'lltype' @@ -164,6 +179,7 @@ assert S1 is S2 + class TestOOtype(BaseTestCanRaise): type_system = 'ootype' Modified: pypy/branch/faster-raise/pypy/translator/benchmark/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/benchmark/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/benchmark/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/benchmark/benchmarks.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/benchmark/benchmarks.py (original) +++ pypy/branch/faster-raise/pypy/translator/benchmark/benchmarks.py Sat Nov 14 15:24:15 2009 @@ -34,7 +34,7 @@ def external_dependency(dirname, svnurl, revision): """Check out (if necessary) a given fixed revision of a svn url.""" - dirpath = py.magic.autopath().dirpath().join(dirname) + dirpath = py.path.local(__file__).dirpath().join(dirname) revtag = dirpath.join('-svn-rev-') if dirpath.check(): if not revtag.check() or int(revtag.read()) != revision: @@ -70,13 +70,13 @@ return get_result(txt, PYSTONE_PATTERN) def run_richards(executable='/usr/local/bin/python', n=5): - richards = py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py') + richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') txt = run_cmd('"%s" %s %s' % (executable, richards, n)) return get_result(txt, RICHARDS_PATTERN) def run_translate(executable='/usr/local/bin/python'): - translate = py.magic.autopath().dirpath().dirpath().join('goal').join('translate.py') - target = py.magic.autopath().dirpath().dirpath().join('goal').join('targetrpystonedalone.py') + translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') + target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' T = time.time() status = os.system(argstr%(executable, translate, target)) @@ -87,7 +87,7 @@ def run_docutils(executable='/usr/local/bin/python'): docutilssvnpath = 'docutils' # subdir of the local dir - translatetxt = py.magic.autopath().dirpath().dirpath().dirpath().join('doc').join('translation.txt') + translatetxt = py.path.local(__file__).dirpath().dirpath().dirpath().join('doc').join('translation.txt') command = """import sys sys.modules['unicodedata'] = sys # docutils need 'import unicodedata' to work, but no more... sys.path[0:0] = ['%s', '%s/extras'] @@ -123,7 +123,7 @@ templess is some simple templating language, to check out use 'svn co -r100 http://johnnydebris.net/templess/trunk templess' """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() pypath = os.path.dirname(os.path.dirname(py.__file__)) templessdir = here.join('templess') testscript = templessdir.join('test/oneshot.py') @@ -143,7 +143,7 @@ def run_gadfly(executable='/usr/local/bin/python'): """ run some tests in the gadfly pure Python database """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() gadfly = here.join('gadfly') testscript = gadfly.join('test', 'testsubset.py') command = 'PYTHONPATH="%s" "%s" "%s"' % (gadfly, executable, testscript) @@ -167,7 +167,7 @@ def run_mako(executable='/usr/local/bin/python'): """ run some tests in the mako templating system """ - here = py.magic.autopath().dirpath() + here = py.path.local(__file__).dirpath() mako = here.join('mako') testscript = mako.join('examples', 'bench', 'basic.py') command = 'PYTHONPATH="%s" "%s" "%s" mako' % (mako.join('lib'), Modified: pypy/branch/faster-raise/pypy/translator/benchmark/jitbench.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/benchmark/jitbench.py (original) +++ pypy/branch/faster-raise/pypy/translator/benchmark/jitbench.py Sat Nov 14 15:24:15 2009 @@ -14,7 +14,7 @@ response.read() def run_richards(executable='python'): - richards = str(py.magic.autopath().dirpath().dirpath().join('goal').join('richards.py')) + richards = str(py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py')) pipe = subprocess.Popen([executable, richards], stdout=subprocess.PIPE, stderr=subprocess.PIPE) return pipe.communicate() Modified: pypy/branch/faster-raise/pypy/translator/c/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_asmgcroot.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_asmgcroot.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_asmgcroot.py Sat Nov 14 15:24:15 2009 @@ -6,18 +6,20 @@ from pypy.annotation.listdef import s_list_of_strings from pypy import conftest -def setup_module(module): - if sys.platform == 'win32': - if not ('mingw' in os.popen('gcc --version').read() and - 'GNU' in os.popen('make --version').read()): - py.test.skip("mingw32 and MSYS are required for asmgcc on Windows") - class AbstractTestAsmGCRoot: # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved # instructions: should_be_moving = False @classmethod + def make_config(cls): + from pypy.config.pypyoption import get_pypy_config + config = get_pypy_config(translating=True) + config.translation.gc = cls.gcpolicy + config.translation.gcrootfinder = "asmgcc" + return config + + @classmethod def _makefunc_str_int(cls, func): def main(argv): arg0 = argv[1] @@ -29,12 +31,7 @@ else: print 'Result: "%s"' % (res,) return 0 - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) - config.translation.gc = cls.gcpolicy - config.translation.gcrootfinder = "asmgcc" - if sys.platform == 'win32': - config.translation.cc = 'mingw32' + config = cls.make_config() t = TranslationContext(config=config) a = t.buildannotator() a.build_types(main, [s_list_of_strings]) @@ -51,7 +48,7 @@ def run(arg0, arg1): lines = [] - print >> sys.stderr, 'RUN: starting', exe_name + print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': redirect = ' 2> NUL' else: @@ -162,6 +159,33 @@ assert res == 4900 +class TestAsmGCRootWithSemiSpaceGC_Mingw32(TestAsmGCRootWithSemiSpaceGC): + # for the individual tests see + # ====> ../../test/test_newgc.py + + @classmethod + def setup_class(cls): + if sys.platform != 'win32': + py.test.skip("mingw32 specific test") + if not ('mingw' in os.popen('gcc --version').read() and + 'GNU' in os.popen('make --version').read()): + py.test.skip("mingw32 and MSYS are required for this test") + + test_newgc.TestSemiSpaceGC.setup_class.im_func(cls) + + @classmethod + def make_config(cls): + config = TestAsmGCRootWithSemiSpaceGC.make_config() + config.translation.cc = 'mingw32' + return config + + + def test_callback_with_collect(self): + py.test.skip("No libffi yet with mingw32") + + def define_callback_with_collect(cls): + return lambda: 0 + class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot, test_newgc.TestHybridTaggedPointers): pass Modified: pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/gcc/test/test_trackgcroot.py Sat Nov 14 15:24:15 2009 @@ -4,11 +4,11 @@ from pypy.translator.c.gcc.trackgcroot import format_callshape from pypy.translator.c.gcc.trackgcroot import LOC_NOWHERE, LOC_REG from pypy.translator.c.gcc.trackgcroot import LOC_EBP_BASED, LOC_ESP_BASED -from pypy.translator.c.gcc.trackgcroot import GcRootTracker -from pypy.translator.c.gcc.trackgcroot import FunctionGcRootTracker +from pypy.translator.c.gcc.trackgcroot import ElfAssemblerParser +from pypy.translator.c.gcc.trackgcroot import DarwinAssemblerParser from pypy.translator.c.gcc.trackgcroot import compress_callshape from pypy.translator.c.gcc.trackgcroot import decompress_callshape -from pypy.translator.c.gcc.trackgcroot import OFFSET_LABELS +from pypy.translator.c.gcc.trackgcroot import PARSERS from StringIO import StringIO this_dir = py.path.local(__file__).dirpath() @@ -63,7 +63,7 @@ \tMORE STUFF """ lines = source.splitlines(True) - parts = list(GcRootTracker().find_functions(iter(lines))) + parts = list(ElfAssemblerParser().find_functions(iter(lines))) assert len(parts) == 5 assert parts[0] == (False, lines[:2]) assert parts[1] == (True, lines[2:5]) @@ -96,7 +96,7 @@ \t.section stuff """ lines = source.splitlines(True) - parts = list(GcRootTracker(format='darwin').find_functions(iter(lines))) + parts = list(DarwinAssemblerParser().find_functions(iter(lines))) assert len(parts) == 7 assert parts[0] == (False, lines[:3]) assert parts[1] == (True, lines[3:7]) @@ -108,7 +108,7 @@ def test_computegcmaptable(): tests = [] - for format in ('elf', 'darwin'): + for format in ('elf', 'darwin', 'msvc'): for path in this_dir.join(format).listdir("track*.s"): n = path.purebasename[5:] try: @@ -120,15 +120,20 @@ for format, _, path in tests: yield check_computegcmaptable, format, path -r_globallabel = re.compile(r"([\w]+)=[.]+") -r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") + +r_expected = re.compile(r"\s*;;\s*expected\s+([{].+[}])") +r_gcroot_constant = re.compile(r";\tmov\t.+, .+_constant_always_one_") def check_computegcmaptable(format, path): + if format == 'msvc': + r_globallabel = re.compile(r"([\w]+)::") + else: + r_globallabel = re.compile(r"([\w]+)=[.]+") print - print path.basename + print path.dirpath().basename + '/' + path.basename lines = path.readlines() expectedlines = lines[:] - tracker = FunctionGcRootTracker(lines, format=format) + tracker = PARSERS[format].FunctionGcRootTracker(lines) table = tracker.computegcmaptable(verbose=sys.maxint) tabledict = {} seen = {} @@ -148,10 +153,22 @@ got = tabledict[label] assert format_callshape(got) == expected seen[label] = True - expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) - expectedlines.insert(i-1, '%s=.+%d\n' % (label, OFFSET_LABELS)) + if format == 'msvc': + expectedlines.insert(i-2, 'PUBLIC\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s::\n' % (label,)) + else: + expectedlines.insert(i-2, '\t.globl\t%s\n' % (label,)) + expectedlines.insert(i-1, '%s=.+%d\n' % (label, + tracker.OFFSET_LABELS)) + if format == 'msvc' and r_gcroot_constant.match(line): + expectedlines[i] = ';' + expectedlines[i] + expectedlines[i+1] = (expectedlines[i+1] + .replace('\timul\t', '\tmov\t') + + '\t; GCROOT\n') prevline = line assert len(seen) == len(tabledict), ( "computed table contains unexpected entries:\n%r" % [key for key in tabledict if key not in seen]) + print lines + print expectedlines assert lines == expectedlines Modified: pypy/branch/faster-raise/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/gcc/trackgcroot.py Sat Nov 14 15:24:15 2009 @@ -1,302 +1,40 @@ #! /usr/bin/env python - +import autopath import re, sys, os, random -LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' -r_functionstart_elf = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") -r_functionend_elf = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") - -# darwin -r_textstart = re.compile(r"\t.text\s*$") -# see -# http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html -OTHERSECTIONS = ['section', 'zerofill', - 'const', 'static_const', 'cstring', - 'literal4', 'literal8', 'literal16', - 'constructor', 'desctructor', - 'symbol_stub', - 'data', 'static_data', - 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', - 'dyld', 'mod_init_func', 'mod_term_func', - 'const_data' - ] -r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") -r_functionstart_darwin = re.compile(r"_(\w+):\s*$") - -if sys.platform != 'darwin': - OFFSET_LABELS = 2**30 -else: - OFFSET_LABELS = 0 - -# inside functions -r_label = re.compile(LABEL+"[:]\s*$") -r_rel_label = re.compile(r"(\d+):\s*$") -r_globl = re.compile(r"\t[.]globl\t"+LABEL+"\s*$") -r_globllabel = re.compile(LABEL+r"=[.][+]%d\s*$"%OFFSET_LABELS) -r_insn = re.compile(r"\t([a-z]\w*)\s") -r_jump = re.compile(r"\tj\w+\s+"+LABEL+"\s*$") -r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") -OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' -r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+")\s*$") -r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+([*]"+OPERAND+")\s*$") -r_jmp_switch = re.compile(r"\tjmp\t[*]"+LABEL+"[(]") -r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") -r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") -r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -r_binaryinsn = re.compile(r"\t[a-z]\w*\s+("+OPERAND+"),\s*("+OPERAND+")\s*$") -LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" -LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" -r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") -r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") -r_localvarnofp = re.compile(LOCALVAR) -r_localvarfp = re.compile(LOCALVARFP) -r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") -r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") - - -class GcRootTracker(object): - - def __init__(self, verbose=0, shuffle=False, format='elf'): - self.verbose = verbose - self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py - self.format = format - self.clear() - - def clear(self): - self.gcmaptable = [] - self.seen_main = False - - def dump_raw_table(self, output): - print >> output, "seen_main = %d" % (self.seen_main,) - for entry in self.gcmaptable: - print >> output, entry - - def reload_raw_table(self, input): - firstline = input.readline() - assert firstline.startswith("seen_main = ") - self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) - for line in input: - entry = eval(line) - assert type(entry) is tuple - self.gcmaptable.append(entry) - - def dump(self, output): - assert self.seen_main - shapes = {} - shapelines = [] - shapeofs = 0 - def _globalname(name): - if self.format in ('darwin', 'mingw32'): - return '_' + name - return name - def _globl(name): - print >> output, "\t.globl %s" % _globalname(name) - def _label(name): - print >> output, "%s:" % _globalname(name) - def _variant(**kwargs): - txt = kwargs[self.format] - print >> output, "\t%s" % txt - - print >> output, "\t.text" - _globl('pypy_asm_stackwalk') - _variant(elf='.type pypy_asm_stackwalk, @function', - darwin='', - mingw32='') - _label('pypy_asm_stackwalk') - print >> output, """\ - /* See description in asmgcroot.py */ - movl 4(%esp), %edx /* my argument, which is the callback */ - movl %esp, %eax /* my frame top address */ - pushl %eax /* ASM_FRAMEDATA[6] */ - pushl %ebp /* ASM_FRAMEDATA[5] */ - pushl %edi /* ASM_FRAMEDATA[4] */ - pushl %esi /* ASM_FRAMEDATA[3] */ - pushl %ebx /* ASM_FRAMEDATA[2] */ - - /* Add this ASM_FRAMEDATA to the front of the circular linked */ - /* list. Let's call it 'self'. */ - movl __gcrootanchor+4, %eax /* next = gcrootanchor->next */ - pushl %eax /* self->next = next */ - pushl $__gcrootanchor /* self->prev = gcrootanchor */ - movl %esp, __gcrootanchor+4 /* gcrootanchor->next = self */ - movl %esp, (%eax) /* next->prev = self */ - - /* note: the Mac OS X 16 bytes aligment must be respected. */ - call *%edx /* invoke the callback */ - - /* Detach this ASM_FRAMEDATA from the circular linked list */ - popl %esi /* prev = self->prev */ - popl %edi /* next = self->next */ - movl %edi, 4(%esi) /* prev->next = next */ - movl %esi, (%edi) /* next->prev = prev */ - - popl %ebx /* restore from ASM_FRAMEDATA[2] */ - popl %esi /* restore from ASM_FRAMEDATA[3] */ - popl %edi /* restore from ASM_FRAMEDATA[4] */ - popl %ebp /* restore from ASM_FRAMEDATA[5] */ - popl %ecx /* ignored ASM_FRAMEDATA[6] */ - /* the return value is the one of the 'call' above, */ - /* because %eax (and possibly %edx) are unmodified */ - ret -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', - darwin='', - mingw32='') - print >> output, '\t.data' - print >> output, '\t.align\t4' - _globl('__gcrootanchor') - _label('__gcrootanchor') - print >> output, """\ - /* A circular doubly-linked list of all */ - /* the ASM_FRAMEDATAs currently alive */ - .long\t__gcrootanchor /* prev */ - .long\t__gcrootanchor /* next */ -""".replace("__gcrootanchor", _globalname("__gcrootanchor")) - _globl('__gcmapstart') - _label('__gcmapstart') - for label, state, is_range in self.gcmaptable: - try: - n = shapes[state] - except KeyError: - n = shapes[state] = shapeofs - bytes = [str(b) for b in compress_callshape(state)] - shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( - shapeofs, - ', '.join(bytes))) - shapeofs += len(bytes) - if is_range: - n = ~ n - print >> output, '\t.long\t%s-%d' % (label, OFFSET_LABELS) - print >> output, '\t.long\t%d' % (n,) - _globl('__gcmapend') - _label('__gcmapend') - _variant(elf='.section\t.rodata', - darwin='.const', - mingw32='') - _globl('__gccallshapes') - _label('__gccallshapes') - output.writelines(shapelines) - - def find_functions(self, iterlines): - _find_functions = getattr(self, '_find_functions_' + self.format) - return _find_functions(iterlines) - - def _find_functions_elf(self, iterlines): - functionlines = [] - in_function = False - for line in iterlines: - if r_functionstart_elf.match(line): - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - in_function = True - functionlines = [] - functionlines.append(line) - if r_functionend_elf.match(line): - assert in_function, ( - "missed the start of the current function") - yield True, functionlines - in_function = False - functionlines = [] - assert not in_function, ( - "missed the end of the previous function") - yield False, functionlines - - def _find_functions_darwin(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - assert not in_text, "unexpected repeated .text start: %d" % n - in_text = True - elif r_sectionstart.match(line): - if in_function: - yield in_function, functionlines - functionlines = [] - in_text = False - in_function = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - - if functionlines: - yield in_function, functionlines - - def _find_functions_mingw32(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if r_textstart.match(line): - in_text = True - elif r_sectionstart.match(line): - in_text = False - elif in_text and r_functionstart_darwin.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - if functionlines: - yield in_function, functionlines - - def process(self, iterlines, newfile, entrypoint='main', filename='?'): - if self.format in ('darwin', 'mingw32'): - entrypoint = '_' + entrypoint - for in_function, lines in self.find_functions(iterlines): - if in_function: - lines = self.process_function(lines, entrypoint, filename) - newfile.writelines(lines) - if self.verbose == 1: - sys.stderr.write('\n') - - def process_function(self, lines, entrypoint, filename): - tracker = FunctionGcRootTracker(lines, filetag=getidentifier(filename), - format=self.format) - is_main = tracker.funcname == entrypoint - tracker.is_stack_bottom = is_main - if self.verbose == 1: - sys.stderr.write('.') - elif self.verbose > 1: - print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, - tracker.funcname) - table = tracker.computegcmaptable(self.verbose) - if self.verbose > 1: - for label, state in table: - print >> sys.stderr, label, '\t', format_callshape(state) - table = compress_gcmaptable(table) - if self.shuffle and random.random() < 0.5: - self.gcmaptable[:0] = table - else: - self.gcmaptable.extend(table) - self.seen_main |= is_main - return tracker.lines - +from pypy.translator.c.gcc.instruction import Insn, Label, InsnCall, InsnRet +from pypy.translator.c.gcc.instruction import InsnFunctionStart, InsnStop +from pypy.translator.c.gcc.instruction import InsnSetLocal, InsnCopyLocal +from pypy.translator.c.gcc.instruction import InsnPrologue, InsnEpilogue +from pypy.translator.c.gcc.instruction import InsnGCROOT +from pypy.translator.c.gcc.instruction import InsnStackAdjust +from pypy.translator.c.gcc.instruction import InsnCannotFollowEsp +from pypy.translator.c.gcc.instruction import LocalVar, somenewvalue, frameloc +from pypy.translator.c.gcc.instruction import LOC_REG, LOC_NOWHERE, LOC_MASK +from pypy.translator.c.gcc.instruction import LOC_EBP_BASED, LOC_ESP_BASED class FunctionGcRootTracker(object): + skip = 0 - def __init__(self, lines, filetag=0, format='elf'): - if format == 'elf': - match = r_functionstart_elf.match(lines[0]) - funcname = match.group(1) - match = r_functionend_elf.match(lines[-1]) - assert funcname == match.group(1) - assert funcname == match.group(2) - elif format == 'darwin': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - elif format == 'mingw32': - match = r_functionstart_darwin.match(lines[0]) - funcname = '_'+match.group(1) - else: - assert False, "unknown format: %s" % format + @classmethod + def init_regexp(cls): + cls.r_label = re.compile(cls.LABEL+"[:]\s*$") + cls.r_globl = re.compile(r"\t[.]globl\t"+cls.LABEL+"\s*$") + cls.r_globllabel = re.compile(cls.LABEL+r"=[.][+]%d\s*$"%cls.OFFSET_LABELS) + + cls.r_insn = re.compile(r"\t([a-z]\w*)\s") + cls.r_unaryinsn = re.compile(r"\t[a-z]\w*\s+("+cls.OPERAND+")\s*$") + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+"),\s*(?P"+cls.OPERAND+")\s*$") + + cls.r_jump = re.compile(r"\tj\w+\s+"+cls.LABEL+"\s*$") + cls.r_jmp_switch = re.compile(r"\tjmp\t[*]"+cls.LABEL+"[(]") + cls.r_jmp_source = re.compile(r"\d*[(](%[\w]+)[,)]") + def __init__(self, funcname, lines, filetag=0): self.funcname = funcname self.lines = lines self.uses_frame_pointer = False - self.r_localvar = r_localvarnofp + self.r_localvar = self.r_localvarnofp self.filetag = filetag # a "stack bottom" function is either main() or a callback from C code self.is_stack_bottom = False @@ -316,6 +54,9 @@ self.dump() return self.gettable() + def replace_symbols(self, operand): + return operand + def gettable(self): """Returns a list [(label_after_call, callshape_tuple)] See format_callshape() for more details about callshape_tuple. @@ -333,7 +74,7 @@ shape = [retaddr] # the first gcroots are always the ones corresponding to # the callee-saved registers - for reg in CALLEE_SAVE_REGISTERS: + for reg in self.CALLEE_SAVE_REGISTERS: shape.append(LOC_NOWHERE) gcroots = [] for localvar, tag in insn.gcroots.items(): @@ -341,17 +82,17 @@ loc = localvar.getlocation(insn.framesize, self.uses_frame_pointer) else: - assert localvar in REG2LOC, "%s: %s" % (self.funcname, - localvar) - loc = REG2LOC[localvar] + assert localvar in self.REG2LOC, "%s: %s" % (self.funcname, + localvar) + loc = self.REG2LOC[localvar] assert isinstance(loc, int) if tag is None: gcroots.append(loc) else: - regindex = CALLEE_SAVE_REGISTERS.index(tag) + regindex = self.CALLEE_SAVE_REGISTERS.index(tag) shape[1 + regindex] = loc if LOC_NOWHERE in shape and not self.is_stack_bottom: - reg = CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] + reg = self.CALLEE_SAVE_REGISTERS[shape.index(LOC_NOWHERE) - 1] raise AssertionError("cannot track where register %s is saved" % (reg,)) gcroots.sort() @@ -362,13 +103,13 @@ def findlabels(self): self.labels = {} # {name: Label()} for lineno, line in enumerate(self.lines): - match = r_label.match(line) + match = self.r_label.match(line) label = None if match: label = match.group(1) else: # labels used by: j* NNNf - match = r_rel_label.match(line) + match = self.r_rel_label.match(line) if match: label = "rel %d" % lineno if label: @@ -388,30 +129,35 @@ lst.append(previnsn) def parse_instructions(self): - self.insns = [InsnFunctionStart()] + self.insns = [InsnFunctionStart(self.CALLEE_SAVE_REGISTERS)] ignore_insns = False for lineno, line in enumerate(self.lines): + if lineno < self.skip: + continue self.currentlineno = lineno insn = [] - match = r_insn.match(line) - if match: + match = self.r_insn.match(line) + + if self.r_bottom_marker.match(line): + self.is_stack_bottom = True + elif match: if not ignore_insns: opname = match.group(1) try: meth = getattr(self, 'visit_' + opname) except AttributeError: - meth = self.find_missing_visit_method(opname) + self.find_missing_visit_method(opname) + meth = getattr(self, 'visit_' + opname) + line = line.rsplit(';', 1)[0] insn = meth(line) - elif r_gcroot_marker.match(line): + elif self.r_gcroot_marker.match(line): insn = self._visit_gcroot_marker(line) - elif r_bottom_marker.match(line): - self.is_stack_bottom = True elif line == '\t/* ignore_in_trackgcroot */\n': ignore_insns = True elif line == '\t/* end_ignore_in_trackgcroot */\n': ignore_insns = False else: - match = r_label.match(line) + match = self.r_label.match(line) if match: insn = self.labels[match.group(1)] @@ -421,18 +167,17 @@ else: self.append_instruction(insn) - del self.currentlineno + del self.currentlineno - def find_missing_visit_method(self, opname): + @classmethod + def find_missing_visit_method(cls, opname): # only for operations that are no-ops as far as we are concerned prefix = opname - while prefix not in self.IGNORE_OPS_WITH_PREFIXES: + while prefix not in cls.IGNORE_OPS_WITH_PREFIXES: prefix = prefix[:-1] if not prefix: raise UnrecognizedOperation(opname) - visit_nop = FunctionGcRootTracker.__dict__['visit_nop'] - setattr(FunctionGcRootTracker, 'visit_' + opname, visit_nop) - return self.visit_nop + setattr(cls, 'visit_' + opname, cls.visit_nop) def list_call_insns(self): return [insn for insn in self.insns if isinstance(insn, InsnCall)] @@ -461,8 +206,8 @@ for insn1, delta1 in deltas.items(): if hasattr(insn1, 'framesize'): size_at_insn.append(insn1.framesize + delta1) - assert len(size_at_insn) > 0, ( - "cannot reach the start of the function??") + if not size_at_insn: + continue size_at_insn = size_at_insn[0] for insn1, delta1 in deltas.items(): size_at_insn1 = size_at_insn - delta1 @@ -480,20 +225,24 @@ elif isinstance(localvar, (list, tuple)): return [fixvar(var) for var in localvar] - match = r_localvar_esp.match(localvar) + match = self.r_localvar_esp.match(localvar) if match: - if localvar == '0(%esp)': # for pushl and popl, by - hint = None # default ebp addressing is - else: # a bit nicer + if localvar == self.TOP_OF_STACK: # for pushl and popl, by + hint = None # default ebp addressing is + else: # a bit nicer hint = 'esp' ofs_from_esp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_esp += int(match.group(2) or '0') localvar = ofs_from_esp - insn.framesize assert localvar != 0 # that's the return address return LocalVar(localvar, hint=hint) elif self.uses_frame_pointer: - match = r_localvar_ebp.match(localvar) + match = self.r_localvar_ebp.match(localvar) if match: ofs_from_ebp = int(match.group(1) or '0') + if self.format == 'msvc': + ofs_from_ebp += int(match.group(2) or '0') localvar = ofs_from_ebp - 4 assert localvar != 0 # that's the return address return LocalVar(localvar, hint='ebp') @@ -553,10 +302,10 @@ # script); otherwise invent a name and add the label to tracker.lines. label = None # this checks for a ".globl NAME" followed by "NAME:" - match = r_globl.match(self.lines[call.lineno+1]) + match = self.r_globl.match(self.lines[call.lineno+1]) if match: label1 = match.group(1) - match = r_globllabel.match(self.lines[call.lineno+2]) + match = self.r_globllabel.match(self.lines[call.lineno+2]) if match: label2 = match.group(1) if label1 == label2: @@ -569,22 +318,26 @@ break k += 1 self.labels[label] = None - # These global symbols are not directly labels pointing to the - # code location because such global labels in the middle of - # functions confuse gdb. Instead, we add to the global symbol's - # value a big constant, which is subtracted again when we need - # the original value for gcmaptable.s. That's a hack. - self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, - OFFSET_LABELS)) - self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) + if self.format == 'msvc': + self.lines.insert(call.lineno+1, '%s::\n' % (label,)) + self.lines.insert(call.lineno+1, 'PUBLIC\t%s\n' % (label,)) + else: + # These global symbols are not directly labels pointing to the + # code location because such global labels in the middle of + # functions confuse gdb. Instead, we add to the global symbol's + # value a big constant, which is subtracted again when we need + # the original value for gcmaptable.s. That's a hack. + self.lines.insert(call.lineno+1, '%s=.+%d\n' % (label, + self.OFFSET_LABELS)) + self.lines.insert(call.lineno+1, '\t.globl\t%s\n' % (label,)) call.global_label = label # ____________________________________________________________ def _visit_gcroot_marker(self, line): - match = r_gcroot_marker.match(line) + match = self.r_gcroot_marker.match(line) loc = match.group(1) - return InsnGCROOT(loc) + return InsnGCROOT(self.replace_symbols(loc)) def visit_nop(self, line): return [] @@ -615,15 +368,15 @@ visit_xorw = visit_nop def visit_addl(self, line, sign=+1): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if target == '%esp': - count = match.group(1) - if not count.startswith('$'): + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if target == self.ESP: + count = self.extract_immediate(source) + if count is None: # strange instruction - I've seen 'subl %eax, %esp' return InsnCannotFollowEsp() - return InsnStackAdjust(sign * int(count[1:])) + return InsnStackAdjust(sign * count) elif self.r_localvar.match(target): return InsnSetLocal(target, [source, target]) else: @@ -633,7 +386,7 @@ return self.visit_addl(line, sign=-1) def unary_insn(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) if self.r_localvar.match(target): return InsnSetLocal(target) @@ -641,14 +394,14 @@ return [] def binary_insn(self, line): - match = r_binaryinsn.match(line) + match = self.r_binaryinsn.match(line) if not match: raise UnrecognizedOperation(line) - source = match.group(1) - target = match.group(2) + source = match.group("source") + target = match.group("target") if self.r_localvar.match(target): return InsnSetLocal(target, [source]) - elif target == '%esp': + elif target == self.ESP: raise UnrecognizedOperation(line) else: return [] @@ -673,9 +426,9 @@ visit_cmovno = binary_insn def visit_andl(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for andl $-16, %esp used to align the stack in main(). # The exact amount of adjutment is not known yet, so we use # an odd-valued estimate to make sure the real value is not used @@ -685,12 +438,12 @@ return self.binary_insn(line) def visit_leal(self, line): - match = r_binaryinsn.match(line) - target = match.group(2) - if target == '%esp': + match = self.r_binaryinsn.match(line) + target = match.group("target") + if target == self.ESP: # only for leal -12(%ebp), %esp in function epilogues - source = match.group(1) - match = r_localvar_ebp.match(source) + source = match.group("source") + match = self.r_localvar_ebp.match(source) if match: if not self.uses_frame_pointer: raise UnrecognizedOperation('epilogue without prologue') @@ -698,7 +451,7 @@ assert ofs_from_ebp <= 0 framesize = 4 - ofs_from_ebp else: - match = r_localvar_esp.match(source) + match = self.r_localvar_esp.match(source) # leal 12(%esp), %esp if match: return InsnStackAdjust(int(match.group(1))) @@ -709,7 +462,9 @@ return self.binary_insn(line) def insns_for_copy(self, source, target): - if source == '%esp' or target == '%esp': + source = self.replace_symbols(source) + target = self.replace_symbols(target) + if source == self.ESP or target == self.ESP: raise UnrecognizedOperation('%s -> %s' % (source, target)) elif self.r_localvar.match(target): if self.r_localvar.match(source): @@ -720,14 +475,14 @@ return [] def visit_movl(self, line): - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) - if source == '%esp' and target == '%ebp': + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") + if source == self.ESP and target == self.EBP: return self._visit_prologue() - elif source == '%ebp' and target == '%esp': + elif source == self.EBP and target == self.ESP: return self._visit_epilogue() - if source == '%esp' and self.funcname.startswith('VALGRIND_'): + if source == self.ESP and self.funcname.startswith('VALGRIND_'): return [] # in VALGRIND_XXX functions, there is a dummy-looking # mov %esp, %eax. Shows up only when compiling with # gcc -fno-unit-at-a-time. @@ -736,25 +491,25 @@ visit_mov = visit_movl def visit_pushl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) source = match.group(1) - return [InsnStackAdjust(-4)] + self.insns_for_copy(source, '0(%esp)') + return [InsnStackAdjust(-4)] + self.insns_for_copy(source, self.TOP_OF_STACK) def visit_pushw(self, line): return [InsnStackAdjust(-2)] # rare but not impossible def _visit_pop(self, target): - return self.insns_for_copy('0(%esp)', target) + [InsnStackAdjust(+4)] + return self.insns_for_copy(self.TOP_OF_STACK, target) + [InsnStackAdjust(+4)] def visit_popl(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) return self._visit_pop(target) def _visit_prologue(self): # for the prologue of functions that use %ebp as frame pointer self.uses_frame_pointer = True - self.r_localvar = r_localvarfp + self.r_localvar = self.r_localvarfp return [InsnPrologue()] def _visit_epilogue(self): @@ -763,38 +518,38 @@ return [InsnEpilogue(4)] def visit_leave(self, line): - return self._visit_epilogue() + self._visit_pop('%ebp') + return self._visit_epilogue() + self._visit_pop(self.EBP) def visit_ret(self, line): - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) def visit_jmp(self, line): tablelabels = [] - match = r_jmp_switch.match(line) + match = self.r_jmp_switch.match(line) if match: # this is a jmp *Label(%index), used for table-based switches. # Assume that the table is just a list of lines looking like # .long LABEL or .long 0, ending in a .text or .section .text.hot. tablelabels.append(match.group(1)) - elif r_unaryinsn_star.match(line): + elif self.r_unaryinsn_star.match(line): # maybe a jmp similar to the above, but stored in a # registry: # movl L9341(%eax), %eax # jmp *%eax - operand = r_unaryinsn_star.match(line).group(1)[1:] + operand = self.r_unaryinsn_star.match(line).group(1) def walker(insn, locs): sources = [] for loc in locs: for s in insn.all_sources_of(loc): # if the source looks like 8(%eax,%edx,4) # %eax is the real source, %edx is an offset. - match = r_jmp_source.match(s) - if match and not r_localvar_esp.match(s): + match = self.r_jmp_source.match(s) + if match and not self.r_localvar_esp.match(s): sources.append(match.group(1)) else: sources.append(s) for source in sources: - label_match = re.compile(LABEL).match(source) + label_match = re.compile(self.LABEL).match(source) if label_match: tablelabels.append(label_match.group(0)) return @@ -809,37 +564,43 @@ assert len(tablelabels) <= 1 if tablelabels: tablelin = self.labels[tablelabels[0]].lineno + 1 - while not r_jmptable_end.match(self.lines[tablelin]): - match = r_jmptable_item.match(self.lines[tablelin]) + while not self.r_jmptable_end.match(self.lines[tablelin]): + # skip empty lines + if (not self.lines[tablelin].strip() + or self.lines[tablelin].startswith(';')): + tablelin += 1 + continue + match = self.r_jmptable_item.match(self.lines[tablelin]) if not match: - raise NoPatternMatch(self.lines[tablelin]) + raise NoPatternMatch(repr(self.lines[tablelin])) label = match.group(1) if label != '0': self.register_jump_to(label) tablelin += 1 return InsnStop() - if r_unaryinsn_star.match(line): + if self.r_unaryinsn_star.match(line): # that looks like an indirect tail-call. # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) try: self.conditional_jump(line) except KeyError: # label not found: check if it's a tail-call turned into a jump - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) target = match.group(1) assert not target.startswith('.') # tail-calls are equivalent to RET for us - return InsnRet() + return InsnRet(self.CALLEE_SAVE_REGISTERS) return InsnStop() def register_jump_to(self, label): - self.labels[label].previous_insns.append(self.insns[-1]) + if not isinstance(self.insns[-1], InsnStop): + self.labels[label].previous_insns.append(self.insns[-1]) def conditional_jump(self, line): - match = r_jump.match(line) + match = self.r_jump.match(line) if not match: - match = r_jump_rel_label.match(line) + match = self.r_jump_rel_label.match(line) if not match: raise UnrecognizedOperation(line) # j* NNNf @@ -878,260 +639,116 @@ def visit_xchgl(self, line): # only support the format used in VALGRIND_DISCARD_TRANSLATIONS # which is to use a marker no-op "xchgl %ebx, %ebx" - match = r_binaryinsn.match(line) - source = match.group(1) - target = match.group(2) + match = self.r_binaryinsn.match(line) + source = match.group("source") + target = match.group("target") if source == target: return [] raise UnrecognizedOperation(line) def visit_call(self, line): - match = r_unaryinsn.match(line) + match = self.r_unaryinsn.match(line) + if match is None: - assert r_unaryinsn_star.match(line) # indirect call - else: - target = match.group(1) - if target in FUNCTIONS_NOT_RETURNING: - return InsnStop() - if sys.platform == 'win32' and target == '__alloca': - # in functions with large stack requirements, windows - # needs a call to _alloca(), to turn reserved pages - # into committed memory. - # With mingw32 gcc at least, %esp is not used before - # this call. So we don't bother to compute the exact - # stack effect. - return [InsnCannotFollowEsp()] - if target in self.labels: - lineoffset = self.labels[target].lineno - self.currentlineno - if lineoffset >= 0: - assert lineoffset in (1,2) - return [InsnStackAdjust(-4)] + assert self.r_unaryinsn_star.match(line) # indirect call + return [InsnCall(self.currentlineno), + InsnSetLocal(self.EAX)] # the result is there + + target = match.group(1) + + if self.format in ('msvc'): + # On win32, the address of a foreign function must be + # computed, the optimizer may store it in a register. We + # could ignore this, except when the function need special + # processing (not returning, __stdcall...) + def find_register(target): + reg = [] + def walker(insn, locs): + sources = [] + for loc in locs: + for s in insn.all_sources_of(loc): + sources.append(s) + for source in sources: + m = re.match("DWORD PTR " + self.LABEL, source) + if m: + reg.append(m.group(1)) + if reg: + return + yield tuple(sources) + insn = InsnStop() + insn.previous_insns = [self.insns[-1]] + self.walk_instructions_backwards(walker, insn, (target,)) + return reg + + if match and self.r_localvarfp.match(target): + sources = find_register(target) + if sources: + target, = sources + + if target in self.FUNCTIONS_NOT_RETURNING: + return [InsnStop(), InsnCannotFollowEsp()] + if self.format == 'mingw32' and target == '__alloca': + # in functions with large stack requirements, windows + # needs a call to _alloca(), to turn reserved pages + # into committed memory. + # With mingw32 gcc at least, %esp is not used before + # this call. So we don't bother to compute the exact + # stack effect. + return [InsnCannotFollowEsp()] + if target in self.labels: + lineoffset = self.labels[target].lineno - self.currentlineno + if lineoffset >= 0: + assert lineoffset in (1,2) + return [InsnStackAdjust(-4)] + insns = [InsnCall(self.currentlineno), - InsnSetLocal('%eax')] # the result is there - if sys.platform == 'win32': + InsnSetLocal(self.EAX)] # the result is there + if self.format in ('mingw32', 'msvc'): # handle __stdcall calling convention: # Stack cleanup is performed by the called function, # Function name is decorated with "@N" where N is the stack size - if match and '@' in target: - insns.append(InsnStackAdjust(int(target.split('@')[1]))) + if '@' in target and not target.startswith('@'): + insns.append(InsnStackAdjust(int(target.rsplit('@', 1)[1]))) + # Some (intrinsic?) functions use the "fastcall" calling convention + # XXX without any declaration, how can we guess the stack effect? + if target in ['__alldiv', '__allrem', '__allmul', '__alldvrm']: + insns.append(InsnStackAdjust(16)) return insns -class UnrecognizedOperation(Exception): - pass - -class NoPatternMatch(Exception): - pass - -class SomeNewValue(object): - pass -somenewvalue = SomeNewValue() +class ElfFunctionGcRootTracker(FunctionGcRootTracker): + format = 'elf' -class LocalVar(object): - # A local variable location at position 'ofs_from_frame_end', - # which is counted from the end of the stack frame (so it is always - # negative, unless it refers to arguments of the current function). - def __init__(self, ofs_from_frame_end, hint=None): - self.ofs_from_frame_end = ofs_from_frame_end - self.hint = hint - - def __repr__(self): - return '<%+d;%s>' % (self.ofs_from_frame_end, self.hint or 'e*p') - - def __hash__(self): - return hash(self.ofs_from_frame_end) - - def __cmp__(self, other): - if isinstance(other, LocalVar): - return cmp(self.ofs_from_frame_end, other.ofs_from_frame_end) - else: - return 1 - - def getlocation(self, framesize, uses_frame_pointer): - if (self.hint == 'esp' or not uses_frame_pointer - or self.ofs_from_frame_end % 2 != 0): - # try to use esp-relative addressing - ofs_from_esp = framesize + self.ofs_from_frame_end - if ofs_from_esp % 2 == 0: - return frameloc(LOC_ESP_BASED, ofs_from_esp) - # we can get an odd value if the framesize is marked as bogus - # by visit_andl() - assert uses_frame_pointer - ofs_from_ebp = self.ofs_from_frame_end + 4 - return frameloc(LOC_EBP_BASED, ofs_from_ebp) - - -class Insn(object): - _args_ = [] - _locals_ = [] - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join([str(getattr(self, name)) - for name in self._args_])) - def requestgcroots(self, tracker): - return {} - - def source_of(self, localvar, tag): - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class Label(Insn): - _args_ = ['label', 'lineno'] - def __init__(self, label, lineno): - self.label = label - self.lineno = lineno - self.previous_insns = [] # all insns that jump (or fallthrough) here - -class InsnFunctionStart(Insn): - framesize = 0 - previous_insns = () - def __init__(self): - self.arguments = {} - for reg in CALLEE_SAVE_REGISTERS: - self.arguments[reg] = somenewvalue - - def source_of(self, localvar, tag): - if localvar not in self.arguments: - if localvar in ('%eax', '%edx', '%ecx'): - # xxx this might show a bug in trackgcroot.py failing to - # figure out which instruction stored a value in these - # registers. However, this case also occurs when the - # the function's calling convention was optimized by gcc: - # the 3 registers above are then used to pass arguments - pass - else: - assert (isinstance(localvar, LocalVar) and - localvar.ofs_from_frame_end > 0), ( - "must come from an argument to the function, got %r" % - (localvar,)) - self.arguments[localvar] = somenewvalue - return self.arguments[localvar] - - def all_sources_of(self, localvar): - return [] - -class InsnSetLocal(Insn): - _args_ = ['target', 'sources'] - _locals_ = ['target', 'sources'] - - def __init__(self, target, sources=()): - self.target = target - self.sources = sources - - def source_of(self, localvar, tag): - if localvar == self.target: - return somenewvalue - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return self.sources - return [localvar] - -class InsnCopyLocal(Insn): - _args_ = ['source', 'target'] - _locals_ = ['source', 'target'] - - def __init__(self, source, target): - self.source = source - self.target = target - - def source_of(self, localvar, tag): - if localvar == self.target: - return self.source - return localvar - - def all_sources_of(self, localvar): - if localvar == self.target: - return [self.source] - return [localvar] - -class InsnStackAdjust(Insn): - _args_ = ['delta'] - def __init__(self, delta): - assert delta % 2 == 0 # should be "% 4", but there is the special - self.delta = delta # case of 'pushw' to handle - -class InsnCannotFollowEsp(InsnStackAdjust): - def __init__(self): - self.delta = -7 # use an odd value as marker - -class InsnStop(Insn): - pass + ESP = '%esp' + EBP = '%ebp' + EAX = '%eax' + CALLEE_SAVE_REGISTERS = ['%ebx', '%esi', '%edi', '%ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + OPERAND = r'(?:[-\w$%+.:@"]+(?:[(][\w%,]+[)])?|[(][\w%,]+[)])' + LABEL = r'([a-zA-Z_$.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 2**30 + TOP_OF_STACK = '0(%esp)' + + r_functionstart = re.compile(r"\t.type\s+"+LABEL+",\s*[@]function\s*$") + r_functionend = re.compile(r"\t.size\s+"+LABEL+",\s*[.]-"+LABEL+"\s*$") + LOCALVAR = r"%eax|%edx|%ecx|%ebx|%esi|%edi|%ebp|\d*[(]%esp[)]" + LOCALVARFP = LOCALVAR + r"|-?\d*[(]%ebp[)]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"(\d*)[(]%esp[)]") + r_localvar_ebp = re.compile(r"(-?\d*)[(]%ebp[)]") + + r_rel_label = re.compile(r"(\d+):\s*$") + r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$") + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\t.long\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL) -class InsnRet(InsnStop): - framesize = 0 - def requestgcroots(self, tracker): - # no need to track the value of these registers in the caller - # function if we are the main(), or if we are flagged as a - # "bottom" function (a callback from C code) - if tracker.is_stack_bottom: - return {} - else: - return dict(zip(CALLEE_SAVE_REGISTERS, CALLEE_SAVE_REGISTERS)) - -class InsnCall(Insn): - _args_ = ['lineno', 'gcroots'] - def __init__(self, lineno): - # 'gcroots' is a dict built by side-effect during the call to - # FunctionGcRootTracker.trackgcroots(). Its meaning is as - # follows: the keys are the locations that contain gc roots - # (register names or LocalVar instances). The value - # corresponding to a key is the "tag", which is None for a - # normal gc root, or else the name of a callee-saved register. - # In the latter case it means that this is only a gc root if the - # corresponding register in the caller was really containing a - # gc pointer. A typical example: - # - # InsnCall({LocalVar(-8)': None, - # '%esi': '%esi', - # LocalVar(-12)': '%ebx'}) - # - # means that the value at -8 from the frame end is a gc root - # across this call; that %esi is a gc root if it was in the - # caller (typically because %esi is not modified at all in the - # current function); and that the value at -12 from the frame - # end is a gc root if %ebx was a gc root in the caller - # (typically because the current function saves and restores - # %ebx from there in the prologue and epilogue). - self.gcroots = {} - self.lineno = lineno - - def source_of(self, localvar, tag): - tag1 = self.gcroots.setdefault(localvar, tag) - assert tag1 == tag, ( - "conflicting entries for InsnCall.gcroots[%s]:\n%r and %r" % ( - localvar, tag1, tag)) - return localvar - - def all_sources_of(self, localvar): - return [localvar] - -class InsnGCROOT(Insn): - _args_ = ['loc'] - _locals_ = ['loc'] - def __init__(self, loc): - self.loc = loc - def requestgcroots(self, tracker): - return {self.loc: None} - -class InsnPrologue(Insn): - def __setattr__(self, attr, value): - if attr == 'framesize': - assert value == 4, ("unrecognized function prologue - " - "only supports push %ebp; movl %esp, %ebp") - Insn.__setattr__(self, attr, value) - -class InsnEpilogue(Insn): - def __init__(self, framesize=None): - if framesize is not None: - self.framesize = framesize + r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/") + r_bottom_marker = re.compile(r"\t/[*] GC_STACK_BOTTOM [*]/") - -if sys.platform != 'win32': FUNCTIONS_NOT_RETURNING = { 'abort': None, '_exit': None, @@ -1139,31 +756,684 @@ '___assert_rtn': None, 'L___assert_rtn$stub': None } -else: + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = match.group(1) + match = self.r_functionend.match(lines[-1]) + assert funcname == match.group(1) + assert funcname == match.group(2) + super(ElfFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def extract_immediate(self, value): + if not value.startswith('$'): + return None + return int(value[1:]) + +ElfFunctionGcRootTracker.init_regexp() + +class DarwinFunctionGcRootTracker(ElfFunctionGcRootTracker): + format = 'darwin' + + r_functionstart = re.compile(r"_(\w+):\s*$") + OFFSET_LABELS = 0 + + def __init__(self, lines, filetag=0): + match = self.r_functionstart.match(lines[0]) + funcname = '_' + match.group(1) + FunctionGcRootTracker.__init__(self, funcname, lines, filetag) + +class Mingw32FunctionGcRootTracker(DarwinFunctionGcRootTracker): + format = 'mingw32' + +class MsvcFunctionGcRootTracker(FunctionGcRootTracker): + format = 'msvc' + ESP = 'esp' + EBP = 'ebp' + EAX = 'eax' + CALLEE_SAVE_REGISTERS = ['ebx', 'esi', 'edi', 'ebp'] + REG2LOC = dict((_reg, LOC_REG | (_i<<2)) + for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS)) + TOP_OF_STACK = 'DWORD PTR [esp]' + + OPERAND = r'(?:(:?WORD|DWORD|BYTE) PTR |OFFSET )?[_\w?:@$]*(?:[-+0-9]+)?(:?\[[-+*\w0-9]+\])?' + LABEL = r'([a-zA-Z_$@.][a-zA-Z0-9_$@.]*)' + OFFSET_LABELS = 0 + + r_functionstart = re.compile(r"; Function compile flags: ") + r_codestart = re.compile(LABEL+r"\s+PROC\s*(:?;.+)?\n$") + r_functionend = re.compile(LABEL+r"\s+ENDP\s*$") + r_symboldefine = re.compile(r"([_A-Za-z0-9$]+) = ([-0-9]+)\s*;.+\n") + + LOCALVAR = r"eax|edx|ecx|ebx|esi|edi|ebp|DWORD PTR [-+]?\d*\[esp[-+]?\d*\]" + LOCALVARFP = LOCALVAR + r"|DWORD PTR -?\d*\[ebp\]" + r_localvarnofp = re.compile(LOCALVAR) + r_localvarfp = re.compile(LOCALVARFP) + r_localvar_esp = re.compile(r"DWORD PTR ([-+]?\d+)?\[esp([-+]?\d+)?\]") + r_localvar_ebp = re.compile(r"DWORD PTR ([-+]?\d+)?\[ebp([-+]?\d+)?\]") + + r_rel_label = re.compile(r"$1") # never matches + r_jump_rel_label = re.compile(r"$1") # never matches + + r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+DWORD PTR ("+OPERAND+")\s*$") + r_jmptable_item = re.compile(r"\tDD\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$") + r_jmptable_end = re.compile(r"[^\t\n;]") + + r_gcroot_marker = re.compile(r"$1") # never matches + r_gcroot_marker_var = re.compile(r"DWORD PTR .+_constant_always_one_.+pypy_asm_gcroot") + r_bottom_marker = re.compile(r"; .+\tpypy_asm_stack_bottom\(\);") + FUNCTIONS_NOT_RETURNING = { '_abort': None, '__exit': None, '__assert': None, '__wassert': None, + '__imp__abort': None, + '__imp___wassert': None, + 'DWORD PTR __imp__abort': None, + 'DWORD PTR __imp___wassert': None, } -CALLEE_SAVE_REGISTERS_NOEBP = ['%ebx', '%esi', '%edi'] -CALLEE_SAVE_REGISTERS = CALLEE_SAVE_REGISTERS_NOEBP + ['%ebp'] + @classmethod + def init_regexp(cls): + super(MsvcFunctionGcRootTracker, cls).init_regexp() + cls.r_binaryinsn = re.compile(r"\t[a-z]\w*\s+(?P"+cls.OPERAND+r"),\s*(?P"+cls.OPERAND+r")\s*(?:;.+)?$") + cls.r_jump = re.compile(r"\tj\w+\s+(?:SHORT |DWORD PTR )?"+cls.LABEL+"\s*$") + + def __init__(self, lines, filetag=0): + self.defines = {} + for i, line in enumerate(lines): + if self.r_symboldefine.match(line): + match = self.r_symboldefine.match(line) + name = match.group(1) + value = int(match.group(2)) + self.defines[name] = value + continue + + match = self.r_codestart.match(line) + if match: + self.skip = i + break + + funcname = match.group(1) + super(MsvcFunctionGcRootTracker, self).__init__( + funcname, lines, filetag) + + def replace_symbols(self, operand): + for name, value in self.defines.items(): + operand = operand.replace(name, str(value)) + return operand + + for name in ''' + push pop mov lea + xor sub add + '''.split(): + locals()['visit_' + name] = getattr(FunctionGcRootTracker, + 'visit_' + name + 'l') + + visit_int = FunctionGcRootTracker.visit_nop + visit_npad = FunctionGcRootTracker.visit_nop + # probably not GC pointers + visit_cdq = FunctionGcRootTracker.visit_nop + + def extract_immediate(self, value): + try: + return int(value) + except ValueError: + return None + + def _visit_gcroot_marker(self, line=None): + # two possible patterns: + # 1. mov reg, DWORD PTR _always_one_ + # imul target, reg + # 2. mov reg, DWORD PTR _always_one_ + # imul reg, target + assert self.lines[self.currentlineno].startswith("\tmov\t") + mov = self.r_binaryinsn.match(self.lines[self.currentlineno]) + assert re.match("DWORD PTR .+_always_one_", mov.group("source")) + reg = mov.group("target") + + self.lines[self.currentlineno] = ";" + self.lines[self.currentlineno] + + # the 'imul' must appear in the same block; the 'reg' must not + # appear in the instructions between + imul = None + lineno = self.currentlineno + 1 + stop = False + while not stop: + line = self.lines[lineno] + if line == '\n': + stop = True + elif line.startswith("\tjmp\t"): + stop = True + elif self.r_gcroot_marker_var.search(line): + stop = True + elif line.startswith("\tmov\t%s," % (reg,)): + # mov reg, + stop = True + elif line.startswith("\txor\t%s, %s" % (reg, reg)): + # xor reg, reg + stop = True + elif line.startswith("\timul\t"): + imul = self.r_binaryinsn.match(line) + imul_arg1 = imul.group("target") + imul_arg2 = imul.group("source") + break + # the register may not appear in other instructions + elif reg in line: + assert False, (line, lineno) + + lineno += 1 + else: + # No imul, the returned value is not used in this function + return [] + + if reg == imul_arg2: + self.lines[lineno] = ";" + self.lines[lineno] + return InsnGCROOT(self.replace_symbols(imul_arg1)) + else: + assert reg == imul_arg1 + self.lines[lineno] = "\tmov\t%s, %s\n" % (imul_arg1, imul_arg2) + if imul_arg2.startswith('OFFSET '): + # ignore static global variables + pass + else: + self.lines[lineno] += "\t; GCROOT\n" + + return [] + + def insns_for_copy(self, source, target): + if self.r_gcroot_marker_var.match(source): + return self._visit_gcroot_marker() + if self.lines[self.currentlineno].endswith("\t; GCROOT\n"): + insns = [InsnGCROOT(self.replace_symbols(source))] + else: + insns = [] + return insns + super(MsvcFunctionGcRootTracker, self).insns_for_copy(source, target) + + +MsvcFunctionGcRootTracker.init_regexp() + +class AssemblerParser(object): + def __init__(self, verbose=0, shuffle=False): + self.verbose = verbose + self.shuffle = shuffle + self.gcmaptable = [] + self.seen_main = False + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + for in_function, lines in self.find_functions(iterlines): + if in_function: + lines = self.process_function(lines, entrypoint, filename) + self.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + + def write_newfile(self, newfile, lines, grist): + newfile.writelines(lines) + + def process_function(self, lines, entrypoint, filename): + tracker = self.FunctionGcRootTracker( + lines, filetag=getidentifier(filename)) + is_main = tracker.funcname == entrypoint + tracker.is_stack_bottom = is_main + if self.verbose == 1: + sys.stderr.write('.') + elif self.verbose > 1: + print >> sys.stderr, '[trackgcroot:%s] %s' % (filename, + tracker.funcname) + table = tracker.computegcmaptable(self.verbose) + if self.verbose > 1: + for label, state in table: + print >> sys.stderr, label, '\t', format_callshape(state) + table = compress_gcmaptable(table) + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = table + else: + self.gcmaptable.extend(table) + self.seen_main |= is_main + return tracker.lines + +class ElfAssemblerParser(AssemblerParser): + format = "elf" + FunctionGcRootTracker = ElfFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + +class DarwinAssemblerParser(AssemblerParser): + format = "darwin" + FunctionGcRootTracker = DarwinFunctionGcRootTracker + + r_textstart = re.compile(r"\t.text\s*$") + + # see + # http://developer.apple.com/documentation/developertools/Reference/Assembler/040-Assembler_Directives/asm_directives.html + OTHERSECTIONS = ['section', 'zerofill', + 'const', 'static_const', 'cstring', + 'literal4', 'literal8', 'literal16', + 'constructor', 'desctructor', + 'symbol_stub', + 'data', 'static_data', + 'non_lazy_symbol_pointer', 'lazy_symbol_pointer', + 'dyld', 'mod_init_func', 'mod_term_func', + 'const_data' + ] + r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + assert not in_text, "unexpected repeated .text start: %d" % n + in_text = True + elif cls.r_sectionstart.match(line): + if in_function: + yield in_function, functionlines + functionlines = [] + in_text = False + in_function = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + + if functionlines: + yield in_function, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(DarwinAssemblerParser, self).process_function( + lines, entrypoint, filename) + +class Mingw32AssemblerParser(DarwinAssemblerParser): + format = "mingw32" + FunctionGcRootTracker = Mingw32FunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_text = False + in_function = False + for n, line in enumerate(iterlines): + if cls.r_textstart.match(line): + in_text = True + elif cls.r_sectionstart.match(line): + in_text = False + elif in_text and cls.FunctionGcRootTracker.r_functionstart.match(line): + yield in_function, functionlines + functionlines = [] + in_function = True + functionlines.append(line) + if functionlines: + yield in_function, functionlines + +class MsvcAssemblerParser(AssemblerParser): + format = "msvc" + FunctionGcRootTracker = MsvcFunctionGcRootTracker + + @classmethod + def find_functions(cls, iterlines): + functionlines = [] + in_function = False + for line in iterlines: + if cls.FunctionGcRootTracker.r_functionstart.match(line): + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + in_function = True + functionlines = [] + functionlines.append(line) + if cls.FunctionGcRootTracker.r_functionend.match(line): + assert in_function, ( + "missed the start of the current function") + yield True, functionlines + in_function = False + functionlines = [] + assert not in_function, ( + "missed the end of the previous function") + yield False, functionlines + + def process_function(self, lines, entrypoint, filename): + entrypoint = '_' + entrypoint + return super(MsvcAssemblerParser, self).process_function( + lines, entrypoint, filename) + + def write_newfile(self, newfile, lines, grist): + newlines = [] + for line in lines: + # truncate long comments + if line.startswith(";"): + line = line[:-1][:500] + '\n' + + # Workaround a bug in the .s files generated by msvc + # compiler: every string or float constant is exported + # with a name built after its value, and will conflict + # with other modules. + if line.startswith("PUBLIC\t__real@"): + line = '; ' + line + elif line.startswith("PUBLIC\t??_C@"): + line = '; ' + line + elif line == "PUBLIC\t__$ArrayPad$\n": + line = '; ' + line + + # Because we insert labels in the code, some "SHORT" jumps + # are now longer than 127 bytes. We turn them all into + # "NEAR" jumps. Note that the assembler allocates space + # for a near jump, but still generates a short jump when + # it can. + line = line.replace('\tjmp\tSHORT ', '\tjmp\t') + line = line.replace('\tjne\tSHORT ', '\tjne\t') + line = line.replace('\tje\tSHORT ', '\tje\t') + + newlines.append(line) + + if line == "\t.model\tflat\n": + newlines.append("\tassume fs:nothing\n") + + newfile.writelines(newlines) + +PARSERS = { + 'elf': ElfAssemblerParser, + 'darwin': DarwinAssemblerParser, + 'mingw32': Mingw32AssemblerParser, + 'msvc': MsvcAssemblerParser, + } + +class GcRootTracker(object): + + def __init__(self, verbose=0, shuffle=False, format='elf'): + self.verbose = verbose + self.shuffle = shuffle # to debug the sorting logic in asmgcroot.py + self.format = format + self.gcmaptable = [] + self.seen_main = False + + def dump_raw_table(self, output): + print >> output, "seen_main = %d" % (self.seen_main,) + for entry in self.gcmaptable: + print >> output, entry + + def reload_raw_table(self, input): + firstline = input.readline() + assert firstline.startswith("seen_main = ") + self.seen_main |= bool(int(firstline[len("seen_main = "):].strip())) + for line in input: + entry = eval(line) + assert type(entry) is tuple + self.gcmaptable.append(entry) + + def dump(self, output): + assert self.seen_main + shapes = {} + shapelines = [] + shapeofs = 0 + def _globalname(name, disp=""): + if self.format in ('darwin', 'mingw32', 'msvc'): + name = '_' + name + + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s + %s" % (name, disp) + else: + return name + + def _globl(name): + if self.format == 'msvc': + print >> output, "PUBLIC %s" % _globalname(name) + else: + print >> output, "\t.globl %s" % _globalname(name) + def _label(name): + print >> output, "%s:" % _globalname(name) + def _variant(**kwargs): + txt = kwargs[self.format] + print >> output, "\t%s" % txt + + def _comment(comment): + if self.format == 'msvc': + print >> output, "; %s" % comment + else: + print >> output, "/* %s */" % comment + + def _movl(source, target, comment): + if self.format == 'msvc': + print >> output, "\tmov\t%s, %s\t\t; %s" % (target, source, comment) + else: + print >> output, "\tmovl\t%s, %s\t\t/* %s */ " % (source, target, comment) + + def _pushl(source, comment): + if self.format == 'msvc': + print >> output, "\tpush\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpushl\t%s\t\t/* %s */ " % (source, comment) + + def _popl(source, comment): + if self.format == 'msvc': + print >> output, "\tpop\t%s\t\t; %s" % (source, comment) + else: + print >> output, "\tpopl\t%s\t\t/* %s */ " % (source, comment) + + + def _register(name, disp=""): + if self.format == 'msvc': + if disp: + return "DWORD PTR [%s+%s]" % (name, disp) + else: + return name + else: + if disp: + return "%s(%%%s)" % (disp, name) + else: + return '%' + name + + def _offset(name): + if self.format == 'msvc': + return "OFFSET %s" % _globalname(name) + else: + return "$%s" % _globalname(name) + + def _call(arg, comment): + if self.format == 'msvc': + print >> output, "\tcall\t%s\t\t;%s" % (arg, comment) + else: + print >> output, "\tcall\t%s\t\t/* %s */" % (arg, comment) + + def _indirectjmp(arg): + if self.format == 'msvc': + return "DWORD PTR " + arg + else: + return "*%" + arg + + if self.format == 'msvc': + print >> output, """\ + TITLE\tgcmaptable.s + .686P + .XMM + .model\tflat + """ + + _variant(elf='\t.text', + darwin='\t.text', + mingw32='\t.text', + msvc='_TEXT\tSEGMENT') + + _globl('pypy_asm_stackwalk') + _variant(elf='.type pypy_asm_stackwalk, @function', + darwin='', + mingw32='', + msvc='') + _label('pypy_asm_stackwalk') + _comment("See description in asmgcroot.py") + _movl(_register("esp", disp="4"), _register("edx"), "my argument, which is the callback") + _movl(_register("esp"), _register("eax"), "my frame top address") + _pushl(_register("eax"), "ASM_FRAMEDATA[6]") + _pushl(_register("ebp"), "ASM_FRAMEDATA[5]") + _pushl(_register("edi"), "ASM_FRAMEDATA[4]") + _pushl(_register("esi"), "ASM_FRAMEDATA[3]") + _pushl(_register("ebx"), "ASM_FRAMEDATA[2]") + + print >> output + _comment("Add this ASM_FRAMEDATA to the front of the circular linked") + _comment("list. Let's call it 'self'.") + print >> output + _movl(_globalname("__gcrootanchor", disp=4), _register("eax"), "next = gcrootanchor->next") + _pushl(_register("eax"), "self->next = next") + _pushl(_offset("__gcrootanchor"), "self->prev = gcrootanchor") + _movl(_register("esp"), _globalname("__gcrootanchor", disp=4), "gcrootanchor->next = self") + _movl(_register("esp"), _register("eax", "0"), "next->prev = self") + print >> output + + _comment("note: the Mac OS X 16 bytes aligment must be respected.") + _call(_indirectjmp("edx"), "invoke the callback") + print >> output + + _comment("Detach this ASM_FRAMEDATA from the circular linked list") + _popl(_register("esi"), "prev = self->prev") + _popl(_register("edi"), "next = self->next") + _movl(_register("edi"), _register("esi", disp="4"), "prev->next = next") + _movl(_register("esi"), _register("edi", disp="0"), "next->prev = prev") + print >> output + + _popl(_register("ebx"), "restore from ASM_FRAMEDATA[2]") + _popl(_register("esi"), "restore from ASM_FRAMEDATA[3]") + _popl(_register("edi"), "restore from ASM_FRAMEDATA[4]") + _popl(_register("ebp"), "restore from ASM_FRAMEDATA[5]") + _popl(_register("ecx"), "ignored ASM_FRAMEDATA[6]") + _comment("the return value is the one of the 'call' above,") + _comment("because %eax (and possibly %edx) are unmodified") + + print >> output, "\tret" + + _variant(elf='.size pypy_asm_stackwalk, .-pypy_asm_stackwalk', + darwin='', + mingw32='', + msvc='') + + if self.format == 'msvc': + for label, state, is_range in self.gcmaptable: + print >> output, "EXTERN %s:NEAR" % label + + if self.format == 'msvc': + print >> output, '_DATA SEGMENT' + + _comment("A circular doubly-linked list of all") + _comment("the ASM_FRAMEDATAs currently alive") + if self.format == 'msvc': + _globl('__gcrootanchor') + print >> output, '%s\tDD FLAT:___gcrootanchor ; prev' % _globalname("__gcrootanchor") + print >> output, '\tDD FLAT:___gcrootanchor ; next' + else: + print >> output, '\t.data' + print >> output, '\t.align\t4' + _globl('__gcrootanchor') + _label('__gcrootanchor') + print >> output, """\ + .long\t__gcrootanchor /* prev */ + .long\t__gcrootanchor /* next */ +""".replace("__gcrootanchor", _globalname("__gcrootanchor")) + + _globl('__gcmapstart') + if self.format == 'msvc': + print >> output, '%s' % _globalname('__gcmapstart'), + else: + _label('__gcmapstart') + for label, state, is_range in self.gcmaptable: + try: + n = shapes[state] + except KeyError: + n = shapes[state] = shapeofs + bytes = [str(b) for b in compress_callshape(state)] + if self.format == 'msvc': + shapelines.append('\tDB\t%s\t;%s\n' % ( + ', '.join(bytes), + shapeofs)) + else: + shapelines.append('\t/*%d*/\t.byte\t%s\n' % ( + shapeofs, + ', '.join(bytes))) + shapeofs += len(bytes) + if is_range: + n = ~ n + if self.format == 'msvc': + print >> output, '\tDD\t%s' % (label,) + print >> output, '\tDD\t%d' % (n,) + else: + print >> output, '\t.long\t%s-%d' % ( + label, + PARSERS[self.format].FunctionGcRootTracker.OFFSET_LABELS) + print >> output, '\t.long\t%d' % (n,) + + _globl('__gcmapend') + if self.format == 'msvc': + print >> output, '%s DD ?' % _globalname('__gcmapend') + else: + _label('__gcmapend') + _variant(elf='.section\t.rodata', + darwin='.const', + mingw32='', + msvc='') + + _globl('__gccallshapes') + if self.format == 'msvc': + print >> output, _globalname('__gccallshapes'), + else: + _label('__gccallshapes') + output.writelines(shapelines) + + if self.format == 'msvc': + print >> output, "_DATA\tENDS" + print >> output, "END" + + def process(self, iterlines, newfile, entrypoint='main', filename='?'): + parser = PARSERS[format](verbose=self.verbose, shuffle=self.shuffle) + for in_function, lines in parser.find_functions(iterlines): + if in_function: + lines = parser.process_function(lines, entrypoint, filename) + parser.write_newfile(newfile, lines, filename.split('.')[0]) + if self.verbose == 1: + sys.stderr.write('\n') + if self.shuffle and random.random() < 0.5: + self.gcmaptable[:0] = parser.gcmaptable + else: + self.gcmaptable.extend(parser.gcmaptable) + self.seen_main |= parser.seen_main + + +class UnrecognizedOperation(Exception): + pass + +class NoPatternMatch(Exception): + pass -LOC_NOWHERE = 0 -LOC_REG = 1 -LOC_EBP_BASED = 2 -LOC_ESP_BASED = 3 -LOC_MASK = 0x03 - -REG2LOC = {} -for _i, _reg in enumerate(CALLEE_SAVE_REGISTERS): - REG2LOC[_reg] = LOC_REG | (_i<<2) - -def frameloc(base, offset): - assert base in (LOC_EBP_BASED, LOC_ESP_BASED) - assert offset % 4 == 0 - return base | offset # __________ debugging output __________ @@ -1179,7 +1449,7 @@ elif kind == LOC_REG: reg = loc >> 2 assert 0 <= reg <= 3 - return CALLEE_SAVE_REGISTERS[reg] + return ElfFunctionGcRootTracker.CALLEE_SAVE_REGISTERS[reg] else: if kind == LOC_EBP_BASED: result = '(%ebp)' @@ -1304,6 +1574,12 @@ verbose = 1 shuffle = False output_raw_table = False + if sys.platform == 'darwin': + format = 'darwin' + elif sys.platform == 'win32': + format = 'mingw32' + else: + format = 'elf' while len(sys.argv) > 1: if sys.argv[1] == '-v': del sys.argv[1] @@ -1314,14 +1590,11 @@ elif sys.argv[1] == '-t': del sys.argv[1] output_raw_table = True + elif sys.argv[1].startswith('-f'): + format = sys.argv[1][2:] + del sys.argv[1] else: break - if sys.platform == 'darwin': - format = 'darwin' - elif sys.platform == 'win32': - format = 'mingw32' - else: - format = 'elf' tracker = GcRootTracker(verbose=verbose, shuffle=shuffle, format=format) for fn in sys.argv[1:]: f = open(fn, 'r') @@ -1332,7 +1605,7 @@ tracker.reload_raw_table(f) f.close() else: - assert fn.endswith('.s') + assert fn.endswith('.s'), fn lblfn = fn[:-2] + '.lbl.s' g = open(lblfn, 'w') try: @@ -1345,6 +1618,5 @@ f.close() if output_raw_table: tracker.dump_raw_table(sys.stdout) - tracker.clear() if not output_raw_table: tracker.dump(sys.stdout) Modified: pypy/branch/faster-raise/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/genc.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/genc.py Sat Nov 14 15:24:15 2009 @@ -488,23 +488,56 @@ mk.rule(*rule) if self.config.translation.gcrootfinder == 'asmgcc': - sfiles = ['%s.s' % (cfile[:-2],) for cfile in mk.cfiles] - lblsfiles = ['%s.lbl.s' % (cfile[:-2],) for cfile in mk.cfiles] - gcmapfiles = ['%s.gcmap' % (cfile[:-2],) for cfile in mk.cfiles] + trackgcfiles = [cfile[:-2] for cfile in mk.cfiles] + if self.translator.platform.name == 'msvc': + trackgcfiles = [f for f in trackgcfiles + if f.startswith(('implement', 'testing', + '../module_cache/module'))] + sfiles = ['%s.s' % (c,) for c in trackgcfiles] + lblsfiles = ['%s.lbl.s' % (c,) for c in trackgcfiles] + gcmapfiles = ['%s.gcmap' % (c,) for c in trackgcfiles] mk.definition('ASMFILES', sfiles) mk.definition('ASMLBLFILES', lblsfiles) mk.definition('GCMAPFILES', gcmapfiles) mk.definition('DEBUGFLAGS', '-O2 -fomit-frame-pointer -g') - mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') - mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + if sys.platform == 'win32': python = sys.executable.replace('\\', '/') + ' ' else: - python = "" - mk.rule('%.lbl.s %.gcmap', '%.s', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') - mk.rule('gcmaptable.s', '$(GCMAPFILES)', - python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') + python = '' + + if self.translator.platform.name == 'msvc': + lblofiles = [] + for cfile in mk.cfiles: + f = cfile[:-2] + if f in trackgcfiles: + ofile = '%s.lbl.obj' % (f,) + else: + ofile = '%s.obj' % (f,) + + lblofiles.append(ofile) + mk.definition('ASMLBLOBJFILES', lblofiles) + mk.definition('OBJECTS', 'gcmaptable.obj $(ASMLBLOBJFILES)') + # /Oi (enable intrinsics) and /Ob1 (some inlining) are mandatory + # even in debug builds + mk.definition('ASM_CFLAGS', '$(CFLAGS) /Oi /Ob1') + mk.rule('.SUFFIXES', '.s', []) + mk.rule('.s.obj', '', + 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') + mk.rule('.c.gcmap', '', + ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + ) + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + 'cmd /c ' + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + + else: + mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') + mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') + mk.rule('%.lbl.s %.gcmap', '%.s', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py -t $< > $*.gcmap') + mk.rule('gcmaptable.s', '$(GCMAPFILES)', + python + '$(PYPYDIR)/translator/c/gcc/trackgcroot.py $(GCMAPFILES) > $@') else: mk.definition('DEBUGFLAGS', '-O1 -g') Modified: pypy/branch/faster-raise/pypy/translator/c/src/char.h ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/src/char.h (original) +++ pypy/branch/faster-raise/pypy/translator/c/src/char.h Sat Nov 14 15:24:15 2009 @@ -6,8 +6,8 @@ /*** binary operations ***/ -#define OP_CHAR_EQ(x,y,r) r = ((x) == (y)) -#define OP_CHAR_NE(x,y,r) r = ((x) != (y)) +#define OP_CHAR_EQ(x,y,r) r = ((unsigned char)(x) == (unsigned char)(y)) +#define OP_CHAR_NE(x,y,r) r = ((unsigned char)(x) != (unsigned char)(y)) #define OP_CHAR_LE(x,y,r) r = ((unsigned char)(x) <= (unsigned char)(y)) #define OP_CHAR_GT(x,y,r) r = ((unsigned char)(x) > (unsigned char)(y)) #define OP_CHAR_LT(x,y,r) r = ((unsigned char)(x) < (unsigned char)(y)) Modified: pypy/branch/faster-raise/pypy/translator/c/src/debug.h ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/src/debug.h (original) +++ pypy/branch/faster-raise/pypy/translator/c/src/debug.h Sat Nov 14 15:24:15 2009 @@ -139,12 +139,18 @@ return 1; } +#if defined(_MSC_VER) || defined(__MINGW32__) +#define PYPY_LONG_LONG_PRINTF_FORMAT "I64" +#else +#define PYPY_LONG_LONG_PRINTF_FORMAT "ll" +#endif + static void display_startstop(const char *prefix, const char *postfix, const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); - fprintf(pypy_debug_file, "%s[%llx] %s%s%s\n%s", + fprintf(pypy_debug_file, "%s[%"PYPY_LONG_LONG_PRINTF_FORMAT"x] %s%s%s\n%s", colors, timestamp, prefix, category, postfix, debug_stop_colors); Modified: pypy/branch/faster-raise/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/src/mem.h (original) +++ pypy/branch/faster-raise/pypy/translator/c/src/mem.h Sat Nov 14 15:24:15 2009 @@ -17,14 +17,18 @@ could make two copies of the local variable (e.g. one in the stack and one in a register), pass one to GCROOT, and later use the other one. In practice the pypy_asm_gcroot() is often a no-op in the final - machine code and doesn't prevent most optimizations. Getting the - asm() right was tricky, though. The asm() is not volatile so that - gcc is free to delete it if the output variable is not used at all. - We need to prevent gcc from moving the asm() *before* the call that - could cause a collection; this is the purpose of the (unused) - __gcnoreorderhack input argument. Any memory input argument would - have this effect: as far as gcc knows the call instruction can modify - arbitrary memory, thus creating the order dependency that we want. */ + machine code and doesn't prevent most optimizations. */ +#ifndef _MSC_VER + +/* With gcc, getting the asm() right was tricky, though. The asm() is + not volatile so that gcc is free to delete it if the output variable + is not used at all. We need to prevent gcc from moving the asm() + *before* the call that could cause a collection; this is the purpose + of the (unused) __gcnoreorderhack input argument. Any memory input + argument would have this effect: as far as gcc knows the call + instruction can modify arbitrary memory, thus creating the order + dependency that we want. */ + #define pypy_asm_gcroot(p) ({void*_r; \ asm ("/* GCROOT %0 */" : "=g" (_r) : \ "0" (p), "m" (__gcnoreorderhack)); \ @@ -36,6 +40,31 @@ /* marker for trackgcroot.py */ #define pypy_asm_stack_bottom() asm volatile ("/* GC_STACK_BOTTOM */" : : ) +#else + +/* With the msvc Microsoft Compiler, the optimizer seems free to move + any code (even asm) that involves local memory (registers and stack). + The _ReadWriteBarrier function has an effect only where the content + of a global variable is *really* used. trackgcroot.py will remove + the extra instructions: the access to _constant_always_one_ is + removed, and the multiplication is replaced with a simple move. */ + +static __forceinline void* +pypy_asm_gcroot(void* _r1) +{ + static volatile int _constant_always_one_ = 1; + (long)_r1 *= _constant_always_one_; + _ReadWriteBarrier(); + return _r1; +} + +#define pypy_asm_keepalive(v) __asm { } +static __declspec(noinline) void pypy_asm_stack_bottom() { } + +#endif + + + #define OP_GC_ASMGCROOT_STATIC(i, r) r = \ i == 0 ? (void*)&__gcmapstart : \ i == 1 ? (void*)&__gcmapend : \ Modified: pypy/branch/faster-raise/pypy/translator/c/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/test/test_extfunc.py Sat Nov 14 15:24:15 2009 @@ -141,7 +141,7 @@ os.unlink(filename) def test_os_access(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_access(path, mode): return os.access(path, mode) f = compile(call_access, [str, int]) @@ -150,7 +150,7 @@ def test_os_stat(): - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) has_blksize = hasattr(os.stat_result, 'st_blksize') has_blocks = hasattr(os.stat_result, 'st_blocks') def call_stat(): @@ -189,7 +189,7 @@ def test_os_fstat(): if os.environ.get('PYPY_CC', '').startswith('tcc'): py.test.skip("segfault with tcc :-(") - filename = str(py.magic.autopath()) + filename = str(py.path.local(__file__)) def call_fstat(): fd = os.open(filename, os.O_RDONLY, 0777) st = os.fstat(fd) Modified: pypy/branch/faster-raise/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/faster-raise/pypy/translator/c/test/test_newgc.py Sat Nov 14 15:24:15 2009 @@ -622,7 +622,7 @@ import gc slong = cast_type_to_ffitype(rffi.LONG) - from pypy.rpython.lltypesystem.ll2ctypes import libc_name + from pypy.rlib.libffi import get_libc_name def callback(ll_args, ll_res, stuff): gc.collect() @@ -637,7 +637,7 @@ res[0] = -1 def f(): - libc = CDLL(libc_name) + libc = CDLL(get_libc_name()) qsort = libc.getpointer('qsort', [ffi_type_pointer, slong, slong, ffi_type_pointer], ffi_type_void) Modified: pypy/branch/faster-raise/pypy/translator/cli/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/cli/conftest.py (original) +++ pypy/branch/faster-raise/pypy/translator/cli/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,5 +1,5 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-cli options") + group = parser.getgroup("pypy-cli options") group.addoption('--source', action="store_true", dest="source", default=False, help="only generate IL source, don't compile") group.addoption('--wd', action="store_true", dest="wd", default=False, Modified: pypy/branch/faster-raise/pypy/translator/cli/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/cli/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/cli/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/driver.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/driver.py (original) +++ pypy/branch/faster-raise/pypy/translator/driver.py Sat Nov 14 15:24:15 2009 @@ -9,7 +9,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.listdef import s_list_of_strings from pypy.annotation import policy as annpolicy -from py.compat import optparse +import optparse from pypy.tool.udir import udir from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS @@ -599,14 +599,25 @@ shutil.copy(os.path.join(usession_path, 'main.il'), dirname) newexename = basename f = file(newexename, 'w') - f.write("""#!/bin/bash + f.write(r"""#!/bin/bash LEDIT=`type -p ledit` EXE=`readlink $0` if [ -z $EXE ] then EXE=$0 fi -if uname -s | grep -iq Cygwin ; then MONO=; else MONO=mono; fi +if uname -s | grep -iq Cygwin +then + MONO= +else + MONO=mono + # workaround for known mono buggy versions + VER=`mono -V | head -1 | sed s/'Mono JIT compiler version \(.*\) (.*'/'\1/'` + if [[ 2.1 < "$VER" && "$VER" < 2.4.3 ]] + then + MONO="mono -O=-branch" + fi +fi $LEDIT $MONO "$(dirname $EXE)/$(basename $EXE)-data/%s" "$@" # XXX doesn't work if it's placed in PATH """ % main_exe_name) f.close() Modified: pypy/branch/faster-raise/pypy/translator/goal/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/goal/targetgbfullprofiling.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/targetgbfullprofiling.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/targetgbfullprofiling.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.profiling.gameboy_profiling_implementation import GameBoyProfiler -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" def entry_point(argv=None): if argv is not None and len(argv) > 1: Modified: pypy/branch/faster-raise/pypy/translator/goal/targetgbimplementation.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/targetgbimplementation.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/targetgbimplementation.py Sat Nov 14 15:24:15 2009 @@ -3,7 +3,7 @@ from pypy.lang.gameboy.gameboy_implementation import GameBoyImplementation -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" print ROM_PATH Modified: pypy/branch/faster-raise/pypy/translator/goal/targetgbrom4.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/targetgbrom4.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/targetgbrom4.py Sat Nov 14 15:24:15 2009 @@ -4,7 +4,7 @@ from pypy.lang.gameboy.gameboy import GameBoy -ROM_PATH = str(py.magic.autopath().dirpath().dirpath().dirpath())+"/lang/gameboy/rom" +ROM_PATH = str(py.path.local(__file__).dirpath().dirpath().dirpath())+"/lang/gameboy/rom" EMULATION_CYCLES = 1<<24 Modified: pypy/branch/faster-raise/pypy/translator/goal/targetpreimportedpypy.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/targetpreimportedpypy.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/targetpreimportedpypy.py Sat Nov 14 15:24:15 2009 @@ -23,7 +23,7 @@ "random", ] -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/branch/faster-raise/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/targetpypystandalone.py Sat Nov 14 15:24:15 2009 @@ -11,7 +11,7 @@ from pypy.tool.option import make_objspace from pypy.translator.goal.nanos import setup_nanos -thisdir = py.magic.autopath().dirpath() +thisdir = py.path.local(__file__).dirpath() try: this_dir = os.path.dirname(__file__) Modified: pypy/branch/faster-raise/pypy/translator/goal/test2/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/test2/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/test2/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/goal/translate.py (original) +++ pypy/branch/faster-raise/pypy/translator/goal/translate.py Sat Nov 14 15:24:15 2009 @@ -84,7 +84,7 @@ import py # we want 2.4 expand_default functionality -optparse = py.compat.optparse +import optparse from pypy.tool.ansi_print import ansi_log log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) Modified: pypy/branch/faster-raise/pypy/translator/interactive.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/interactive.py (original) +++ pypy/branch/faster-raise/pypy/translator/interactive.py Sat Nov 14 15:24:15 2009 @@ -1,4 +1,4 @@ -from py.compat import optparse +import optparse import autopath from pypy.translator.translator import TranslationContext Modified: pypy/branch/faster-raise/pypy/translator/jvm/conftest.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/jvm/conftest.py (original) +++ pypy/branch/faster-raise/pypy/translator/jvm/conftest.py Sat Nov 14 15:24:15 2009 @@ -1,6 +1,6 @@ def pytest_addoption(parser): - group = parser.addgroup("pypy-jvm options") + group = parser.getgroup("pypy-jvm options") group.addoption('--java', action='store', dest='java', default='java', help='Define the java executable to use') group.addoption('--javac', action='store', dest='javac', Modified: pypy/branch/faster-raise/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/faster-raise/pypy/translator/jvm/genjvm.py Sat Nov 14 15:24:15 2009 @@ -83,7 +83,7 @@ self.jasmin_files = None # Determine various paths: - self.thisdir = py.magic.autopath().dirpath() + self.thisdir = py.path.local(__file__).dirpath() self.rootdir = self.thisdir.join('src') self.srcdir = self.rootdir.join('pypy') self.jnajar = self.rootdir.join('jna.jar') Modified: pypy/branch/faster-raise/pypy/translator/microbench/pybench/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/microbench/pybench/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/microbench/pybench/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/platform/__init__.py (original) +++ pypy/branch/faster-raise/pypy/translator/platform/__init__.py Sat Nov 14 15:24:15 2009 @@ -6,7 +6,7 @@ import sys, py, os from pypy.tool.ansi_print import ansi_log -from py.__.code import safe_repr +from py.impl.code.code import safe_repr log = py.log.Producer("platform") py.log.setconsumer("platform", ansi_log) @@ -33,9 +33,9 @@ def __repr__(self): if self.err: - return "" % safe_repr._repr(self.err) + return "" % safe_repr(self.err) else: - return "" % safe_repr._repr(self.out) + return "" % safe_repr(self.out) __str__ = __repr__ @@ -108,7 +108,7 @@ def _handle_error(self, returncode, stderr, stdout, outname): if returncode != 0: errorfile = outname.new(ext='errors') - errorfile.write(stderr.encode("utf-8")) + errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines[:50]: log.ERROR(line) Modified: pypy/branch/faster-raise/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/faster-raise/pypy/translator/platform/test/test_darwin.py Sat Nov 14 15:24:15 2009 @@ -32,7 +32,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(frameworks=('Cocoa',), include_dirs=(includedir,)) executable = self.platform.compile([objcfile], eci) Modified: pypy/branch/faster-raise/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/branch/faster-raise/pypy/translator/platform/test/test_maemo.py Sat Nov 14 15:24:15 2009 @@ -26,7 +26,7 @@ return 0; } ''') - includedir = py.magic.autopath().dirpath().join('include') + includedir = py.path.local(__file__).dirpath().join('include') eci = ExternalCompilationInfo(include_dirs=(includedir,)) executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) Modified: pypy/branch/faster-raise/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/platform/windows.py (original) +++ pypy/branch/faster-raise/pypy/translator/platform/windows.py Sat Nov 14 15:24:15 2009 @@ -96,6 +96,15 @@ # Probably not a msvc compiler... self.version = 0 + # Try to find a masm assembler + returncode, stdout, stderr = _run_subprocess('ml.exe', '', + env=self.c_environ) + r = re.search('Macro Assembler', stderr) + if r is None and os.path.exists('c:/masm32/bin/ml.exe'): + self.masm = 'c:/masm32/bin/ml.exe' + else: + self.masm = 'ml.exe' + # Install debug options only when interpreter is in debug mode if sys.executable.lower().endswith('_d.exe'): self.cflags = ['/MDd', '/Z7', '/Od'] @@ -133,6 +142,12 @@ def _args_for_shared(self, args): return ['/dll'] + args + def check___thread(self): + # __declspec(thread) does not seem to work when using assembler. + # Returning False will cause the program to use TlsAlloc functions. + # see src/thread_nt.h + return False + def _link_args_from_eci(self, eci, standalone): # Windows needs to resolve all symbols even for DLLs args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) @@ -177,10 +192,8 @@ errorfile = outname.new(ext='errors') errorfile.write(stderr) stderrlines = stderr.splitlines() - for line in stderrlines[:5]: + for line in stderrlines: log.ERROR(line) - if len(stderrlines) > 5: - log.ERROR('...') raise CompilationError(stdout, stderr) @@ -225,20 +238,30 @@ ('INCLUDEDIRS', self._includedirs(rel_includedirs)), ('CFLAGS', self.cflags + list(eci.compile_extra)), ('LDFLAGS', self.link_flags + list(eci.link_extra)), - ('CC', self.cc) + ('CC', self.cc), + ('CC_LINK', self.link), + ('MASM', self.masm), ] for args in definitions: m.definition(*args) rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS)'), - ('%.obj', '%.c', '$(CC) $(CFLAGS) -o $@ -c $< $(INCLUDEDIRS)'), + ('.c.obj', '', '$(CC) /nologo $(CFLAGS) /Fo$@ /c $< $(INCLUDEDIRS)'), ] for rule in rules: m.rule(*rule) + if self.version < 80: + m.rule('$(TARGET)', '$(OBJECTS)', + '$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + else: + m.rule('$(TARGET)', '$(OBJECTS)', + ['$(CC_LINK) /nologo $(LDFLAGS) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS) /MANIFESTFILE:$*.manifest', + 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', + ]) + return m def execute_makefile(self, path_to_makefile): @@ -251,14 +274,29 @@ try: returncode, stdout, stderr = _run_subprocess( 'nmake', - ['/f', str(path.join('Makefile'))]) + ['/nologo', '/f', str(path.join('Makefile'))]) finally: oldcwd.chdir() self._handle_error(returncode, stdout, stderr, path.join('make')) class NMakefile(posix.GnuMakefile): - pass # for the moment + def write(self, out=None): + # nmake expands macros when it parses rules. + # Write all macros before the rules. + if out is None: + f = self.makefile_dir.join('Makefile').open('w') + else: + f = out + for line in self.lines: + if not isinstance(line, posix.Rule): + line.write(f) + for line in self.lines: + if isinstance(line, posix.Rule): + line.write(f) + f.flush() + if out is None: + f.close() class MingwPlatform(posix.BasePosix): Modified: pypy/branch/faster-raise/pypy/translator/sandbox/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/sandbox/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/sandbox/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/sandbox/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/sandbox/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/sandbox/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/test/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/test/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/test/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break Modified: pypy/branch/faster-raise/pypy/translator/test/test_driver.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/test/test_driver.py (original) +++ pypy/branch/faster-raise/pypy/translator/test/test_driver.py Sat Nov 14 15:24:15 2009 @@ -1,7 +1,7 @@ import py from pypy.translator.driver import TranslationDriver -from py.compat import optparse +import optparse def test_ctr(): td = TranslationDriver() Modified: pypy/branch/faster-raise/pypy/translator/tool/autopath.py ============================================================================== --- pypy/branch/faster-raise/pypy/translator/tool/autopath.py (original) +++ pypy/branch/faster-raise/pypy/translator/tool/autopath.py Sat Nov 14 15:24:15 2009 @@ -37,8 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - # check if "../py/__init__.py" exists - checkfile = os.path.join(partdir, os.pardir, 'py', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break From cfbolz at codespeak.net Sat Nov 14 15:54:57 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 14 Nov 2009 15:54:57 +0100 (CET) Subject: [pypy-svn] r69288 - in pypy/trunk/pypy: annotation translator/c Message-ID: <20091114145457.0B3F916802D@codespeak.net> Author: cfbolz Date: Sat Nov 14 15:54:56 2009 New Revision: 69288 Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/translator/c/node.py Log: fix deprecation warnings Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Sat Nov 14 15:54:56 2009 @@ -552,7 +552,7 @@ if self.is_exception_class(): if self.pyobj.__module__ == 'exceptions': return True - if self.pyobj is py.magic.AssertionError: + if self.pyobj is py.code._AssertionError: return True return False Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Sat Nov 14 15:54:56 2009 @@ -925,7 +925,7 @@ if isinstance(value, (type, types.ClassType)): if (issubclass(value, BaseException) and (value.__module__ == 'exceptions' - or value is py.magic.AssertionError)): + or value is py.code._AssertionError)): return 'PyExc_' + value.__name__ raise Exception("don't know how to simply render py object: %r" % (value, )) From cfbolz at codespeak.net Sat Nov 14 15:56:33 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 14 Nov 2009 15:56:33 +0100 (CET) Subject: [pypy-svn] r69289 - in pypy/trunk/pypy: module/pypyjit/test objspace/std objspace/std/test Message-ID: <20091114145633.564B116802D@codespeak.net> Author: cfbolz Date: Sat Nov 14 15:56:31 2009 New Revision: 69289 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py pypy/trunk/pypy/objspace/std/callmethod.py pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/test/test_versionedtype.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/objspace/std/typetype.py Log: clean up some code in typeobject to make sure that the version_tag of builtin objects cannot change. Get rid of a few guards here and there. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 14 15:56:31 2009 @@ -187,7 +187,7 @@ assert len(ops) == 2 assert not ops[0].get_opnames("call") assert not ops[0].get_opnames("new") - assert len(ops[0].get_opnames("guard")) <= 8 + assert len(ops[0].get_opnames("guard")) <= 7 assert not ops[1] # second LOOKUP_METHOD folded away ops = self.get_by_bytecode("CALL_METHOD") @@ -233,6 +233,7 @@ while i < n: a = A() assert isinstance(a, A) + assert not isinstance(a, int) a.x = 2 i = i + a.x return i @@ -240,13 +241,42 @@ ([20], 20), ([31], 32)) - callA, callisinstance = self.get_by_bytecode("CALL_FUNCTION") + callA, callisinstance1, callisinstance2 = ( + self.get_by_bytecode("CALL_FUNCTION")) assert not callA.get_opnames("call") assert not callA.get_opnames("new") - assert len(callA.get_opnames("guard")) <= 9 - assert not callisinstance.get_opnames("call") - assert not callisinstance.get_opnames("new") - assert len(callisinstance.get_opnames("guard")) <= 2 + assert len(callA.get_opnames("guard")) <= 8 + assert not callisinstance1.get_opnames("call") + assert not callisinstance1.get_opnames("new") + assert len(callisinstance1.get_opnames("guard")) <= 2 + # calling isinstance on a builtin type gives zero guards + # because the version_tag of a builtin type is immutable + assert not len(callisinstance1.get_opnames("guard")) + + + bytecode, = self.get_by_bytecode("STORE_ATTR") + assert bytecode.get_opnames() == [] + + def test_load_attr(self): + self.run_source(''' + class A(object): + pass + a = A() + a.x = 2 + def main(n): + i = 0 + while i < n: + i = i + a.x + return i + ''', + ([20], 20), + ([31], 32)) + + load = self.get_by_bytecode("LOAD_ATTR") + # 1 guard_value for the class + # 1 guard_value for the version_tag + # 1 guard_value for the structure + assert len(load.get_opnames("guard")) <= 3 bytecode, = self.get_by_bytecode("STORE_ATTR") assert bytecode.get_opnames() == [] Modified: pypy/trunk/pypy/objspace/std/callmethod.py ============================================================================== --- pypy/trunk/pypy/objspace/std/callmethod.py (original) +++ pypy/trunk/pypy/objspace/std/callmethod.py Sat Nov 14 15:56:31 2009 @@ -33,22 +33,8 @@ w_name = f.getname_w(nameindex) w_value = None - if space.config.objspace.std.getattributeshortcut: - w_type = space.type(w_obj) - fastpath = w_type.uses_object_getattribute - # conservatively, 'uses_object_getattribute' can be False - # even if __getattribute__ was not overridden. In this - # case, the code below calls space.getattr(), which will - # set 'uses_object_getattribute' to True for the next time. - else: - w_getattribute = space.lookup(w_obj, '__getattribute__') - if w_getattribute is object_getattribute(space): - w_type = space.type(w_obj) - fastpath = True - else: - fastpath = False - - if fastpath: + w_type = space.type(w_obj) + if w_type.has_object_getattribute(): name = space.str_w(w_name) w_descr = w_type.lookup(name) if w_descr is None: @@ -87,8 +73,8 @@ """An optimized version of space.call_method() based on the same principle as above. """ - w_getattribute = space.lookup(w_obj, '__getattribute__') - if w_getattribute is object_getattribute(space): + w_type = space.type(w_obj) + if w_type.has_object_getattribute(): w_descr = space.lookup(w_obj, methname) typ = type(w_descr) if typ is function.Function or typ is function.FunctionWithFixedCode: Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Sat Nov 14 15:56:31 2009 @@ -393,7 +393,7 @@ def __init__(self, space, w_type): StrDictImplementation.__init__(self, space) self.w_type = w_type - self.original_version_tag = w_type.version_tag + self.original_version_tag = w_type.version_tag() if self.original_version_tag is None: self._shadows_anything = True else: @@ -419,7 +419,7 @@ def impl_shadows_anything(self): return (self._shadows_anything or - self.w_type.version_tag is not self.original_version_tag) + self.w_type.version_tag() is not self.original_version_tag) def impl_set_shadows_anything(self): self._shadows_anything = True Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Sat Nov 14 15:56:31 2009 @@ -692,13 +692,8 @@ from pypy.objspace.descroperation import raiseattrerror from pypy.objspace.descroperation import object_getattribute w_type = self.type(w_obj) - if not w_type.uses_object_getattribute: - # slow path: look for a custom __getattribute__ on the class - w_descr = w_type.lookup('__getattribute__') - # if it was not actually overriden in the class, we remember this - # fact for the next time. - if w_descr is object_getattribute(self): - w_type.uses_object_getattribute = True + w_descr = w_type.getattribute_if_not_from_object() + if w_descr is not None: return self._handle_getattribute(w_descr, w_obj, w_name) # fast path: XXX this is duplicating most of the logic Modified: pypy/trunk/pypy/objspace/std/test/test_versionedtype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_versionedtype.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_versionedtype.py Sat Nov 14 15:56:31 2009 @@ -23,59 +23,100 @@ def test_tag_changes(self): space = self.space w_A, w_B, w_C = self.get_three_classes() - atag = w_A.version_tag - btag = w_B.version_tag + atag = w_A.version_tag() + btag = w_B.version_tag() assert atag is not None assert btag is not None - assert w_C.version_tag is None + assert w_C.version_tag() is None assert atag is not btag w_types = space.appexec([w_A, w_B], """(A, B): B.g = lambda self: None """) - assert w_B.version_tag is not btag - assert w_A.version_tag is atag - btag = w_B.version_tag + assert w_B.version_tag() is not btag + assert w_A.version_tag() is atag + btag = w_B.version_tag() w_types = space.appexec([w_A, w_B], """(A, B): A.f = lambda self: None """) - assert w_B.version_tag is not btag - assert w_A.version_tag is not atag - atag = w_A.version_tag - btag = w_B.version_tag + assert w_B.version_tag() is not btag + assert w_A.version_tag() is not atag + atag = w_A.version_tag() + btag = w_B.version_tag() assert atag is not btag w_types = space.appexec([w_A, w_B], """(A, B): del A.f """) - assert w_B.version_tag is not btag - assert w_A.version_tag is not atag - atag = w_A.version_tag - btag = w_B.version_tag + assert w_B.version_tag() is not btag + assert w_A.version_tag() is not atag + atag = w_A.version_tag() + btag = w_B.version_tag() assert atag is not btag def test_tag_changes_when_bases_change(self): space = self.space w_A, w_B, w_C = self.get_three_classes() - atag = w_A.version_tag - btag = w_B.version_tag + atag = w_A.version_tag() + btag = w_B.version_tag() w_types = space.appexec([w_A, w_B, w_C], """(A, B, C): class D(object): pass B.__bases__ = (D, ) """) - assert w_B.version_tag is not btag + assert w_B.version_tag() is not btag + + def test_tag_changes_only_when_dict_changes(self): + space = self.space + w_A, w_B, w_C = self.get_three_classes() + atag = w_A.version_tag() + # setting a descriptor does not change the version_tag + w_types = space.appexec([w_A, w_B, w_C], """(A, B, C): + A.__name__ = "hello" + """) + + assert w_A.version_tag() is atag + # deleting via a descriptor does not change the version_tag + w_types = space.appexec([w_A, w_B, w_C], """(A, B, C): + try: + del A.__name__ + except AttributeError: + pass + """) + assert w_A.version_tag() is atag + + # deleting a non-existing key does not change the version_tag + w_types = space.appexec([w_A, w_B, w_C], """(A, B, C): + try: + del A.abc + except AttributeError: + pass + """) + assert w_A.version_tag() is atag + + def test_tag_changes_When_module_changes(self): + space = self.space + w_A, w_B, w_C = self.get_three_classes() + atag = w_A.version_tag() + # setting a descriptor does not change the version_tag + w_types = space.appexec([w_A, w_B, w_C], """(A, B, C): + A.__module__ = "hello" + """) + + assert w_A.version_tag() is not atag + + def test_version_tag_of_builtin_types(self): space = self.space - assert space.w_list.version_tag is not None - assert space.w_dict.version_tag is not None - assert space.type(space.sys).version_tag is None - assert space.w_type.version_tag is None + assert space.w_list.version_tag() is not None + assert space.w_dict.version_tag() is not None + assert space.type(space.sys).version_tag() is None + assert space.w_type.version_tag() is None w_function = space.appexec([], """(): def f(): pass return type(f) """) - assert w_function.version_tag is None + assert w_function.version_tag() is None def test_version_tag_of_subclasses_of_builtin_types(self): space = self.space @@ -95,11 +136,11 @@ """) (w_LIST, w_DICT, w_TYPE, w_MODULE, w_OBJECT) = space.unpackiterable(w_types) - assert w_LIST.version_tag is not None - assert w_DICT.version_tag is not None - assert w_TYPE.version_tag is None - assert w_MODULE.version_tag is None - assert w_OBJECT.version_tag is not None + assert w_LIST.version_tag() is not None + assert w_DICT.version_tag() is not None + assert w_TYPE.version_tag() is None + assert w_MODULE.version_tag() is None + assert w_OBJECT.version_tag() is not None class AppTestVersionedType(test_typeobject.AppTestTypeObject): def setup_class(cls): Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Sat Nov 14 15:56:31 2009 @@ -48,7 +48,10 @@ from pypy.objspace.std.typetype import type_typedef as typedef lazyloaders = {} # can be overridden by specific instances - version_tag = None + + # the version_tag changes if the dict or the inheritance hierarchy changes + # other changes to the type (e.g. the name) leave it unchanged + _version_tag = None _immutable_fields_ = ["__flags__", 'needsdel', @@ -92,10 +95,11 @@ if w_self.instancetypedef.hasdict or custom_metaclass: pass else: - w_self.version_tag = VersionTag() + w_self._version_tag = VersionTag() def mutated(w_self): space = w_self.space + assert w_self.is_heaptype() or not space.config.objspace.std.immutable_builtintypes if (not space.config.objspace.std.withtypeversion and not space.config.objspace.std.getattributeshortcut and not space.config.objspace.std.newshortcut): @@ -109,14 +113,52 @@ w_self.w_bltin_new = None if (space.config.objspace.std.withtypeversion - and w_self.version_tag is not None): - w_self.version_tag = VersionTag() + and w_self._version_tag is not None): + w_self._version_tag = VersionTag() subclasses_w = w_self.get_subclasses() for w_subclass in subclasses_w: assert isinstance(w_subclass, W_TypeObject) w_subclass.mutated() + def version_tag(w_self): + if (not we_are_jitted() or w_self.is_heaptype() or not + w_self.space.config.objspace.std.immutable_builtintypes): + return w_self._version_tag + # pure objects cannot get their version_tag changed + w_self = hint(w_self, promote=True) + return w_self._pure_version_tag() + + def getattribute_if_not_from_object(w_self): + """ this method returns the applevel __getattribute__ if that is not + the one from object, in which case it returns None """ + from pypy.objspace.descroperation import object_getattribute + if not we_are_jitted(): + shortcut = w_self.space.config.objspace.std.getattributeshortcut + if not shortcut or not w_self.uses_object_getattribute: + # slow path: look for a custom __getattribute__ on the class + w_descr = w_self.lookup('__getattribute__') + # if it was not actually overriden in the class, we remember this + # fact for the next time. + if w_descr is object_getattribute(w_self.space): + if shortcut: + w_self.uses_object_getattribute = True + else: + return w_descr + return None + # in the JIT case, just use a lookup, because it is folded away + # correctly using the version_tag + w_descr = w_self.lookup('__getattribute__') + if w_descr is not object_getattribute(w_self.space): + return w_descr + + def has_object_getattribute(w_self): + return w_self.getattribute_if_not_from_object() is None + + @purefunction + def _pure_version_tag(w_self): + return w_self._version_tag + def ready(w_self): for w_base in w_self.bases_w: if not isinstance(w_base, W_TypeObject): @@ -195,21 +237,11 @@ return w_class, w_value return None, None - @purefunction - def _pure_lookup_where_builtin_type(w_self, name): - assert not w_self.is_heaptype() - return w_self._lookup_where(name) - def lookup_where_with_method_cache(w_self, name): space = w_self.space w_self = hint(w_self, promote=True) assert space.config.objspace.std.withmethodcache - if (space.config.objspace.std.immutable_builtintypes and - we_are_jitted() and not w_self.is_heaptype()): - w_self = hint(w_self, promote=True) - name = hint(name, promote=True) - return w_self._pure_lookup_where_builtin_type(name) - version_tag = w_self.version_tag + version_tag = w_self.version_tag() version_tag = hint(version_tag, promote=True) if version_tag is None: tup = w_self._lookup_where(name) @@ -594,7 +626,15 @@ else: return space.type(w_obj) # invoke the __new__ of the type - w_bltin_new = w_type.w_bltin_new + if not we_are_jitted(): + # note that the annotator will figure out that w_type.w_bltin_new can + # only be None if the newshortcut config option is not set + w_bltin_new = w_type.w_bltin_new + else: + # for the JIT it is better to take the slow path because normal lookup + # is nicely optimized, but the w_type.w_bltin_new attribute is not + # known to the JIT + w_bltin_new = None call_init = True if w_bltin_new is not None: w_newobject = space.call_obj_args(w_bltin_new, w_type, __args__) @@ -602,6 +642,7 @@ w_newtype, w_newdescr = w_type.lookup_where('__new__') w_newfunc = space.get(w_newdescr, w_type) if (space.config.objspace.std.newshortcut and + not we_are_jitted() and isinstance(w_newtype, W_TypeObject) and not w_newtype.is_heaptype() and not space.is_w(w_newtype, space.w_type)): @@ -622,10 +663,6 @@ return w_type2 in w_type1.mro_w @purefunction -def _pure_issubtype_builtin(w_type1, w_type2): - return _issubtype(w_type1, w_type2) - - at purefunction def _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2): return _issubtype(w_type1, w_type2) @@ -633,19 +670,13 @@ w_type1 = hint(w_type1, promote=True) w_type2 = hint(w_type2, promote=True) if space.config.objspace.std.withtypeversion and we_are_jitted(): - if (space.config.objspace.std.immutable_builtintypes and - not w_type1.is_heaptype() and - not w_type2.is_heaptype()): - res = _pure_issubtype_builtin(w_type1, w_type2) + version_tag1 = w_type1.version_tag() + version_tag2 = w_type2.version_tag() + version_tag1 = hint(version_tag1, promote=True) + version_tag2 = hint(version_tag2, promote=True) + if version_tag1 is not None and version_tag2 is not None: + res = _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2) return space.newbool(res) - else: - version_tag1 = w_type1.version_tag - version_tag2 = w_type2.version_tag - version_tag1 = hint(version_tag1, promote=True) - version_tag2 = hint(version_tag2, promote=True) - if version_tag1 is not None and version_tag2 is not None: - res = _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2) - return space.newbool(res) res = _issubtype(w_type1, w_type2) return space.newbool(res) @@ -684,7 +715,6 @@ # Note. This is exactly the same thing as descroperation.descr__setattr__, # but it is needed at bootstrap to avoid a call to w_type.getdict() which # would un-lazify the whole type. - w_type.mutated() name = space.str_w(w_name) w_descr = space.lookup(w_type, name) if w_descr is not None: @@ -699,10 +729,10 @@ if name == "__del__" and name not in w_type.dict_w: msg = "a __del__ method added to an existing type will not be called" space.warn(msg, space.w_RuntimeWarning) + w_type.mutated() w_type.dict_w[name] = w_value def delattr__Type_ANY(space, w_type, w_name): - w_type.mutated() if w_type.lazyloaders: w_type._freeze_() # force un-lazification name = space.str_w(w_name) @@ -717,9 +747,11 @@ raise OperationError(space.w_TypeError, space.wrap(msg)) try: del w_type.dict_w[name] - return except KeyError: raise OperationError(space.w_AttributeError, w_name) + else: + w_type.mutated() + return # ____________________________________________________________ Modified: pypy/trunk/pypy/objspace/std/typetype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typetype.py (original) +++ pypy/trunk/pypy/objspace/std/typetype.py Sat Nov 14 15:56:31 2009 @@ -194,6 +194,7 @@ raise OperationError(space.w_TypeError, space.wrap("can't set %s.__module__" % w_type.name)) + w_type.mutated() w_type.dict_w['__module__'] = w_value def descr___subclasses__(space, w_type): From afa at codespeak.net Sat Nov 14 16:47:05 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 14 Nov 2009 16:47:05 +0100 (CET) Subject: [pypy-svn] r69290 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091114154705.C4BAE16802C@codespeak.net> Author: afa Date: Sat Nov 14 16:47:04 2009 New Revision: 69290 Added: pypy/trunk/pypy/module/oracle/ (props changed) pypy/trunk/pypy/module/oracle/__init__.py (contents, props changed) pypy/trunk/pypy/module/oracle/app_oracle.py (contents, props changed) pypy/trunk/pypy/module/oracle/config.py (contents, props changed) pypy/trunk/pypy/module/oracle/interp_connect.py (contents, props changed) pypy/trunk/pypy/module/oracle/interp_cursor.py (contents, props changed) pypy/trunk/pypy/module/oracle/interp_environ.py (contents, props changed) pypy/trunk/pypy/module/oracle/interp_error.py (contents, props changed) pypy/trunk/pypy/module/oracle/interp_variable.py (contents, props changed) pypy/trunk/pypy/module/oracle/roci.py (contents, props changed) pypy/trunk/pypy/module/oracle/test/ (props changed) pypy/trunk/pypy/module/oracle/test/test_connect.py (contents, props changed) pypy/trunk/pypy/module/oracle/test/test_cursor.py (contents, props changed) pypy/trunk/pypy/module/oracle/test/test_select.py (contents, props changed) pypy/trunk/pypy/module/oracle/transform.py (contents, props changed) Log: Added: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/__init__.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,21 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevel_name = 'cx_Oracle' + + interpleveldefs = { + 'connect': 'interp_connect.W_Connection', + 'UNICODE': 'interp_variable.VT_NationalCharString', + 'NUMBER': 'interp_variable.VT_Float', + 'Variable': 'interp_variable.W_Variable', + } + + appleveldefs = { + 'version': 'app_oracle.version', + 'makedsn': 'app_oracle.makedsn', + } + for name in """DataError DatabaseError Error IntegrityError InterfaceError + InternalError NotSupportedError OperationalError + ProgrammingError Warning""".split(): + appleveldefs[name] = "app_oracle.%s" % (name,) + Added: pypy/trunk/pypy/module/oracle/app_oracle.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/app_oracle.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,37 @@ +version = '5.0.0' + +class Warning(StandardError): + pass + +class Error(StandardError): + pass + +class InterfaceError(Error): + pass + +class DatabaseError(Error): + pass + +class DataError(DatabaseError): + pass + +class OperationalError(DatabaseError): + pass + +class IntegrityError(DatabaseError): + pass + +class InternalError(DatabaseError): + pass + +class ProgrammingError(DatabaseError): + pass + +class NotSupportedError(DatabaseError): + pass + + +def makedsn(host, port, sid): + return ("(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=" + "(PROTOCOL=TCP)(HOST=%s)(PORT=%s)))" + "(CONNECT_DATA=(SID=%s)))" % (host, port, sid)) Added: pypy/trunk/pypy/module/oracle/config.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/config.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,36 @@ +from pypy.rpython.lltypesystem import rffi, lltype + +WITH_UNICODE = False + +MAX_STRING_CHARS = 4000 + +if WITH_UNICODE: + CHARSETID = roci.OCI_UTF16ID + BYTES_PER_CHAR = 2 + def string_w(space, w_obj): + return space.unicode_w(w_obj) +else: + def string_w(space, w_obj): + return space.str_w(w_obj) + def w_string(space, buf, len=-1): + if len < 0: + return space.wrap(rffi.charp2str(buf)) + else: + return space.wrap(rffi.charpsize2str(buf, len)) + CHARSETID = 0 + BYTES_PER_CHAR = 1 + + class StringBuffer: + def __init__(self): + pass + + def fill(self, space, w_string): + if w_string is None or space.is_w(w_string, space.w_None): + self.clear() + else: + self.ptr = string_w(space, w_string) + self.size = len(self.ptr) + + def clear(self): + self.ptr = None + self.size = 0 Added: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,339 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped +from pypy.interpreter.typedef import (TypeDef, interp_attrproperty_w, + GetSetProperty) +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import rffi, lltype + +Null = NoneNotWrapped + +from pypy.module.oracle import roci, interp_error +from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS +from pypy.module.oracle.interp_environ import Environment +from pypy.module.oracle.interp_cursor import W_Cursor +from pypy.module.oracle.interp_variable import VT_String + +class W_Connection(Wrappable): + def __init__(self): + self.commitMode = roci.OCI_DEFAULT + self.environment = None + self.autocommit = False + + self.inputTypeHandler = None + self.outputTypeHandler = None + + self.w_version = None + + def descr_new(space, w_subtype, + w_user=NoneNotWrapped, + w_password=NoneNotWrapped, + w_dsn=NoneNotWrapped, + mode=roci.OCI_DEFAULT, + handle=0, # XXX should be a ptr type + w_pool=Null, + threaded=False, + twophase=False, + events=False, + w_cclass=Null, + purity=False, + w_newpassword=Null): + self = space.allocate_instance(W_Connection, w_subtype) + W_Connection.__init__(self) + + # set up the environment + if w_pool: + pool = space.instance_w(W_Pool, w_pool) + self.environment = pool.environment.clone() + else: + self.environment = Environment(space, threaded, events) + + self.w_username = w_user + self.w_password = w_password + self.w_tnsentry = w_dsn + + # perform some parsing, if necessary + if (self.w_username and not self.w_password and + space.is_true(space.contains(self.w_username, space.wrap('/')))): + (self.w_username, self.w_password) = space.unpackiterable( + space.call_method(self.w_username, 'split', + space.wrap('/'), space.wrap(1))) + + if (self.w_password and not self.w_tnsentry and + space.is_true(space.contains(self.w_password, space.wrap('@')))): + (self.w_password, self.w_tnsentry) = space.unpackiterable( + space.call_method(self.w_password, 'split', + space.wrap('@'), space.wrap(1))) + + self.connect(space, mode, twophase) + return space.wrap(self) + + descr_new.unwrap_spec = [ObjSpace, W_Root, + W_Root, W_Root, W_Root, + int, int, + W_Root, + bool, bool, bool, + W_Root, + bool, + W_Root] + + def connect(self, space, mode, twophase): + stringBuffer = StringBuffer() + + # allocate the server handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIServer).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_SERVER, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "Connection_Connect(): allocate server handle") + self.serverHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # attach to the server + stringBuffer.fill(space, self.w_tnsentry) + try: + status = roci.OCIServerAttach( + self.serverHandle, + self.environment.errorHandle, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_DEFAULT) + self.environment.checkForError( + status, "Connection_Connect(): server attach") + finally: + stringBuffer.clear() + + # allocate the service context handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCISvcCtx).TO, + 1, flavor='raw') + + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_SVCCTX, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "Connection_Connect(): allocate service context handle") + self.handle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # set attribute for server handle + status = roci.OCIAttrSet( + self.handle, roci.OCI_HTYPE_SVCCTX, + self.serverHandle, 0, + roci.OCI_ATTR_SERVER, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_Connect(): set server handle") + + # set the internal and external names; these are needed for global + # transactions but are limited in terms of the lengths of the strings + + + # allocate the session handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCISession).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_SESSION, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "Connection_Connect(): allocate session handle") + self.sessionHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # set user name in session handle + stringBuffer.fill(space, self.w_username) + try: + if stringBuffer.size > 0: + credentialType = roci.OCI_CRED_RDBMS + status = roci.OCIAttrSet( + self.sessionHandle, + roci.OCI_HTYPE_SESSION, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_USERNAME, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_Connect(): set user name") + finally: + stringBuffer.clear() + + # set password in session handle + stringBuffer.fill(space, self.w_password) + try: + if stringBuffer.size > 0: + credentialType = roci.OCI_CRED_RDBMS + status = roci.OCIAttrSet( + self.sessionHandle, + roci.OCI_HTYPE_SESSION, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_PASSWORD, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_Connect(): set password") + finally: + stringBuffer.clear() + + # set the session handle on the service context handle + status = roci.OCIAttrSet( + self.handle, roci.OCI_HTYPE_SVCCTX, + self.sessionHandle, 0, + roci.OCI_ATTR_SESSION, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_Connect(): set session handle") + + # if a new password has been specified, change it which will also + # establish the session + + # begin the session + status = roci.OCISessionBegin( + self.handle, self.environment.errorHandle, + self.sessionHandle, credentialType, mode) + try: + self.environment.checkForError( + status, "Connection_Connect(): begin session") + except: + self.sessionHandle = None + raise + + def _checkConnected(self, space): + if not self.handle: + raise OperationError( + interp_error.get(space).w_InterfaceError, + space.wrap("not connected")) + + def close(self, space): + # make sure we are actually connnected + self._checkConnected(space) + + # perform a rollback + status = roci.OCITransRollback( + self.handle, self.environment.errorHandle, + roci.OCI_DEFAULT) + self.environment.checkForError( + status, "Connection_Close(): rollback") + + # logoff of the server + if self.sessionHandle: + status = roci.OCISessionEnd( + self.handle, self.environment.errorHandle, + self.sessionHandle, roci.OCI_DEFAULT) + self.environment.checkForError( + status, "Connection_Close(): end session") + roci.OCIHandleFree(self.handle, roci.OCI_HTYPE_SVCCTX) + + self.handle = None + close.unwrap_spec = ['self', ObjSpace] + + def commit(self, space): + # make sure we are actually connected + self._checkConnected(space) + + status = roci.OCITransCommit( + self.handle, self.environment.errorHandle, + self.commitMode) + self.environment.checkForError( + status, "Connection_Commit()") + + self.commitMode = roci.OCI_DEFAULT + commit.unwrap_spec = ['self', ObjSpace] + + def rollback(self, space): + # make sure we are actually connected + self._checkConnected(space) + + status = roci.OCITransRollback( + self.handle, self.environment.errorHandle, + self.OCI_DEFAULT) + self.environment.checkForError( + status, "Connection_Rollback()") + rollback.unwrap_spec = ['self', ObjSpace] + + def newCursor(self, space): + return space.wrap(W_Cursor(space, self)) + newCursor.unwrap_spec = ['self', ObjSpace] + + def _getCharacterSetName(self, space, attribute): + # get character set id + status = roci.OCIAttrGet( + self.environment.handle, roci.HTYPE_ENV, + charsetId, None, + attribute, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetCharacterSetName(): get charset id") + + # get character set name + status = roci.OCINlsCharsetIdToName( + self.environmentHandle, + charsetNameBuf.buf, charsetNameBuf.size, + charsetIdPtr[0]) + self.environment.checkForError( + status, "Connection_GetCharacterSetName(): get Oracle charset name") + + # get IANA character set name + status = roci.OCINlsNameMap( + self.environmentHandle, + ianaCharsetNameBuf.buf, inaCharsetNameBuf.size, + charsetNameBuf.buf, roci.OCI_NLS_CS_ORA_TO_IANA) + self.environment.checkForError( + status, "Connection_GetCharacterSetName(): translate NLS charset") + + return space.wrap(ianaCharsetName) + + def get_encoding(space, self): + return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) + def get_nationalencoding(space, self): + return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) + + def get_version(space, self): + # if version has already been determined, no need to determine again + if self.w_version: + return self.w_version + + # allocate a cursor to retrieve the version + cursor = self.newCursor(space) + + # allocate version and compatibility variables + versionVar = VT_String(cursor, cursor.arraySize, MAX_STRING_CHARS) + compatVar = VT_String(cursor, cursor.arraySize, MAX_STRING_CHARS) + + # call stored procedure + cursor._call(space, "dbms_utility.db_version", + None, [space.wrap(versionVar), + space.wrap(compatVar)]) + + # retrieve value + self.w_version = versionVar.getValue(space, 0) + return self.w_version + +W_Connection.typedef = TypeDef( + "Connection", + __new__ = interp2app(W_Connection.descr_new.im_func, + unwrap_spec=W_Connection.descr_new.unwrap_spec), + username = interp_attrproperty_w('w_username', W_Connection), + password = interp_attrproperty_w('w_password', W_Connection), + tnsentry = interp_attrproperty_w('w_tnsentry', W_Connection), + + close = interp2app(W_Connection.close, + unwrap_spec=W_Connection.close.unwrap_spec), + commit = interp2app(W_Connection.commit, + unwrap_spec=W_Connection.commit.unwrap_spec), + rollback = interp2app(W_Connection.rollback, + unwrap_spec=W_Connection.rollback.unwrap_spec), + + cursor = interp2app(W_Connection.newCursor, + unwrap_spec=W_Connection.newCursor.unwrap_spec), + + encoding = GetSetProperty(W_Connection.get_encoding), + nationalencoding = GetSetProperty(W_Connection.get_nationalencoding), + version = GetSetProperty(W_Connection.get_version), + ) Added: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,700 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped +from pypy.interpreter.argument import Arguments +from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w +from pypy.interpreter.typedef import GetSetProperty +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import rffi, lltype + +from pypy.module.oracle import roci, interp_error +from pypy.module.oracle.config import w_string, string_w, StringBuffer +from pypy.module.oracle import interp_variable +from pypy.module.oracle.interp_error import get + +class W_Cursor(Wrappable): + def __init__(self, space, connection): + self.connection = connection + self.environment = connection.environment + + self.w_statement = None + self.statementType = -1 + self.handle = None + self.isOpen = True + + self.setInputSizes = False + self.arraySize = 50 + self.fetchArraySize = 50 + self.bindArraySize = 1 + self.bindList = None + self.bindDict = None + self.numbersAsStrings = False + + self.inputTypeHandler = None + self.outputTypeHandler = None + self.w_rowFactory = None + + def execute(self, space, w_stmt, __args__): + args_w, kw_w = __args__.unpack() + + if space.is_w(w_stmt, space.w_None): + w_stmt = None + + if len(args_w) > 1: + raise OperationError( + space.w_TypeError, + space.wrap("Too many arguments")) + elif len(args_w) == 1: + if len(kw_w) > 0: + raise OperationError( + interp.error.get(space, 'InterfaceError'), + space.wrap( + "expecting argument or keyword arguments, not both")) + w_args = args_w[0] + vars_w = space.unpackiterable(w_args) + elif len(kw_w) > 0: + vars_w = kw_w + else: + vars_w = None + + # make sure the cursor is open + self._checkOpen(space) + + return self._execute(space, w_stmt, vars_w) + execute.unwrap_spec = ['self', ObjSpace, W_Root, Arguments] + + def prepare(self, space, w_stmt, w_tag=None): + # make sure the cursor is open + self._checkOpen(space) + + # prepare the statement + self._internalPrepare(space, w_stmt, w_tag) + prepare.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + + def _execute(self, space, w_stmt, vars_w): + + # prepare the statement, if applicable + self._internalPrepare(space, w_stmt, None) + + # perform binds + if vars_w is not None: + self._setBindVariablesByPos(space, vars_w, 1, 0, 0) + self._performBind(space) + + # execute the statement + isQuery = self.statementType == roci.OCI_STMT_SELECT + if isQuery: + numIters = 0 + else: + numIters = 1 + self._internalExecute(space, numIters=numIters) + + # perform defines, if necessary + if isQuery and self.fetchVariables is None: + self._performDefine() + + # reset the values of setoutputsize() + self.outputSize = -1 + self.outputSizeColumn = -1 + + # for queries, return the cursor for convenience + if isQuery: + return space.wrap(self) + + # for all other statements, simply return None + return space.w_None + + def close(self, space): + # make sure we are actually open + self._checkOpen(space) + + # close the cursor + self._freeHandle(space, raiseError=True) + + self.isOpen = False + self.handle = None + close.unwrap_spec = ['self', ObjSpace] + + def callfunc(self, space, name, w_returnType, w_parameters=None): + retvar = interp_variable.newVariableByType(space, self, w_returnType, 1) + if space.is_w(w_parameters, space.w_None): + args_w = None + else: + args_w = space.unpackiterable(w_parameters) + + return self._call(space, name, retvar, args_w) + callfunc.unwrap_spec = ['self', ObjSpace, str, W_Root, W_Root] + + def _call(self, space, name, retvar, args_w): + # determine the number of arguments passed + if args_w: + numArguments = len(args_w) + else: + numArguments = 0 + + # make sure we are actually open + self._checkOpen(space) + + # add the return value, if applicable + if retvar: + offset = 1 + if args_w: + vars_w = [retvar] + args_w + else: + vars_w = [retvar] + else: + offset = 0 + vars_w = args_w + + # build up the statement + args = ', '.join(':%d' % (i + offset + 1,) + for i in range(numArguments)) + if retvar: + stmt = "begin :1 := %s(%s); end;" % (name, args) + else: + stmt = "begin %s(%s); end;" % (name, args) + + self._execute(space, space.wrap(stmt), vars_w) + + if retvar: + return retvar.getValue(space, 0) + + def _checkOpen(self, space): + if not self.isOpen: + raise OperationError( + interp_error.get(space).w_InterfaceError, + space.wrap("not open")) + + def _freeHandle(self, space, raiseError=True): + if not self.handle: + return + if self.isOwned: + roci.OciHandleFree(self.handle, OCI_HTYPE_STMT) + elif self.connection.handle: + tagBuffer = StringBuffer() + tagBuffer.fill(space, self.w_statementTag) + try: + status = roci.OCIStmtRelease( + self.handle, self.environment.errorHandle, + tagBuffer.ptr, tagBuffer.size, + roci.OCI_DEFAULT) + self.environment.checkForError( + status, "Cursor_FreeHandle()") + finally: + tagBuffer.clear() + + def _internalPrepare(self, space, w_stmt, w_tag): + # make sure we don't get a situation where nothing is to be executed + if w_stmt is None and self.w_statement is None: + raise OperationError( + interp_error.get(space).w_ProgrammingError, + space.wrap("no statement specified " + "and no prior statement prepared")) + + # nothing to do if the statement is identical to the one already stored + # but go ahead and prepare anyway for create, alter and drop statments + if w_stmt is None or w_stmt == self.w_statement: + if self.statementType not in (roci.OCI_STMT_CREATE, + roci.OCI_STMT_DROP, + roci.OCI_STMT_ALTER): + return + w_stmt = self.w_statement + else: + self.w_statement = w_stmt + + # release existing statement, if necessary + self.w_statementTag = w_tag + self._freeHandle(space) + + # prepare statement + self.isOwned = False + handleptr = lltype.malloc(roci.Ptr(roci.OCIStmt).TO, + 1, flavor='raw') + stmtBuffer = StringBuffer() + tagBuffer = StringBuffer() + stmtBuffer.fill(space, w_stmt) + tagBuffer.fill(space, w_tag) + try: + status = roci.OCIStmtPrepare2( + self.connection.handle, handleptr, + self.environment.errorHandle, + stmtBuffer.ptr, stmtBuffer.size, + tagBuffer.ptr, tagBuffer.size, + roci.OCI_NTV_SYNTAX, roci.OCI_DEFAULT) + + self.environment.checkForError( + status, "Connection_InternalPrepare(): prepare") + self.handle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + stmtBuffer.clear() + tagBuffer.clear() + + # clear bind variables, if applicable + if not self.setInputSizes: + self.bindList = None + self.bindDict = None + + # clear row factory, if applicable + self.rowFactory = None + + # determine if statement is a query + self._getStatementType() + + def _setErrorOffset(self, space, e): + if e.match(space, get(space).w_DatabaseError): + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PARSE_ERROR_OFFSET, + self.environment.errorHandle) + e.offset = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + def _internalExecute(self, space, numIters): + if self.connection.autocommit: + mode = roci.OCI_COMMIT_ON_SUCCESS + else: + mode = roci.OCI_DEFAULT + + status = roci.OCIStmtExecute( + self.connection.handle, + self.handle, + self.environment.errorHandle, + numIters, 0, + lltype.nullptr(roci.OCISnapshot.TO), + lltype.nullptr(roci.OCISnapshot.TO), + mode) + try: + self.environment.checkForError( + status, "Cursor_InternalExecute()") + except OperationError, e: + self._setErrorOffset(space, e) + raise + finally: + self._setRowCount() + + def _getStatementType(self): + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_STMT_TYPE, + self.environment.errorHandle) + + self.environment.checkForError( + status, "Cursor_GetStatementType()") + self.statementType = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + self.fetchVariables = None + + def _setBindVariablesByPos(self, space, + vars_w, numElements, arrayPos, defer): + # make sure positional and named binds are not being intermixed + if self.bindDict is not None: + raise OperationalError( + get(space).w_ProgrammingErrorException, + space.wrap("positional and named binds cannot be intermixed")) + + self.bindList = [] + for i, w_value in enumerate(vars_w): + if i < len(self.bindList): + origVar = self.bindList[i] + else: + origVar = None + newVar = self._setBindVariableHelper(space, w_value, origVar, + numElements, arrayPos, defer) + if newVar: + if i < len(self.bindList): + self.bindList[i] = newVar + else: + assert i == len(self.bindList) + self.bindList.append(newVar) + + def _setBindVariableHelper(self, space, w_value, origVar, + numElements, arrayPos, defer): + + valueIsVariable = space.is_true(space.isinstance(w_value, get(space).w_Variable)) + + # handle case where variable is already bound + if origVar: + + # if the value is a variable object, rebind it if necessary + if valueIsVariable: + newVar = space.interp_w(interp_variable.W_Variable, w_value) + if newVar == origVar: + newVar = None + + # if the number of elements has changed, create a new variable + # this is only necessary for executemany() since execute() always + # passes a value of 1 for the number of elements + elif numElements > origVar.allocatedElements: + newVar = type(origVar)(numElements, origVar.size) + newVar.setValue(space, arrayPos, w_value) + + # otherwise, attempt to set the value + else: + newVar = None + try: + origVar.setValue(space, arrayPos, value) + except OperationError, e: + # executemany() should simply fail after the first element + if arrayPos > 0: + raise + # anything other than IndexError or TypeError should fail + if (not e.match(space, space.w_IndexError) and + not e.match(space, space.w_TypeError)): + raise + # catch the exception and try to create a new variable + origVar = None + + if not origVar: + # if the value is a variable object, bind it directly + if valueIsVariable: + newVar = space.interp_w(interp_variable.W_Variable, w_value) + newVar.boundPos = 0 + newVar.boundName = None + + # otherwise, create a new variable, unless the value is None and + # we wish to defer type assignment + elif not space.is_w(w_value, space.w_None) or not defer: + newVar = interp_variable.newVariableByValue(space, self, + w_value, + numElements) + newVar.setValue(space, arrayPos, w_value) + + return newVar + + def _setBindVariablesByName(self, space, + vars_w, numElements, arrayPos, defer): + XXX + + def _performBind(self, space): + # set values and perform binds for all bind variables + if self.bindList: + for i, var in enumerate(self.bindList): + var.bind(space, self, None, i + 1) + if self.bindDict: + for key, var in self.bindDict.itervalues(): + var.bind(space, self, key, 0) + + # ensure that input sizes are reset + self.setInputSizes = False + + def _setRowCount(self): + if self.statementType == roci.OCI_STMT_SELECT: + self.rowCount = 0 + self.actualRows = -1 + self.rowNum = 0 + elif self.statementType in (roci.OCI_STMT_INSERT, + roci.OCI_STMT_UPDATE, + roci.OCI_STMT_DELETE): + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_ROW_COUNT, + self.environment.errorHandle) + + self.environment.checkForError( + status, "Cursor_SetRowCount()") + self.rowCount = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + else: + self.rowCount = -1 + + def _performDefine(self): + # determine number of items in select-list + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PARAM_COUNT, + self.environment.errorHandle) + + self.environment.checkForError( + status, "Cursor_PerformDefine()") + numParams = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + self.fetchVariables = [] + + # define a variable for each select-item + self.fetchArraySize = self.arraySize + for i in range(numParams): + var = interp_variable.define(self, i+1, self.fetchArraySize) + self.fetchVariables.append(var) + + def _verifyFetch(self, space): + # make sure the cursor is open + self._checkOpen(space) + + # fixup bound cursor, if necessary + self._fixupBoundCursor() + + # make sure the cursor is for a query + if self.statementType != roci.OCI_STMT_SELECT: + raise OperationError( + get(space).w_InterfaceError, + space.wrap("not a query")) + + def _fixupBoundCursor(self): + if self.handle and self.statementType < 0: + self._getStatementType() + if self.statementType == roci.OCI_STMT_SELECT: + self._performDefine() + self._setRowCount() + + def fetchone(self, space): + # verify fetch can be performed + self._verifyFetch(space) + + # setup return value + if self._moreRows(space): + return self._createRow(space) + + return space.w_None + fetchone.unwrap_spec = ['self', ObjSpace] + + def fetchall(self, space): + # verify fetch can be performed + self._verifyFetch(space) + + return self._multiFetch(space, limit=None) + fetchall.unwrap_spec = ['self', ObjSpace] + + def descr_iter(self, space): + self._verifyFetch(space) + return space.wrap(self) + descr_iter.unwrap_spec = ['self', ObjSpace] + + def descr_next(self, space): + # verify fetch can be performed + self._verifyFetch(space) + + # setup return value + if self._moreRows(space): + return self._createRow(space) + + raise OperationError(space.w_StopIteration, space.w_None) + descr_next.unwrap_spec = ['self', ObjSpace] + + def _moreRows(self, space): + if self.rowNum < self.actualRows: + return True + if self.actualRows < 0 or self.actualRows == self.fetchArraySize: + self._internalFetch(space, self.fetchArraySize) + if self.rowNum < self.actualRows: + return True + + return False + + def _internalFetch(self, space, numRows): + if not self.fetchVariables: + raise OperationError( + get(space).w_InterfaceError, + space.wrap("query not executed")) + + status = roci.OCIStmtFetch( + self.handle, + self.environment.errorHandle, + numRows, + roci.OCI_FETCH_NEXT, + roci.OCI_DEFAULT) + + if status != roci.OCI_NO_DATA: + self.environment.checkForError( + status, + "Cursor_InternalFetch(): fetch") + + for var in self.fetchVariables: + var.internalFetchNum += 1 + + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_ROW_COUNT, + self.environment.errorHandle) + + self.environment.checkForError( + status, "Cursor_InternalFetch(): row count") + + self.actualRows = attrptr[0] - self.rowCount + self.rowNum = 0 + finally: + lltype.free(attrptr, flavor='raw') + + def _multiFetch(self, space, limit=None): + results_w = [] + rowNum = 0 + + # fetch as many rows as possible + while limit is None or rowNum < limit: + if not self._moreRows(space): + break + w_row = self._createRow(space) + results_w.append(w_row) + return space.newlist(results_w) + + def _createRow(self, space): + items_w = [] + numItems = len(self.fetchVariables) + + # acquire the value for each item + for var in self.fetchVariables: + w_item = var.getValue(space, self.rowNum) + items_w.append(w_item) + + # increment row counters + self.rowNum += 1 + self.rowCount += 1 + + w_row = space.newtuple(items_w) + + # if a row factory is defined, call it + if self.w_rowFactory: + w_row = space.call(self.w_rowFactory, w_row) + + return w_row + + def _get_bind_info(self, space, numElements): + # avoid bus errors on 64bit platforms + numElements = numElements + (rffi.sizeof(roci.dvoidp) - + numElements % rffi.sizeof(roci.dvoidp)) + # initialize the buffers + bindNames = lltype.malloc(roci.Ptr(roci.oratext).TO, + numElements, flavor='raw') + bindNameLengths = lltype.malloc(roci.Ptr(roci.ub1).TO, + numElements, flavor='raw') + indicatorNames = lltype.malloc(roci.Ptr(roci.oratext).TO, + numElements, flavor='raw') + indicatorNameLengths = lltype.malloc(roci.Ptr(roci.ub1).TO, + numElements, flavor='raw') + duplicate = lltype.malloc(roci.Ptr(roci.ub1).TO, + numElements, flavor='raw') + bindHandles = lltype.malloc(roci.Ptr(roci.OCIBind).TO, + numElements, flavor='raw') + + foundElementsPtr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, + flavor='raw') + + try: + status = roci.OCIStmtGetBindInfo( + self.handle, + self.environment.errorHandle, + numElements, + 1, + foundElementsPtr, + bindNames, bindNameLengths, + indicatorNames, indicatorNameLengths, + duplicate, bindHandles) + if status != roci.OCI_NO_DATA: + self.environment.checkForError( + status, "Cursor_GetBindNames()") + + # Too few elements allocated + if foundElementsPtr[0] < 0: + return -foundElementsPtr[0], None + + names_w = [] + # process the bind information returned + for i in range(foundElementsPtr[0]): + if duplicate[i]: + continue + names_w.append( + w_string(space, bindNames[i], bindNameLengths[i])) + + return 0, names_w + finally: + lltype.free(bindNames, flavor='raw') + lltype.free(bindNameLengths, flavor='raw') + lltype.free(indicatorNames, flavor='raw') + lltype.free(indicatorNameLengths, flavor='raw') + lltype.free(duplicate, flavor='raw') + lltype.free(bindHandles, flavor='raw') + lltype.free(foundElementsPtr, flavor='raw') + + def bindnames(self, space): + # make sure the cursor is open + self._checkOpen(space) + + # ensure that a statement has already been prepared + if not self.w_statement: + raise OperationError(get(space).w_ProgrammingError, + space.wrap("statement must be prepared first")) + + nbElements, names = self._get_bind_info(space, 8) + if nbElements: + _, names = self._get_bind_info(space, nbElements) + return space.newlist(names) + bindnames.unwrap_spec = ['self', ObjSpace] + + def var(self, space, w_type, size=0, w_arraysize=None, + w_inconverter=None, w_outconverter=None): + if space.is_w(w_arraysize, space.w_None): + arraySize = self.bindArraySize + else: + arraySize = space.int_w(w_arraysize) + + # determine the type of variable + varType = interp_variable.typeByPythonType(space, self, w_type) + if varType.isVariableLength and size == 0: + size = varType.size + + # create the variable + var = varType(self, arraySize, size) + var.w_inconverter = w_inconverter + var.w_outconverter = w_outconverter + + return space.wrap(var) + var.unwrap_spec = ['self', ObjSpace, W_Root, int, W_Root, W_Root, W_Root] + +def cursor_arraysize_get(space, obj): + return space.wrap(space.arraySize) +def cursor_arraysize_set(space, obj, w_value): + space.arraySize = space.int_w(w_value) + +W_Cursor.typedef = TypeDef( + 'Cursor', + execute = interp2app(W_Cursor.execute, + unwrap_spec=W_Cursor.execute.unwrap_spec), + prepare = interp2app(W_Cursor.prepare, + unwrap_spec=W_Cursor.prepare.unwrap_spec), + fetchone = interp2app(W_Cursor.fetchone, + unwrap_spec=W_Cursor.fetchone.unwrap_spec), + fetchall = interp2app(W_Cursor.fetchall, + unwrap_spec=W_Cursor.fetchall.unwrap_spec), + close = interp2app(W_Cursor.close, + unwrap_spec=W_Cursor.close.unwrap_spec), + bindnames = interp2app(W_Cursor.bindnames, + unwrap_spec=W_Cursor.bindnames.unwrap_spec), + callfunc = interp2app(W_Cursor.callfunc, + unwrap_spec=W_Cursor.callfunc.unwrap_spec), + var = interp2app(W_Cursor.var, + unwrap_spec=W_Cursor.var.unwrap_spec), + + __iter__ = interp2app(W_Cursor.descr_iter), + next = interp2app(W_Cursor.descr_next), + + arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), +) Added: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,83 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.oracle import roci, config +from pypy.interpreter.error import OperationError + +from pypy.module.oracle.interp_error import W_Error, get + +class Environment: + def __init__(self, space, threaded, events): + self.space = space + mode = roci.OCI_OBJECT + if threaded: + mode |= roci.OCI_THREADED + if events: + mode |= roci.OCI_EVENTS + + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIEnv).TO, + 1, flavor='raw') + + try: + + status = roci.OCIEnvNlsCreate( + handleptr, mode, + None, + lltype.nullptr(lltype.FuncType( # malocfp + [roci.dvoidp, roci.size_t], roci.dvoidp)), + lltype.nullptr(lltype.FuncType( # ralocfp + [roci.dvoidp, roci.dvoidp, roci.size_t], roci.dvoidp)), + lltype.nullptr(lltype.FuncType( # mfreefp + [roci.dvoidp, roci.dvoidp], lltype.Void)), + 0, lltype.nullptr(rffi.CArray(roci.dvoidp)), + config.CHARSETID, config.CHARSETID) + + if not handleptr[0] or status not in (roci.OCI_SUCCESS, + roci.OCI_SUCCESS_WITH_INFO): + raise OperationError( + w_InterfaceErrorException, + self.space.wrap( + "Unable to acquire Oracle environment handle")) + + self.handle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + + self.maxBytesPerCharacter = config.BYTES_PER_CHAR + self.maxStringBytes = config.BYTES_PER_CHAR * config.MAX_STRING_CHARS + + # create the error handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIError).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.handle, + handleptr, roci.OCI_HTYPE_ERROR, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.checkForError( + status, "Environment_New(): create error handle") + self.errorHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + + def checkForError(self, status, context): + if status in (roci.OCI_SUCCESS, roci.OCI_SUCCESS_WITH_INFO): + return + + if status != roci.OCI_INVALID_HANDLE: + # At this point it is assumed that the Oracle + # environment is fully initialized + error = W_Error(self.space, self, context, 1) + if error.code in (1, 1400, 2290, 2291, 2292): + w_type = w_IntegrityErrorException + elif error.code in (1012, 1033, 1034, 1089, 3113, 3114, + 12203, 12500, 12571): + w_type = get(self.space).w_OperationalError + else: + w_type = get(self.space).w_DatabaseError + raise OperationError(w_type, self.space.wrap(error)) + + error = W_Error(self.space, self, context, 0) + error.code = 0 + error.message = self.space.wrap("Invalid handle!") + raise OperationError(w_DatabaseErrorException, self.space.wrap(error)) Added: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_error.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,70 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.gateway import interp2app +from pypy.module.oracle import roci, config + +class State: + def __init__(self, space): + w_module = space.getbuiltinmodule('cx_Oracle') + def get(name): + return space.getattr(w_module, space.wrap(name)) + + self.w_DatabaseError = get('DatabaseError') + self.w_OperationalError = get('OperationalError') + self.w_InterfaceError = get('InterfaceError') + self.w_ProgrammingError = get('ProgrammingError') + self.w_NotSupportedError = get('NotSupportedError') + self.w_Variable = get('Variable') + self.w_NUMBER = get('NUMBER') + +def get(space): + return space.fromcache(State) + +class W_Error(Wrappable): + def __init__(self, space, environment, context, retrieveError): + self.context = context + if retrieveError: + if environment.errorHandle: + handle = environment.errorHandle + handleType = roci.OCI_HTYPE_ERROR + else: + handle = environment.handle + handleType = roci.OCI_HTYPE_ENV + + codeptr = lltype.malloc(rffi.CArray(roci.sb4), 1, flavor='raw') + BUFSIZE = 1024 + textbuf, text = rffi.alloc_buffer(BUFSIZE) + + try: + status = roci.OCIErrorGet( + handle, 1, lltype.nullptr(roci.oratext.TO), codeptr, + textbuf, BUFSIZE, handleType) + if status != roci.OCI_SUCCESS: + raise OperationError( + w_InternalErrorException, + space.wrap("No Oracle error?")) + + self.code = codeptr[0] + self.message = config.w_string( + space, + rffi.str_from_buffer(textbuf, text, + BUFSIZE, BUFSIZE)) + finally: + lltype.free(codeptr, flavor='raw') + rffi.keep_buffer_alive_until_here(textbuf, text) + + if config.WITH_UNICODE: + # XXX remove double zeros at the end + pass + + def desc_str(self): + return self.message + +W_Error.typedef = TypeDef( + 'Error', + __str__ = interp2app(W_Error.desc_str), + code = interp_attrproperty('code', W_Error), + message = interp_attrproperty('message', W_Error)) + + Added: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,697 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import ObjSpace, W_Root +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rlib.rarithmetic import ovfcheck + +import sys +from pypy.module.oracle import roci, config, transform +from pypy.module.oracle.interp_error import get +from pypy.module.oracle.config import string_w, StringBuffer + +def define(cursor, position, numElements): + paramptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, 1, flavor='raw') + try: + status = roci.OCIParamGet( + cursor.handle, roci.OCI_HTYPE_STMT, + cursor.environment.errorHandle, + paramptr, position) + cursor.environment.checkForError( + status, + "Variable_Define(): parameter") + + param = paramptr[0] + finally: + lltype.free(paramptr, flavor='raw') + + # call the helper to do the actual work + var = _defineHelper(cursor, param, position, numElements) + + roci.OCIDescriptorFree(param, roci.OCI_DTYPE_PARAM) + + return var + +def _defineHelper(cursor, param, position, numElements): + # determine data type + varType = _typeByOracleDescriptor(param, cursor.environment) + if cursor.numbersAsStrings and varType is VT_Float: + varType = VT_NumberAsString + + # retrieve size of the parameter + size = varType.size + if varType.isVariableLength: + + # determine the maximum length from Oracle + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_DATA_SIZE, + cursor.environment.errorHandle) + + cursor.environment.checkForError( + status, "Variable_Define(): data size") + sizeFromOracle = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + # use the length from Oracle directly if available + if sizeFromOracle: + size = sizeFromOracle + + # otherwise, use the value set with the setoutputsize() parameter + elif cursor.outputSize >= 0: + if (cursor.outputSizeColumn < 0 or + position == cursor.outputSizeColumn): + size = cursor.outputSize + + # create a variable of the correct type + if cursor.outputTypeHandler: + var = _newByOutputTypeHandler( + cursor, param, + cursor.outputTypeHandler, + varType, size, numElements) + elif cursor.connection.outputTypeHandler: + var = _newByOutputTypeHandler( + cursor, param, + cursor.connection.outputTypeHandler, + varType, size, numElements) + else: + var = varType(cursor, numElements, size) + + # call the procedure to set values prior to define + var.preDefine(param) + + # perform the define + handleptr = lltype.malloc(roci.Ptr(roci.OCIDefine).TO, 1, flavor='raw') + try: + status = roci.OCIDefineByPos( + cursor.handle, + handleptr, + var.environment.errorHandle, + position, + var.data, var.bufferSize, + var.oracleType, + rffi.cast(roci.dvoidp, var.indicator), + var.actualLength, + var.returnCode, + roci.OCI_DEFAULT) + var.environment.checkForError( + status, + "Variable_Define(): define") + var.defineHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # call the procedure to set values after define + var.postDefine() + + return var + + +class W_Variable(Wrappable): + charsetForm = roci.SQLCS_IMPLICIT + isVariableLength = False + + def __init__(self, cursor, numElements, size=0): + self.environment = cursor.environment + self.boundCursorHandle = lltype.nullptr(roci.OCIStmt.TO) + self.isArray = False + self.actualElementsPtr = lltype.malloc(roci.Ptr(roci.ub4).TO, 1, + zero=True, flavor='raw') + + if numElements < 1: + self.allocatedElements = 1 + else: + self.allocatedElements = numElements + self.internalFetchNum = 0 + self.actualLength = lltype.nullptr(rffi.CArrayPtr(roci.ub2).TO) + self.returnCode = lltype.nullptr(rffi.CArrayPtr(roci.ub2).TO) + + # set the maximum length of the variable, ensure that a minimum of + # 2 bytes is allocated to ensure that the array size check works + if self.isVariableLength: + size = max(size, rffi.sizeof(roci.ub2)) + if self.size != size: + self.size = size + + # allocate the data for the variable + self.allocateData() + + # allocate the indicator for the variable + self.indicator = lltype.malloc(rffi.CArrayPtr(roci.sb2).TO, + self.allocatedElements, + flavor='raw', zero=True) # XXX + + # ensure that all variable values start out NULL + for i in range(self.allocatedElements): + self.indicator[i] = rffi.cast(roci.sb2, roci.OCI_IND_NULL) + + # for variable length data, also allocate the return code + if self.isVariableLength: + self.returnCode = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, + self.allocatedElements, + flavor='raw', zero=True) # XXX + + # perform extended initialization + self.initialize(cursor) + + def __del__(self): + lltype.free(self.actualElementsPtr, flavor='raw') + if self.actualLength: + lltype.free(self.actualLength, flavor='raw') + if self.data: + lltype.free(self.data, flavor='raw') + + def getBufferSize(self): + return self.size + + def allocateData(self): + # set the buffer size for the variable + self.bufferSize = self.getBufferSize() + + # allocate the data as long as it is small enough + dataLength = ovfcheck(self.allocatedElements * self.bufferSize) + if dataLength > sys.maxint: + raise ValueError("array size too large") + + self.data = lltype.malloc(rffi.CCHARP.TO, int(dataLength), + flavor='raw', zero=True) + + def initialize(self, cursor): + pass + + def preDefine(self, param): + pass + + def postDefine(self): + pass + + def bind(self, space, cursor, name, pos): + # nothing to do if already bound + if (cursor.handle == self.boundCursorHandle and + name == self.boundName and pos == self.boundPos): + return + + # set the instance variables specific for binding + self.boundCursorHandle = cursor.handle + self.boundPos = pos + self.boundName = name + + # perform the bind + self._internalBind() + + def _internalBind(self): + bindHandlePtr = lltype.malloc(roci.Ptr(roci.OCIBind).TO, 1, + flavor='raw') + if self.isArray: + allocatedElements = self.allocatedElements + actualElementsPtr = self.actualElementsPtr + else: + allocatedElements = 0 + actualElementsPtr = lltype.nullptr(roci.Ptr(roci.ub4).TO) + + try: + if self.boundName: + nameBuffer = config.StringBuffer() + nameBuffer.fill(self.boundName) + status = roci.OCIBindByName( + self.boundCursorHandle, bindHandlePtr, + self.environment.errorHandle, + nameBuffer.ptr, nameBuffer.size, + self.data, self.bufferSize, + self.oracleType, + rffi.cast(roci.dvoidp, self.indicator), + self.actualLength, self.returnCode, + allocatedElements, actualElementsPtr, + roci.OCI_DEFAULT) + else: + status = roci.OCIBindByPos( + self.boundCursorHandle, bindHandlePtr, + self.environment.errorHandle, + self.boundPos, + self.data, self.bufferSize, + self.oracleType, + rffi.cast(roci.dvoidp, self.indicator), + self.actualLength, self.returnCode, + allocatedElements, actualElementsPtr, + roci.OCI_DEFAULT) + + self.environment.checkForError( + status, "NumberVar_InternalBind()") + self.bindHandle = bindHandlePtr[0] + finally: + lltype.free(bindHandlePtr, flavor='raw') + + def isNull(self, pos): + return self.indicator[pos] == roci.OCI_IND_NULL + + def verifyFetch(self, space, pos): + # Verifies that truncation or other problems did not take place on + # retrieve. + if self.isVariableLength: + if self.returnCode[pos] != 0: + error = W_Error(space, self.environment, + "Variable_VerifyFetch()", 0) + error.code = self.returnCode[pos] + error.message = self.space.wrap( + "column at array pos %d fetched with error: %d" % + (pos, self.returnCode[pos])) + w_error = get(self.space).w_DatabaseError + + raise OperationError(get(self.space).w_DatabaseError, + self.space.wrap(error)) + + def getSingleValue(self, space, pos): + # ensure we do not exceed the number of allocated elements + if pos >= self.allocatedElements: + raise OperationError( + space.w_PyExc_IndexError, + space.wrap("Variable_GetSingleValue: array size exceeded")) + + # check for a NULL value + if self.isNull(pos): + return space.wrap(None) + + # check for truncation or other problems on retrieve + self.verifyFetch(space, pos) + + # calculate value to return + value = self.getValueProc(space, pos) + # XXX outConverter + return value + + def getValue(self, space, pos=0): + if self.isArray: + return self.getArrayValue(self, self.actualElements) + return self.getSingleValue(space, pos) + getValue.unwrap_spec = ['self', ObjSpace, int] + + def setSingleValue(self, space, pos, w_value): + # ensure we do not exceed the number of allocated elements + if pos >= self.allocatedElements: + raise OperationError( + space.w_PyExc_IndexError, + space.wrap("Variable_SetSingleValue: array size exceeded")) + + # convert value, if necessary + # XXX inConverter + + # check for a NULL value + if space.is_w(w_value, space.w_None): + self.indicator[pos] = roci.OCI_IND_NULL + return + + self.indicator[pos] = roci.OCI_IND_NOTNULL + if self.isVariableLength: + self.returnCode[pos] = rffi.cast(roci.ub2, 0) + + self.setValueProc(space, pos, w_value) + + def setValue(self, space, pos, w_value): + if self.isArray: + self.setArrayValue(self, w_value) + self.setSingleValue(space, pos, w_value) + setValue.unwrap_spec = ['self', ObjSpace, int, W_Root] + + +W_Variable.typedef = TypeDef( + 'Variable', + getValue = interp2app(W_Variable.getValue, + unwrap_spec=W_Variable.getValue.unwrap_spec), + setValue = interp2app(W_Variable.setValue, + unwrap_spec=W_Variable.setValue.unwrap_spec), + ) + +class VT_String(W_Variable): + oracleType = roci.SQLT_CHR + size = config.MAX_STRING_CHARS + isVariableLength = True + + def getBufferSize(self): + if config.WITH_UNICODE: + return self.size * BYTES_PER_CHAR + else: + if self.charsetForm == roci.SQLCS_IMPLICIT: + return self.size * self.environment.maxBytesPerCharacter + else: + return self.size * 2 + + def initialize(self, cursor): + self.actualLength = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, + self.allocatedElements, + zero=True, flavor='raw') + + def getValueProc(self, space, pos): + offset = pos * self.bufferSize + length = self.actualLength[pos] + + l = [] + i = 0 + if config.WITH_UNICODE: + if isinstance(self, VT_Binary): + while i < length: + l.append(self.data[offset + i]) + i += 1 + return space.wrap(''.join(l)) + else: + while i < length: + l.append(unichr((ord(self.data[offset + i]) << 8) + + ord(self.data[offset + i + 1]))) + i += 2 + return space.wrap(u''.join(l)) + else: + if self.charsetForm == roci.SQLCS_IMPLICIT: + while i < length: + l.append(self.data[offset + i]) + i += 1 + return space.wrap(''.join(l)) + else: + while i < length: + l.append(unichr((ord(self.data[offset + i]) << 8) + + ord(self.data[offset + i + 1]))) + i += 2 + return space.wrap(u''.join(l)) + + def setValueProc(self, space, pos, w_value): + if config.WITH_UNICODE: + wantBytes = not self.isCharacterData + else: + wantBytes = self.charsetForm == roci.SQLCS_IMPLICIT + + if wantBytes: + if space.is_true(space.isinstance(w_value, space.w_str)): + buf = config.StringBuffer() + buf.fill(space, w_value) + size = buf.size + + try: + if buf.size > self.environment.maxStringBytes: + raise OperationError( + space.w_ValueError, + space.wrap("string data too large")) + + # ensure that the buffer is large enough + if buf.size > self.bufferSize: + self.resize(size) + + # keep a copy of the string + self.actualLength[pos] = rffi.cast(roci.ub2, buf.size) + offset = pos * self.bufferSize + for index in range(buf.size): + self.data[offset + index] = buf.ptr[index] + finally: + buf.clear() + +class VT_FixedChar(VT_String): + oracleType = roci.SQLT_AFC + size = 2000 + +class VT_NationalCharString(W_Variable): + pass + +class VT_LongString(W_Variable): + pass + +class VT_FixedNationalChar(W_Variable): + pass + +class VT_Rowid(W_Variable): + pass + +class VT_Binary(W_Variable): + pass + +class VT_LongBinary(W_Variable): + pass + +class VT_NativeFloat(W_Variable): + pass + +class VT_Float(W_Variable): + oracleType = roci.SQLT_VNU + size = rffi.sizeof(roci.OCINumber) + + def getValueProc(self, space, pos): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCINumber), self.data), + pos) + if isinstance(self, VT_Integer): + status = roci.OCINumberToInt( + self.environment.errorHandle, + dataptr, + rffi.sizeof(rffi.LONG), + roci.OCI_NUMBER_SIGNED, + intergerValuePtr) + self.environment.checkForError( + status, "NumberVar_GetValue(): as integer") + if isinstance(self, VT_Boolean): + return space.wrap(integerValuePtr[0]) + else: + return space.w_bool(integerValuePtr[0]) + elif isinstance(self, (VT_NumberAsString, VT_LongInteger)): + XXX = NumberAsString, LongInteger + else: + return transform.OracleNumberToPythonFloat( + self.environment, dataptr) + + def setValueProc(self, space, pos, w_value): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCINumber), self.data), + pos) + + if space.is_true(space.isinstance(w_value, space.w_int)): + integerValuePtr = lltype.malloc(roci.Ptr(lltype.Signed).TO, 1, + flavor='raw') + try: + integerValuePtr[0] = space.int_w(w_value) + status = roci.OCINumberFromInt( + self.environment.errorHandle, + rffi.cast(roci.dvoidp, integerValuePtr), + rffi.sizeof(lltype.Signed), + roci.OCI_NUMBER_SIGNED, + dataptr) + self.environment.checkForError( + status, "NumberVar_SetValue(): from integer") + finally: + lltype.free(integerValuePtr, flavor='raw') + return + raise TypeError("expecting numeric data") + +class VT_Integer(VT_Float): + pass + +class VT_Boolean(VT_Integer): + pass + +class VT_NumberAsString(VT_Float): + pass + +class VT_LongInteger(VT_Float): + pass + +class VT_DateTime(W_Variable): + oracleType = roci.SQLT_ODT + size = rffi.sizeof(roci.OCIDate) + + def getValueProc(self, space, pos): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDate), self.data), + pos) + dataptr = rffi.cast(roci.Ptr(roci.OCIDate), self.data) + return transform.OracleDateToPythonDateTime(self.environment, dataptr) + +class VT_Date(W_Variable): + oracleType = roci.SQLT_ODT + size = rffi.sizeof(roci.OCIDate) + + def getValueProc(self, space, pos): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDate), self.data), + pos) + return transform.OracleDateToPythonDate(self.environment, dataptr) + +class VT_Timestamp(W_Variable): + pass + +class VT_Interval(W_Variable): + pass + +class VT_CLOB(W_Variable): + pass + +class VT_NCLOB(W_Variable): + pass + +class VT_BLOB(W_Variable): + pass + +class VT_BFILE(W_Variable): + pass + +class VT_Cursor(W_Variable): + pass + +class VT_Object(W_Variable): + pass + +for cls in (VT_NationalCharString, VT_String): + cls.typedef = TypeDef( + cls.__name__, W_Variable.typedef, + ) + +def _typeByOracleDescriptor(param, environment): + # retrieve datatype of the parameter + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_DATA_TYPE, + environment.errorHandle) + environment.checkForError( + status, + "Variable_TypeByOracleDescriptor(): data type") + dataType = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + # retrieve character set form of the parameter + if dataType not in (roci.SQLT_CHR, roci.SQLT_AFC, roci.SQLT_CLOB): + charsetForm = roci.SQLCS_IMPLICIT + else: + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub1).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_CHARSET_FORM, + environment.errorHandle) + environment.checkForError( + status, + "Variable_TypeByOracleDescriptor(): charset form") + charsetForm = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + return _typeByOracleDataType(dataType, charsetForm) + +variableType = { + roci.SQLT_LNG: VT_LongString, + roci.SQLT_AFC: VT_FixedChar, + roci.SQLT_CHR: VT_String, + roci.SQLT_RDD: VT_Rowid, + roci.SQLT_BIN: VT_Binary, + roci.SQLT_LBI: VT_LongBinary, + roci.SQLT_BFLOAT: VT_NativeFloat, + roci.SQLT_IBFLOAT: VT_NativeFloat, + roci.SQLT_BDOUBLE: VT_NativeFloat, + roci.SQLT_IBDOUBLE: VT_NativeFloat, + roci.SQLT_NUM: VT_Float, + roci.SQLT_VNU: VT_Float, + roci.SQLT_DAT: VT_DateTime, + roci.SQLT_ODT: VT_DateTime, + roci.SQLT_DATE: VT_Timestamp, + roci.SQLT_TIMESTAMP: VT_Timestamp, + roci.SQLT_TIMESTAMP_TZ: VT_Timestamp, + roci.SQLT_TIMESTAMP_LTZ: VT_Timestamp, + roci.SQLT_INTERVAL_DS: VT_Interval, + roci.SQLT_CLOB: VT_CLOB, + roci.SQLT_BLOB: VT_BLOB, + roci.SQLT_BFILE: VT_BFILE, + roci.SQLT_RSET: VT_Cursor, + roci.SQLT_NTY: VT_Object, + } +variableTypeNChar = { + roci.SQLT_AFC: VT_FixedNationalChar, + roci.SQLT_CHR: VT_NationalCharString, + roci.SQLT_CLOB: VT_NCLOB, + } + +def _typeByOracleDataType(dataType, charsetForm): + if charsetForm == roci.SQLCS_NCHAR: + varType = variableTypeNChar.get(dataType) + else: + varType = variableType.get(dataType) + + if varType is None: + raise ValueError("Variable_TypeByOracleDataType: " + "unhandled data type %d" % (dataType,)) + + return varType + +def typeByPythonType(space, cursor, w_type): + """Return a variable type given a Python type object""" + moduledict = get(space) + if space.is_w(w_type, moduledict.w_NUMBER): + return VT_Float + raise OperationError( + moduledict.w_NotSupportedError, + space.wrap("Variable_TypeByPythonType(): unhandled data type")) + +def typeByValue(space, w_value, numElements): + "Return a variable type given a Python object" + moduledict = get(space) + + # handle scalars + if space.is_w(w_value, space.w_None): + return VT_String, 1, numElements + + if space.is_true(space.isinstance(w_value, space.w_str)): + size = space.int_w(space.len(w_value)) + return VT_String, size, numElements + + # XXX Unicode + + if space.is_true(space.isinstance(w_value, space.w_int)): + return VT_Integer, 0, numElements + + raise OperationError( + moduledict.w_NotSupportedError, + space.wrap("Variable_TypeByValue(): unhandled data type %s" % + (space.type(w_value).getname(space, '?'),))) + +def newVariableByValue(space, cursor, w_value, numElements): + if cursor.inputTypeHandler: + return newByInputTypeHandler( + cursor, cursor.inputTypeHandler, + w_value, numElements) + elif cursor.connection.inputTypeHandler: + return newByInputTypeHandler( + cursor, cursor.connection.inputTypeHandler, + w_value, numElements) + else: + varType, size, numElements = typeByValue(space, w_value, numElements) + var = varType(cursor, numElements, size) + if space.is_true(space.isinstance(w_value, space.w_list)): + var.makeArray() + return var + +def newVariableByType(space, cursor, w_value, numElements): + # passing an integer is assumed to be a string + if space.is_true(space.isinstance(w_value, space.w_int)): + size = space.int_w(w_value) + if size > MAX_STRING_CHARS: + varType = VT_LongString + else: + varType = VT_String + return varType(cursor, numElements, size) + + # passing an array of two elements define an array + if space.is_true(space.isinstance(w_value, space.w_list)): + XXX + + # handle directly bound variables + if space.is_true(space.isinstance(w_value, + get(space).w_Variable)): + return space.interp_w(W_Variable, w_value) + + # everything else ought to be a Python type + varType = typeByPythonType(space, cursor, w_value) + return varType(cursor, numElements) Added: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/roci.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,327 @@ +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rpython.tool import rffi_platform as platform +from pypy.rpython.lltypesystem import rffi, lltype +import py + + +ORACLE_HOME = py.path.local('c:/oraclexe/app/oracle/product/10.2.0/server') + +eci = ExternalCompilationInfo( + includes = ['oci.h'], + include_dirs = [str(ORACLE_HOME.join('OCI', 'include'))], + libraries = ['oci'], + library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))], + ) + +class CConfig: + _compilation_info_ = eci + + ub1 = platform.SimpleType('ub1', rffi.UINT) + ub2 = platform.SimpleType('ub2', rffi.UINT) + sb2 = platform.SimpleType('sb2', rffi.INT) + ub4 = platform.SimpleType('ub4', rffi.UINT) + sb4 = platform.SimpleType('sb4', rffi.INT) + sword = platform.SimpleType('sword', rffi.INT) + uword = platform.SimpleType('sword', rffi.UINT) + + OCINumber = platform.Struct('OCINumber', []) + OCITime = platform.Struct('OCITime', + [('OCITimeHH', rffi.INT), + ('OCITimeMI', rffi.INT), + ('OCITimeSS', rffi.INT), + ]) + OCIDate = platform.Struct('OCIDate', + [('OCIDateYYYY', rffi.INT), + ('OCIDateMM', rffi.INT), + ('OCIDateDD', rffi.INT), + ('OCIDateTime', OCITime), + ]) + + constants = ''' + OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS + OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA + OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE + OCI_DTYPE_PARAM + OCI_CRED_RDBMS + OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD + OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT + OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM + OCI_ATTR_PARSE_ERROR_OFFSET + OCI_NTV_SYNTAX + OCI_FETCH_NEXT + OCI_IND_NULL OCI_IND_NOTNULL + OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER + OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE + SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI + SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE + SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP + SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS + SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY + SQLCS_IMPLICIT SQLCS_NCHAR + OCI_NUMBER_SIGNED + '''.split() + + for c in constants: + locals()[c] = platform.ConstantInteger(c) + +globals().update(platform.configure(CConfig)) + +OCI_IND_NOTNULL = rffi.cast(rffi.SHORT, OCI_IND_NOTNULL) + +OCISvcCtx = rffi.VOIDP +OCIEnv = rffi.VOIDP +OCIError = rffi.VOIDP +OCIServer = rffi.VOIDP +OCISession = rffi.VOIDP +OCIStmt = rffi.VOIDP +OCIParam = rffi.VOIDP +OCIBind = rffi.VOIDP +OCIDefine = rffi.VOIDP +OCISnapshot = rffi.VOIDP + +void = lltype.Void +dvoidp = rffi.VOIDP +dvoidpp = rffi.VOIDPP +size_t = rffi.SIZE_T +oratext = rffi.CCHARP +Ptr = rffi.CArrayPtr + +def external(name, args, result): + return rffi.llexternal(name, args, result, compilation_info=eci) + +# Connect, Authorize, and Initialize Functions + +OCIEnvNlsCreate = external( + 'OCIEnvNlsCreate', + [rffi.CArrayPtr(OCIEnv), # envhpp + ub4, # mode + dvoidp, # ctxp + rffi.CCallback( # malocfp + [dvoidp, size_t], dvoidp), + rffi.CCallback( # ralocfp + [dvoidp, dvoidp, size_t], dvoidp), + rffi.CCallback( # mfreefp + [dvoidp, dvoidp], lltype.Void), + size_t, # xtramemsz + dvoidpp, # usermempp + ub2, # charset + ub2], # ncharset + sword) + +OCIServerAttach = external( + 'OCIServerAttach', + [OCIServer, # srvhp + OCIError, # errhp + oratext, # dblink + sb4, # dblink_len + ub4], # mode + sword) + +OCISessionBegin = external( + 'OCISessionBegin', + [OCISvcCtx, OCIError, OCISession, ub4, ub4], + sword) + +OCISessionEnd = external( + 'OCISessionEnd', + [OCISvcCtx, # svchp + OCIError, # errhp + OCISession, # usrhp + ub4], # mode + sword) + +# Handle and Descriptor Functions + +OCIAttrGet = external( + 'OCIAttrGet', + [dvoidp, # trgthndlp + ub4, # trghndltyp + dvoidp, # attributep + Ptr(ub4), # sizep + ub4, # attrtype + OCIError], # errhp + sword) + +OCIAttrSet = external( + 'OCIAttrSet', + [dvoidp, # trgthndlp + ub4, # trghndltyp + dvoidp, # attributep + ub4, # size + ub4, # attrtype + OCIError], # errhp + sword) + +OCIDescriptorFree = external( + 'OCIDescriptorFree', + [dvoidp, # descp + ub4], # type + sword) + +OCIHandleAlloc = external( + 'OCIHandleAlloc', + [dvoidp, # parenth + dvoidpp, # handlepp + ub4, # type + size_t, # xtramem_sz + dvoidpp], # usermempp + sword) + +OCIHandleFree = external( + 'OCIHandleFree', + [dvoidp, # handlp + ub4], # type + sword) + +OCIParamGet = external( + 'OCIParamGet', + [dvoidp, # hndlp + ub4, # htype + OCIError, # errhp + dvoidpp, # parmdpp + ub4], # pos + sword) + +# Bind, Define, and Describe Functions + +OCIBindByPos = external( + 'OCIBindByPos', + [OCIStmt, # stmtp + Ptr(OCIBind), # bindpp + OCIError, # errhp + ub4, # position + dvoidp, # valuep + sb4, # value_sz + ub2, # dty + dvoidp, # indp + Ptr(ub2), # alenp + Ptr(ub2), # rcodep + ub4, # maxarr_len + Ptr(ub4), # curelep + ub4], # mode + sword) + +OCIDefineByPos = external( + 'OCIDefineByPos', + [OCIStmt, # stmtp + Ptr(OCIDefine), # defnpp + OCIError, # errhp + ub4, # position + dvoidp, # valuep + sb4, # value_sz + ub2, # dty + dvoidp, # indp + Ptr(ub2), # rlenp + Ptr(ub2), # rcodep + ub4], # mode + sword) + +OCIStmtGetBindInfo = external( + 'OCIStmtGetBindInfo', + [OCIStmt, # stmtp + OCIError, # errhp + ub4, # size + ub4, # startloc + Ptr(sb4), # found + Ptr(oratext), # bvnp + Ptr(ub1), # bvnl + Ptr(oratext), # invp + Ptr(ub1), # inpl + Ptr(ub1), # dupl + Ptr(OCIBind)], # hndl + sword) + +# Statement Functions + +OCIStmtExecute = external( + 'OCIStmtExecute', + [OCISvcCtx, # svchp + OCIStmt, # stmtp + OCIError, # errhp + ub4, # iters + ub4, # rowoff + OCISnapshot, # snap_in + OCISnapshot, # snap_out + ub4], # mode + sword) + +OCIStmtFetch = external( + 'OCIStmtFetch', + [OCIStmt, # stmtp + OCIError, # errhp + ub4, # nrows + ub2, # orientation + ub4], # mode + sword) + +OCIStmtPrepare2 = external( + 'OCIStmtPrepare2', + [OCISvcCtx, # svchp + Ptr(OCIStmt), # stmthp + OCIError, # errhp + oratext, # stmttext + ub4, # stmt_len + oratext, # key + ub4, # keylen + ub4, # language + ub4], # mode + sword) + +OCIStmtRelease = external( + 'OCIStmtRelease', + [OCIStmt, # stmthp + OCIError, # errp + oratext, # key + ub4, # keylen + ub4], # mode + sword) + +# Transaction Functions + +OCITransCommit = external( + 'OCITransCommit', + [OCISvcCtx, # svchp + OCIError, # errhp + ub4], # mode + sword) + +OCITransRollback = external( + 'OCITransRollback', + [OCISvcCtx, # svchp + OCIError, # errhp + ub4], # mode + sword) + +# Miscellaneous Functions + +OCIErrorGet = external( + 'OCIErrorGet', + [dvoidp, # hndlp + ub4, # recordno + oratext, # sqlstate + Ptr(sb4) , # errcodep + oratext, # bufp + ub4, # bufsize + ub4], # type + sword) + +# OCI Number Functions + +OCINumberFromInt = external( + 'OCINumberFromInt', + [OCIError, # err + dvoidp, # inum + uword, # inum_length + uword, # inum_s_flag + Ptr(OCINumber)], # number + sword) + +OCINumberToReal = external( + 'OCINumberToReal', + [OCIError, # err + Ptr(OCINumber), # number + uword, # rsl_length + dvoidp], # rsl + sword) + Added: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,80 @@ +from pypy.conftest import gettestobjspace +import py + +from pypy.rpython.tool.rffi_platform import CompilationError +try: + from pypy.module.oracle import roci +except (CompilationError, ImportError): + py.test.skip("Oracle client not available") + +class AppTestConnection: + + def setup_class(cls): + space = gettestobjspace(usemodules=('oracle',)) + cls.space = space + space.setitem(space.builtin.w_dict, space.wrap('oracle'), + space.getbuiltinmodule('cx_Oracle')) + cls.w_username = space.wrap('cx_oracle') + cls.w_password = space.wrap('dev') + cls.w_tnsentry = space.wrap('xe') + + def test_connect(self): + cnx = oracle.connect(self.username, self.password, + self.tnsentry, threaded=True) + assert cnx.username == self.username + assert cnx.password == self.password + assert cnx.tnsentry == self.tnsentry + assert isinstance(cnx.version, str) + + def test_singleArg(self): + cnx = oracle.connect("%s/%s@%s" % (self.username, self.password, + self.tnsentry)) + assert cnx.username == self.username + assert cnx.password == self.password + assert cnx.tnsentry == self.tnsentry + + def test_connect_badPassword(self): + raises(oracle.DatabaseError, oracle.connect, + self.username, self.password + 'X', self.tnsentry) + + def test_connect_badConnectString(self): + raises(oracle.DatabaseError, oracle.connect, + self.username) + raises(oracle.DatabaseError, oracle.connect, + self.username + "@" + self.tnsentry) + raises(oracle.DatabaseError, oracle.connect, + self.username + "@" + self.tnsentry + "/" + self.password) + + def test_exceptionOnClose(self): + connection = oracle.connect(self.username, self.password, + self.tnsentry) + connection.close() + raises(oracle.InterfaceError, connection.rollback) + + def test_makedsn(self): + formatString = ("(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)" + "(HOST=%s)(PORT=%d)))(CONNECT_DATA=(SID=%s)))") + args = ("hostname", 1521, "TEST") + result = oracle.makedsn(*args) + assert result == formatString % args + + def test_rollbackOnClose(self): + connection = oracle.connect(self.username, self.password, + self.tnsentry) + cursor = connection.cursor() + try: + cursor.execute("drop table pypy_test_temp") + except oracle.DatabaseError: + pass + cursor.execute("create table pypy_test_temp (n number)") + + otherConnection = oracle.connect(self.username, self.password, + self.tnsentry) + otherCursor = otherConnection.cursor() + otherCursor.execute("insert into pypy_test_temp (n) values (1)") + otherConnection.close() + cursor.execute("select count(*) from pypy_test_temp") + count, = cursor.fetchone() + assert count == 0 + + Added: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,77 @@ +from pypy.conftest import gettestobjspace +import py + +from pypy.rpython.tool.rffi_platform import CompilationError +try: + from pypy.module.oracle import roci +except (CompilationError, ImportError): + py.test.skip("Oracle client not available") + +class AppTestCursor: + + def setup_class(cls): + space = gettestobjspace(usemodules=('oracle',)) + cls.space = space + space.setitem(space.builtin.w_dict, space.wrap('oracle'), + space.getbuiltinmodule('cx_Oracle')) + cls.w_username = space.wrap('cx_oracle') + cls.w_password = space.wrap('dev') + cls.w_tnsentry = space.wrap('') + cls.w_cnx = space.appexec( + [cls.w_username, cls.w_password, cls.w_tnsentry], + """(username, password, tnsentry): + import cx_Oracle + return cx_Oracle.connect(username, password, tnsentry) + """) + + def test_bindNames(self): + cur = self.cnx.cursor() + raises(oracle.ProgrammingError, cur.bindnames) + cur.prepare("begin null; end;") + assert cur.bindnames() == [] + cur.prepare("begin :retval := :inval + 5; end;") + assert cur.bindnames() == ["RETVAL", "INVAL"] + cur.prepare("begin :retval := :a * :a + :b * :b; end;") + assert cur.bindnames() == ["RETVAL", "A", "B"] + cur.prepare("begin :a := :b + :c + :d + :e + :f + :g + " + ":h + :i + :j + :k + :l; end;") + assert cur.bindnames() == ["A", "B", "C", "D", "E", "F", + "G", "H", "I", "J", "K", "L"] + + def test_bind_out(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + var.setValue(0, 5) + assert var.getValue(0) == 5 + assert var.getValue() == 5 + cur.execute("begin :1 := 3; end;", (var,)) + value = var.getValue(0) + assert value == 3 + assert isinstance(value, float) + + def test_callFunc0(self): + cur = self.cnx.cursor() + try: + cur.execute("drop function pypy_temp_function") + except oracle.DatabaseError: + pass + cur.execute("create function pypy_temp_function " + "return number as " + "begin return 42; end;") + assert cur.callfunc("pypy_temp_function", + oracle.NUMBER) == 42 + + def test_callFunc1(self): + cur = self.cnx.cursor() + try: + cur.execute("drop function pypy_temp_function") + except oracle.DatabaseError: + pass + cur.execute("create function pypy_temp_function " + "(x varchar2, y number) return number as " + "begin return length(x) + y; end;") + res = cur.callfunc("pypy_temp_function", + oracle.NUMBER, ("Hi", 5)) + assert res == 7 + + Added: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,58 @@ +from pypy.conftest import gettestobjspace +import py + +from pypy.rpython.tool.rffi_platform import CompilationError +try: + from pypy.module.oracle import roci +except (CompilationError, ImportError): + py.test.skip("Oracle client not available") + +class AppTestSelect: + + def setup_class(cls): + space = gettestobjspace(usemodules=('oracle',)) + cls.space = space + cls.w_username = space.wrap('cx_oracle') + cls.w_password = space.wrap('dev') + cls.w_tnsentry = space.wrap('') + cls.w_cnx = space.appexec( + [cls.w_username, cls.w_password, cls.w_tnsentry], + """(username, password, tnsentry): + import cx_Oracle + return cx_Oracle.connect(username, password, tnsentry) + """) + + def test_fetchone(self): + cur = self.cnx.cursor() + cur.execute("select 42, 'Hello' from dual") + row = cur.fetchone() + assert isinstance(row[0], float) + assert isinstance(row[1], str) + assert row == (42, 'Hello') + + def test_sysdate(self): + import datetime + cur = self.cnx.cursor() + cur.execute("select sysdate from dual") + row = cur.fetchone() + sysdate = row[0] + assert isinstance(sysdate, datetime.datetime) + delta = abs(sysdate - datetime.datetime.now()) + assert delta < datetime.timedelta(seconds=2) + + def test_fetchall(self): + cur = self.cnx.cursor() + # An Oracle trick to retrieve 42 lines + cur.execute("select level-1 from dual connect by level-1<42") + rows = cur.fetchall() + assert rows == zip(range(42)) + + def test_iterator(self): + cur = self.cnx.cursor() + # An Oracle trick to retrieve 42 lines + cur.execute("select level-1 from dual connect by level-1<42") + for i, row in enumerate(cur): + assert row == (i,) + assert i == 41 + + Added: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/transform.py Sat Nov 14 16:47:04 2009 @@ -0,0 +1,60 @@ +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.oracle import roci, config + +def OracleNumberToPythonFloat(environment, valueptr): + "Return a Python float object given an oracle number" + doubleptr = lltype.malloc(roci.Ptr(rffi.DOUBLE).TO, 1, flavor='raw') + try: + status = roci.OCINumberToReal( + environment.errorHandle, + valueptr, + rffi.sizeof(rffi.DOUBLE), + rffi.cast(roci.dvoidp, doubleptr)) + environment.checkForError(status, "OracleNumberToPythonFloat()") + return environment.space.wrap(doubleptr[0]) + finally: + lltype.free(doubleptr, flavor='raw') + +def OracleDateToPythonDate(environment, valueptr): + print valueptr.OCIDateYYYY, valueptr.OCIDateMM, valueptr.OCIDateDD + yearptr = lltype.malloc(roci.Ptr(roci.sb2).TO, 1, flavor='raw') + monthptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + dayptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + + try: + roci.OCIDateGetDate( + valueptr, + yearptr, + monthptr, + dayptr) + + space = environment.space + w = space.wrap + + return space.call_w(w_date, [w(yearptr[0]), w(monthptr[0]), w(dayptr[0])]) + finally: + lltype.free(yearptr, flavor='raw') + lltype.free(monthptr, flavor='raw') + lltype.free(dayptr, flavor='raw') + +def OracleDateToPythonDateTime(environment, valueptr): + space = environment.space + w = space.wrap + + # XXX check that this does not copy the whole structure + date = valueptr[0] + time = date.c_OCIDateTime + + w_datetime = space.getattr( + space.getbuiltinmodule('datetime'), + w('datetime')) + + return space.call_function( + w_datetime, + w(date.c_OCIDateYYYY), + w(date.c_OCIDateMM), + w(date.c_OCIDateDD), + w(time.c_OCITimeHH), + w(time.c_OCITimeMI), + w(time.c_OCITimeSS)) + From cfbolz at codespeak.net Sat Nov 14 18:29:53 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 14 Nov 2009 18:29:53 +0100 (CET) Subject: [pypy-svn] r69291 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091114172953.BDDDE168027@codespeak.net> Author: cfbolz Date: Sat Nov 14 18:29:52 2009 New Revision: 69291 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: forgot a guard Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 14 18:29:52 2009 @@ -272,11 +272,12 @@ ([20], 20), ([31], 32)) - load = self.get_by_bytecode("LOAD_ATTR") + load, = self.get_by_bytecode("LOAD_ATTR") # 1 guard_value for the class # 1 guard_value for the version_tag # 1 guard_value for the structure - assert len(load.get_opnames("guard")) <= 3 + # 1 guard_nonnull_class for the result since it is used later + assert len(load.get_opnames("guard")) <= 4 bytecode, = self.get_by_bytecode("STORE_ATTR") assert bytecode.get_opnames() == [] From cfbolz at codespeak.net Sat Nov 14 18:35:14 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 14 Nov 2009 18:35:14 +0100 (CET) Subject: [pypy-svn] r69292 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091114173514.1DB1B168027@codespeak.net> Author: cfbolz Date: Sat Nov 14 18:35:13 2009 New Revision: 69292 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: grmpf, copy-paste mistake Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 14 18:35:13 2009 @@ -279,9 +279,6 @@ # 1 guard_nonnull_class for the result since it is used later assert len(load.get_opnames("guard")) <= 4 - bytecode, = self.get_by_bytecode("STORE_ATTR") - assert bytecode.get_opnames() == [] - def test_mixed_type_loop(self): self.run_source(''' class A(object): From benjamin at codespeak.net Sat Nov 14 18:46:18 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 14 Nov 2009 18:46:18 +0100 (CET) Subject: [pypy-svn] r69293 - pypy/trunk/pypy/interpreter/astcompiler Message-ID: <20091114174618.62611168027@codespeak.net> Author: benjamin Date: Sat Nov 14 18:46:17 2009 New Revision: 69293 Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py Log: remove extra quote Modified: pypy/trunk/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/assemble.py Sat Nov 14 18:46:17 2009 @@ -227,7 +227,7 @@ self.lineno_set = False def _resolve_block_targets(self, blocks): - """"Compute the arguments of jump instructions.""" + """Compute the arguments of jump instructions.""" last_extended_arg_count = 0 # The reason for this loop is extended jumps. EXTENDED_ARG extends the # bytecode size, so it might invalidate the offsets we've already given. From fijal at codespeak.net Sun Nov 15 10:15:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 10:15:28 +0100 (CET) Subject: [pypy-svn] r69294 - pypy/trunk/pypy/module/oracle Message-ID: <20091115091528.8A5E916802F@codespeak.net> Author: fijal Date: Sun Nov 15 10:15:26 2009 New Revision: 69294 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py Log: no tabs please Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Sun Nov 15 10:15:26 2009 @@ -66,7 +66,7 @@ space.wrap('@'), space.wrap(1))) self.connect(space, mode, twophase) - return space.wrap(self) + return space.wrap(self) descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, From fijal at codespeak.net Sun Nov 15 10:19:09 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 10:19:09 +0100 (CET) Subject: [pypy-svn] r69295 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20091115091909.D6F6316802F@codespeak.net> Author: fijal Date: Sun Nov 15 10:19:09 2009 New Revision: 69295 Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Log: Fix this test Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Sun Nov 15 10:19:09 2009 @@ -149,7 +149,8 @@ pass metainterp_sd = FakeMetaInterpSd() metainterp_sd.info_from_codewriter(None, None, None, - [(123, "a"), (456, "b")]) + [(123, "a"), (456, "b")], + None) assert metainterp_sd.get_name_from_address(123) == 'a' assert metainterp_sd.get_name_from_address(456) == 'b' assert metainterp_sd.get_name_from_address(789) == '' From fijal at codespeak.net Sun Nov 15 10:20:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 10:20:55 +0100 (CET) Subject: [pypy-svn] r69296 - pypy/trunk/pypy/doc/config Message-ID: <20091115092055.6F1EE16802F@codespeak.net> Author: fijal Date: Sun Nov 15 10:20:54 2009 New Revision: 69296 Added: pypy/trunk/pypy/doc/config/objspace.usemodules.oracle.txt (contents, props changed) Log: Document oracle module Added: pypy/trunk/pypy/doc/config/objspace.usemodules.oracle.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/objspace.usemodules.oracle.txt Sun Nov 15 10:20:54 2009 @@ -0,0 +1,2 @@ +Use the 'oracle' module. +This module is off by default, requires oracle client installed. From fijal at codespeak.net Sun Nov 15 10:47:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 10:47:12 +0100 (CET) Subject: [pypy-svn] r69297 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091115094712.8373F16802C@codespeak.net> Author: fijal Date: Sun Nov 15 10:47:11 2009 New Revision: 69297 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Bah. People do pass random crap as arguments to IOError Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sun Nov 15 10:47:11 2009 @@ -335,12 +335,13 @@ if (not space.is_w(self.w_errno, space.w_None) and not space.is_w(self.w_strerror, space.w_None)): if not space.is_w(self.w_filename, space.w_None): - return space.wrap("[Errno %d] %s: %s" % ( - space.int_w(self.w_errno), + return space.wrap("[Errno %s] %s: %s" % ( + space.str_w(space.str(self.w_errno)), space.str_w(self.w_strerror), space.str_w(space.repr(self.w_filename)))) - return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), - space.str_w(self.w_strerror))) + return space.wrap("[Errno %s] %s" % + (space.str_w(space.str(self.w_errno)), + space.str_w(self.w_strerror))) return W_BaseException.descr_str(self, space) descr_str.unwrap_spec = ['self', ObjSpace] Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sun Nov 15 10:47:11 2009 @@ -56,6 +56,7 @@ assert isinstance(Exception(), Exception) assert isinstance(Exception(), BaseException) assert repr(Exception(3, "x")) == "Exception(3, 'x')" + assert str(IOError("foo", "bar")) == "[Errno foo] bar" def test_custom_class(self): from exceptions import Exception, BaseException, LookupError From fijal at codespeak.net Sun Nov 15 11:33:41 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 11:33:41 +0100 (CET) Subject: [pypy-svn] r69298 - pypy/branch/faster-raise/lib-python/modified-2.5.2/test Message-ID: <20091115103341.7E0C016802C@codespeak.net> Author: fijal Date: Sun Nov 15 11:33:39 2009 New Revision: 69298 Added: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py (contents, props changed) Log: Modify a test, so errors while parsing arguments can be decoding errors as well Added: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py ============================================================================== --- (empty file) +++ pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py Sun Nov 15 11:33:39 2009 @@ -0,0 +1,801 @@ +import test.test_support, unittest +import sys, codecs, htmlentitydefs, unicodedata + +class PosReturn: + # this can be used for configurable callbacks + + def __init__(self): + self.pos = 0 + + def handle(self, exc): + oldpos = self.pos + realpos = oldpos + if realpos<0: + realpos = len(exc.object) + realpos + # if we don't advance this time, terminate on the next call + # otherwise we'd get an endless loop + if realpos <= exc.start: + self.pos = len(exc.object) + return (u"", oldpos) + +# A UnicodeEncodeError object with a bad start attribute +class BadStartUnicodeEncodeError(UnicodeEncodeError): + def __init__(self): + UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad") + self.start = [] + +# A UnicodeEncodeError object with a bad object attribute +class BadObjectUnicodeEncodeError(UnicodeEncodeError): + def __init__(self): + UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad") + self.object = [] + +# A UnicodeDecodeError object without an end attribute +class NoEndUnicodeDecodeError(UnicodeDecodeError): + def __init__(self): + UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad") + del self.end + +# A UnicodeDecodeError object with a bad object attribute +class BadObjectUnicodeDecodeError(UnicodeDecodeError): + def __init__(self): + UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad") + self.object = [] + +# A UnicodeTranslateError object without a start attribute +class NoStartUnicodeTranslateError(UnicodeTranslateError): + def __init__(self): + UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + del self.start + +# A UnicodeTranslateError object without an end attribute +class NoEndUnicodeTranslateError(UnicodeTranslateError): + def __init__(self): + UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + del self.end + +# A UnicodeTranslateError object without an object attribute +class NoObjectUnicodeTranslateError(UnicodeTranslateError): + def __init__(self): + UnicodeTranslateError.__init__(self, u"", 0, 1, "bad") + del self.object + +class CodecCallbackTest(unittest.TestCase): + + def test_xmlcharrefreplace(self): + # replace unencodable characters which numeric character entities. + # For ascii, latin-1 and charmaps this is completely implemented + # in C and should be reasonably fast. + s = u"\u30b9\u30d1\u30e2 \xe4nd eggs" + self.assertEqual( + s.encode("ascii", "xmlcharrefreplace"), + "スパモ änd eggs" + ) + self.assertEqual( + s.encode("latin-1", "xmlcharrefreplace"), + "スパモ \xe4nd eggs" + ) + + def test_xmlcharnamereplace(self): + # This time use a named character entity for unencodable + # characters, if one is available. + + def xmlcharnamereplace(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [] + for c in exc.object[exc.start:exc.end]: + try: + l.append(u"&%s;" % htmlentitydefs.codepoint2name[ord(c)]) + except KeyError: + l.append(u"&#%d;" % ord(c)) + return (u"".join(l), exc.end) + + codecs.register_error( + "test.xmlcharnamereplace", xmlcharnamereplace) + + sin = u"\xab\u211c\xbb = \u2329\u1234\u20ac\u232a" + sout = "«ℜ» = ⟨ሴ€⟩" + self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout) + sout = "\xabℜ\xbb = ⟨ሴ€⟩" + self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout) + sout = "\xabℜ\xbb = ⟨ሴ\xa4⟩" + self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout) + + def test_uninamereplace(self): + # We're using the names from the unicode database this time, + # and we're doing "syntax highlighting" here, i.e. we include + # the replaced text in ANSI escape sequences. For this it is + # useful that the error handler is not called for every single + # unencodable character, but for a complete sequence of + # unencodable characters, otherwise we would output many + # unneccessary escape sequences. + + def uninamereplace(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [] + for c in exc.object[exc.start:exc.end]: + l.append(unicodedata.name(c, u"0x%x" % ord(c))) + return (u"\033[1m%s\033[0m" % u", ".join(l), exc.end) + + codecs.register_error( + "test.uninamereplace", uninamereplace) + + sin = u"\xac\u1234\u20ac\u8000" + sout = "\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m" + self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout) + + sout = "\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m" + self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout) + + sout = "\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m" + self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout) + + def test_backslashescape(self): + # Does the same as the "unicode-escape" encoding, but with different + # base encodings. + sin = u"a\xac\u1234\u20ac\u8000" + if sys.maxunicode > 0xffff: + sin += unichr(sys.maxunicode) + sout = "a\\xac\\u1234\\u20ac\\u8000" + if sys.maxunicode > 0xffff: + sout += "\\U%08x" % sys.maxunicode + self.assertEqual(sin.encode("ascii", "backslashreplace"), sout) + + sout = "a\xac\\u1234\\u20ac\\u8000" + if sys.maxunicode > 0xffff: + sout += "\\U%08x" % sys.maxunicode + self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout) + + sout = "a\xac\\u1234\xa4\\u8000" + if sys.maxunicode > 0xffff: + sout += "\\U%08x" % sys.maxunicode + self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) + + def test_decoderelaxedutf8(self): + # This is the test for a decoding callback handler, + # that relaxes the UTF-8 minimal encoding restriction. + # A null byte that is encoded as "\xc0\x80" will be + # decoded as a null byte. All other illegal sequences + # will be handled strictly. + def relaxedutf8(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + if exc.object[exc.start:exc.end].startswith("\xc0\x80"): + return (u"\x00", exc.start+2) # retry after two bytes + else: + raise exc + + codecs.register_error( + "test.relaxedutf8", relaxedutf8) + + sin = "a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80" + sout = u"a\x00b\x00c\xfc\x00\x00" + self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout) + sin = "\xc0\x80\xc0\x81" + self.assertRaises(UnicodeError, sin.decode, "utf-8", "test.relaxedutf8") + + def test_charmapencode(self): + # For charmap encodings the replacement string will be + # mapped through the encoding again. This means, that + # to be able to use e.g. the "replace" handler, the + # charmap has to have a mapping for "?". + charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"]) + sin = u"abc" + sout = "AABBCC" + self.assertEquals(codecs.charmap_encode(sin, "strict", charmap)[0], sout) + + sin = u"abcA" + self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap) + + charmap[ord("?")] = "XYZ" + sin = u"abcDEF" + sout = "AABBCCXYZXYZXYZ" + self.assertEquals(codecs.charmap_encode(sin, "replace", charmap)[0], sout) + + charmap[ord("?")] = u"XYZ" + self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) + + charmap[ord("?")] = u"XYZ" + self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) + + def test_decodeunicodeinternal(self): + self.assertRaises( + UnicodeDecodeError, + "\x00\x00\x00\x00\x00".decode, + "unicode-internal", + ) + if sys.maxunicode > 0xffff: + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + return (u"\x01", 1) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), + u"\u0000" + ) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"), + u"\u0000\ufffd" + ) + + codecs.register_error("test.hui", handler_unicodeinternal) + + self.assertEqual( + "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"), + u"\u0000\u0001\u0000" + ) + + def test_callbacks(self): + def handler1(exc): + if not isinstance(exc, UnicodeEncodeError) \ + and not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)] + return (u"[%s]" % u"".join(l), exc.end) + + codecs.register_error("test.handler1", handler1) + + def handler2(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)] + return (u"[%s]" % u"".join(l), exc.end+1) # skip one character + + codecs.register_error("test.handler2", handler2) + + s = "\x00\x81\x7f\x80\xff" + + self.assertEqual( + s.decode("ascii", "test.handler1"), + u"\x00[<129>]\x7f[<128>][<255>]" + ) + self.assertEqual( + s.decode("ascii", "test.handler2"), + u"\x00[<129>][<128>]" + ) + + self.assertEqual( + "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"), + u"\u3042[<92><117><51><120>]xx" + ) + + self.assertEqual( + "\\u3042\u3xx".decode("unicode-escape", "test.handler1"), + u"\u3042[<92><117><51><120><120>]" + ) + + self.assertEqual( + codecs.charmap_decode("abc", "test.handler1", {ord("a"): u"z"})[0], + u"z[<98>][<99>]" + ) + + self.assertEqual( + u"g\xfc\xdfrk".encode("ascii", "test.handler1"), + u"g[<252><223>]rk" + ) + + self.assertEqual( + u"g\xfc\xdf".encode("ascii", "test.handler1"), + u"g[<252><223>]" + ) + + def test_longstrings(self): + # test long strings to check for memory overflow problems + errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", "backslashreplace"] + # register the handlers under different names, + # to prevent the codec from recognizing the name + for err in errors: + codecs.register_error("test." + err, codecs.lookup_error(err)) + l = 1000 + errors += [ "test." + err for err in errors ] + for uni in [ s*l for s in (u"x", u"\u3042", u"a\xe4") ]: + for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15", "utf-8", "utf-7", "utf-16"): + for err in errors: + try: + uni.encode(enc, err) + except UnicodeError: + pass + + def check_exceptionobjectargs(self, exctype, args, msg): + # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion + # check with one missing argument + self.assertRaises(TypeError, exctype, *args[:-1]) + # check with one argument too much + self.assertRaises(TypeError, exctype, *(args + ["too much"])) + # check with one argument of the wrong type + wrongargs = [ "spam", u"eggs", 42, 1.0, None ] + for i in xrange(len(args)): + for wrongarg in wrongargs: + if type(wrongarg) is type(args[i]): + continue + # build argument array + callargs = [] + for j in xrange(len(args)): + if i==j: + callargs.append(wrongarg) + else: + callargs.append(args[i]) + errors = (UnicodeDecodeError, UnicodeEncodeError, TypeError) + self.assertRaises(errors, exctype, *callargs) + + # check with the correct number and type of arguments + exc = exctype(*args) + self.assertEquals(str(exc), msg) + + def test_unicodeencodeerror(self): + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"g\xfcrk", 1, 2, "ouch"], + "'ascii' codec can't encode character u'\\xfc' in position 1: ouch" + ) + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"g\xfcrk", 1, 4, "ouch"], + "'ascii' codec can't encode characters in position 1-3: ouch" + ) + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"\xfcx", 0, 1, "ouch"], + "'ascii' codec can't encode character u'\\xfc' in position 0: ouch" + ) + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"\u0100x", 0, 1, "ouch"], + "'ascii' codec can't encode character u'\\u0100' in position 0: ouch" + ) + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"\uffffx", 0, 1, "ouch"], + "'ascii' codec can't encode character u'\\uffff' in position 0: ouch" + ) + if sys.maxunicode > 0xffff: + self.check_exceptionobjectargs( + UnicodeEncodeError, + ["ascii", u"\U00010000x", 0, 1, "ouch"], + "'ascii' codec can't encode character u'\\U00010000' in position 0: ouch" + ) + + def test_unicodedecodeerror(self): + self.check_exceptionobjectargs( + UnicodeDecodeError, + ["ascii", "g\xfcrk", 1, 2, "ouch"], + "'ascii' codec can't decode byte 0xfc in position 1: ouch" + ) + self.check_exceptionobjectargs( + UnicodeDecodeError, + ["ascii", "g\xfcrk", 1, 3, "ouch"], + "'ascii' codec can't decode bytes in position 1-2: ouch" + ) + + def test_unicodetranslateerror(self): + self.check_exceptionobjectargs( + UnicodeTranslateError, + [u"g\xfcrk", 1, 2, "ouch"], + "can't translate character u'\\xfc' in position 1: ouch" + ) + self.check_exceptionobjectargs( + UnicodeTranslateError, + [u"g\u0100rk", 1, 2, "ouch"], + "can't translate character u'\\u0100' in position 1: ouch" + ) + self.check_exceptionobjectargs( + UnicodeTranslateError, + [u"g\uffffrk", 1, 2, "ouch"], + "can't translate character u'\\uffff' in position 1: ouch" + ) + if sys.maxunicode > 0xffff: + self.check_exceptionobjectargs( + UnicodeTranslateError, + [u"g\U00010000rk", 1, 2, "ouch"], + "can't translate character u'\\U00010000' in position 1: ouch" + ) + self.check_exceptionobjectargs( + UnicodeTranslateError, + [u"g\xfcrk", 1, 3, "ouch"], + "can't translate characters in position 1-2: ouch" + ) + + def test_badandgoodstrictexceptions(self): + # "strict" complains about a non-exception passed in + self.assertRaises( + TypeError, + codecs.strict_errors, + 42 + ) + # "strict" complains about the wrong exception type + self.assertRaises( + Exception, + codecs.strict_errors, + Exception("ouch") + ) + + # If the correct exception is passed in, "strict" raises it + self.assertRaises( + UnicodeEncodeError, + codecs.strict_errors, + UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch") + ) + + def test_badandgoodignoreexceptions(self): + # "ignore" complains about a non-exception passed in + self.assertRaises( + TypeError, + codecs.ignore_errors, + 42 + ) + # "ignore" complains about the wrong exception type + self.assertRaises( + TypeError, + codecs.ignore_errors, + UnicodeError("ouch") + ) + # If the correct exception is passed in, "ignore" returns an empty replacement + self.assertEquals( + codecs.ignore_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")), + (u"", 1) + ) + self.assertEquals( + codecs.ignore_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")), + (u"", 1) + ) + self.assertEquals( + codecs.ignore_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")), + (u"", 1) + ) + + def test_badandgoodreplaceexceptions(self): + # "replace" complains about a non-exception passed in + self.assertRaises( + TypeError, + codecs.replace_errors, + 42 + ) + # "replace" complains about the wrong exception type + self.assertRaises( + TypeError, + codecs.replace_errors, + UnicodeError("ouch") + ) + self.assertRaises( + TypeError, + codecs.replace_errors, + BadObjectUnicodeEncodeError() + ) + self.assertRaises( + TypeError, + codecs.replace_errors, + BadObjectUnicodeDecodeError() + ) + # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement + self.assertEquals( + codecs.replace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")), + (u"?", 1) + ) + self.assertEquals( + codecs.replace_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")), + (u"\ufffd", 1) + ) + self.assertEquals( + codecs.replace_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")), + (u"\ufffd", 1) + ) + + def test_badandgoodxmlcharrefreplaceexceptions(self): + # "xmlcharrefreplace" complains about a non-exception passed in + self.assertRaises( + TypeError, + codecs.xmlcharrefreplace_errors, + 42 + ) + # "xmlcharrefreplace" complains about the wrong exception types + self.assertRaises( + TypeError, + codecs.xmlcharrefreplace_errors, + UnicodeError("ouch") + ) + # "xmlcharrefreplace" can only be used for encoding + self.assertRaises( + TypeError, + codecs.xmlcharrefreplace_errors, + UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch") + ) + self.assertRaises( + TypeError, + codecs.xmlcharrefreplace_errors, + UnicodeTranslateError(u"\u3042", 0, 1, "ouch") + ) + # Use the correct exception + cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042) + s = "".join(unichr(c) for c in cs) + self.assertEquals( + codecs.xmlcharrefreplace_errors( + UnicodeEncodeError("ascii", s, 0, len(s), "ouch") + ), + (u"".join(u"&#%d;" % ord(c) for c in s), len(s)) + ) + + def test_badandgoodbackslashreplaceexceptions(self): + # "backslashreplace" complains about a non-exception passed in + self.assertRaises( + TypeError, + codecs.backslashreplace_errors, + 42 + ) + # "backslashreplace" complains about the wrong exception types + self.assertRaises( + TypeError, + codecs.backslashreplace_errors, + UnicodeError("ouch") + ) + # "backslashreplace" can only be used for encoding + self.assertRaises( + TypeError, + codecs.backslashreplace_errors, + UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch") + ) + self.assertRaises( + TypeError, + codecs.backslashreplace_errors, + UnicodeTranslateError(u"\u3042", 0, 1, "ouch") + ) + # Use the correct exception + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")), + (u"\\u3042", 1) + ) + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\x00", 0, 1, "ouch")), + (u"\\x00", 1) + ) + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\xff", 0, 1, "ouch")), + (u"\\xff", 1) + ) + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u0100", 0, 1, "ouch")), + (u"\\u0100", 1) + ) + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\uffff", 0, 1, "ouch")), + (u"\\uffff", 1) + ) + if sys.maxunicode>0xffff: + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U00010000", 0, 1, "ouch")), + (u"\\U00010000", 1) + ) + self.assertEquals( + codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U0010ffff", 0, 1, "ouch")), + (u"\\U0010ffff", 1) + ) + + def test_badhandlerresults(self): + results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) + encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15") + + for res in results: + codecs.register_error("test.badhandler", lambda: res) + for enc in encs: + self.assertRaises( + TypeError, + u"\u3042".encode, + enc, + "test.badhandler" + ) + for (enc, bytes) in ( + ("ascii", "\xff"), + ("utf-8", "\xff"), + ("utf-7", "+x-"), + ("unicode-internal", "\x00"), + ): + self.assertRaises( + TypeError, + bytes.decode, + enc, + "test.badhandler" + ) + + def test_lookup(self): + self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict")) + self.assertEquals(codecs.ignore_errors, codecs.lookup_error("ignore")) + self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict")) + self.assertEquals( + codecs.xmlcharrefreplace_errors, + codecs.lookup_error("xmlcharrefreplace") + ) + self.assertEquals( + codecs.backslashreplace_errors, + codecs.lookup_error("backslashreplace") + ) + + def test_unencodablereplacement(self): + def unencrepl(exc): + if isinstance(exc, UnicodeEncodeError): + return (u"\u4242", exc.end) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error("test.unencreplhandler", unencrepl) + for enc in ("ascii", "iso-8859-1", "iso-8859-15"): + self.assertRaises( + UnicodeEncodeError, + u"\u4242".encode, + enc, + "test.unencreplhandler" + ) + + def test_badregistercall(self): + # enhance coverage of: + # Modules/_codecsmodule.c::register_error() + # Python/codecs.c::PyCodec_RegisterError() + self.assertRaises(TypeError, codecs.register_error, 42) + self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42) + + def test_badlookupcall(self): + # enhance coverage of: + # Modules/_codecsmodule.c::lookup_error() + self.assertRaises(TypeError, codecs.lookup_error) + + def test_unknownhandler(self): + # enhance coverage of: + # Modules/_codecsmodule.c::lookup_error() + self.assertRaises(LookupError, codecs.lookup_error, "test.unknown") + + def test_xmlcharrefvalues(self): + # enhance coverage of: + # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors() + # and inline implementations + v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000) + if sys.maxunicode>=100000: + v += (100000, 500000, 1000000) + s = u"".join([unichr(x) for x in v]) + codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors) + for enc in ("ascii", "iso-8859-15"): + for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"): + s.encode(enc, err) + + def test_decodehelper(self): + # enhance coverage of: + # Objects/unicodeobject.c::unicode_decode_call_errorhandler() + # and callers + self.assertRaises(LookupError, "\xff".decode, "ascii", "test.unknown") + + def baddecodereturn1(exc): + return 42 + codecs.register_error("test.baddecodereturn1", baddecodereturn1) + self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1") + self.assertRaises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1") + self.assertRaises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1") + self.assertRaises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1") + self.assertRaises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1") + self.assertRaises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1") + + def baddecodereturn2(exc): + return (u"?", None) + codecs.register_error("test.baddecodereturn2", baddecodereturn2) + self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2") + + handler = PosReturn() + codecs.register_error("test.posreturn", handler.handle) + + # Valid negative position + handler.pos = -1 + self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"0") + + # Valid negative position + handler.pos = -2 + self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"") + + # Negative position out of bounds + handler.pos = -3 + self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn") + + # Valid positive position + handler.pos = 1 + self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"0") + + # Largest valid positive position (one beyond end of input) + handler.pos = 2 + self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"") + + # Invalid positive position + handler.pos = 3 + self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn") + + # Restart at the "0" + handler.pos = 6 + self.assertEquals("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"0") + + class D(dict): + def __getitem__(self, key): + raise ValueError + self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None}) + self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D()) + self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1}) + + def test_encodehelper(self): + # enhance coverage of: + # Objects/unicodeobject.c::unicode_encode_call_errorhandler() + # and callers + self.assertRaises(LookupError, u"\xff".encode, "ascii", "test.unknown") + + def badencodereturn1(exc): + return 42 + codecs.register_error("test.badencodereturn1", badencodereturn1) + self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn1") + + def badencodereturn2(exc): + return (u"?", None) + codecs.register_error("test.badencodereturn2", badencodereturn2) + self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2") + + handler = PosReturn() + codecs.register_error("test.posreturn", handler.handle) + + # Valid negative position + handler.pos = -1 + self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "0") + + # Valid negative position + handler.pos = -2 + self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "") + + # Negative position out of bounds + handler.pos = -3 + self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn") + + # Valid positive position + handler.pos = 1 + self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "0") + + # Largest valid positive position (one beyond end of input + handler.pos = 2 + self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "") + + # Invalid positive position + handler.pos = 3 + self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn") + + handler.pos = 0 + + class D(dict): + def __getitem__(self, key): + raise ValueError + for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"): + self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None}) + self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D()) + self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300}) + + def test_translatehelper(self): + # enhance coverage of: + # Objects/unicodeobject.c::unicode_encode_call_errorhandler() + # and callers + # (Unfortunately the errors argument is not directly accessible + # from Python, so we can't test that much) + class D(dict): + def __getitem__(self, key): + raise ValueError + self.assertRaises(ValueError, u"\xff".translate, D()) + self.assertRaises(TypeError, u"\xff".translate, {0xff: sys.maxunicode+1}) + self.assertRaises(TypeError, u"\xff".translate, {0xff: ()}) + + def test_bug828737(self): + charmap = { + ord("&"): u"&", + ord("<"): u"<", + ord(">"): u">", + ord('"'): u""", + } + + for n in (1, 10, 100, 1000): + text = u'abcghi'*n + text.translate(charmap) + +def test_main(): + test.test_support.run_unittest(CodecCallbackTest) + +if __name__ == "__main__": + test_main() From fijal at codespeak.net Sun Nov 15 11:36:16 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 11:36:16 +0100 (CET) Subject: [pypy-svn] r69299 - pypy/branch/faster-raise/lib-python/modified-2.5.2/test Message-ID: <20091115103616.B0CB416802C@codespeak.net> Author: fijal Date: Sun Nov 15 11:36:15 2009 New Revision: 69299 Modified: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py Log: We don't really care if creation or replacing causes TypeErrors Modified: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py ============================================================================== --- pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py (original) +++ pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py Sun Nov 15 11:36:15 2009 @@ -447,6 +447,14 @@ (u"", 1) ) + def check_double_raise(self, tp, func, arg): + try: + arg() + except tp: + pass + else: + self.assertRaises(tp, func, arg) + def test_badandgoodreplaceexceptions(self): # "replace" complains about a non-exception passed in self.assertRaises( @@ -460,15 +468,15 @@ codecs.replace_errors, UnicodeError("ouch") ) - self.assertRaises( + self.check_double_raise( TypeError, codecs.replace_errors, - BadObjectUnicodeEncodeError() + BadObjectUnicodeEncodeError, ) - self.assertRaises( + self.check_double_raise( TypeError, codecs.replace_errors, - BadObjectUnicodeDecodeError() + BadObjectUnicodeDecodeError, ) # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement self.assertEquals( From cfbolz at codespeak.net Sun Nov 15 11:54:43 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 15 Nov 2009 11:54:43 +0100 (CET) Subject: [pypy-svn] r69300 - in pypy/extradoc: planning sprintinfo/ddorf2009 Message-ID: <20091115105443.5737316802C@codespeak.net> Author: cfbolz Date: Sun Nov 15 11:54:42 2009 New Revision: 69300 Modified: pypy/extradoc/planning/jit.txt pypy/extradoc/sprintinfo/ddorf2009/planning.txt Log: migrate tasks Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sun Nov 15 11:54:42 2009 @@ -1,20 +1,22 @@ TASKS ----- +- compress the virtuals part of resume data more + - sort out a benchmark infrastructure. graphs! - lose less information across residual calls - improve on predictability: don't trace into signals ... but produce just a conditional call -- we need to be able to not inline when the traces are too long, this would - need being able to start tracing from function starts, the latter - should probably not be done unconditionally but only for functions - seen along too long traces - - we should think about merging several consecutive guards, to make the assembler smaller and to save resume data space +- directly call assembler for residual portal calls + +- make the assembler produced by generate_failure smaller + + - jit/asmgcc + threads? - think about code memory management @@ -28,10 +30,12 @@ - raising an exception tends to escape frames, due to the traceback capturing - prevent jitting really general */** calls +- put the class into the structure to get only one promote when using an + instance +- look into failing pypy-c-jit apptests, pypy-c-jit translate.py + - improve test running, compile only once -- module/__builtin__/app_inspect.py forces frames for globals() and locals(). - It's a horrible idea, should be fixed (and rest reviewed). META ----- Modified: pypy/extradoc/sprintinfo/ddorf2009/planning.txt ============================================================================== --- pypy/extradoc/sprintinfo/ddorf2009/planning.txt (original) +++ pypy/extradoc/sprintinfo/ddorf2009/planning.txt Sun Nov 15 11:54:42 2009 @@ -16,9 +16,9 @@ instance - look into failing pypy-c-jit apptests, pypy-c-jit translate.py POSTPONED - - write blog post (Samuele, Carl Friedrich) + - write blog post DONE - - at the end of the sprint, migrate tasks to jit.txt (Carl Friedrich) + - at the end of the sprint, migrate tasks to jit.txt DONE Talks ----- From cfbolz at codespeak.net Sun Nov 15 11:56:01 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 15 Nov 2009 11:56:01 +0100 (CET) Subject: [pypy-svn] r69301 - pypy/extradoc/planning Message-ID: <20091115105601.21D1216802C@codespeak.net> Author: cfbolz Date: Sun Nov 15 11:56:00 2009 New Revision: 69301 Modified: pypy/extradoc/planning/jit.txt Log: clean up the things that were done during the sprint Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sun Nov 15 11:56:00 2009 @@ -5,18 +5,12 @@ - sort out a benchmark infrastructure. graphs! -- lose less information across residual calls - - improve on predictability: don't trace into signals ... but produce just a conditional call -- we should think about merging several consecutive guards, to make the - assembler smaller and to save resume data space - - directly call assembler for residual portal calls - make the assembler produced by generate_failure smaller - - jit/asmgcc + threads? - think about code memory management @@ -27,7 +21,6 @@ - goal: on average <=5 guards per original bytecode -- raising an exception tends to escape frames, due to the traceback capturing - prevent jitting really general */** calls - put the class into the structure to get only one promote when using an @@ -55,7 +48,6 @@ - at some point we need to merge the tails of loops, to avoid exponential explosion -- need to remember when we gave up - tracing aggressively will put pressure on the speed of tracing - what should we do about recursive calls? - connecting compiled loops accross a call? From arigo at codespeak.net Sun Nov 15 12:36:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 15 Nov 2009 12:36:17 +0100 (CET) Subject: [pypy-svn] r69302 - pypy/trunk/pypy/doc Message-ID: <20091115113617.EE27916802C@codespeak.net> Author: arigo Date: Sun Nov 15 12:36:17 2009 New Revision: 69302 Modified: pypy/trunk/pypy/doc/index.txt Log: Uh? It seems that I modified doc/index.txt by accident by running fixeol. Revert. Modified: pypy/trunk/pypy/doc/index.txt ============================================================================== --- pypy/trunk/pypy/doc/index.txt (original) +++ pypy/trunk/pypy/doc/index.txt Sun Nov 15 12:36:17 2009 @@ -1,331 +1,59 @@ -================================================= -PyPy - a Python_ implementation written in Python -================================================= -.. _Python: http://www.python.org/dev/doc/maint24/ref/ref.html +The PyPy project aims at producing a flexible and fast Python_ +implementation. The guiding idea is to translate a Python-level +description of the Python language itself to lower level languages. +Rumors have it that the secret goal is being faster-than-C which is +nonsense, isn't it? `more...`_ -.. sectnum:: -.. contents:: :depth: 1 +Getting into PyPy ... +============================================= +* `Release 1.1`_: the latest official release -PyPy User Documentation -=============================================== +* `PyPy Blog`_: news and status info about PyPy -`getting started`_ provides hands-on instructions -including a two-liner to run the PyPy Python interpreter -on your system, examples on advanced features and -entry points for using PyPy's translation tool chain. +* `Documentation`_: extensive documentation and papers_ about PyPy. -`FAQ`_ contains some frequently asked questions. +* `Getting Started`_: Getting started and playing with PyPy. -New features of PyPy's Python Interpreter and -Translation Framework: +Mailing lists, bug tracker, IRC channel +============================================= - * `What PyPy can do for your objects`_ - * `Stackless and coroutines`_ - * `JIT Generation in PyPy`_ - * `Sandboxing Python code`_ +* `Development mailing list`_: development and conceptual + discussions. -`PyPy Prolog Interpreter`_ describes an implementation of -Prolog that makes use of our Translation Tool chain. +* `Subversion commit mailing list`_: updates to code and + documentation. -Status_ of the project. +* `Development bug/feature tracker`_: filing bugs and feature requests. +* `Sprint mailing list`_: mailing list for organising upcoming sprints. -Project Documentation -===================================== +* **IRC channel #pypy on freenode**: Many of the core developers are hanging out + at #pypy on irc.freenode.net. You are welcome to join and ask questions + (if they are not already developed in the FAQ_). + You can find logs of the channel here_. -PyPy was funded by the EU for several years. See the `web site of the EU -project`_ for more details. - -.. _`web site of the EU project`: http://pypy.org - -architecture_ gives a complete view of PyPy's basic design. - -`coding guide`_ helps you to write code for PyPy (especially also describes -coding in RPython a bit). - -`sprint reports`_ lists reports written at most of our sprints, from -2003 to the present. - -`talks and related projects`_ lists presentations -and related projects. - -`ideas for PyPy related projects`_ which might be a good way to get -into PyPy. - -`PyPy video documentation`_ is a page linking to the videos (e.g. of talks and -introductions) that are available. - -`Technical reports`_ is a page that contains links to the -reports that we submitted to the European Union. - -`development methodology`_ describes our sprint-driven approach. - -`license`_ contains licensing details (basically a straight MIT-license). - -`Glossary`_ of PyPy words to help you align your inner self with -the PyPy universe. - - -Status -=================================== - -PyPy can be used to run Python programs on Linux, OS/X, -Windows, on top of .NET, and on top of Java. -It is recommended to try out the current Subversion HEAD, -which contains `major improvements`__ since the last release. - -.. __: http://codespeak.net/pipermail/pypy-dev/2007q4/004103.html - -PyPy is mainly developed on Linux and Mac OS X. Windows is supported, -but platform-specific bugs tend to take longer before we notice and fix -them. Linux 64-bit machines are supported (though it may also take some -time before we notice and fix bugs). - -PyPy's own tests `summary`_, daily updated, run through BuildBot infrastructure. -You can also find CPython's compliance tests run with compiled ``pypy-c`` -exeuctables there. - -information dating from early 2007: - -`PyPy LOC statistics`_ shows LOC statistics about PyPy. - -`PyPy statistics`_ is a page with various statistics about the PyPy project. - -`compatibility matrix`_ is a diagram that shows which of the various features -of the PyPy interpreter work together with which other features. - - -Source Code Documentation -=============================================== - -`object spaces`_ discusses the object space interface -and several implementations. - -`bytecode interpreter`_ explains the basic mechanisms -of the bytecode interpreter and virtual machine. - -`interpreter optimizations`_ describes our various strategies for -improving the performance of our interpreter, including alternative -object implementations (for strings, dictionaries and lists) in the -standard object space. - -`translation`_ is a detailed overview of our translation process. The -rtyper_ is the largest component of our translation process. - -`dynamic-language translation`_ is a paper that describes -the translation process, especially the flow object space -and the annotator in detail. This document is also part -of the `EU reports`_. - -`low-level encapsulation`_ describes how our approach hides -away a lot of low level details. This document is also part -of the `EU reports`_. - -`translation aspects`_ describes how we weave different -properties into our interpreter during the translation -process. This document is also part of the `EU reports`_. - -`garbage collector`_ strategies that can be used by the virtual -machines produced by the translation process. - -`parser`_ contains (outdated, unfinished) documentation about -the parser. - -`rlib`_ describes some modules that can be used when implementing programs in -RPython. - -`configuration documentation`_ describes the various configuration options that -allow you to customize PyPy. - -`CLI backend`_ describes the details of the .NET backend. - -`JIT Generation in PyPy`_ describes how we produce the Python Just-in-time Compiler -from our Python interpreter. +.. XXX play1? +Meeting PyPy developers +======================= +The PyPy developers are organising sprints and presenting results at +conferences all year round. They will be happy to meet in person with +anyone interested in the project. Watch out for sprint announcements +on the `development mailing list`_. +.. _Python: http://docs.python.org/index.html +.. _`more...`: architecture.html#mission-statement +.. _`PyPy blog`: http://morepypy.blogspot.com/ +.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _here: http://tismerysoft.de/pypy/irc-logs/pypy +.. _`sprint mailing list`: http://codespeak.net/mailman/listinfo/pypy-sprint +.. _`subversion commit mailing list`: http://codespeak.net/mailman/listinfo/pypy-svn +.. _`development mailing list`: http://codespeak.net/mailman/listinfo/pypy-dev .. _`FAQ`: faq.html -.. _Glossary: glossary.html -.. _`PyPy video documentation`: video-index.html -.. _parser: parser.html -.. _`development methodology`: dev_method.html -.. _`sprint reports`: sprint-reports.html -.. _`talks and related projects`: extradoc.html -.. _`license`: ../../LICENSE -.. _`PyPy LOC statistics`: http://codespeak.net/~hpk/pypy-stat/ -.. _`PyPy statistics`: http://codespeak.net/pypy/trunk/pypy/doc/statistic -.. _`object spaces`: objspace.html -.. _`interpreter optimizations`: interpreter-optimizations.html -.. _`translation`: translation.html -.. _`dynamic-language translation`: dynamic-language-translation.html -.. _`low-level encapsulation`: low-level-encapsulation.html -.. _`translation aspects`: translation-aspects.html -.. _`configuration documentation`: config/ -.. _`coding guide`: coding-guide.html -.. _`architecture`: architecture.html -.. _`getting started`: getting-started.html -.. _`theory`: theory.html -.. _`bytecode interpreter`: interpreter.html -.. _`EU reports`: index-report.html -.. _`Technical reports`: index-report.html -.. _`summary`: http://codespeak.net:8099/summary -.. _`ideas for PyPy related projects`: project-ideas.html -.. _`Nightly builds and benchmarks`: http://tuatara.cs.uni-duesseldorf.de/benchmark.html -.. _`directory reference`: -.. _`rlib`: rlib.html -.. _`Sandboxing Python code`: sandbox.html - -PyPy directory cross-reference ------------------------------- - -Here is a fully referenced alphabetical two-level deep -directory overview of PyPy: - -============================ =========================================== -Directory explanation/links -============================ =========================================== -`annotation/`_ `type inferencing code`_ for `RPython`_ programs - -`bin/`_ command-line scripts, mainly `py.py`_ and `translatorshell.py`_ - -`config/`_ handles the numerous options for building and running PyPy - -`doc/`_ text versions of PyPy developer documentation - -`doc/config/`_ documentation for the numerous translation options - -`doc/discussion/`_ drafts of ideas and documentation - -``doc/*/`` other specific documentation topics or tools - -`interpreter/`_ `bytecode interpreter`_ and related objects - (frames, functions, modules,...) - -`interpreter/pyparser/`_ interpreter-level Python source parser - -`interpreter/astcompiler/`_ interpreter-level bytecode compiler, via an AST - representation - -`lang/`_ interpreters for non-Python languages, written in RPython_ - -`lang/js/`_ a JavaScript interpreter (in-progress) - -`lang/prolog/`_ a `Prolog interpreter`_ - -`lib/`_ PyPy's wholesale reimplementations of CPython modules_ - and experimental new application-level modules - -`lib/app_test/`_ tests for the reimplementations, running on top of CPython - -`lib/distributed/`_ distributed execution prototype, based on `transparent proxies`_ - -`module/`_ contains `mixed modules`_ implementing core modules with - both application and interpreter level code. - Not all are finished and working. Use the ``--withmod-xxx`` - or ``--allworkingmodules`` translation options. - -`objspace/`_ `object space`_ implementations - -`objspace/trace.py`_ the `trace object space`_ monitoring bytecode and space operations - -`objspace/dump.py`_ the dump object space saves a large, searchable log file - with all operations - -`objspace/taint.py`_ the `taint object space`_, providing object tainting - -`objspace/thunk.py`_ the `thunk object space`_, providing unique object features - -`objspace/flow/`_ the FlowObjSpace_ implementing `abstract interpretation` - -`objspace/std/`_ the StdObjSpace_ implementing CPython's objects and types - -`rlib/`_ a `"standard library"`_ for RPython_ programs - -`rpython/`_ the `RPython Typer`_ - -`rpython/lltypesystem/`_ the `low-level type system`_ for C-like backends - -`rpython/ootypesystem/`_ the `object-oriented type system`_ for OO backends - -`rpython/memory/`_ the `garbage collector`_ construction framework - -`tool/`_ various utilities and hacks used from various places - -`tool/algo/`_ general-purpose algorithmic and mathematic - tools - -`tool/pytest/`_ support code for our `testing methods`_ - -`translator/`_ translation_ backends and support code - -`translator/backendopt/`_ general optimizations that run before a backend generates code - -`translator/c/`_ the `GenC backend`_, producing C code from an - RPython program (generally via the rtyper_) - -`translator/cli/`_ the `CLI backend`_ for `.NET`_ (Microsoft CLR or Mono_) - -`translator/goal/`_ our `main PyPy-translation scripts`_ live here - -`translator/jvm/`_ the Java backend - -`translator/stackless/`_ the `Stackless Transform`_ - -`translator/tool/`_ helper tools for translation, including the Pygame - `graph viewer`_ - -``*/test/`` many directories have a test subdirectory containing test - modules (see `Testing in PyPy`_) - -``_cache/`` holds cache files from internally `translating application - level to interpreterlevel`_ code. -============================ =========================================== - -.. _`bytecode interpreter`: interpreter.html -.. _`translating application level to interpreterlevel`: geninterp.html -.. _documentation: index.html -.. _`Testing in PyPy`: coding-guide.html#testing-in-pypy -.. _`mixed modules`: coding-guide.html#mixed-modules -.. _`modules`: coding-guide.html#modules -.. _`basil`: http://people.cs.uchicago.edu/~jriehl/BasilTalk.pdf -.. _`object space`: objspace.html -.. _FlowObjSpace: objspace.html#the-flow-object-space -.. _`trace object space`: objspace.html#the-trace-object-space -.. _`taint object space`: objspace-proxies.html#taint -.. _`thunk object space`: objspace-proxies.html#thunk -.. _`transparent proxies`: objspace-proxies.html#tproxy -.. _`What PyPy can do for your objects`: objspace-proxies.html -.. _`Stackless and coroutines`: stackless.html -.. _StdObjSpace: objspace.html#the-standard-object-space -.. _`abstract interpretation`: theory.html#abstract-interpretation -.. _`rpython`: coding-guide.html#rpython -.. _`type inferencing code`: translation.html#the-annotation-pass -.. _`RPython Typer`: translation.html#rpython-typer -.. _`testing methods`: coding-guide.html#testing-in-pypy -.. _`translation`: translation.html -.. _`GenC backend`: translation.html#genc -.. _`CLI backend`: cli-backend.html -.. _`py.py`: getting-started.html#main-entry-point -.. _`translatorshell.py`: getting-started.html#try-out-the-translator -.. _JIT: jit/index.html -.. _`JIT Generation in PyPy`: jit/index.html -.. _`just-in-time compiler generator`: jit/index.html -.. _rtyper: rtyper.html -.. _`low-level type system`: rtyper.html#low-level-type -.. _`object-oriented type system`: rtyper.html#oo-type -.. _`garbage collector`: garbage_collection.html -.. _`Stackless Transform`: translation.html#the-stackless-transform -.. _`PyPy Prolog Interpreter`: prolog-interpreter.html -.. _`Prolog Interpreter`: prolog-interpreter.html -.. _`main PyPy-translation scripts`: getting-started.html#translating-the-pypy-python-interpreter -.. _`translate.py`: getting-started.html#translating-the-pypy-python-interpreter -.. _`.NET`: http://www.microsoft.com/net/ -.. _Mono: http://www.mono-project.com/ -.. _`"standard library"`: rlib.html -.. _`graph viewer`: getting-started.html#try-out-the-translator -.. _`compatibility matrix`: image/compat-matrix.png - -.. include:: _ref.txt - +.. _`Documentation`: docindex.html +.. _`Getting Started`: getting-started.html +.. _papers: extradoc.html +.. _`Release 1.1`: release-1.1.0.html From cfbolz at codespeak.net Sun Nov 15 13:45:17 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 15 Nov 2009 13:45:17 +0100 (CET) Subject: [pypy-svn] r69303 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091115124517.1B9AE16802F@codespeak.net> Author: cfbolz Date: Sun Nov 15 13:45:16 2009 New Revision: 69303 Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py Log: kill a non-used variable Modified: pypy/trunk/pypy/rpython/lltypesystem/rlist.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rlist.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rlist.py Sun Nov 15 13:45:16 2009 @@ -185,8 +185,6 @@ l.length to newsize. Note that l.items may change, and even if newsize is less than l.length on entry. """ - allocated = len(l.items) - # This over-allocates proportional to the list size, making room # for additional growth. The over-allocation is mild, but is # enough to give linear-time amortized behavior over a long From fijal at codespeak.net Sun Nov 15 17:05:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 15 Nov 2009 17:05:45 +0100 (CET) Subject: [pypy-svn] r69304 - pypy/extradoc/planning Message-ID: <20091115160545.63A07168008@codespeak.net> Author: fijal Date: Sun Nov 15 17:05:43 2009 New Revision: 69304 Modified: pypy/extradoc/planning/jit.txt Log: Two things that come to mind while running larger examples Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Sun Nov 15 17:05:43 2009 @@ -17,6 +17,14 @@ - forcing virtualizables should only force fields affected, not everything +- think out looking into functions or not, based on arguments, + for example contains__Tuple should be unrolled if tuple is of constant + length + +- look at example of storing small strings in large lists (any sane templating + engine would do it) and not spend all the time in + _trace_and_drag_out_of_nursery + Python interpreter: - goal: on average <=5 guards per original bytecode From antocuni at codespeak.net Sun Nov 15 20:26:09 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 15 Nov 2009 20:26:09 +0100 (CET) Subject: [pypy-svn] r69305 - in pypy/trunk/pypy: jit/metainterp jit/metainterp/test rpython rpython/ootypesystem translator/cli Message-ID: <20091115192609.7E0E1168008@codespeak.net> Author: antocuni Date: Sun Nov 15 20:26:06 2009 New Revision: 69305 Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_basic.py pypy/trunk/pypy/rpython/llinterp.py pypy/trunk/pypy/rpython/ootypesystem/rdict.py pypy/trunk/pypy/translator/cli/function.py Log: put all oonewcustomdict operations into its own function and mark it as dont_look_inside, so that the jit does not need to know about custom dicts Modified: pypy/trunk/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/codewriter.py (original) +++ pypy/trunk/pypy/jit/metainterp/codewriter.py Sun Nov 15 20:26:06 2009 @@ -924,8 +924,7 @@ self.register_var(op.result) def serialize_op_oonewcustomdict(self, op): - self.emit('not_implemented') - self.register_var(op.result) + assert False, 'should not be seen by the jit' def serialize_op_instanceof(self, op): v, c_TYPE = op.args Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Sun Nov 15 20:26:06 2009 @@ -885,10 +885,6 @@ return self.metainterp.finishframe_exception(self.exception_box, self.exc_value_box) - @arguments() - def opimpl_not_implemented(self): - raise NotImplementedError - # ------------------------------ def setup_call(self, argboxes): Modified: pypy/trunk/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_basic.py Sun Nov 15 20:26:06 2009 @@ -1242,10 +1242,6 @@ res = self.interp_operations(f, [0]) assert res == ord('A') + ord('B') - def test_r_dict(self): - py.test.skip('in-progress') - - def test_subclassof(self): A = ootype.Instance("A", ootype.ROOT) B = ootype.Instance("B", A) Modified: pypy/trunk/pypy/rpython/llinterp.py ============================================================================== --- pypy/trunk/pypy/rpython/llinterp.py (original) +++ pypy/trunk/pypy/rpython/llinterp.py Sun Nov 15 20:26:06 2009 @@ -1336,7 +1336,7 @@ # obj is an instance, we want to call 'method_name' on it assert fn is None self_arg = [obj] - func_graph = obj._TYPE._methods[method_name].graph + func_graph = obj._TYPE._methods[method_name._str].graph return wrap_graph(llinterpreter, func_graph, self_arg) Modified: pypy/trunk/pypy/rpython/ootypesystem/rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/ootypesystem/rdict.py (original) +++ pypy/trunk/pypy/rpython/ootypesystem/rdict.py Sun Nov 15 20:26:06 2009 @@ -229,26 +229,28 @@ hop.exception_is_here() return r_dict.send_message(hop, 'll_contains', v_args=vlist) -def _get_call_vars(hop, r_func, arg, params_annotation): +def dum_ignore_me(): pass + +def _get_call_args(hop, r_func, arg, params_annotation): if isinstance(r_func, AbstractFunctionsPBCRepr): - v_fn = r_func.get_unique_llfn() + fn = r_func.get_unique_llfn().value v_obj = hop.inputconst(ootype.Void, None) - c_method_name = hop.inputconst(ootype.Void, None) + methodname = None elif isinstance(r_func, AbstractMethodsPBCRepr): + fn = None + v_obj = hop.inputarg(r_func, arg=arg) s_pbc_fn = hop.args_s[arg] methodname = r_func._get_method_name("simple_call", s_pbc_fn, params_annotation) - v_fn = hop.inputconst(ootype.Void, None) - v_obj = hop.inputarg(r_func, arg=arg) - c_method_name = hop.inputconst(ootype.Void, methodname) elif isinstance(r_func, MethodOfFrozenPBCRepr): r_impl, nimplicitarg = r_func.get_r_implfunc() - v_fn = r_impl.get_unique_llfn() - v_obj = hop.inputarg(r_func, arg=arg) - c_method_name = hop.inputconst(ootype.Void, None) - - return v_fn, v_obj, c_method_name + fn = r_impl.get_unique_llfn().value + v_obj = hop.inputconst(ootype.Void, dum_ignore_me) + methodname = None + return fn, v_obj, methodname def rtype_r_dict(hop): + from pypy.rlib import jit + r_dict = hop.r_result if not r_dict.custom_eq_hash: raise TyperError("r_dict() call does not return an r_dict instance") @@ -258,24 +260,29 @@ # the signature of oonewcustomdict is a bit complicated because we # can have three different ways to pass the equal (and hash) # callables: - # 1. pass a plain function: v_eqfn is a StaticMethod, v_eqobj - # and c_eq_method_name are None - # 2. pass a bound method: v_eqfn is None, v_eqobj is the - # instance, c_method_name is the name of the method, - # 3. pass a method of a frozen PBC: v_eqfn is a StaticMethod, - # v_eqobj is the PBC to be pushed in front of the StaticMethod, - # c_eq_method_name is None + # 1. pass a plain function: eqfn is a StaticMethod, v_eqobj + # and eq_method_name are None + # 2. pass a bound method: eqfn is None, v_eqobj is the + # instance, eq_method_name is the name of the method, + # 3. pass a method of a frozen PBC: eqfn is a StaticMethod, v_eqobj is a + # non-None Void value (to be ignored by all the backends except the + # llinterp), eq_method_name is None s_key = r_dict.dictkey.s_value - v_eqfn, v_eqobj, c_eq_method_name =\ - _get_call_vars(hop, r_dict.r_rdict_eqfn, 0, [s_key, s_key]) - v_hashfn, v_hashobj, c_hash_method_name =\ - _get_call_vars(hop, r_dict.r_rdict_hashfn, 1, [s_key]) - - return hop.genop("oonewcustomdict", [cDICT, - v_eqfn, v_eqobj, c_eq_method_name, - v_hashfn, v_hashobj, c_hash_method_name], - resulttype=hop.r_result.lowleveltype) + eqfn, v_eqobj, eq_method_name =\ + _get_call_args(hop, r_dict.r_rdict_eqfn, 0, [s_key, s_key]) + hashfn, v_hashobj, hash_method_name =\ + _get_call_args(hop, r_dict.r_rdict_hashfn, 1, [s_key]) + + @jit.dont_look_inside + def ll_newcustomdict(DICT, v_eqobj, v_hashobj): + from pypy.rpython.lltypesystem.lloperation import llop + return llop.oonewcustomdict(DICT, DICT, + eqfn, v_eqobj, eq_method_name, + hashfn, v_hashobj, hash_method_name) + ll_newcustomdict._dont_inline_ = True + + return hop.gendirectcall(ll_newcustomdict, cDICT, v_eqobj, v_hashobj) def ll_newdict(DICT): return ootype.new(DICT) Modified: pypy/trunk/pypy/translator/cli/function.py ============================================================================== --- pypy/trunk/pypy/translator/cli/function.py (original) +++ pypy/trunk/pypy/translator/cli/function.py Sun Nov 15 20:26:06 2009 @@ -1,4 +1,5 @@ from pypy.objspace.flow import model as flowmodel +from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rpython.lltypesystem.lltype import Void from pypy.rpython.ootypesystem import ootype from pypy.translator.oosupport.treebuilder import SubOperation @@ -148,7 +149,7 @@ from pypy.translator.cli.dotnet import NativeInstance if isinstance(to_load, flowmodel.Constant): value = to_load.value - is_null = not value + is_null = (not isinstance(value, CDefinedIntSymbolic)) and (not value) T = ootype.typeOf(to_load.value) if isinstance(T, NativeInstance) and T._is_value_type and is_null: return True From afa at codespeak.net Sun Nov 15 22:15:06 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 15 Nov 2009 22:15:06 +0100 (CET) Subject: [pypy-svn] r69306 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091115211506.511A1168007@codespeak.net> Author: afa Date: Sun Nov 15 22:15:04 2009 New Revision: 69306 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: oracle module: handle named binds Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Sun Nov 15 22:15:04 2009 @@ -66,7 +66,7 @@ space.wrap('@'), space.wrap(1))) self.connect(space, mode, twophase) - return space.wrap(self) + return space.wrap(self) descr_new.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root, W_Root, Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Sun Nov 15 22:15:04 2009 @@ -47,7 +47,7 @@ elif len(args_w) == 1: if len(kw_w) > 0: raise OperationError( - interp.error.get(space, 'InterfaceError'), + interp_error.get(space).w_InterfaceError, space.wrap( "expecting argument or keyword arguments, not both")) w_args = args_w[0] @@ -77,7 +77,11 @@ self._internalPrepare(space, w_stmt, None) # perform binds - if vars_w is not None: + if vars_w is None: + pass + elif isinstance(vars_w, dict): + self._setBindVariablesByName(space, vars_w, 1, 0, 0) + else: self._setBindVariablesByPos(space, vars_w, 1, 0, 0) self._performBind(space) @@ -298,6 +302,7 @@ def _setBindVariablesByPos(self, space, vars_w, numElements, arrayPos, defer): + "handle positional binds" # make sure positional and named binds are not being intermixed if self.bindDict is not None: raise OperationalError( @@ -319,6 +324,24 @@ assert i == len(self.bindList) self.bindList.append(newVar) + def _setBindVariablesByName(self, space, + vars_w, numElements, arrayPos, defer): + "handle named binds" + # make sure positional and named binds are not being intermixed + if self.bindList is not None: + raise OperationalError( + get(space).w_ProgrammingErrorException, + space.wrap("positional and named binds cannot be intermixed")) + + self.bindDict = {} + + for key, w_value in vars_w.iteritems(): + origVar = self.bindDict.get(key, None) + newVar = self._setBindVariableHelper(space, w_value, origVar, + numElements, arrayPos, defer) + if newVar: + self.bindDict[key] = newVar + def _setBindVariableHelper(self, space, w_value, origVar, numElements, arrayPos, defer): @@ -373,17 +396,13 @@ return newVar - def _setBindVariablesByName(self, space, - vars_w, numElements, arrayPos, defer): - XXX - def _performBind(self, space): # set values and perform binds for all bind variables if self.bindList: for i, var in enumerate(self.bindList): var.bind(space, self, None, i + 1) if self.bindDict: - for key, var in self.bindDict.itervalues(): + for key, var in self.bindDict.iteritems(): var.bind(space, self, key, 0) # ensure that input sizes are reset Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Sun Nov 15 22:15:04 2009 @@ -203,9 +203,9 @@ self.boundName = name # perform the bind - self._internalBind() + self._internalBind(space) - def _internalBind(self): + def _internalBind(self, space): bindHandlePtr = lltype.malloc(roci.Ptr(roci.OCIBind).TO, 1, flavor='raw') if self.isArray: @@ -218,7 +218,7 @@ try: if self.boundName: nameBuffer = config.StringBuffer() - nameBuffer.fill(self.boundName) + nameBuffer.fill(space, space.wrap(self.boundName)) status = roci.OCIBindByName( self.boundCursorHandle, bindHandlePtr, self.environment.errorHandle, @@ -321,9 +321,9 @@ W_Variable.typedef = TypeDef( 'Variable', - getValue = interp2app(W_Variable.getValue, + getvalue = interp2app(W_Variable.getValue, unwrap_spec=W_Variable.getValue.unwrap_spec), - setValue = interp2app(W_Variable.setValue, + setvalue = interp2app(W_Variable.setValue, unwrap_spec=W_Variable.setValue.unwrap_spec), ) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Sun Nov 15 22:15:04 2009 @@ -202,6 +202,24 @@ ub4], # mode sword) +OCIBindByName = external( + 'OCIBindByName', + [OCIStmt, # stmtp + Ptr(OCIBind), # bindpp + OCIError, # errhp + oratext, # placeholder + sb4, # placeh_len + dvoidp, # valuep + sb4, # value_sz + ub2, # dty + dvoidp, # indp + Ptr(ub2), # alenp + Ptr(ub2), # rcodep + ub4, # maxarr_len + Ptr(ub4), # curelep + ub4], # mode + sword) + OCIDefineByPos = external( 'OCIDefineByPos', [OCIStmt, # stmtp Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Sun Nov 15 22:15:04 2009 @@ -41,14 +41,20 @@ def test_bind_out(self): cur = self.cnx.cursor() var = cur.var(oracle.NUMBER) - var.setValue(0, 5) - assert var.getValue(0) == 5 - assert var.getValue() == 5 + var.setvalue(0, 5) + assert var.getvalue(0) == 5 + assert var.getvalue() == 5 cur.execute("begin :1 := 3; end;", (var,)) - value = var.getValue(0) + value = var.getvalue(0) assert value == 3 assert isinstance(value, float) + def test_execute_with_keywords(self): + cur = self.cnx.cursor() + simpleVar = cur.var(oracle.NUMBER) + cur.execute("begin :p := 5; end;", p=simpleVar) + assert simpleVar.getvalue() == 5 + def test_callFunc0(self): cur = self.cnx.cursor() try: From afa at codespeak.net Sun Nov 15 22:33:34 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 15 Nov 2009 22:33:34 +0100 (CET) Subject: [pypy-svn] r69307 - pypy/trunk/pypy/module/oracle Message-ID: <20091115213334.4D4A9168007@codespeak.net> Author: afa Date: Sun Nov 15 22:33:33 2009 New Revision: 69307 Modified: pypy/trunk/pypy/module/oracle/roci.py Log: Give the module a chance to run on other machines (windows for now...) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Sun Nov 15 22:33:33 2009 @@ -1,10 +1,14 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem import rffi, lltype +import os import py - -ORACLE_HOME = py.path.local('c:/oraclexe/app/oracle/product/10.2.0/server') +try: + ORACLE_HOME = py.path.local(os.environ['ORACLE_HOME']) +except KeyError: + raise ImportError( + "Please set ORACLE_HOME to the root of an Oracle client installation") eci = ExternalCompilationInfo( includes = ['oci.h'], From afa at codespeak.net Sun Nov 15 23:04:49 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 15 Nov 2009 23:04:49 +0100 (CET) Subject: [pypy-svn] r69308 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091115220449.D32D2168006@codespeak.net> Author: afa Date: Sun Nov 15 23:04:49 2009 New Revision: 69308 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: implement Cursor.callproc Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Sun Nov 15 23:04:49 2009 @@ -126,9 +126,27 @@ else: args_w = space.unpackiterable(w_parameters) - return self._call(space, name, retvar, args_w) + self._call(space, name, retvar, args_w) + + # determine the results + return retvar.getValue(space, 0) callfunc.unwrap_spec = ['self', ObjSpace, str, W_Root, W_Root] + def callproc(self, space, name, w_parameters=None): + if space.is_w(w_parameters, space.w_None): + args_w = None + else: + args_w = space.unpackiterable(w_parameters) + + self._call(space, name, None, args_w) + + # create the return value + print "AFA", self.bindList + ret_w = [v.getValue(space, 0) for v in self.bindList] + return space.newlist(ret_w) + + callproc.unwrap_spec = ['self', ObjSpace, str, W_Root] + def _call(self, space, name, retvar, args_w): # determine the number of arguments passed if args_w: @@ -160,9 +178,6 @@ self._execute(space, space.wrap(stmt), vars_w) - if retvar: - return retvar.getValue(space, 0) - def _checkOpen(self, space): if not self.isOpen: raise OperationError( @@ -709,6 +724,8 @@ unwrap_spec=W_Cursor.bindnames.unwrap_spec), callfunc = interp2app(W_Cursor.callfunc, unwrap_spec=W_Cursor.callfunc.unwrap_spec), + callproc = interp2app(W_Cursor.callproc, + unwrap_spec=W_Cursor.callproc.unwrap_spec), var = interp2app(W_Cursor.var, unwrap_spec=W_Cursor.var.unwrap_spec), Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Sun Nov 15 23:04:49 2009 @@ -441,18 +441,23 @@ rffi.cast(roci.Ptr(roci.OCINumber), self.data), pos) if isinstance(self, VT_Integer): - status = roci.OCINumberToInt( - self.environment.errorHandle, - dataptr, - rffi.sizeof(rffi.LONG), - roci.OCI_NUMBER_SIGNED, - intergerValuePtr) - self.environment.checkForError( - status, "NumberVar_GetValue(): as integer") - if isinstance(self, VT_Boolean): - return space.wrap(integerValuePtr[0]) - else: - return space.w_bool(integerValuePtr[0]) + integerValuePtr = lltype.malloc(roci.Ptr(lltype.Signed).TO, 1, + flavor='raw') + try: + status = roci.OCINumberToInt( + self.environment.errorHandle, + dataptr, + rffi.sizeof(rffi.LONG), + roci.OCI_NUMBER_SIGNED, + rffi.cast(roci.dvoidp, integerValuePtr)) + self.environment.checkForError( + status, "NumberVar_GetValue(): as integer") + if isinstance(self, VT_Boolean): + return space.newbool(integerValuePtr[0]) + else: + return space.wrap(integerValuePtr[0]) + finally: + lltype.free(integerValuePtr, flavor='raw') elif isinstance(self, (VT_NumberAsString, VT_LongInteger)): XXX = NumberAsString, LongInteger else: Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Sun Nov 15 23:04:49 2009 @@ -189,12 +189,13 @@ # Bind, Define, and Describe Functions -OCIBindByPos = external( - 'OCIBindByPos', +OCIBindByName = external( + 'OCIBindByName', [OCIStmt, # stmtp Ptr(OCIBind), # bindpp OCIError, # errhp - ub4, # position + oratext, # placeholder + sb4, # placeh_len dvoidp, # valuep sb4, # value_sz ub2, # dty @@ -206,13 +207,12 @@ ub4], # mode sword) -OCIBindByName = external( - 'OCIBindByName', +OCIBindByPos = external( + 'OCIBindByPos', [OCIStmt, # stmtp Ptr(OCIBind), # bindpp OCIError, # errhp - oratext, # placeholder - sb4, # placeh_len + ub4, # position dvoidp, # valuep sb4, # value_sz ub2, # dty @@ -339,10 +339,19 @@ Ptr(OCINumber)], # number sword) +OCINumberToInt = external( + 'OCINumberToInt', + [OCIError, # err + Ptr(OCINumber), # number + uword, # rsl_length + uword, # rsl_flag + dvoidp], # rsl + sword) + OCINumberToReal = external( 'OCINumberToReal', [OCIError, # err - Ptr(OCINumber), # number + Ptr(OCINumber), # number uword, # rsl_length dvoidp], # rsl sword) Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Sun Nov 15 23:04:49 2009 @@ -66,7 +66,7 @@ "begin return 42; end;") assert cur.callfunc("pypy_temp_function", oracle.NUMBER) == 42 - + def test_callFunc1(self): cur = self.cnx.cursor() try: @@ -80,4 +80,18 @@ oracle.NUMBER, ("Hi", 5)) assert res == 7 - + def test_callProc(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + try: + cur.execute("drop procedure pypy_temp_procedure") + except oracle.DatabaseError: + pass + cur.execute("create procedure pypy_temp_procedure " + "(x varchar2, y in out number, z out number) as " + "begin " + " y := 10;" + " z := 2;" + "end;") + results = cur.callproc("pypy_temp_procedure", ("hi", 5, var)) + assert results == ["hi", 10, 2.0] From afa at codespeak.net Sun Nov 15 23:59:34 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 15 Nov 2009 23:59:34 +0100 (CET) Subject: [pypy-svn] r69309 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091115225934.F3BC2168007@codespeak.net> Author: afa Date: Sun Nov 15 23:59:33 2009 New Revision: 69309 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: implement cursor.executemany, at least for a list of lists Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Sun Nov 15 23:59:33 2009 @@ -108,6 +108,45 @@ # for all other statements, simply return None return space.w_None + def executemany(self, space, w_stmt, w_list_of_args): + if space.is_w(w_stmt, space.w_None): + w_stmt = None + if not space.is_true(space.isinstance(w_list_of_args, space.w_list)): + raise OperationError( + space.w_TypeError, + space.wrap("list expected")) + + # make sure the cursor is open + self._checkOpen(space) + + # prepare the statement + self._internalPrepare(space, w_stmt, None) + + # queries are not supported as the result is undefined + if self.statementType == roci.OCI_STMT_SELECT: + raise OperationError( + w_NotSupportedErrorException, + space.wrap("queries not supported: results undefined")) + + # perform binds + numrows = space.int_w(space.len(w_list_of_args)) + for i, arguments in enumerate(space.viewiterable(w_list_of_args)): + deferred = i < numrows - 1 + if space.is_true(space.isinstance(arguments, space.w_dict)): + self._setBindVariablesByName( + space, arguments, numrows, i, deferred) + else: + args_w = space.viewiterable(arguments) + self._setBindVariablesByPos( + space, args_w, numrows, i, deferred) + self._performBind(space) + + # execute the statement, but only if the number of rows is greater than + # zero since Oracle raises an error otherwise + if numrows > 0: + self._internalExecute(space, numIters=numrows) + executemany.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] + def close(self, space): # make sure we are actually open self._checkOpen(space) @@ -141,7 +180,6 @@ self._call(space, name, None, args_w) # create the return value - print "AFA", self.bindList ret_w = [v.getValue(space, 0) for v in self.bindList] return space.newlist(ret_w) @@ -324,7 +362,9 @@ get(space).w_ProgrammingErrorException, space.wrap("positional and named binds cannot be intermixed")) - self.bindList = [] + if self.bindList is None: + self.bindList = [] + for i, w_value in enumerate(vars_w): if i < len(self.bindList): origVar = self.bindList[i] @@ -348,7 +388,8 @@ get(space).w_ProgrammingErrorException, space.wrap("positional and named binds cannot be intermixed")) - self.bindDict = {} + if self.bindDict is None: + self.bindDict = {} for key, w_value in vars_w.iteritems(): origVar = self.bindDict.get(key, None) @@ -382,7 +423,7 @@ else: newVar = None try: - origVar.setValue(space, arrayPos, value) + origVar.setValue(space, arrayPos, w_value) except OperationError, e: # executemany() should simply fail after the first element if arrayPos > 0: @@ -712,6 +753,8 @@ 'Cursor', execute = interp2app(W_Cursor.execute, unwrap_spec=W_Cursor.execute.unwrap_spec), + executemany = interp2app(W_Cursor.executemany, + unwrap_spec=W_Cursor.executemany.unwrap_spec), prepare = interp2app(W_Cursor.prepare, unwrap_spec=W_Cursor.prepare.unwrap_spec), fetchone = interp2app(W_Cursor.fetchone, Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Sun Nov 15 23:59:33 2009 @@ -33,7 +33,7 @@ if not handleptr[0] or status not in (roci.OCI_SUCCESS, roci.OCI_SUCCESS_WITH_INFO): raise OperationError( - w_InterfaceErrorException, + get(self.space).w_InterfaceError, self.space.wrap( "Unable to acquire Oracle environment handle")) @@ -69,7 +69,7 @@ # environment is fully initialized error = W_Error(self.space, self, context, 1) if error.code in (1, 1400, 2290, 2291, 2292): - w_type = w_IntegrityErrorException + w_type = get(self.space).w_IntegrityError elif error.code in (1012, 1033, 1034, 1089, 3113, 3114, 12203, 12500, 12571): w_type = get(self.space).w_OperationalError @@ -80,4 +80,5 @@ error = W_Error(self.space, self, context, 0) error.code = 0 error.message = self.space.wrap("Invalid handle!") - raise OperationError(w_DatabaseErrorException, self.space.wrap(error)) + raise OperationError(get(self.space).w_DatabaseError, + self.space.wrap(error)) Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Sun Nov 15 23:59:33 2009 @@ -15,6 +15,7 @@ self.w_InterfaceError = get('InterfaceError') self.w_ProgrammingError = get('ProgrammingError') self.w_NotSupportedError = get('NotSupportedError') + self.w_IntegrityError = get('IntegrityError') self.w_Variable = get('Variable') self.w_NUMBER = get('NUMBER') Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Sun Nov 15 23:59:33 2009 @@ -95,3 +95,18 @@ "end;") results = cur.callproc("pypy_temp_procedure", ("hi", 5, var)) assert results == ["hi", 10, 2.0] + + def test_executemanyByPosition(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (intcol number)") + rows = [ [n] for n in range(23) ] + cur.arraysize = 10 + statement = "insert into TestExecuteMany (IntCol) values (:1)" + cur.executemany(statement, rows) + cur.execute("select count(*) from TestExecuteMany") + count, = cur.fetchone() + assert count == len(rows) From afa at codespeak.net Mon Nov 16 00:08:52 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 16 Nov 2009 00:08:52 +0100 (CET) Subject: [pypy-svn] r69310 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091115230852.38F70168007@codespeak.net> Author: afa Date: Mon Nov 16 00:08:50 2009 New Revision: 69310 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: fix cursor.callproc when no argument at all is given Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 16 00:08:50 2009 @@ -180,8 +180,11 @@ self._call(space, name, None, args_w) # create the return value - ret_w = [v.getValue(space, 0) for v in self.bindList] - return space.newlist(ret_w) + if self.bindList: + ret_w = [v.getValue(space, 0) for v in self.bindList] + return space.newlist(ret_w) + else: + return space.newlist([]) callproc.unwrap_spec = ['self', ObjSpace, str, W_Root] Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Mon Nov 16 00:08:50 2009 @@ -80,7 +80,7 @@ oracle.NUMBER, ("Hi", 5)) assert res == 7 - def test_callProc(self): + def test_callproc(self): cur = self.cnx.cursor() var = cur.var(oracle.NUMBER) try: @@ -96,6 +96,17 @@ results = cur.callproc("pypy_temp_procedure", ("hi", 5, var)) assert results == ["hi", 10, 2.0] + def test_callproc_noargs(self): + cur = self.cnx.cursor() + try: + cur.execute("drop procedure pypy_temp_procedure") + except oracle.DatabaseError: + pass + cur.execute("create procedure pypy_temp_procedure as " + "begin null; end;") + results = cur.callproc("pypy_temp_procedure", ()) + assert results == [] + def test_executemanyByPosition(self): cur = self.cnx.cursor() try: From tobami at codespeak.net Mon Nov 16 01:01:03 2009 From: tobami at codespeak.net (tobami at codespeak.net) Date: Mon, 16 Nov 2009 01:01:03 +0100 (CET) Subject: [pypy-svn] r69311 - in codespeed: . documentation pyspeed/codespeed pyspeed/fixtures tools Message-ID: <20091116000103.A26ED168007@codespeak.net> Author: tobami Date: Mon Nov 16 01:01:01 2009 New Revision: 69311 Added: codespeed/documentation/ codespeed/documentation/Backend_DB.mwb (contents, props changed) codespeed/documentation/Backend_DB.png - copied unchanged from r69108, codespeed/Backend_DB.png codespeed/tools/ codespeed/tools/save_result.py Removed: codespeed/Backend_DB.png codespeed/pyspeed/fixtures/fixtures.json Modified: codespeed/pyspeed/codespeed/admin.py codespeed/pyspeed/codespeed/models.py codespeed/pyspeed/codespeed/tests.py codespeed/pyspeed/codespeed/urls.py codespeed/pyspeed/codespeed/views.py Log: Add view to save result data. Add unittest for the view Add script to save result data Added: codespeed/documentation/Backend_DB.mwb ============================================================================== Binary file. No diff available. Modified: codespeed/pyspeed/codespeed/admin.py ============================================================================== --- codespeed/pyspeed/codespeed/admin.py (original) +++ codespeed/pyspeed/codespeed/admin.py Mon Nov 16 01:01:01 2009 @@ -3,7 +3,7 @@ from django.contrib import admin class RevisionAdmin(admin.ModelAdmin): - list_display = ('number', 'project', 'tag') + list_display = ('number', 'project', 'tag', 'date') admin.site.register(Revision, RevisionAdmin) Modified: codespeed/pyspeed/codespeed/models.py ============================================================================== --- codespeed/pyspeed/codespeed/models.py (original) +++ codespeed/pyspeed/codespeed/models.py Mon Nov 16 01:01:01 2009 @@ -26,6 +26,7 @@ TYPES = ( ('T', 'Trunk'), ('D', 'Debug'), + ('P', 'Python'), ('M', 'Multilanguage'), ) def __unicode__(self): Modified: codespeed/pyspeed/codespeed/tests.py ============================================================================== --- codespeed/pyspeed/codespeed/tests.py (original) +++ codespeed/pyspeed/codespeed/tests.py Mon Nov 16 01:01:01 2009 @@ -1,35 +1,79 @@ # -*- coding: utf-8 -*- -""" -This file demonstrates two different styles of tests (one doctest and one -unittest). These will both pass when you run "manage.py test". - -Replace these with more appropriate tests for your application. -""" - from django.test import TestCase -import urllib, urllib2 -import simplejson from datetime import datetime +from django.test.client import Client +from pyspeed.codespeed.models import Benchmark, Revision, Interpreter, Environment, Result class AddResultTest(TestCase): + def setUp(self): + self.path = '/result/add/' + self.client = Client() + self.e = Environment(name='Dual Core Linux', cpu='Core 2 Duo 8200') + self.e.save() + self.cdate = datetime.today() + self.data = { + 'revision_number': '23232', + 'revision_project': 'pypy', + 'interpreter_name': 'pypy-c', + 'interpreter_coptions': 'gc=B?hm', + 'benchmark_name': 'Richards', + 'environment': 'Dual Core Linux', + 'result_key': 'total', + 'result_value': 456, + 'result_date': self.cdate, + } def test_add_result(self): """ Add result data """ - data = { - 'revision_number': '23232', - 'revision_project': 'pypy', - 'interpreter_name': 'pypy-c', - 'interpreter_coptions': 'gc=Boehm', - 'benchmark_name': 'Richards', - 'environment': 1, - 'result_key': 'total', - 'result_value': 456, - 'result_date': datetime.today(), - } - params = urllib.urlencode(data) - f = urllib2.urlopen('http://localhost:8000/pypy/result/add/', params) - data = f.read() - print "Server response:", data - f.close() + response = self.client.post(self.path, self.data) + self.assertEquals(response.status_code, 200) + self.assertEquals(response.content, "Result data saved succesfully") + e = Environment.objects.get(name='Dual Core Linux') + b = Benchmark.objects.get(name='Richards') + r = Revision.objects.get(number='23232', project='pypy') + i = Interpreter.objects.get(name='pypy-c', coptions='gc=B?hm') + self.assertTrue(Result.objects.get( + key='total', + value=456, + date=self.cdate, + revision=r, + interpreter=i, + benchmark=b, + environment=e + )) + def test_bad_environment(self): + """ + Add result associated with non-existing environment + """ + no_name = 'Dual Core Linux1' + self.data['environment'] = no_name + response = self.client.post(self.path, self.data) + self.assertEquals(response.status_code, 404) + self.assertEquals(response.content, "Environment " + no_name + " not found") + self.data['environment'] = 'Dual Core Linux' + + def test_empty_argument(self): + """ + Make POST request with an empty argument. + """ + for key in self.data: + backup = self.data[key] + self.data[key] = "" + response = self.client.post(self.path, self.data) + self.assertEquals(response.status_code, 400) + self.assertEquals(response.content, 'Key "' + key + '" empty in request') + self.data[key] = backup + + def test_missing_argument(self): + """ + Make POST request with a missing argument. + """ + for key in self.data: + backup = self.data[key] + del(self.data[key]) + response = self.client.post(self.path, self.data) + self.assertEquals(response.status_code, 400) + self.assertEquals(response.content, 'Key "' + key + '" missing from request') + self.data[key] = backup Modified: codespeed/pyspeed/codespeed/urls.py ============================================================================== --- codespeed/pyspeed/codespeed/urls.py (original) +++ codespeed/pyspeed/codespeed/urls.py Mon Nov 16 01:01:01 2009 @@ -33,7 +33,7 @@ (r'^pypy/result/$', list_detail.object_list, result_list), (r'^pypy/revision/$', list_detail.object_list, revision_list), # URL interface for adding results - (r'^pypy/result/add/$', 'addresult'), + (r'^result/add/$', 'addresult'), ) if settings.DEBUG: Modified: codespeed/pyspeed/codespeed/views.py ============================================================================== --- codespeed/pyspeed/codespeed/views.py (original) +++ codespeed/pyspeed/codespeed/views.py Mon Nov 16 01:01:01 2009 @@ -1,20 +1,53 @@ # -*- coding: utf-8 -*- -#from django.shortcuts import render_to_response -#from django.views.generic.list_detail import object_list -from pyspeed.codespeed.models import Revision, Result, Interpreter, Benchmark -#from pyspeed import settings -from django.http import HttpResponse, HttpResponseNotAllowed from django.shortcuts import get_object_or_404 +from pyspeed.codespeed.models import Revision, Result, Interpreter, Benchmark, Environment +from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseNotFound def addresult(request): if request.method != 'POST': return HttpResponseNotAllowed('POST') data = request.POST - print data - ben = get_object_or_404(Benchmark, name=data["benchmark_name"]) - print "benchmark:", ben - rev = get_object_or_404(Revision, number=data["revision_number"], project=data["revision_project"]) - print "revision object:", rev - inter = get_object_or_404(Interpreter, name=data["interpreter_name"], coptions=data["interpreter_coptions"]) - print inter - return HttpResponse("Data saved succesfully") + + mandatory_data = [ + 'revision_number', + 'revision_project', + 'interpreter_name', + 'interpreter_coptions', + 'benchmark_name', + 'environment', + 'result_key', + 'result_value', + 'result_date', + ] + for key in mandatory_data: + if data.has_key(key): + if data[key] == "": + return HttpResponseBadRequest('Key "' + key + '" empty in request') + else: return HttpResponseBadRequest('Key "' + key + '" missing from request') + b, created = Benchmark.objects.get_or_create(name=data["benchmark_name"]) + if data.has_key('benchmark_type'): + b.benchmark_type = data['benchmark_type'] + b.save() + rev, created = Revision.objects.get_or_create(number=data["revision_number"], project=data["revision_project"]) + if data.has_key('revision_date'): + rev.date = data['revision_date'] + rev.save() + inter, created = Interpreter.objects.get_or_create(name=data["interpreter_name"], coptions=data["interpreter_coptions"]) + try: + e = get_object_or_404(Environment, name=data["environment"]) + except Http404: + return HttpResponseNotFound("Environment " + data["environment"] + " not found") + result_type = "T" + if data.has_key('result_type'): + result_type = data['result_type'] + r, created = Result.objects.get_or_create( + key=data["result_key"], + value=data["result_value"], + result_type=result_type, + date=data["result_date"], + revision=rev, + interpreter=inter, + benchmark=b, + environment=e + ) + return HttpResponse("Result data saved succesfully") Added: codespeed/tools/save_result.py ============================================================================== --- (empty file) +++ codespeed/tools/save_result.py Mon Nov 16 01:01:01 2009 @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import urllib, urllib2 + +BASEURL = 'http://localhost:8080/' + +def add_result(): + data = { + 'revision_number': '23238', + 'revision_project': 'pypy', + 'revision_date': "2009-11-15 18:11:29", # Optional + 'interpreter_name': 'pypy-c-jit', + 'interpreter_coptions': 'gc=Boehm', + 'benchmark_name': 'Richards', + 'benchmark_type': 'P',# Optional. Default Trunk. (Trunk, Debug, Python, Multilanguage) + 'environment': "Dual Core Linux", + 'result_key': 'total', + 'result_value': 400, + 'result_type': 'M',# Optional. Default Time in milliseconds. (Time, Memory, Score) + 'result_date': datetime.today(), + } + + # TODO add HTTPError try + params = urllib.urlencode(data) + f = urllib2.urlopen(BASEURL + 'result/add/', params) + response = f.read() + print "Server response:", response + f.close() + +if __name__ == "__main__": + add_result() From afa at codespeak.net Mon Nov 16 01:07:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 16 Nov 2009 01:07:43 +0100 (CET) Subject: [pypy-svn] r69312 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091116000743.6B28C168007@codespeak.net> Author: afa Date: Mon Nov 16 01:07:42 2009 New Revision: 69312 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: Turn the "bindDict" into a applevel dict, and fix executemany() with a list of dicts Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 16 01:07:42 2009 @@ -50,17 +50,18 @@ interp_error.get(space).w_InterfaceError, space.wrap( "expecting argument or keyword arguments, not both")) - w_args = args_w[0] - vars_w = space.unpackiterable(w_args) + w_vars = args_w[0] elif len(kw_w) > 0: - vars_w = kw_w + w_vars = space.newdict() + for key, w_value in kw_w.iteritems(): + space.setitem(w_vars, space.wrap(key), w_value) else: - vars_w = None + w_vars = None # make sure the cursor is open self._checkOpen(space) - return self._execute(space, w_stmt, vars_w) + return self._execute(space, w_stmt, w_vars) execute.unwrap_spec = ['self', ObjSpace, W_Root, Arguments] def prepare(self, space, w_stmt, w_tag=None): @@ -71,18 +72,18 @@ self._internalPrepare(space, w_stmt, w_tag) prepare.unwrap_spec = ['self', ObjSpace, W_Root, W_Root] - def _execute(self, space, w_stmt, vars_w): + def _execute(self, space, w_stmt, w_vars): # prepare the statement, if applicable self._internalPrepare(space, w_stmt, None) # perform binds - if vars_w is None: + if w_vars is None: pass - elif isinstance(vars_w, dict): - self._setBindVariablesByName(space, vars_w, 1, 0, 0) + elif space.is_true(space.isinstance(w_vars, space.w_dict)): + self._setBindVariablesByName(space, w_vars, 1, 0, 0) else: - self._setBindVariablesByPos(space, vars_w, 1, 0, 0) + self._setBindVariablesByPos(space, w_vars, 1, 0, 0) self._performBind(space) # execute the statement @@ -136,9 +137,8 @@ self._setBindVariablesByName( space, arguments, numrows, i, deferred) else: - args_w = space.viewiterable(arguments) self._setBindVariablesByPos( - space, args_w, numrows, i, deferred) + space, arguments, numrows, i, deferred) self._performBind(space) # execute the statement, but only if the number of rows is greater than @@ -217,7 +217,7 @@ else: stmt = "begin %s(%s); end;" % (name, args) - self._execute(space, space.wrap(stmt), vars_w) + self._execute(space, space.wrap(stmt), space.newlist(vars_w)) def _checkOpen(self, space): if not self.isOpen: @@ -357,7 +357,7 @@ self.fetchVariables = None def _setBindVariablesByPos(self, space, - vars_w, numElements, arrayPos, defer): + w_vars, numElements, arrayPos, defer): "handle positional binds" # make sure positional and named binds are not being intermixed if self.bindDict is not None: @@ -368,7 +368,7 @@ if self.bindList is None: self.bindList = [] - for i, w_value in enumerate(vars_w): + for i, w_value in enumerate(space.viewiterable(w_vars)): if i < len(self.bindList): origVar = self.bindList[i] else: @@ -383,7 +383,7 @@ self.bindList.append(newVar) def _setBindVariablesByName(self, space, - vars_w, numElements, arrayPos, defer): + w_vars, numElements, arrayPos, defer): "handle named binds" # make sure positional and named binds are not being intermixed if self.bindList is not None: @@ -392,14 +392,16 @@ space.wrap("positional and named binds cannot be intermixed")) if self.bindDict is None: - self.bindDict = {} + self.bindDict = space.newdict() - for key, w_value in vars_w.iteritems(): - origVar = self.bindDict.get(key, None) + items = space.viewiterable(space.call_method(w_vars, "iteritems")) + for item in items: + w_key, w_value = space.unpackiterable(item) + origVar = space.finditem(self.bindDict, w_key) newVar = self._setBindVariableHelper(space, w_value, origVar, numElements, arrayPos, defer) if newVar: - self.bindDict[key] = newVar + space.setitem(self.bindDict, w_key, newVar) def _setBindVariableHelper(self, space, w_value, origVar, numElements, arrayPos, defer): @@ -461,8 +463,11 @@ for i, var in enumerate(self.bindList): var.bind(space, self, None, i + 1) if self.bindDict: - for key, var in self.bindDict.iteritems(): - var.bind(space, self, key, 0) + items = space.viewiterable( + space.call_method(self.bindDict, "iteritems")) + for item in items: + w_key, var = space.unpackiterable(item) + var.bind(space, self, w_key, 0) # ensure that input sizes are reset self.setInputSizes = False Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 16 01:07:42 2009 @@ -191,16 +191,16 @@ def postDefine(self): pass - def bind(self, space, cursor, name, pos): + def bind(self, space, cursor, w_name, pos): # nothing to do if already bound if (cursor.handle == self.boundCursorHandle and - name == self.boundName and pos == self.boundPos): + w_name == self.w_boundName and pos == self.boundPos): return # set the instance variables specific for binding self.boundCursorHandle = cursor.handle self.boundPos = pos - self.boundName = name + self.w_boundName = w_name # perform the bind self._internalBind(space) @@ -216,9 +216,9 @@ actualElementsPtr = lltype.nullptr(roci.Ptr(roci.ub4).TO) try: - if self.boundName: + if self.w_boundName: nameBuffer = config.StringBuffer() - nameBuffer.fill(space, space.wrap(self.boundName)) + nameBuffer.fill(space, self.w_boundName) status = roci.OCIBindByName( self.boundCursorHandle, bindHandlePtr, self.environment.errorHandle, Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Mon Nov 16 01:07:42 2009 @@ -107,7 +107,7 @@ results = cur.callproc("pypy_temp_procedure", ()) assert results == [] - def test_executemanyByPosition(self): + def test_executemany_bypos(self): cur = self.cnx.cursor() try: cur.execute("drop table pypy_temp_table") @@ -116,8 +116,24 @@ cur.execute("create table pypy_temp_table (intcol number)") rows = [ [n] for n in range(23) ] cur.arraysize = 10 - statement = "insert into TestExecuteMany (IntCol) values (:1)" + statement = "insert into pypy_temp_table (intcol) values (:1)" cur.executemany(statement, rows) - cur.execute("select count(*) from TestExecuteMany") + cur.execute("select count(*) from pypy_temp_table") count, = cur.fetchone() assert count == len(rows) + + def test_executemany_byname(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (intcol number)") + rows = [ { "value" : n } for n in range(23) ] + cur.arraysize = 10 + statement = "insert into pypy_temp_table (intcol) values (:value)" + cur.executemany(statement, rows) + cur.execute("select count(*) from pypy_temp_table") + count, = cur.fetchone() + assert count == len(rows) + From tobami at codespeak.net Mon Nov 16 01:14:35 2009 From: tobami at codespeak.net (tobami at codespeak.net) Date: Mon, 16 Nov 2009 01:14:35 +0100 (CET) Subject: [pypy-svn] r69313 - codespeed/tools Message-ID: <20091116001435.C75F816800B@codespeak.net> Author: tobami Date: Mon Nov 16 01:14:35 2009 New Revision: 69313 Modified: codespeed/tools/save_result.py Log: Added some comments Modified: codespeed/tools/save_result.py ============================================================================== --- codespeed/tools/save_result.py (original) +++ codespeed/tools/save_result.py Mon Nov 16 01:14:35 2009 @@ -2,21 +2,21 @@ from datetime import datetime import urllib, urllib2 -BASEURL = 'http://localhost:8080/' +BASEURL = 'http://localhost:8080/'# This will be pyspeed.pypy.org/ def add_result(): data = { 'revision_number': '23238', 'revision_project': 'pypy', - 'revision_date': "2009-11-15 18:11:29", # Optional + 'revision_date': "2009-11-15 18:11:29", # Optional. Make mandatory? 'interpreter_name': 'pypy-c-jit', 'interpreter_coptions': 'gc=Boehm', 'benchmark_name': 'Richards', - 'benchmark_type': 'P',# Optional. Default Trunk. (Trunk, Debug, Python, Multilanguage) + 'benchmark_type': 'P',# Optional. Default is T for Trunk. (Trunk, Debug, Python, Multilanguage) 'environment': "Dual Core Linux", 'result_key': 'total', 'result_value': 400, - 'result_type': 'M',# Optional. Default Time in milliseconds. (Time, Memory, Score) + 'result_type': 'M',# Optional. Default is 'T' for Time in milliseconds. (Time, Memory, Score) 'result_date': datetime.today(), } From afa at codespeak.net Mon Nov 16 01:20:03 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 16 Nov 2009 01:20:03 +0100 (CET) Subject: [pypy-svn] r69314 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091116002003.A8C4E168008@codespeak.net> Author: afa Date: Mon Nov 16 01:20:03 2009 New Revision: 69314 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: Test and fix Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 16 01:20:03 2009 @@ -161,11 +161,9 @@ def callfunc(self, space, name, w_returnType, w_parameters=None): retvar = interp_variable.newVariableByType(space, self, w_returnType, 1) if space.is_w(w_parameters, space.w_None): - args_w = None - else: - args_w = space.unpackiterable(w_parameters) + w_parameters = None - self._call(space, name, retvar, args_w) + self._call(space, name, retvar, w_parameters) # determine the results return retvar.getValue(space, 0) @@ -173,11 +171,9 @@ def callproc(self, space, name, w_parameters=None): if space.is_w(w_parameters, space.w_None): - args_w = None - else: - args_w = space.unpackiterable(w_parameters) + w_parameters = None - self._call(space, name, None, args_w) + self._call(space, name, None, w_parameters) # create the return value if self.bindList: @@ -188,10 +184,10 @@ callproc.unwrap_spec = ['self', ObjSpace, str, W_Root] - def _call(self, space, name, retvar, args_w): + def _call(self, space, name, retvar, w_args): # determine the number of arguments passed - if args_w: - numArguments = len(args_w) + if w_args: + numArguments = space.int_w(space.len(w_args)) else: numArguments = 0 @@ -201,13 +197,12 @@ # add the return value, if applicable if retvar: offset = 1 - if args_w: - vars_w = [retvar] + args_w - else: - vars_w = [retvar] + w_vars = space.newlist([retvar]) + if w_args: + space.call_method(w_vars, "extend", w_args) else: offset = 0 - vars_w = args_w + w_vars = w_args # build up the statement args = ', '.join(':%d' % (i + offset + 1,) @@ -217,7 +212,7 @@ else: stmt = "begin %s(%s); end;" % (name, args) - self._execute(space, space.wrap(stmt), space.newlist(vars_w)) + self._execute(space, space.wrap(stmt), w_vars) def _checkOpen(self, space): if not self.isOpen: Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Mon Nov 16 01:20:03 2009 @@ -106,6 +106,8 @@ "begin null; end;") results = cur.callproc("pypy_temp_procedure", ()) assert results == [] + results = cur.callproc("pypy_temp_procedure") + assert results == [] def test_executemany_bypos(self): cur = self.cnx.cursor() From afa at codespeak.net Mon Nov 16 01:37:58 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 16 Nov 2009 01:37:58 +0100 (CET) Subject: [pypy-svn] r69315 - pypy/trunk/pypy/module/oracle Message-ID: <20091116003758.E1850168007@codespeak.net> Author: afa Date: Mon Nov 16 01:37:58 2009 New Revision: 69315 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: more fixes Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 16 01:37:58 2009 @@ -416,7 +416,7 @@ # this is only necessary for executemany() since execute() always # passes a value of 1 for the number of elements elif numElements > origVar.allocatedElements: - newVar = type(origVar)(numElements, origVar.size) + newVar = type(origVar)(self, numElements, origVar.size) newVar.setValue(space, arrayPos, w_value) # otherwise, attempt to set the value Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 16 01:37:58 2009 @@ -485,7 +485,9 @@ finally: lltype.free(integerValuePtr, flavor='raw') return - raise TypeError("expecting numeric data") + raise OperationError( + space.w_TypeError, + space.wrap("expecting numeric data")) class VT_Integer(VT_Float): pass Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Mon Nov 16 01:37:58 2009 @@ -72,6 +72,7 @@ globals().update(platform.configure(CConfig)) OCI_IND_NOTNULL = rffi.cast(rffi.SHORT, OCI_IND_NOTNULL) +OCI_IND_NULL = rffi.cast(rffi.SHORT, OCI_IND_NULL) OCISvcCtx = rffi.VOIDP OCIEnv = rffi.VOIDP From cfbolz at codespeak.net Mon Nov 16 11:31:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 16 Nov 2009 11:31:46 +0100 (CET) Subject: [pypy-svn] r69317 - in pypy/trunk/pypy: interpreter objspace/std rlib rlib/test Message-ID: <20091116103146.15BC6168008@codespeak.net> Author: cfbolz Date: Mon Nov 16 11:31:45 2009 New Revision: 69317 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/objspace/std/sharingdict.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rlib/test/test_jit.py Log: A new decorator, purefunction_promote that is like purefunction but also promotes all its arguments. Use it in a few places. Maybe should be renamed to jit_compile_time? Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Mon Nov 16 11:31:45 2009 @@ -16,7 +16,7 @@ funccallunrolling = unrolling_iterable(range(4)) - at jit.purefunction + at jit.purefunction_promote def _get_immutable_code(func): assert not func.can_change_code return func.code @@ -58,7 +58,6 @@ def getcode(self): if jit.we_are_jitted(): if not self.can_change_code: - self = jit.hint(self, promote=True) return _get_immutable_code(self) return jit.hint(self.code, promote=True) return self.code Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Mon Nov 16 11:31:45 2009 @@ -1,6 +1,6 @@ from pypy.objspace.std.dictmultiobject import IteratorImplementation from pypy.objspace.std.dictmultiobject import W_DictMultiObject, _is_sane_hash -from pypy.rlib.jit import purefunction, hint, we_are_jitted, unroll_safe +from pypy.rlib.jit import purefunction_promote, hint, we_are_jitted, unroll_safe from pypy.rlib.rweakref import RWeakValueDictionary NUM_DIGITS = 4 @@ -32,41 +32,22 @@ self.other_structs.set(added_key, new_structure) return new_structure + @purefunction_promote def lookup_position(self, key): - # jit helper - self = hint(self, promote=True) - key = hint(key, promote=True) - return _lookup_position_shared(self, key) + return self.keys.get(key, -1) + @purefunction_promote def get_next_structure(self, key): - # jit helper - self = hint(self, promote=True) - key = hint(key, promote=True) - newstruct = _get_next_structure_shared(self, key) - if not we_are_jitted(): - self._size_estimate -= self.size_estimate() - self._size_estimate += newstruct.size_estimate() + new_structure = self.other_structs.get(key) + if new_structure is None: + new_structure = self.new_structure(key) + self._size_estimate -= self.size_estimate() + self._size_estimate += newstruct.size_estimate() return newstruct + @purefunction_promote def size_estimate(self): - self = hint(self, promote=True) - return _size_estimate(self) - - at purefunction -def _lookup_position_shared(self, key): - return self.keys.get(key, -1) - - at purefunction -def _get_next_structure_shared(self, key): - new_structure = self.other_structs.get(key) - if new_structure is None: - new_structure = self.new_structure(key) - return new_structure - - at purefunction -def _size_estimate(self): - return self._size_estimate >> NUM_DIGITS - + return self._size_estimate >> NUM_DIGITS class State(object): def __init__(self, space): Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Mon Nov 16 11:31:45 2009 @@ -7,7 +7,8 @@ from pypy.objspace.std.dictproxyobject import W_DictProxyObject from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash -from pypy.rlib.jit import hint, purefunction, we_are_jitted, dont_look_inside +from pypy.rlib.jit import hint, purefunction_promote, we_are_jitted +from pypy.rlib.jit import dont_look_inside from pypy.rlib.rarithmetic import intmask, r_uint from copy_reg import _HEAPTYPE @@ -126,7 +127,6 @@ w_self.space.config.objspace.std.immutable_builtintypes): return w_self._version_tag # pure objects cannot get their version_tag changed - w_self = hint(w_self, promote=True) return w_self._pure_version_tag() def getattribute_if_not_from_object(w_self): @@ -155,7 +155,7 @@ def has_object_getattribute(w_self): return w_self.getattribute_if_not_from_object() is None - @purefunction + @purefunction_promote def _pure_version_tag(w_self): return w_self._version_tag @@ -242,14 +242,12 @@ w_self = hint(w_self, promote=True) assert space.config.objspace.std.withmethodcache version_tag = w_self.version_tag() - version_tag = hint(version_tag, promote=True) if version_tag is None: tup = w_self._lookup_where(name) return tup - name = hint(name, promote=True) return w_self._pure_lookup_where_with_method_cache(name, version_tag) - @purefunction + @purefunction_promote def _pure_lookup_where_with_method_cache(w_self, name, version_tag): space = w_self.space SHIFT = r_uint.BITS - space.config.objspace.std.methodcachesizeexp @@ -662,7 +660,7 @@ def _issubtype(w_type1, w_type2): return w_type2 in w_type1.mro_w - at purefunction + at purefunction_promote def _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2): return _issubtype(w_type1, w_type2) @@ -672,8 +670,6 @@ if space.config.objspace.std.withtypeversion and we_are_jitted(): version_tag1 = w_type1.version_tag() version_tag2 = w_type2.version_tag() - version_tag1 = hint(version_tag1, promote=True) - version_tag2 = hint(version_tag2, promote=True) if version_tag1 is not None and version_tag2 is not None: res = _pure_issubtype(w_type1, w_type2, version_tag1, version_tag2) return space.newbool(res) Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Mon Nov 16 11:31:45 2009 @@ -1,3 +1,4 @@ +import py import sys from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.objectmodel import CDefinedIntSymbolic @@ -18,6 +19,23 @@ func._jit_unroll_safe_ = True return func +def purefunction_promote(func): + import inspect + purefunction(func) + args, varargs, varkw, defaults = inspect.getargspec(func) + assert varargs is None and varkw is None + assert not defaults + argstring = ", ".join(args) + code = ["def f(%s):\n" % (argstring, )] + for arg in args: + code.append(" %s = hint(%s, promote=True)\n" % (arg, arg)) + code.append(" return func(%s)\n" % (argstring, )) + d = {"func": func, "hint": hint} + exec py.code.Source("\n".join(code)).compile() in d + result = d["f"] + result.func_name = func.func_name + "_promote" + return result + class Entry(ExtRegistryEntry): _about_ = hint Modified: pypy/trunk/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_jit.py (original) +++ pypy/trunk/pypy/rlib/test/test_jit.py Mon Nov 16 11:31:45 2009 @@ -1,5 +1,5 @@ import py -from pypy.rlib.jit import hint, we_are_jitted, JitDriver +from pypy.rlib.jit import hint, we_are_jitted, JitDriver, purefunction_promote from pypy.translator.translator import TranslationContext, graphof from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rpython.lltypesystem import lltype @@ -23,6 +23,14 @@ res = self.interpret(f, [4]) assert res == 5 + def test_purefunction_promote(self): + @purefunction_promote + def g(x): + return x + 1 + def f(x): + return g(x * 2) + res = self.interpret(f, [2]) + assert res == 5 def test_annotate_hooks(self): From cfbolz at codespeak.net Mon Nov 16 13:46:07 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 16 Nov 2009 13:46:07 +0100 (CET) Subject: [pypy-svn] r69319 - pypy/trunk/pypy/objspace/std Message-ID: <20091116124607.4ADEC168008@codespeak.net> Author: cfbolz Date: Mon Nov 16 13:46:06 2009 New Revision: 69319 Modified: pypy/trunk/pypy/objspace/std/sharingdict.py Log: Ouch, didn't run enough tests. Sorry about that. Modified: pypy/trunk/pypy/objspace/std/sharingdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sharingdict.py (original) +++ pypy/trunk/pypy/objspace/std/sharingdict.py Mon Nov 16 13:46:06 2009 @@ -42,8 +42,8 @@ if new_structure is None: new_structure = self.new_structure(key) self._size_estimate -= self.size_estimate() - self._size_estimate += newstruct.size_estimate() - return newstruct + self._size_estimate += new_structure.size_estimate() + return new_structure @purefunction_promote def size_estimate(self): From tobami at codespeak.net Mon Nov 16 13:47:41 2009 From: tobami at codespeak.net (tobami at codespeak.net) Date: Mon, 16 Nov 2009 13:47:41 +0100 (CET) Subject: [pypy-svn] r69320 - codespeed/pyspeed/codespeed Message-ID: <20091116124741.83B83168008@codespeak.net> Author: tobami Date: Mon Nov 16 13:47:41 2009 New Revision: 69320 Modified: codespeed/pyspeed/codespeed/tests.py Log: made tests use reverse for url path Modified: codespeed/pyspeed/codespeed/tests.py ============================================================================== --- codespeed/pyspeed/codespeed/tests.py (original) +++ codespeed/pyspeed/codespeed/tests.py Mon Nov 16 13:47:41 2009 @@ -3,10 +3,11 @@ from datetime import datetime from django.test.client import Client from pyspeed.codespeed.models import Benchmark, Revision, Interpreter, Environment, Result +from django.core.urlresolvers import reverse class AddResultTest(TestCase): def setUp(self): - self.path = '/result/add/' + self.path = reverse('pyspeed.codespeed.views.addresult') self.client = Client() self.e = Environment(name='Dual Core Linux', cpu='Core 2 Duo 8200') self.e.save() From cfbolz at codespeak.net Mon Nov 16 13:52:23 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 16 Nov 2009 13:52:23 +0100 (CET) Subject: [pypy-svn] r69321 - in pypy/trunk/pypy/rlib: . test Message-ID: <20091116125223.C1BB1168008@codespeak.net> Author: cfbolz Date: Mon Nov 16 13:52:23 2009 New Revision: 69321 Modified: pypy/trunk/pypy/rlib/jit.py pypy/trunk/pypy/rlib/test/test_jit.py Log: be more careful about names Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Mon Nov 16 13:52:23 2009 @@ -23,6 +23,7 @@ import inspect purefunction(func) args, varargs, varkw, defaults = inspect.getargspec(func) + args = ["v%s" % (i, ) for i in range(len(args))] assert varargs is None and varkw is None assert not defaults argstring = ", ".join(args) Modified: pypy/trunk/pypy/rlib/test/test_jit.py ============================================================================== --- pypy/trunk/pypy/rlib/test/test_jit.py (original) +++ pypy/trunk/pypy/rlib/test/test_jit.py Mon Nov 16 13:52:23 2009 @@ -25,8 +25,8 @@ def test_purefunction_promote(self): @purefunction_promote - def g(x): - return x + 1 + def g(func): + return func + 1 def f(x): return g(x * 2) res = self.interpret(f, [2]) From pedronis at codespeak.net Mon Nov 16 14:08:54 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 16 Nov 2009 14:08:54 +0100 (CET) Subject: [pypy-svn] r69322 - pypy/extradoc/planning Message-ID: <20091116130854.0C11C168006@codespeak.net> Author: pedronis Date: Mon Nov 16 14:08:54 2009 New Revision: 69322 Modified: pypy/extradoc/planning/jit.txt Log: moved into tasks now Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 16 14:08:54 2009 @@ -33,6 +33,7 @@ - put the class into the structure to get only one promote when using an instance + - look into failing pypy-c-jit apptests, pypy-c-jit translate.py - improve test running, compile only once @@ -68,10 +69,6 @@ - virtualizables are not finished: fix 'test_external_read_sometimes' -- save space on all the guard operations (see resume.py): - - think of ways to share information about virtuals - - backend: - speed of backend? From pedronis at codespeak.net Mon Nov 16 14:10:20 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 16 Nov 2009 14:10:20 +0100 (CET) Subject: [pypy-svn] r69323 - pypy/extradoc/planning Message-ID: <20091116131020.7C2C4168008@codespeak.net> Author: pedronis Date: Mon Nov 16 14:10:19 2009 New Revision: 69323 Modified: pypy/extradoc/planning/jit.txt Log: something about tracing (in the python sense) Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 16 14:10:19 2009 @@ -38,6 +38,7 @@ - improve test running, compile only once +- we would probably enabling tracing to leave the jit instead of just being ignored META ----- From afa at codespeak.net Mon Nov 16 15:27:07 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 16 Nov 2009 15:27:07 +0100 (CET) Subject: [pypy-svn] r69324 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091116142707.42493168008@codespeak.net> Author: afa Date: Mon Nov 16 15:27:05 2009 New Revision: 69324 Added: pypy/trunk/pypy/module/oracle/conftest.py (contents, props changed) pypy/trunk/pypy/module/oracle/test/__init__.py (contents, props changed) Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py pypy/trunk/pypy/module/oracle/test/test_cursor.py pypy/trunk/pypy/module/oracle/test/test_select.py Log: Add command line options to run oracle tests: Oracle module options: --oracle-home=ORACLE_HOME Home directory of Oracle client installation --oracle-connect=ORACLE_CONNECT connect string (user/pwd at db) used for tests Added: pypy/trunk/pypy/module/oracle/conftest.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/conftest.py Mon Nov 16 15:27:05 2009 @@ -0,0 +1,11 @@ +from pypy.rpython.tool.rffi_platform import CompilationError +import py +import os + +def pytest_addoption(parser): + group = parser.getgroup("Oracle module options") + group.addoption('--oracle-home', dest="oracle_home", + help="Home directory of Oracle client installation", + default=os.environ.get("ORACLE_HOME")) + group.addoption('--oracle-connect', dest="oracle_connect", + help="connect string (user/pwd at db) used for tests") Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Mon Nov 16 15:27:05 2009 @@ -308,8 +308,8 @@ # call stored procedure cursor._call(space, "dbms_utility.db_version", - None, [space.wrap(versionVar), - space.wrap(compatVar)]) + None, space.newlist([space.wrap(versionVar), + space.wrap(compatVar)])) # retrieve value self.w_version = versionVar.getValue(space, 0) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Mon Nov 16 15:27:05 2009 @@ -1,12 +1,13 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem import rffi, lltype +from pypy.conftest import option import os import py -try: - ORACLE_HOME = py.path.local(os.environ['ORACLE_HOME']) -except KeyError: +if option.oracle_home: + ORACLE_HOME = py.path.local(option.oracle_home) +else: raise ImportError( "Please set ORACLE_HOME to the root of an Oracle client installation") Added: pypy/trunk/pypy/module/oracle/test/__init__.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/__init__.py Mon Nov 16 15:27:05 2009 @@ -0,0 +1 @@ +# empty Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Mon Nov 16 15:27:05 2009 @@ -1,37 +1,51 @@ from pypy.conftest import gettestobjspace -import py - +from pypy.conftest import option from pypy.rpython.tool.rffi_platform import CompilationError -try: - from pypy.module.oracle import roci -except (CompilationError, ImportError): - py.test.skip("Oracle client not available") +import py -class AppTestConnection: +class OracleTestBase(object): + @classmethod def setup_class(cls): + try: + from pypy.module.oracle import roci + except ImportError: + py.test.skip("Oracle client not available") + space = gettestobjspace(usemodules=('oracle',)) cls.space = space space.setitem(space.builtin.w_dict, space.wrap('oracle'), space.getbuiltinmodule('cx_Oracle')) - cls.w_username = space.wrap('cx_oracle') - cls.w_password = space.wrap('dev') - cls.w_tnsentry = space.wrap('xe') + oracle_connect = option.oracle_connect + if not oracle_connect: + py.test.skip( + "Please set --oracle-connect to a valid connect string") + usrpwd, tnsentry = oracle_connect.rsplit('@', 1) + username, password = usrpwd.split('/', 1) + cls.w_username = space.wrap(username) + cls.w_password = space.wrap(password) + cls.w_tnsentry = space.wrap(tnsentry) + +class AppTestConnection(OracleTestBase): + + def teardown_method(self, func): + if hasattr(self, 'cnx'): + self.cnx.close() def test_connect(self): - cnx = oracle.connect(self.username, self.password, - self.tnsentry, threaded=True) - assert cnx.username == self.username - assert cnx.password == self.password - assert cnx.tnsentry == self.tnsentry - assert isinstance(cnx.version, str) + self.cnx = oracle.connect(self.username, self.password, + self.tnsentry, threaded=True) + assert self.cnx.username == self.username + assert self.cnx.password == self.password + assert self.cnx.tnsentry == self.tnsentry + assert isinstance(self.cnx.version, str) def test_singleArg(self): - cnx = oracle.connect("%s/%s@%s" % (self.username, self.password, - self.tnsentry)) - assert cnx.username == self.username - assert cnx.password == self.password - assert cnx.tnsentry == self.tnsentry + self.cnx = oracle.connect("%s/%s@%s" % (self.username, self.password, + self.tnsentry)) + assert self.cnx.username == self.username + assert self.cnx.password == self.password + assert self.cnx.tnsentry == self.tnsentry def test_connect_badPassword(self): raises(oracle.DatabaseError, oracle.connect, @@ -59,9 +73,9 @@ assert result == formatString % args def test_rollbackOnClose(self): - connection = oracle.connect(self.username, self.password, + self.cnx = oracle.connect(self.username, self.password, self.tnsentry) - cursor = connection.cursor() + cursor = self.cnx.cursor() try: cursor.execute("drop table pypy_test_temp") except oracle.DatabaseError: Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Mon Nov 16 15:27:05 2009 @@ -1,29 +1,20 @@ -from pypy.conftest import gettestobjspace -import py +from pypy.module.oracle.test.test_connect import OracleTestBase -from pypy.rpython.tool.rffi_platform import CompilationError -try: - from pypy.module.oracle import roci -except (CompilationError, ImportError): - py.test.skip("Oracle client not available") - -class AppTestCursor: +class AppTestCursor(OracleTestBase): + @classmethod def setup_class(cls): - space = gettestobjspace(usemodules=('oracle',)) - cls.space = space - space.setitem(space.builtin.w_dict, space.wrap('oracle'), - space.getbuiltinmodule('cx_Oracle')) - cls.w_username = space.wrap('cx_oracle') - cls.w_password = space.wrap('dev') - cls.w_tnsentry = space.wrap('') - cls.w_cnx = space.appexec( + super(AppTestCursor, cls).setup_class() + cls.w_cnx = cls.space.appexec( [cls.w_username, cls.w_password, cls.w_tnsentry], """(username, password, tnsentry): import cx_Oracle return cx_Oracle.connect(username, password, tnsentry) """) + def teardown_class(cls): + cls.space.call_method(cls.w_cnx, "close") + def test_bindNames(self): cur = self.cnx.cursor() raises(oracle.ProgrammingError, cur.bindnames) Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Mon Nov 16 15:27:05 2009 @@ -1,27 +1,20 @@ -from pypy.conftest import gettestobjspace -import py +from pypy.module.oracle.test.test_connect import OracleTestBase -from pypy.rpython.tool.rffi_platform import CompilationError -try: - from pypy.module.oracle import roci -except (CompilationError, ImportError): - py.test.skip("Oracle client not available") - -class AppTestSelect: +class AppTestSelect(OracleTestBase): + @classmethod def setup_class(cls): - space = gettestobjspace(usemodules=('oracle',)) - cls.space = space - cls.w_username = space.wrap('cx_oracle') - cls.w_password = space.wrap('dev') - cls.w_tnsentry = space.wrap('') - cls.w_cnx = space.appexec( + super(AppTestSelect, cls).setup_class() + cls.w_cnx = cls.space.appexec( [cls.w_username, cls.w_password, cls.w_tnsentry], """(username, password, tnsentry): import cx_Oracle return cx_Oracle.connect(username, password, tnsentry) """) + def teardown_class(cls): + cls.space.call_method(cls.w_cnx, "close") + def test_fetchone(self): cur = self.cnx.cursor() cur.execute("select 42, 'Hello' from dual") From antocuni at codespeak.net Mon Nov 16 16:10:09 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 16 Nov 2009 16:10:09 +0100 (CET) Subject: [pypy-svn] r69327 - pypy/trunk/pypy/translator/jvm Message-ID: <20091116151009.BCC9B168008@codespeak.net> Author: antocuni Date: Mon Nov 16 16:10:08 2009 New Revision: 69327 Modified: pypy/trunk/pypy/translator/jvm/metavm.py Log: fix test_objectmodel.test_rtype_r_dict_bm, which was broken by the change of signature of oonewcustomdict in 69305 Modified: pypy/trunk/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/metavm.py (original) +++ pypy/trunk/pypy/translator/jvm/metavm.py Mon Nov 16 16:10:08 2009 @@ -78,7 +78,7 @@ # object, using the "bind()" static method that bound # method wrapper classes have. INSTANCE = obj.concretetype - method_name = method_name.value + method_name = method_name.value._str ty = db.record_delegate_bound_method_impl(INSTANCE, method_name) gen.load(obj) gen.emit(ty.bind_method) From fijal at codespeak.net Tue Nov 17 08:51:17 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 17 Nov 2009 08:51:17 +0100 (CET) Subject: [pypy-svn] r69329 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091117075117.0C079168048@codespeak.net> Author: fijal Date: Tue Nov 17 08:51:16 2009 New Revision: 69329 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: A test and a fix, hopefully last Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Tue Nov 17 08:51:16 2009 @@ -331,6 +331,18 @@ self.args_w = [args_w[0], args_w[1]] descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] + # since we rebind args_w, we need special reduce, grump + def descr_reduce(self, space): + if not space.is_w(self.w_filename, space.w_None): + lst = [self.getclass(space), space.newtuple( + self.args_w + [self.w_filename])] + else: + lst = [self.getclass(space), space.newtuple(self.args_w)] + if self.w_dict is not None and space.is_true(self.w_dict): + lst = lst + [self.w_dict] + return space.newtuple(lst) + descr_reduce.unwrap_spec = ['self', ObjSpace] + def descr_str(self, space): if (not space.is_w(self.w_errno, space.w_None) and not space.is_w(self.w_strerror, space.w_None)): @@ -351,6 +363,7 @@ __doc__ = W_EnvironmentError.__doc__, __module__ = 'exceptions', __new__ = _new(W_EnvironmentError), + __reduce__ = interp2app(W_EnvironmentError.descr_reduce), __init__ = interp2app(W_EnvironmentError.descr_init), __str__ = interp2app(W_EnvironmentError.descr_str), errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Tue Nov 17 08:51:16 2009 @@ -198,12 +198,14 @@ assert e.__module__ == 'exceptions', e def test_reduce(self): - from exceptions import LookupError + from exceptions import LookupError, EnvironmentError le = LookupError(1, 2, "a") assert le.__reduce__() == (LookupError, (1, 2, "a")) le.xyz = (1, 2) assert le.__reduce__() == (LookupError, (1, 2, "a"), {"xyz": (1, 2)}) + ee = EnvironmentError(1, 2, "a") + assert ee.__reduce__() == (EnvironmentError, (1, 2, "a")) def test_setstate(self): from exceptions import FutureWarning @@ -214,3 +216,4 @@ fw.__setstate__({'z': 1}) assert fw.z == 1 assert fw.xyz == (1, 2) + From fijal at codespeak.net Tue Nov 17 08:56:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 17 Nov 2009 08:56:12 +0100 (CET) Subject: [pypy-svn] r69330 - in pypy/trunk/pypy: config translator/backendopt translator/backendopt/test Message-ID: <20091117075612.B5D4C318139@codespeak.net> Author: fijal Date: Tue Nov 17 08:56:11 2009 New Revision: 69330 Added: pypy/trunk/pypy/translator/backendopt/storesink.py (contents, props changed) pypy/trunk/pypy/translator/backendopt/test/test_storesink.py (contents, props changed) Modified: pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/translator/backendopt/all.py Log: implement extremely dummy store sinking (well, for starters reading only) Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Nov 17 08:56:11 2009 @@ -246,7 +246,7 @@ "Tranform graphs in SSI form into graphs tailored for " "stack based virtual machines (only for backends that support it)", default=True), - + BoolOption("storesink", "Perform store sinking", default=False), BoolOption("none", "Do not run any backend optimizations", requires=[('translation.backendopt.inline', False), Modified: pypy/trunk/pypy/translator/backendopt/all.py ============================================================================== --- pypy/trunk/pypy/translator/backendopt/all.py (original) +++ pypy/trunk/pypy/translator/backendopt/all.py Tue Nov 17 08:56:11 2009 @@ -10,6 +10,7 @@ from pypy.translator.backendopt.removeassert import remove_asserts from pypy.translator.backendopt.support import log from pypy.translator.backendopt.checkvirtual import check_virtual_methods +from pypy.translator.backendopt.storesink import storesink_graph from pypy.objspace.flow.model import checkgraph INLINE_THRESHOLD_FOR_TEST = 33 @@ -104,6 +105,9 @@ print "after clever inlining and malloc removal" print_statistics(translator.graphs[0], translator) + if config.storesink: + for graph in graphs: + storesink_graph(graph) if config.profile_based_inline and not secondary: threshold = config.profile_based_inline_threshold Added: pypy/trunk/pypy/translator/backendopt/storesink.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/backendopt/storesink.py Tue Nov 17 08:56:11 2009 @@ -0,0 +1,49 @@ + +from pypy.rpython.lltypesystem.lloperation import llop + +def has_side_effects(op): + if op.opname == 'debug_assert': + return False + try: + return getattr(llop, op.opname).sideeffects + except AttributeError: + return True + +def storesink_graph(graph): + def rename(op, renaming): + for i, arg in enumerate(op.args): + r = renaming.get(arg, None) + if r is not None: + op.args[i] = r + return op + + def clear_cache_for(cache, concretetype, fieldname): + for k in cache.keys(): + if k[0].concretetype == concretetype and k[1] == fieldname: + del cache[k] + + for block in graph.iterblocks(): + newops = [] + cache = {} + renaming = {} + for op in block.operations: + if op.opname == 'getfield': + tup = (op.args[0], op.args[1].value) + res = cache.get(tup, None) + if res is not None: + renaming[op.result] = res + continue + cache[tup] = op.result + elif op.opname in ['setarrayitem', 'setinteriorfield']: + pass + elif op.opname == 'setfield': + clear_cache_for(cache, op.args[0].concretetype, + op.args[1].value) + elif has_side_effects(op): + cache = {} + newops.append(rename(op, renaming)) + if block.operations: + block.operations = newops + for exit in block.exits: + rename(exit, renaming) + Added: pypy/trunk/pypy/translator/backendopt/test/test_storesink.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/backendopt/test/test_storesink.py Tue Nov 17 08:56:11 2009 @@ -0,0 +1,121 @@ + +import py +from pypy.translator.translator import TranslationContext, graphof +from pypy.translator.backendopt.storesink import storesink_graph +from pypy.objspace.flow.model import last_exception, checkgraph +from pypy.conftest import option + +class TestStoreSink(object): + # not sure if it makes any sense on ootype, maybe worth trying + type_system = 'lltype' + + def translate(self, func, argtypes): + t = TranslationContext() + t.buildannotator().build_types(func, argtypes) + t.buildrtyper(type_system=self.type_system).specialize() + return t + + def check(self, f, argtypes, no_getfields=0): + t = self.translate(f, argtypes) + getfields = 0 + graph = graphof(t, f) + checkgraph(graph) + storesink_graph(graph) + checkgraph(graph) + if option.view: + t.view() + for block in graph.iterblocks(): + for op in block.operations: + if op.opname == 'getfield': + getfields += 1 + if no_getfields != getfields: + py.test.fail("Expected %d, got %d getfields" % + (no_getfields, getfields)) + + def test_infrastructure(self): + class A(object): + pass + + def f(i): + a = A() + a.x = i + return a.x + + self.check(f, [int], 1) + + def test_simple(self): + class A(object): + pass + + def f(i): + a = A() + a.x = i + return a.x + a.x + + self.check(f, [int], 1) + + def test_irrelevant_setfield(self): + class A(object): + pass + + def f(i): + a = A() + a.x = i + one = a.x + a.y = 3 + two = a.x + return one + two + + self.check(f, [int], 1) + + def test_relevant_setfield(self): + class A(object): + pass + + def f(i): + a = A() + b = A() + a.x = i + b.x = i + 1 + one = a.x + b.x = i + two = a.x + return one + two + + self.check(f, [int], 2) + + def test_different_concretetype(self): + class A(object): + pass + + class B(object): + pass + + def f(i): + a = A() + b = B() + a.x = i + one = a.x + b.x = i + 1 + two = a.x + return one + two + + self.check(f, [int], 1) + + def test_subclass(self): + class A(object): + pass + + class B(A): + pass + + def f(i): + a = A() + b = B() + a.x = i + one = a.x + b.x = i + 1 + two = a.x + return one + two + + self.check(f, [int], 2) From fijal at codespeak.net Tue Nov 17 08:59:19 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 17 Nov 2009 08:59:19 +0100 (CET) Subject: [pypy-svn] r69331 - in pypy/trunk/pypy/rpython/memory/gctransform: . test Message-ID: <20091117075919.8606C168048@codespeak.net> Author: fijal Date: Tue Nov 17 08:59:18 2009 New Revision: 69331 Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Log: Implement write barrier shortcut for the case: x = l[i] l[j] = x In which case we don't need a write barrier. To work correctly with lists (as opposed to just GcArrays), it requires storesinking on. Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Nov 17 08:59:18 2009 @@ -52,7 +52,7 @@ # a bit of a hackish analysis: if a block contains a malloc and check that # the result is not zero, then the block following the True link will # usually initialize the newly allocated object - result = {} + result = set() def find_in_block(block, mallocvars): for i, op in enumerate(block.operations): if op.opname in ("cast_pointer", "same_as"): @@ -63,7 +63,7 @@ if (op.args[0] in mallocvars and isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == "gc"): - result[op] = True + result.add(op) else: if collect_analyzer.analyze(op): return @@ -110,6 +110,20 @@ # print "found %s initializing stores in %s" % (len(result), graph.name) return result +def find_clean_setarrayitems(collect_analyzer, graph): + result = set() + for block in graph.iterblocks(): + cache = set() + for op in block.operations: + if op.opname == 'getarrayitem': + cache.add((op.args[0], op.result)) + elif op.opname == 'setarrayitem': + if (op.args[0], op.args[2]) in cache: + result.add(op) + elif collect_analyzer.analyze(op): + cache = set() + return result + class FrameworkGCTransformer(GCTransformer): use_stackless = False root_stack_depth = 163840 @@ -544,11 +558,12 @@ def transform_graph(self, graph): if self.write_barrier_ptr: - self.initializing_stores = find_initializing_stores( - self.collect_analyzer, graph) + self.clean_sets = ( + find_clean_setarrayitems(self.collect_analyzer, graph).union( + find_initializing_stores(self.collect_analyzer, graph))) super(FrameworkGCTransformer, self).transform_graph(graph) if self.write_barrier_ptr: - self.initializing_stores = None + self.clean_sets = None def gct_direct_call(self, hop): if self.collect_analyzer.analyze(hop.spaceop): @@ -879,7 +894,7 @@ if (self.write_barrier_ptr is not None and not isinstance(v_newvalue, Constant) and v_struct.concretetype.TO._gckind == "gc" - and hop.spaceop not in self.initializing_stores): + and hop.spaceop not in self.clean_sets): self.write_barrier_calls += 1 v_newvalue = hop.genop("cast_ptr_to_adr", [v_newvalue], resulttype = llmemory.Address) Modified: pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/test/test_framework.py Tue Nov 17 08:59:18 2009 @@ -5,13 +5,14 @@ rtype_and_transform from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, \ - CollectAnalyzer, find_initializing_stores + CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems from pypy.rpython.lltypesystem import lltype from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof from pypy.translator.unsimplify import varoftype from pypy.translator.exceptiontransform import ExceptionTransformer +from pypy.translator.backendopt.all import backend_optimizations from pypy import conftest import py @@ -87,7 +88,7 @@ assert can_collect class WriteBarrierTransformer(FrameworkGCTransformer): - initializing_stores = {} + clean_sets = {} GC_PARAMS = {} class GCClass(MarkSweepGC): needs_write_barrier = True @@ -188,3 +189,88 @@ collect_analyzer = CollectAnalyzer(t) init_stores = find_initializing_stores(collect_analyzer, t.graphs[0]) assert len(init_stores) == 5 + +def test_find_clean_setarrayitems(): + S = lltype.GcStruct('S') + A = lltype.GcArray(lltype.Ptr(S)) + + def f(): + l = lltype.malloc(A, 3) + l[0] = lltype.malloc(S) + l[1] = lltype.malloc(S) + l[2] = lltype.malloc(S) + x = l[1] + l[0] = x + return len(l) + + t = rtype(f, []) + etrafo = ExceptionTransformer(t) + graph = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, + t.graphs[0]) + assert len(clean_setarrayitems) == 1 + +def test_find_clean_setarrayitems_2(): + S = lltype.GcStruct('S') + A = lltype.GcArray(lltype.Ptr(S)) + + def f(): + l = lltype.malloc(A, 3) + l[0] = lltype.malloc(S) + l[1] = lltype.malloc(S) + l[2] = lltype.malloc(S) + x = l[1] + l[2] = lltype.malloc(S) # <- this can possibly collect + l[0] = x + return len(l) + + t = rtype(f, []) + etrafo = ExceptionTransformer(t) + graph = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, + t.graphs[0]) + assert len(clean_setarrayitems) == 0 + +def test_find_clean_setarrayitems_3(): + S = lltype.GcStruct('S') + A = lltype.GcArray(lltype.Ptr(S)) + + def f(): + l = lltype.malloc(A, 3) + l[0] = lltype.malloc(S) + l[1] = lltype.malloc(S) + l[2] = lltype.malloc(S) + l2 = lltype.malloc(A, 4) + x = l[1] + l2[0] = x # <- different list + return len(l) + + t = rtype(f, []) + etrafo = ExceptionTransformer(t) + graph = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, + t.graphs[0]) + assert len(clean_setarrayitems) == 0 + +def test_list_operations(): + + class A(object): + pass + + def f(): + l = [A(), A()] + l.append(A()) + l[1] = l[0] + return len(l) + + t = rtype(f, []) + backend_optimizations(t, clever_malloc_removal=False, storesink=True) + etrafo = ExceptionTransformer(t) + graph = etrafo.transform_completely() + collect_analyzer = CollectAnalyzer(t) + clean_setarrayitems = find_clean_setarrayitems(collect_analyzer, + t.graphs[0]) + assert len(clean_setarrayitems) == 1 From arigo at codespeak.net Tue Nov 17 11:16:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 11:16:48 +0100 (CET) Subject: [pypy-svn] r69335 - pypy/trunk/pypy/config Message-ID: <20091117101648.90A89168048@codespeak.net> Author: arigo Date: Tue Nov 17 11:16:48 2009 New Revision: 69335 Modified: pypy/trunk/pypy/config/translationoption.py Log: As expected, PyPy translates and seems to run fine with both --jit and --thread. Remove the disabling. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Nov 17 11:16:48 2009 @@ -98,7 +98,6 @@ # JIT generation: use -Ojit to enable it BoolOption("jit", "generate a JIT", default=False, - requires=[("translation.thread", False)], suggests=[("translation.gc", "hybrid"), # or "boehm" ("translation.gcrootfinder", "asmgcc"), ("translation.list_comprehension_operations", True)]), From cfbolz at codespeak.net Tue Nov 17 13:47:31 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Nov 2009 13:47:31 +0100 (CET) Subject: [pypy-svn] r69341 - pypy/trunk/pypy/lang/prolog/interpreter Message-ID: <20091117124731.7C582168048@codespeak.net> Author: cfbolz Date: Tue Nov 17 13:47:31 2009 New Revision: 69341 Modified: pypy/trunk/pypy/lang/prolog/interpreter/term.py Log: remnants from the old jit: use a for-loop Modified: pypy/trunk/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/trunk/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/trunk/pypy/lang/prolog/interpreter/term.py Tue Nov 17 13:47:31 2009 @@ -449,11 +449,9 @@ return intmask(compute_hash(self.signature) << TAGBITS | self.TAG) def unify_hash_of_children(self, heap): - unify_hash = [] - i = 0 - while i < len(self.args): - unify_hash.append(self.args[i].get_unify_hash(heap)) - i += 1 + unify_hash = [0] * len(self.args) + for i in range(len(self.args)): + unify_hash[i] = self.args[i].get_unify_hash(heap) return unify_hash def get_prolog_signature(self): From tobami at codespeak.net Tue Nov 17 14:03:06 2009 From: tobami at codespeak.net (tobami at codespeak.net) Date: Tue, 17 Nov 2009 14:03:06 +0100 (CET) Subject: [pypy-svn] r69342 - codespeed/pyspeed/codespeed Message-ID: <20091117130306.2821C31813D@codespeak.net> Author: tobami Date: Tue Nov 17 14:03:05 2009 New Revision: 69342 Modified: codespeed/pyspeed/codespeed/urls.py codespeed/pyspeed/codespeed/views.py Log: changed url paths and import error Modified: codespeed/pyspeed/codespeed/urls.py ============================================================================== --- codespeed/pyspeed/codespeed/urls.py (original) +++ codespeed/pyspeed/codespeed/urls.py Tue Nov 17 14:03:05 2009 @@ -30,8 +30,8 @@ urlpatterns = patterns('pyspeed.codespeed.views', #(r'^$', 'index'), - (r'^pypy/result/$', list_detail.object_list, result_list), - (r'^pypy/revision/$', list_detail.object_list, revision_list), + (r'^result/$', list_detail.object_list, result_list), + (r'^revision/$', list_detail.object_list, revision_list), # URL interface for adding results (r'^result/add/$', 'addresult'), ) Modified: codespeed/pyspeed/codespeed/views.py ============================================================================== --- codespeed/pyspeed/codespeed/views.py (original) +++ codespeed/pyspeed/codespeed/views.py Tue Nov 17 14:03:05 2009 @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.shortcuts import get_object_or_404 from pyspeed.codespeed.models import Revision, Result, Interpreter, Benchmark, Environment -from django.http import HttpResponse, Http404, HttpResponseBadRequest, HttpResponseNotFound +from django.http import HttpResponse, Http404, HttpResponseNotAllowed, HttpResponseBadRequest, HttpResponseNotFound def addresult(request): if request.method != 'POST': From arigo at codespeak.net Tue Nov 17 14:22:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 14:22:43 +0100 (CET) Subject: [pypy-svn] r69343 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091117132243.E6DA2318139@codespeak.net> Author: arigo Date: Tue Nov 17 14:22:43 2009 New Revision: 69343 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Store unwrapped attributes. Helps fix errors shown by test_codeccallbacks.py. Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Tue Nov 17 14:22:43 2009 @@ -558,18 +558,25 @@ class W_UnicodeDecodeError(W_UnicodeError): """Unicode decoding error.""" - encoding = '' - object = '' - start = 0 - end = 0 - reason = '' + w_encoding = None + w_object = None + w_start = None + w_end = None + w_reason = None def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): - self.encoding = space.str_w(w_encoding) - self.object = space.str_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.str_w(w_encoding) + space.str_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.str_w(w_reason) + # assign attributes + self.w_encoding = w_encoding + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_encoding, w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, @@ -594,11 +601,11 @@ __new__ = _new(W_UnicodeDecodeError), __init__ = interp2app(W_UnicodeDecodeError.descr_init), __str__ = interp2app(W_UnicodeDecodeError.descr_str), - encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), - object = readwrite_attrproperty('object', W_UnicodeDecodeError, 'str_w'), - start = readwrite_attrproperty('start', W_UnicodeDecodeError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeDecodeError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeDecodeError, 'str_w'), + encoding = readwrite_attrproperty_w('w_encoding', W_UnicodeDecodeError), + object = readwrite_attrproperty_w('w_object', W_UnicodeDecodeError), + start = readwrite_attrproperty_w('w_start', W_UnicodeDecodeError), + end = readwrite_attrproperty_w('w_end', W_UnicodeDecodeError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeDecodeError), ) W_TypeError = _new_exception('TypeError', W_StandardError, @@ -643,19 +650,25 @@ class W_UnicodeEncodeError(W_UnicodeError): """Unicode encoding error.""" - - encoding = '' - object = u'' - start = 0 - end = 0 - reason = '' + w_encoding = None + w_object = None + w_start = None + w_end = None + w_reason = None def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): - self.encoding = space.str_w(w_encoding) - self.object = space.unicode_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.str_w(w_encoding) + space.unicode_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.str_w(w_reason) + # assign attributes + self.w_encoding = w_encoding + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_encoding, w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, @@ -686,9 +699,9 @@ __new__ = _new(W_UnicodeEncodeError), __init__ = interp2app(W_UnicodeEncodeError.descr_init), __str__ = interp2app(W_UnicodeEncodeError.descr_str), - encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), - object = readwrite_attrproperty('object', W_UnicodeEncodeError, 'unicode_w'), - start = readwrite_attrproperty('start', W_UnicodeEncodeError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeEncodeError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeEncodeError, 'str_w'), + encoding = readwrite_attrproperty_w('w_encoding', W_UnicodeEncodeError), + object = readwrite_attrproperty_w('w_object', W_UnicodeEncodeError), + start = readwrite_attrproperty_w('w_start', W_UnicodeEncodeError), + end = readwrite_attrproperty_w('w_end', W_UnicodeEncodeError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeEncodeError), ) Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Tue Nov 17 14:22:43 2009 @@ -164,6 +164,8 @@ assert str(ue) == "'x' codec can't encode characters in position 1-4: bah" ue.end = 2 assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" + ue.object = [] + assert ue.object == [] def test_multiple_inheritance(self): from exceptions import LookupError, ValueError, Exception From pedronis at codespeak.net Tue Nov 17 14:27:35 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 17 Nov 2009 14:27:35 +0100 (CET) Subject: [pypy-svn] r69344 - pypy/extradoc/planning Message-ID: <20091117132735.56AE916804B@codespeak.net> Author: pedronis Date: Tue Nov 17 14:27:34 2009 New Revision: 69344 Modified: pypy/extradoc/planning/jit.txt Log: fix the sentence Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 17 14:27:34 2009 @@ -38,7 +38,7 @@ - improve test running, compile only once -- we would probably enabling tracing to leave the jit instead of just being ignored +- we would like probably enabling tracing to leave the jit instead of just being ignored META ----- From arigo at codespeak.net Tue Nov 17 14:33:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 14:33:14 +0100 (CET) Subject: [pypy-svn] r69345 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091117133314.D3BB5318139@codespeak.net> Author: arigo Date: Tue Nov 17 14:33:14 2009 New Revision: 69345 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Also fix UnicodeTranslateError, removing the last use of readwrite_attrproperty. Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Tue Nov 17 14:33:14 2009 @@ -78,13 +78,6 @@ from pypy.interpreter.error import OperationError from pypy.rlib import rwin32 -def readwrite_attrproperty(name, cls, unwrapname): - def fget(space, obj): - return space.wrap(getattr(obj, name)) - def fset(space, obj, w_val): - setattr(obj, name, getattr(space, unwrapname)(w_val)) - return GetSetProperty(fget, fset, cls=cls) - def readwrite_attrproperty_w(name, cls): def fget(space, obj): return getattr(obj, name) @@ -244,16 +237,22 @@ class W_UnicodeTranslateError(W_UnicodeError): """Unicode translation error.""" - object = u'' - start = 0 - end = 0 - reason = '' + object = None + start = None + end = None + reason = None def descr_init(self, space, w_object, w_start, w_end, w_reason): - self.object = space.unicode_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.unicode_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.str_w(w_reason) + # assign attributes + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, @@ -280,10 +279,10 @@ __new__ = _new(W_UnicodeTranslateError), __init__ = interp2app(W_UnicodeTranslateError.descr_init), __str__ = interp2app(W_UnicodeTranslateError.descr_str), - object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), - start = readwrite_attrproperty('start', W_UnicodeTranslateError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeTranslateError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeTranslateError, 'str_w'), + object = readwrite_attrproperty_w('w_object', W_UnicodeTranslateError), + start = readwrite_attrproperty_w('w_start', W_UnicodeTranslateError), + end = readwrite_attrproperty_w('w_end', W_UnicodeTranslateError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeTranslateError), ) W_LookupError = _new_exception('LookupError', W_StandardError, Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Tue Nov 17 14:33:14 2009 @@ -88,6 +88,8 @@ ut.start = 4 ut.object = u'012345' assert str(ut) == "can't translate character u'\\x34' in position 4: bah" + ut.object = [] + assert ut.object == [] def test_key_error(self): from exceptions import KeyError From arigo at codespeak.net Tue Nov 17 14:36:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 14:36:34 +0100 (CET) Subject: [pypy-svn] r69346 - in pypy/branch/faster-raise/pypy: interpreter module/exceptions module/exceptions/test Message-ID: <20091117133634.DEE4A318139@codespeak.net> Author: arigo Date: Tue Nov 17 14:36:34 2009 New Revision: 69346 Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Test and fix. Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Tue Nov 17 14:36:34 2009 @@ -1033,6 +1033,14 @@ buffer = self.buffer_w(w_obj) return buffer.as_str() + def realunicode_w(self, w_obj): + # Like unicode_w, but only works if w_obj is really of type + # 'unicode'. + if not self.is_true(self.isinstance(w_obj, self.w_unicode)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a unicode')) + return self.unicode_w(w_obj) + def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Tue Nov 17 14:36:34 2009 @@ -244,7 +244,7 @@ def descr_init(self, space, w_object, w_start, w_end, w_reason): # typechecking - space.unicode_w(w_object) + space.realunicode_w(w_object) space.int_w(w_start) space.int_w(w_end) space.str_w(w_reason) @@ -658,7 +658,7 @@ def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): # typechecking space.str_w(w_encoding) - space.unicode_w(w_object) + space.realunicode_w(w_object) space.int_w(w_start) space.int_w(w_end) space.str_w(w_reason) Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Tue Nov 17 14:36:34 2009 @@ -168,6 +168,7 @@ assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" ue.object = [] assert ue.object == [] + raises(TypeError, UnicodeEncodeError, "x", "y", 1, 5, "bah") def test_multiple_inheritance(self): from exceptions import LookupError, ValueError, Exception From arigo at codespeak.net Tue Nov 17 14:37:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 14:37:20 +0100 (CET) Subject: [pypy-svn] r69347 - pypy/branch/faster-raise/lib-python/modified-2.5.2/test Message-ID: <20091117133720.9DD53318139@codespeak.net> Author: arigo Date: Tue Nov 17 14:37:20 2009 New Revision: 69347 Removed: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py Log: Remove the changes done to this test. From afa at codespeak.net Tue Nov 17 14:57:56 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 14:57:56 +0100 (CET) Subject: [pypy-svn] r69348 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117135756.791E4318139@codespeak.net> Author: afa Date: Tue Nov 17 14:57:55 2009 New Revision: 69348 Added: pypy/trunk/pypy/module/oracle/test/test_numbervar.py (contents, props changed) Modified: pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py pypy/trunk/pypy/module/oracle/test/test_cursor.py pypy/trunk/pypy/module/oracle/test/test_select.py pypy/trunk/pypy/module/oracle/transform.py Log: oracle module: implement binds for float and Decimal values Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 17 14:57:55 2009 @@ -19,6 +19,12 @@ self.w_Variable = get('Variable') self.w_NUMBER = get('NUMBER') + w_import = space.builtin.get('__import__') + w_decimal = space.call(w_import, space.newlist( + [space.wrap('decimal')])) + self.w_DecimalType = space.getattr(w_decimal, space.wrap("Decimal")) + + def get(space): return space.fromcache(State) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 17 14:57:55 2009 @@ -485,6 +485,35 @@ finally: lltype.free(integerValuePtr, flavor='raw') return + elif space.is_true(space.isinstance(w_value, space.w_float)): + doubleValuePtr = lltype.malloc(roci.Ptr(lltype.Float).TO, 1, + flavor='raw') + try: + doubleValuePtr[0] = space.float_w(w_value) + status = roci.OCINumberFromReal( + self.environment.errorHandle, + rffi.cast(roci.dvoidp, doubleValuePtr), + rffi.sizeof(lltype.Float), + dataptr) + self.environment.checkForError( + status, "NumberVar_SetValue(): from float") + finally: + lltype.free(doubleValuePtr, flavor='raw') + return + elif space.is_true(space.isinstance(w_value, get(space).w_DecimalType)): + w_text, w_format = transform.DecimalToFormatAndText(self.environment, w_value) + text_buf = config.StringBuffer() + text_buf.fill(space, w_text) + format_buf = config.StringBuffer() + format_buf.fill(space, w_format) + status = roci.OCINumberFromText( + self.environment.errorHandle, + text_buf.ptr, text_buf.size, + format_buf.ptr, format_buf.size, + None, 0, dataptr) + self.environment.checkForError( + status, "NumberVar_SetValue(): from decimal") + return raise OperationError( space.w_TypeError, space.wrap("expecting numeric data")) @@ -659,6 +688,26 @@ if space.is_true(space.isinstance(w_value, space.w_int)): return VT_Integer, 0, numElements + # XXX long + + if space.is_true(space.isinstance(w_value, space.w_float)): + return VT_Float, 0, numElements + + # XXX cxBinary + + # XXX bool + + # XXX datetime + + # XXX date + + # XXX Delta + + # XXX cursorType + + if space.is_true(space.isinstance(w_value, get(space).w_DecimalType)): + return VT_NumberAsString, 0, numElements + raise OperationError( moduledict.w_NotSupportedError, space.wrap("Variable_TypeByValue(): unhandled data type %s" % Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Tue Nov 17 14:57:55 2009 @@ -5,8 +5,10 @@ import os import py -if option.oracle_home: - ORACLE_HOME = py.path.local(option.oracle_home) +oracle_home = getattr(option, 'oracle_home', + os.environ.get("ORACLE_HOME")) +if oracle_home: + ORACLE_HOME = py.path.local(oracle_home) else: raise ImportError( "Please set ORACLE_HOME to the root of an Oracle client installation") @@ -341,6 +343,26 @@ Ptr(OCINumber)], # number sword) +OCINumberFromReal = external( + 'OCINumberFromReal', + [OCIError, # err + dvoidp, # rnum + uword, # rnum_length + Ptr(OCINumber)], # number + sword) + +OCINumberFromText = external( + 'OCINumberFromText', + [OCIError, # err + oratext, # str + ub4, # str_length + oratext, # fmt + ub4, # fmt_length + oratext, # nls_params + ub4, # nls_p_length + Ptr(OCINumber)], # number + sword) + OCINumberToInt = external( 'OCINumberToInt', [OCIError, # err Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Tue Nov 17 14:57:55 2009 @@ -3,7 +3,7 @@ from pypy.rpython.tool.rffi_platform import CompilationError import py -class OracleTestBase(object): +class OracleNotConnectedTestBase(object): @classmethod def setup_class(cls): @@ -26,7 +26,21 @@ cls.w_password = space.wrap(password) cls.w_tnsentry = space.wrap(tnsentry) -class AppTestConnection(OracleTestBase): +class OracleTestBase(OracleNotConnectedTestBase): + @classmethod + def setup_class(cls): + super(OracleTestBase, cls).setup_class() + cls.w_cnx = cls.space.appexec( + [cls.w_username, cls.w_password, cls.w_tnsentry], + """(username, password, tnsentry): + import cx_Oracle + return cx_Oracle.connect(username, password, tnsentry) + """) + + def teardown_class(cls): + cls.space.call_method(cls.w_cnx, "close") + +class AppTestConnection(OracleNotConnectedTestBase): def teardown_method(self, func): if hasattr(self, 'cnx'): Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Tue Nov 17 14:57:55 2009 @@ -2,19 +2,6 @@ class AppTestCursor(OracleTestBase): - @classmethod - def setup_class(cls): - super(AppTestCursor, cls).setup_class() - cls.w_cnx = cls.space.appexec( - [cls.w_username, cls.w_password, cls.w_tnsentry], - """(username, password, tnsentry): - import cx_Oracle - return cx_Oracle.connect(username, password, tnsentry) - """) - - def teardown_class(cls): - cls.space.call_method(cls.w_cnx, "close") - def test_bindNames(self): cur = self.cnx.cursor() raises(oracle.ProgrammingError, cur.bindnames) Added: pypy/trunk/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_numbervar.py Tue Nov 17 14:57:55 2009 @@ -0,0 +1,18 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class AppTestSelect(OracleTestBase): + + def test_float(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + cur.execute("begin :a := :b*2.5; end;", a=var, + b=0.5) + assert var.getvalue() == 1.25 + + def test_decimal(self): + import decimal + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + cur.execute("begin :a := :b*2.5; end;", a=var, + b=decimal.Decimal("0.5")) + assert var.getvalue() == 1.25 Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Tue Nov 17 14:57:55 2009 @@ -2,19 +2,6 @@ class AppTestSelect(OracleTestBase): - @classmethod - def setup_class(cls): - super(AppTestSelect, cls).setup_class() - cls.w_cnx = cls.space.appexec( - [cls.w_username, cls.w_password, cls.w_tnsentry], - """(username, password, tnsentry): - import cx_Oracle - return cx_Oracle.connect(username, password, tnsentry) - """) - - def teardown_class(cls): - cls.space.call_method(cls.w_cnx, "close") - def test_fetchone(self): cur = self.cnx.cursor() cur.execute("select 42, 'Hello' from dual") Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Tue Nov 17 14:57:55 2009 @@ -58,3 +58,39 @@ w(time.c_OCITimeMI), w(time.c_OCITimeSS)) +def DecimalToFormatAndText(environment, w_value): + space = environment.space + w_tuple_value = space.call_method(w_value, "as_tuple") + + # acquire basic information from the value tuple + w_sign, w_digits, w_scale = space.viewiterable(w_tuple_value) + + text = '' + format = '' + + digits_w = space.viewiterable(w_digits) + num_digits = len(digits_w) + scale = space.int_w(w_scale) + + # allocate memory for the string and format to use in conversion + if space.is_true(w_sign): + text += '-' + for i in xrange(0, num_digits + scale): + format += '9' + if i < numdigits: + digit = space.int_w(digits_w[i]) + text += "0123456789"[digit] + else: + text += '0' + if scale < 0: + format += 'D' + text += '.' + for i in xrange(scale, 0): + format += '9' + if num_digits + i < 0: + text += '0' + else: + digit = space.int_w(digits_w[num_digits + i]) + text += "0123456789"[digit] + + return space.wrap(text), space.wrap(format) From afa at codespeak.net Tue Nov 17 15:06:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 15:06:29 +0100 (CET) Subject: [pypy-svn] r69349 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117140629.430AB318139@codespeak.net> Author: afa Date: Tue Nov 17 15:06:28 2009 New Revision: 69349 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_select.py Log: Add cursor.rowcount Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 17 15:06:28 2009 @@ -1,8 +1,8 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.interpreter.argument import Arguments -from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w -from pypy.interpreter.typedef import GetSetProperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype @@ -779,4 +779,5 @@ next = interp2app(W_Cursor.descr_next), arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), + rowcount = interp_attrproperty('rowCount', W_Cursor), ) Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Tue Nov 17 15:06:28 2009 @@ -26,6 +26,7 @@ cur.execute("select level-1 from dual connect by level-1<42") rows = cur.fetchall() assert rows == zip(range(42)) + assert cur.rowcount == 42 def test_iterator(self): cur = self.cnx.cursor() From afa at codespeak.net Tue Nov 17 15:09:46 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 15:09:46 +0100 (CET) Subject: [pypy-svn] r69350 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117140946.B5CBD318139@codespeak.net> Author: afa Date: Tue Nov 17 15:09:46 2009 New Revision: 69350 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: add cursor.statement Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 17 15:09:46 2009 @@ -2,7 +2,7 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.typedef import interp_attrproperty +from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype @@ -780,4 +780,5 @@ arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), rowcount = interp_attrproperty('rowCount', W_Cursor), + statement = interp_attrproperty_w('w_statement', W_Cursor), ) Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Tue Nov 17 15:09:46 2009 @@ -117,3 +117,11 @@ count, = cur.fetchone() assert count == len(rows) + def test_attributes(self): + cur = self.cnx.cursor() + stmt = "select 1 from dual" + cur.execute(stmt) + assert cur.rowcount == 0 + cur.fetchall() + assert cur.rowcount == 1 + assert cur.statement == stmt From afa at codespeak.net Tue Nov 17 16:16:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 16:16:11 +0100 (CET) Subject: [pypy-svn] r69352 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117151611.A8028318139@codespeak.net> Author: afa Date: Tue Nov 17 16:16:10 2009 New Revision: 69352 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: more cursor methods: setinputsizes(), bindvars Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Tue Nov 17 16:16:10 2009 @@ -293,6 +293,8 @@ return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) def get_nationalencoding(space, self): return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) + def get_maxbytespercharacter(space, self): + return space.wrap(self.environment.maxBytesPerCharacter) def get_version(space, self): # if version has already been determined, no need to determine again @@ -335,5 +337,6 @@ encoding = GetSetProperty(W_Connection.get_encoding), nationalencoding = GetSetProperty(W_Connection.get_nationalencoding), + maxBytesPerCharacter = GetSetProperty(W_Connection.get_maxbytespercharacter), version = GetSetProperty(W_Connection.get_version), ) Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 17 16:16:10 2009 @@ -747,10 +747,60 @@ return space.wrap(var) var.unwrap_spec = ['self', ObjSpace, W_Root, int, W_Root, W_Root, W_Root] + def setinputsizes(self, space, __args__): + args_w, kw_w = __args__.unpack() + + # only expect keyword arguments or positional arguments, not both + if args_w and kw_w: + raise OperationError( + interp_error.get(space).w_InterfaceError, + space.wrap( + "expecting argument or keyword arguments, not both")) + + # make sure the cursor is open + self._checkOpen(space) + + # eliminate existing bind variables + self.bindList = None + self.bindDict = None + + self.setInputSizes = 1 + + # process each input + if kw_w: + self.bindDict = space.newdict() + for key, w_value in kw_w.iteritems(): + var = interp_variable.newVariableByType( + space, self, w_value, self.bindArraySize) + space.setitem(self.bindDict, space.wrap(key), var) + return self.bindDict + else: + self.bindList = [None] * len(args_w) + for i, w_value in enumerate(args_w): + if space.is_w(w_value, space.w_None): + var = space.w_None + else: + var = interp_variable.newVariableByType( + space, self, w_value, self.bindArraySize) + self.bindList[i] = var + return space.newlist(self.bindList) + setinputsizes.unwrap_spec = ['self', ObjSpace, Arguments] + def cursor_arraysize_get(space, obj): - return space.wrap(space.arraySize) + return space.wrap(obj.arraySize) def cursor_arraysize_set(space, obj, w_value): - space.arraySize = space.int_w(w_value) + obj.arraySize = space.int_w(w_value) + +def cursor_bindarraysize_get(space, obj): + return space.wrap(obj.bindArraySize) +def cursor_bindarraysize_set(space, obj, w_value): + obj.bindArraySize = space.int_w(w_value) + +def cursor_bindvars_get(space, obj): + if obj.bindList: + return space.newlist(obj.bindList) + if obj.bindDict: + return obj.bindDict W_Cursor.typedef = TypeDef( 'Cursor', @@ -774,11 +824,15 @@ unwrap_spec=W_Cursor.callproc.unwrap_spec), var = interp2app(W_Cursor.var, unwrap_spec=W_Cursor.var.unwrap_spec), + setinputsizes = interp2app(W_Cursor.setinputsizes, + unwrap_spec=W_Cursor.setinputsizes.unwrap_spec), __iter__ = interp2app(W_Cursor.descr_iter), next = interp2app(W_Cursor.descr_next), arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), + bindarraysize = GetSetProperty(cursor_bindarraysize_get, cursor_bindarraysize_set), rowcount = interp_attrproperty('rowCount', W_Cursor), statement = interp_attrproperty_w('w_statement', W_Cursor), + bindvars = GetSetProperty(cursor_bindvars_get) ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 17 16:16:10 2009 @@ -1,5 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.gateway import ObjSpace, W_Root from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -325,6 +326,9 @@ unwrap_spec=W_Variable.getValue.unwrap_spec), setvalue = interp2app(W_Variable.setValue, unwrap_spec=W_Variable.setValue.unwrap_spec), + + maxlength = interp_attrproperty('bufferSize', W_Variable), + ) class VT_String(W_Variable): @@ -667,6 +671,8 @@ moduledict = get(space) if space.is_w(w_type, moduledict.w_NUMBER): return VT_Float + if space.is_w(w_type, space.w_int): + return VT_Integer raise OperationError( moduledict.w_NotSupportedError, space.wrap("Variable_TypeByPythonType(): unhandled data type")) @@ -733,7 +739,7 @@ # passing an integer is assumed to be a string if space.is_true(space.isinstance(w_value, space.w_int)): size = space.int_w(w_value) - if size > MAX_STRING_CHARS: + if size > config.MAX_STRING_CHARS: varType = VT_LongString else: varType = VT_String Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Tue Nov 17 16:16:10 2009 @@ -2,6 +2,15 @@ class AppTestCursor(OracleTestBase): + def test_attributes(self): + cur = self.cnx.cursor() + stmt = "select 1 from dual" + cur.execute(stmt) + assert cur.rowcount == 0 + cur.fetchall() + assert cur.rowcount == 1 + assert cur.statement == stmt + def test_bindNames(self): cur = self.cnx.cursor() raises(oracle.ProgrammingError, cur.bindnames) @@ -117,11 +126,27 @@ count, = cur.fetchone() assert count == len(rows) - def test_attributes(self): + def test_executemany_withresize(self): cur = self.cnx.cursor() - stmt = "select 1 from dual" - cur.execute(stmt) - assert cur.rowcount == 0 - cur.fetchall() - assert cur.rowcount == 1 - assert cur.statement == stmt + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table" + "(intcol number, stringcol varchar2(10))") + rows = [ ( 1, "First" ), + ( 2, "Second" ), + ( 3, "Third" ), + ( 4, "Fourth" ), + ( 5, "Fifth" ), + ( 6, "Sixth" ), + ( 7, "Seventh" ) ] + cur.bindarraysize = 5 + cur.setinputsizes(int, 100) + sql = "insert into pypy_temp_table (intcol, stringcol) values (:1, :2)" + cur.executemany(sql, rows) + var = cur.bindvars[1] + cur.execute("select count(*) from pypy_temp_table") + count, = cur.fetchone() + assert count == len(rows) + assert var.maxlength == 100 * self.cnx.maxBytesPerCharacter From arigo at codespeak.net Tue Nov 17 16:16:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 16:16:41 +0100 (CET) Subject: [pypy-svn] r69353 - in pypy/branch/faster-raise/pypy: interpreter module/exceptions module/exceptions/test Message-ID: <20091117151641.1D00C318139@codespeak.net> Author: arigo Date: Tue Nov 17 16:16:40 2009 New Revision: 69353 Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Yet another fix for test_codeccallbacks.py. Modified: pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/baseobjspace.py Tue Nov 17 16:16:40 2009 @@ -1033,6 +1033,13 @@ buffer = self.buffer_w(w_obj) return buffer.as_str() + def realstr_w(self, w_obj): + # Like str_w, but only works if w_obj is really of type 'str'. + if not self.is_true(self.isinstance(w_obj, self.w_str)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a string')) + return self.str_w(w_obj) + def realunicode_w(self, w_obj): # Like unicode_w, but only works if w_obj is really of type # 'unicode'. Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Tue Nov 17 16:16:40 2009 @@ -247,7 +247,7 @@ space.realunicode_w(w_object) space.int_w(w_start) space.int_w(w_end) - space.str_w(w_reason) + space.realstr_w(w_reason) # assign attributes self.w_object = w_object self.w_start = w_start @@ -565,11 +565,11 @@ def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): # typechecking - space.str_w(w_encoding) - space.str_w(w_object) + space.realstr_w(w_encoding) + space.realstr_w(w_object) space.int_w(w_start) space.int_w(w_end) - space.str_w(w_reason) + space.realstr_w(w_reason) # assign attributes self.w_encoding = w_encoding self.w_object = w_object @@ -657,11 +657,11 @@ def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): # typechecking - space.str_w(w_encoding) + space.realstr_w(w_encoding) space.realunicode_w(w_object) space.int_w(w_start) space.int_w(w_end) - space.str_w(w_reason) + space.realstr_w(w_reason) # assign attributes self.w_encoding = w_encoding self.w_object = w_object Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Tue Nov 17 16:16:40 2009 @@ -169,6 +169,7 @@ ue.object = [] assert ue.object == [] raises(TypeError, UnicodeEncodeError, "x", "y", 1, 5, "bah") + raises(TypeError, UnicodeEncodeError, u"x", u"y", 1, 5, "bah") def test_multiple_inheritance(self): from exceptions import LookupError, ValueError, Exception From arigo at codespeak.net Tue Nov 17 17:07:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 17 Nov 2009 17:07:29 +0100 (CET) Subject: [pypy-svn] r69354 - in pypy/trunk/pypy/doc: jit statistic Message-ID: <20091117160729.36090318139@codespeak.net> Author: arigo Date: Tue Nov 17 17:07:28 2009 New Revision: 69354 Added: pypy/trunk/pypy/doc/jit/__init__.py (contents, props changed) pypy/trunk/pypy/doc/statistic/__init__.py (contents, props changed) Log: Add some __init__.py, as recommended by the error message. Seems to help. Added: pypy/trunk/pypy/doc/jit/__init__.py ============================================================================== Added: pypy/trunk/pypy/doc/statistic/__init__.py ============================================================================== From cfbolz at codespeak.net Tue Nov 17 18:06:55 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 17 Nov 2009 18:06:55 +0100 (CET) Subject: [pypy-svn] r69355 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091117170655.97FB131813D@codespeak.net> Author: cfbolz Date: Tue Nov 17 18:06:55 2009 New Revision: 69355 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Ignore bridges altogether for now. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Tue Nov 17 18:06:55 2009 @@ -72,7 +72,7 @@ parts = logparser.extract_category(log, 'jit-log-opt-') # skip entry bridges, they can contain random things self.loops = [parse(part, no_namespace=True) for part in parts - if "entry bridge" not in part] + if "bridge" not in part.lower()] self.sliced_loops = [] # contains all bytecodes of all loops for loop in self.loops: for op in loop.operations: From afa at codespeak.net Tue Nov 17 18:43:49 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 18:43:49 +0100 (CET) Subject: [pypy-svn] r69356 - pypy/trunk/pypy/module/oracle Message-ID: <20091117174349.A0B89318141@codespeak.net> Author: afa Date: Tue Nov 17 18:43:49 2009 New Revision: 69356 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: Expose the STRING variable type. Now all tests pass in cx_Oracle's in Cursor.py. I'm not sure that "variableTypeByTypedef" will translate though. But how can I get the interp-level class from its applevel object? Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Tue Nov 17 18:43:49 2009 @@ -7,6 +7,7 @@ 'connect': 'interp_connect.W_Connection', 'UNICODE': 'interp_variable.VT_NationalCharString', 'NUMBER': 'interp_variable.VT_Float', + 'STRING': 'interp_variable.VT_String', 'Variable': 'interp_variable.W_Variable', } Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 17 18:43:49 2009 @@ -366,6 +366,8 @@ for i, w_value in enumerate(space.viewiterable(w_vars)): if i < len(self.bindList): origVar = self.bindList[i] + if space.is_w(origVar, space.w_None): + origVar = None else: origVar = None newVar = self._setBindVariableHelper(space, w_value, origVar, Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 17 18:43:49 2009 @@ -17,7 +17,6 @@ self.w_NotSupportedError = get('NotSupportedError') self.w_IntegrityError = get('IntegrityError') self.w_Variable = get('Variable') - self.w_NUMBER = get('NUMBER') w_import = space.builtin.get('__import__') w_decimal = space.call(w_import, space.newlist( Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 17 18:43:49 2009 @@ -579,11 +579,15 @@ class VT_Object(W_Variable): pass -for cls in (VT_NationalCharString, VT_String): +variableTypeByTypedef = {} +for name, cls in globals().items(): + if not name.startswith('VT_') or not isinstance(cls, type): + continue cls.typedef = TypeDef( cls.__name__, W_Variable.typedef, ) - + variableTypeByTypedef[cls.typedef] = cls + def _typeByOracleDescriptor(param, environment): # retrieve datatype of the parameter attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, flavor='raw') @@ -669,8 +673,8 @@ def typeByPythonType(space, cursor, w_type): """Return a variable type given a Python type object""" moduledict = get(space) - if space.is_w(w_type, moduledict.w_NUMBER): - return VT_Float + if w_type.instancetypedef in variableTypeByTypedef: + return variableTypeByTypedef[w_type.instancetypedef] if space.is_w(w_type, space.w_int): return VT_Integer raise OperationError( From afa at codespeak.net Tue Nov 17 22:31:36 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 17 Nov 2009 22:31:36 +0100 (CET) Subject: [pypy-svn] r69358 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117213136.85BC6318140@codespeak.net> Author: afa Date: Tue Nov 17 22:31:34 2009 New Revision: 69358 Added: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py (contents, props changed) Modified: pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: Support of datetime.datetime in queries Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 17 22:31:34 2009 @@ -22,6 +22,9 @@ w_decimal = space.call(w_import, space.newlist( [space.wrap('decimal')])) self.w_DecimalType = space.getattr(w_decimal, space.wrap("Decimal")) + w_datetime = space.call(w_import, space.newlist( + [space.wrap('datetime')])) + self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) def get(space): Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 17 22:31:34 2009 @@ -545,6 +545,33 @@ dataptr = rffi.cast(roci.Ptr(roci.OCIDate), self.data) return transform.OracleDateToPythonDateTime(self.environment, dataptr) + def setValueProc(self, space, pos, w_value): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDate), self.data), + pos) + if space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): + year = space.int_w(space.getattr(w_value, space.wrap('year'))) + month = space.int_w(space.getattr(w_value, space.wrap('month'))) + day = space.int_w(space.getattr(w_value, space.wrap('day'))) + hour = space.int_w(space.getattr(w_value, space.wrap('hour'))) + minute = space.int_w(space.getattr(w_value, space.wrap('minute'))) + second = space.int_w(space.getattr(w_value, space.wrap('second'))) + elif space.is_true(space.isinstance(w_value, get(space).w_DateType)): + XXX + else: + raise OperationError( + space.w_TypeError, + space.wrap("expecting date data")) + + # store a copy of the value + timePart = dataptr[0].c_OCIDateTime + rffi.setintfield(timePart, 'c_OCITimeHH', hour) + rffi.setintfield(timePart, 'c_OCITimeMI', minute) + rffi.setintfield(timePart, 'c_OCITimeSS', second) + rffi.setintfield(dataptr[0], 'c_OCIDateYYYY', year) + rffi.setintfield(dataptr[0], 'c_OCIDateMM', month) + rffi.setintfield(dataptr[0], 'c_OCIDateDD', day) + class VT_Date(W_Variable): oracleType = roci.SQLT_ODT size = rffi.sizeof(roci.OCIDate) @@ -707,7 +734,8 @@ # XXX bool - # XXX datetime + if space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): + return VT_DateTime, 0, numElements # XXX date Added: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Tue Nov 17 22:31:34 2009 @@ -0,0 +1,12 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class AppTestDatetime(OracleTestBase): + + def test_bind_date(self): + import datetime + cur = self.cnx.cursor() + cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", + d=datetime.datetime(2002, 12, 13, 9, 36, 15)) + data = cur.fetchall() + assert data == [('20021213-093615',)] + From afa at codespeak.net Wed Nov 18 00:03:31 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 00:03:31 +0100 (CET) Subject: [pypy-svn] r69359 - pypy/trunk/pypy/module/oracle Message-ID: <20091117230331.D3C52318140@codespeak.net> Author: afa Date: Wed Nov 18 00:03:30 2009 New Revision: 69359 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/app_oracle.py pypy/trunk/pypy/module/oracle/transform.py Log: Expose the DATETIME variable type. Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Nov 18 00:03:30 2009 @@ -8,12 +8,14 @@ 'UNICODE': 'interp_variable.VT_NationalCharString', 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', + 'DATETIME': 'interp_variable.VT_DateTime', 'Variable': 'interp_variable.W_Variable', } appleveldefs = { 'version': 'app_oracle.version', 'makedsn': 'app_oracle.makedsn', + 'TimestampFromTicks': 'app_oracle.TimestampFromTicks', } for name in """DataError DatabaseError Error IntegrityError InterfaceError InternalError NotSupportedError OperationalError Modified: pypy/trunk/pypy/module/oracle/app_oracle.py ============================================================================== --- pypy/trunk/pypy/module/oracle/app_oracle.py (original) +++ pypy/trunk/pypy/module/oracle/app_oracle.py Wed Nov 18 00:03:30 2009 @@ -35,3 +35,7 @@ return ("(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=" "(PROTOCOL=TCP)(HOST=%s)(PORT=%s)))" "(CONNECT_DATA=(SID=%s)))" % (host, port, sid)) + +def TimestampFromTicks(*args): + import datetime + return datetime.datetime.fromtimestamp(*args) Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Wed Nov 18 00:03:30 2009 @@ -94,3 +94,4 @@ text += "0123456789"[digit] return space.wrap(text), space.wrap(format) + From afa at codespeak.net Wed Nov 18 00:27:05 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 00:27:05 +0100 (CET) Subject: [pypy-svn] r69360 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091117232705.AC0AB49843F@codespeak.net> Author: afa Date: Wed Nov 18 00:27:04 2009 New Revision: 69360 Added: pypy/trunk/pypy/module/oracle/test/test_stringvar.py (contents, props changed) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_numbervar.py Log: Implement the rowid data type Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 00:27:04 2009 @@ -424,8 +424,10 @@ class VT_FixedNationalChar(W_Variable): pass -class VT_Rowid(W_Variable): - pass +class VT_Rowid(VT_String): + oracleType = roci.SQLT_CHR + size = 18 + isVariableLength = False class VT_Binary(W_Variable): pass Modified: pypy/trunk/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_numbervar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_numbervar.py Wed Nov 18 00:27:04 2009 @@ -1,6 +1,6 @@ from pypy.module.oracle.test.test_connect import OracleTestBase -class AppTestSelect(OracleTestBase): +class AppTestNumberVar(OracleTestBase): def test_float(self): cur = self.cnx.cursor() Added: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Wed Nov 18 00:27:04 2009 @@ -0,0 +1,14 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class AppTestStringVar(OracleTestBase): + + def test_rowid(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + cur.execute("select rowid from dual") + rowid, = cur.fetchone() + cur.execute("select * from dual where rowid = :r", + r=rowid) + cur.fetchall() + assert cur.rowcount == 1 + From afa at codespeak.net Wed Nov 18 11:17:12 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 11:17:12 +0100 (CET) Subject: [pypy-svn] r69361 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118101712.996B731813D@codespeak.net> Author: afa Date: Wed Nov 18 11:17:11 2009 New Revision: 69361 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Add support for array variables Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Wed Nov 18 11:17:11 2009 @@ -393,7 +393,7 @@ items = space.viewiterable(space.call_method(w_vars, "iteritems")) for item in items: - w_key, w_value = space.unpackiterable(item) + w_key, w_value = space.viewiterable(item, 2) origVar = space.finditem(self.bindDict, w_key) newVar = self._setBindVariableHelper(space, w_value, origVar, numElements, arrayPos, defer) @@ -463,7 +463,7 @@ items = space.viewiterable( space.call_method(self.bindDict, "iteritems")) for item in items: - w_key, var = space.unpackiterable(item) + w_key, var = space.viewiterable(item, 2) var.bind(space, self, w_key, 0) # ensure that input sizes are reset Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 11:17:11 2009 @@ -117,6 +117,7 @@ class W_Variable(Wrappable): charsetForm = roci.SQLCS_IMPLICIT isVariableLength = False + canBeInArray = True def __init__(self, cursor, numElements, size=0): self.environment = cursor.environment @@ -183,6 +184,14 @@ self.data = lltype.malloc(rffi.CCHARP.TO, int(dataLength), flavor='raw', zero=True) + def makeArray(self, space): + if not self.canBeInArray: + raise OperationError( + get(space).w_NotSupportedError, + space.wrap( + "Variable_MakeArray(): type does not support arrays")) + self.isArray = True + def initialize(self, cursor): pass @@ -296,7 +305,7 @@ # ensure we do not exceed the number of allocated elements if pos >= self.allocatedElements: raise OperationError( - space.w_PyExc_IndexError, + space.w_IndexError, space.wrap("Variable_SetSingleValue: array size exceeded")) # convert value, if necessary @@ -313,10 +322,31 @@ self.setValueProc(space, pos, w_value) + def setArrayValue(self, space, w_value): + # ensure we have an array to set + if not space.is_true(space.isinstance(w_value, space.w_list)): + raise OperationError( + space.w_TypeError, + space.wrap("expecting array data")) + + elements_w = space.viewiterable(w_value) + + # ensure we haven't exceeded the number of allocated elements + if len(elements_w) > self.allocatedElements: + raise OperationError( + space.w_IndexError, + space.wrap("Variable_SetArrayValue: array size exceeded")) + + # set all of the values + self.actualElementsPtr[0] = rffi.cast(lltype.Unsigned, len(elements_w)) + for i, w_element in enumerate(elements_w): + self.setSingleValue(space, i, w_element) + def setValue(self, space, pos, w_value): if self.isArray: - self.setArrayValue(self, w_value) - self.setSingleValue(space, pos, w_value) + self.setArrayValue(space, w_value) + else: + self.setSingleValue(space, pos, w_value) setValue.unwrap_spec = ['self', ObjSpace, int, W_Root] @@ -392,6 +422,10 @@ buf = config.StringBuffer() buf.fill(space, w_value) size = buf.size + else: + raise OperationError( + space.w_TypeError, + space.wrap("expecting string or buffer data")) try: if buf.size > self.environment.maxStringBytes: @@ -603,10 +637,10 @@ pass class VT_Cursor(W_Variable): - pass + canBeInArray = False class VT_Object(W_Variable): - pass + canBeInArray = False variableTypeByTypedef = {} for name, cls in globals().items(): @@ -748,6 +782,17 @@ if space.is_true(space.isinstance(w_value, get(space).w_DecimalType)): return VT_NumberAsString, 0, numElements + # handle arrays + if space.is_true(space.isinstance(w_value, space.w_list)): + elements_w = space.viewiterable(w_value) + for w_element in elements_w: + if not space.is_w(w_element, space.w_None): + break + else: + w_element = space.w_None + varType, size, _ = typeByValue(space, w_element, numElements) + return varType, size, len(elements_w) + raise OperationError( moduledict.w_NotSupportedError, space.wrap("Variable_TypeByValue(): unhandled data type %s" % @@ -766,9 +811,21 @@ varType, size, numElements = typeByValue(space, w_value, numElements) var = varType(cursor, numElements, size) if space.is_true(space.isinstance(w_value, space.w_list)): - var.makeArray() + var.makeArray(space) return var +def newArrayVariableByType(space, cursor, w_value): + "Allocate a new PL/SQL array by looking at the Python data type." + + w_type, w_numElements = space.viewiterable(w_value, 2) + + numElements = space.int_w(w_numElements) + varType = typeByPythonType(space, cursor, w_type) + + var = varType(cursor, numElements) + var.makeArray(space) + return var + def newVariableByType(space, cursor, w_value, numElements): # passing an integer is assumed to be a string if space.is_true(space.isinstance(w_value, space.w_int)): @@ -781,7 +838,7 @@ # passing an array of two elements define an array if space.is_true(space.isinstance(w_value, space.w_list)): - XXX + return newArrayVariableByType(space, cursor, w_value) # handle directly bound variables if space.is_true(space.isinstance(w_value, Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Wed Nov 18 11:17:11 2009 @@ -12,3 +12,41 @@ cur.fetchall() assert cur.rowcount == 1 + def test_array(self): + cur = self.cnx.cursor() + array = map(str, range(20)) + tablelen = cur.var(oracle.NUMBER) + output = cur.var(oracle.STRING) + statement = """ + declare + array dbms_utility.uncl_array := :array; + begin + dbms_utility.table_to_comma( + array, :tablelen, :output); + end;""" + cur.execute(statement, + array=array, + tablelen=tablelen, + output=output) + assert tablelen.getvalue() == 20 + assert output.getvalue() == ','.join(array) + + def test_array_bysize(self): + cur = self.cnx.cursor() + array = map(str, range(20)) + tablelen = cur.var(oracle.NUMBER) + output = cur.var(oracle.STRING) + cur.setinputsizes(array=[oracle.STRING, 10]) + statement = """ + declare + array dbms_utility.uncl_array := :array; + begin + dbms_utility.table_to_comma( + array, :tablelen, :output); + end;""" + cur.execute(statement, + array=array, + tablelen=tablelen, + output=output) + assert tablelen.getvalue() == 20 + assert output.getvalue() == ','.join(array) From afa at codespeak.net Wed Nov 18 11:29:00 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 11:29:00 +0100 (CET) Subject: [pypy-svn] r69362 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118102900.E5BE531813D@codespeak.net> Author: afa Date: Wed Nov 18 11:29:00 2009 New Revision: 69362 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Add cursor.arrayvar() Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Wed Nov 18 11:29:00 2009 @@ -749,6 +749,33 @@ return space.wrap(var) var.unwrap_spec = ['self', ObjSpace, W_Root, int, W_Root, W_Root, W_Root] + def arrayvar(self, space, w_type, w_value, size=0): + # determine the type of variable + varType = interp_variable.typeByPythonType(space, self, w_type) + if varType.isVariableLength and size == 0: + size = varType.size + + # determine the number of elements to create + if space.is_true(space.isinstance(w_value, space.w_list)): + numElements = space.int_w(space.len(w_value)) + elif space.is_true(space.isinstance(w_value, space.w_int)): + numElements = space.int_w(w_value) + else: + raise OperationError( + w_NotSupportedErrorException, + space.wrap("expecting integer or list of values")) + + # create the variable + var = varType(self, numElements, size) + var.makeArray(space) + + # set the value, if applicable + if space.is_true(space.isinstance(w_value, space.w_list)): + var.setArrayValue(space, w_value) + + return var + arrayvar.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, int] + def setinputsizes(self, space, __args__): args_w, kw_w = __args__.unpack() @@ -826,6 +853,8 @@ unwrap_spec=W_Cursor.callproc.unwrap_spec), var = interp2app(W_Cursor.var, unwrap_spec=W_Cursor.var.unwrap_spec), + arrayvar = interp2app(W_Cursor.arrayvar, + unwrap_spec=W_Cursor.arrayvar.unwrap_spec), setinputsizes = interp2app(W_Cursor.setinputsizes, unwrap_spec=W_Cursor.setinputsizes.unwrap_spec), Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Wed Nov 18 11:29:00 2009 @@ -50,3 +50,24 @@ output=output) assert tablelen.getvalue() == 20 assert output.getvalue() == ','.join(array) + + def test_arrayvar(self): + cur = self.cnx.cursor() + array = map(str, range(20)) + tablelen = cur.var(oracle.NUMBER) + output = cur.var(oracle.STRING) + arrayvar = cur.arrayvar(oracle.STRING, array) + arrayvar.setvalue(0, array) + statement = """ + declare + array dbms_utility.uncl_array := :array; + begin + dbms_utility.table_to_comma( + array, :tablelen, :output); + end;""" + cur.execute(statement, + array=arrayvar, + tablelen=tablelen, + output=output) + assert tablelen.getvalue() == 20 + assert output.getvalue() == ','.join(array) From pedronis at codespeak.net Wed Nov 18 11:53:48 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 11:53:48 +0100 (CET) Subject: [pypy-svn] r69365 - pypy/build/bot2/pypybuildbot Message-ID: <20091118105348.B0B9C49843F@codespeak.net> Author: pedronis Date: Wed Nov 18 11:53:46 2009 New Revision: 69365 Modified: pypy/build/bot2/pypybuildbot/summary.py Log: experiment in having links to switch between just and filter nothing Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 11:53:46 2009 @@ -783,7 +783,21 @@ total_time = time.time()-t0 page.add_comment('t=%.2f; %s' % (total_time, outcome_set_cache.stats())) - return page.render() + + if request.args: + trunk_vs_any_text = "filter nothing" + trunk_vs_any_query = "" + else: + trunk_vs_any_text = "all " + trunk_vs_any_query = "?branch=" + + trunk_vs_any_anchor = html.a(trunk_vs_any_text, + href="/summary%s" % + trunk_vs_any_query, + class_="failSummary branch") + trunk_vs_any = html.div(html.h2(trunk_vs_any_anchor), + style="position: absolute; right: 5%;") + return trunk_vs_any.unicode() + page.render() def head(self, request): return """ From afa at codespeak.net Wed Nov 18 11:58:36 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 11:58:36 +0100 (CET) Subject: [pypy-svn] r69366 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118105836.2DBF031813D@codespeak.net> Author: afa Date: Wed Nov 18 11:58:35 2009 New Revision: 69366 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: implement getvalue() for array variables Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 11:58:35 2009 @@ -295,9 +295,14 @@ # XXX outConverter return value + def getArrayValue(self, space, numElements): + return space.newlist( + [self.getSingleValue(space, i) + for i in range(numElements)]) + def getValue(self, space, pos=0): if self.isArray: - return self.getArrayValue(self, self.actualElements) + return self.getArrayValue(space, self.actualElementsPtr[0]) return self.getSingleValue(space, pos) getValue.unwrap_spec = ['self', ObjSpace, int] Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Wed Nov 18 11:58:35 2009 @@ -14,7 +14,7 @@ def test_array(self): cur = self.cnx.cursor() - array = map(str, range(20)) + array = ["n%d" % d for d in range(20)] tablelen = cur.var(oracle.NUMBER) output = cur.var(oracle.STRING) statement = """ @@ -33,7 +33,7 @@ def test_array_bysize(self): cur = self.cnx.cursor() - array = map(str, range(20)) + array = ["n%d" % d for d in range(20)] tablelen = cur.var(oracle.NUMBER) output = cur.var(oracle.STRING) cur.setinputsizes(array=[oracle.STRING, 10]) @@ -53,7 +53,7 @@ def test_arrayvar(self): cur = self.cnx.cursor() - array = map(str, range(20)) + array = ["n%d" % d for d in range(20)] tablelen = cur.var(oracle.NUMBER) output = cur.var(oracle.STRING) arrayvar = cur.arrayvar(oracle.STRING, array) @@ -71,3 +71,24 @@ output=output) assert tablelen.getvalue() == 20 assert output.getvalue() == ','.join(array) + + def test_arrayvar_out(self): + cur = self.cnx.cursor() + array = ["n%d" % d for d in range(20)] + tablelen = cur.var(oracle.NUMBER) + arrayvar = cur.arrayvar(oracle.STRING, 25) + statement = """ + declare + array dbms_utility.uncl_array; + begin + dbms_utility.comma_to_table( + :input, :tablelen, array); + :array := array; + end;""" + cur.execute(statement, + input=','.join(array), + tablelen=tablelen, + array=arrayvar) + assert tablelen.getvalue() == 20 + # dbms_utility.comma_to_table returns a 'NULL-terminated' table + assert arrayvar.getvalue() == array + [None] From pedronis at codespeak.net Wed Nov 18 12:02:09 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:02:09 +0100 (CET) Subject: [pypy-svn] r69367 - in pypy/build/bot2: codespeak-html pypybuildbot Message-ID: <20091118110209.76C8C31813D@codespeak.net> Author: pedronis Date: Wed Nov 18 12:02:08 2009 New Revision: 69367 Modified: pypy/build/bot2/codespeak-html/buildbot.css pypy/build/bot2/pypybuildbot/summary.py Log: styling Modified: pypy/build/bot2/codespeak-html/buildbot.css ============================================================================== --- pypy/build/bot2/codespeak-html/buildbot.css (original) +++ pypy/build/bot2/codespeak-html/buildbot.css Wed Nov 18 12:02:08 2009 @@ -103,6 +103,10 @@ color: black; } +a.failSummary.trunkVsAny, a:visited.failSummary.trunkVsAny { + color: #a9a9aa9; +} + a.failSummary.builderquery, a:visited.failSummary.builderquery { color: black; } Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 12:02:08 2009 @@ -794,7 +794,7 @@ trunk_vs_any_anchor = html.a(trunk_vs_any_text, href="/summary%s" % trunk_vs_any_query, - class_="failSummary branch") + class_="failSummary trunkVsAny") trunk_vs_any = html.div(html.h2(trunk_vs_any_anchor), style="position: absolute; right: 5%;") return trunk_vs_any.unicode() + page.render() From pedronis at codespeak.net Wed Nov 18 12:03:52 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:03:52 +0100 (CET) Subject: [pypy-svn] r69368 - pypy/build/bot2/codespeak-html Message-ID: <20091118110352.C982931813D@codespeak.net> Author: pedronis Date: Wed Nov 18 12:03:52 2009 New Revision: 69368 Modified: pypy/build/bot2/codespeak-html/buildbot.css Log: oops Modified: pypy/build/bot2/codespeak-html/buildbot.css ============================================================================== --- pypy/build/bot2/codespeak-html/buildbot.css (original) +++ pypy/build/bot2/codespeak-html/buildbot.css Wed Nov 18 12:03:52 2009 @@ -104,7 +104,7 @@ } a.failSummary.trunkVsAny, a:visited.failSummary.trunkVsAny { - color: #a9a9aa9; + color: #a9a9a9; } a.failSummary.builderquery, a:visited.failSummary.builderquery { From pedronis at codespeak.net Wed Nov 18 12:09:12 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:09:12 +0100 (CET) Subject: [pypy-svn] r69369 - in pypy/build/bot2: codespeak-html pypybuildbot Message-ID: <20091118110912.B2A5D31813D@codespeak.net> Author: pedronis Date: Wed Nov 18 12:09:12 2009 New Revision: 69369 Modified: pypy/build/bot2/codespeak-html/index.html pypy/build/bot2/pypybuildbot/summary.py Log: tweaks Modified: pypy/build/bot2/codespeak-html/index.html ============================================================================== --- pypy/build/bot2/codespeak-html/index.html (original) +++ pypy/build/bot2/codespeak-html/index.html Wed Nov 18 12:09:12 2009 @@ -9,8 +9,11 @@

Welcome to codespeak's PyPy Buildbot!

    +
  • the Summary Display <trunk> will give you a + failure-oriented summary for recent revisions (<trunk> only).
    • +
  • the Summary Display will give you a - failure-oriented summary for recent revisions.
    • + failure-oriented summary for recent revisions (all branches).
  • the Waterfall Display will give you a time-oriented summary of recent buildbot activity.
    • Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 12:09:12 2009 @@ -795,7 +795,7 @@ href="/summary%s" % trunk_vs_any_query, class_="failSummary trunkVsAny") - trunk_vs_any = html.div(html.h2(trunk_vs_any_anchor), + trunk_vs_any = html.div(trunk_vs_any_anchor, style="position: absolute; right: 5%;") return trunk_vs_any.unicode() + page.render() From pedronis at codespeak.net Wed Nov 18 12:19:27 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:19:27 +0100 (CET) Subject: [pypy-svn] r69370 - pypy/build/bot2/pypybuildbot Message-ID: <20091118111927.A59A716803A@codespeak.net> Author: pedronis Date: Wed Nov 18 12:19:25 2009 New Revision: 69370 Modified: pypy/build/bot2/pypybuildbot/master.py pypy/build/bot2/pypybuildbot/summary.py Log: tweaks Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Nov 18 12:19:25 2009 @@ -37,7 +37,9 @@ status.putChild('summary', summary.Summary(['own', 'applevel', 'lib-python', 'jit', 'stackless', - 'windows', 'mac', 'maemo', 'other'])) + 'windows', 'mac', + 'benchmark-run', + 'maemo', 'other'])) pypybuilds = load('pypybuildbot.builds') @@ -159,7 +161,7 @@ "slavenames": ["bigdogvm2"], "builddir": JITBENCH, "factory": pypyJITBenchmarkFactory, - "category": 'benchmark', + "category": 'benchmark-run', }, ], Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 12:19:25 2009 @@ -144,7 +144,8 @@ run_info = {'URL': run_url, 'elapsed': pytest_elapsed or None, 'times': build.getTimes()} outcome_set = RevisionOutcomeSet(rev, key, run_info) - someresult = False + # "*-run" categories means the build is not a test build! + someresult = builderStatus.category.endswith("-run") if pytest_logs: for stepName, resultLog in pytest_logs: if resultLog.hasContents(): From pedronis at codespeak.net Wed Nov 18 12:23:21 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:23:21 +0100 (CET) Subject: [pypy-svn] r69371 - pypy/build/bot2/pypybuildbot Message-ID: <20091118112321.E795B16803A@codespeak.net> Author: pedronis Date: Wed Nov 18 12:23:20 2009 New Revision: 69371 Modified: pypy/build/bot2/pypybuildbot/master.py pypy/build/bot2/pypybuildbot/summary.py Log: last checkin was unintended, revert Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Nov 18 12:23:20 2009 @@ -37,9 +37,7 @@ status.putChild('summary', summary.Summary(['own', 'applevel', 'lib-python', 'jit', 'stackless', - 'windows', 'mac', - 'benchmark-run', - 'maemo', 'other'])) + 'windows', 'mac', 'maemo', 'other'])) pypybuilds = load('pypybuildbot.builds') @@ -161,7 +159,7 @@ "slavenames": ["bigdogvm2"], "builddir": JITBENCH, "factory": pypyJITBenchmarkFactory, - "category": 'benchmark-run', + "category": 'benchmark', }, ], Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 12:23:20 2009 @@ -144,8 +144,7 @@ run_info = {'URL': run_url, 'elapsed': pytest_elapsed or None, 'times': build.getTimes()} outcome_set = RevisionOutcomeSet(rev, key, run_info) - # "*-run" categories means the build is not a test build! - someresult = builderStatus.category.endswith("-run") + someresult = False if pytest_logs: for stepName, resultLog in pytest_logs: if resultLog.hasContents(): From pedronis at codespeak.net Wed Nov 18 12:24:10 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 12:24:10 +0100 (CET) Subject: [pypy-svn] r69372 - pypy/build/bot2/pypybuildbot Message-ID: <20091118112410.95D4616803A@codespeak.net> Author: pedronis Date: Wed Nov 18 12:24:09 2009 New Revision: 69372 Modified: pypy/build/bot2/pypybuildbot/master.py pypy/build/bot2/pypybuildbot/summary.py Log: experiment to avoid the benchmarks run to be marked as failed, sort them differently too Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Wed Nov 18 12:24:09 2009 @@ -37,7 +37,9 @@ status.putChild('summary', summary.Summary(['own', 'applevel', 'lib-python', 'jit', 'stackless', - 'windows', 'mac', 'maemo', 'other'])) + 'windows', 'mac', + 'benchmark-run', + 'maemo', 'other'])) pypybuilds = load('pypybuildbot.builds') @@ -159,7 +161,7 @@ "slavenames": ["bigdogvm2"], "builddir": JITBENCH, "factory": pypyJITBenchmarkFactory, - "category": 'benchmark', + "category": 'benchmark-run', }, ], Modified: pypy/build/bot2/pypybuildbot/summary.py ============================================================================== --- pypy/build/bot2/pypybuildbot/summary.py (original) +++ pypy/build/bot2/pypybuildbot/summary.py Wed Nov 18 12:24:09 2009 @@ -145,6 +145,9 @@ 'times': build.getTimes()} outcome_set = RevisionOutcomeSet(rev, key, run_info) someresult = False + # "*-run" categories mean the build is not a test build! + if builderStatus.category: + someresult = builderStatus.category.endswith("-run") if pytest_logs: for stepName, resultLog in pytest_logs: if resultLog.hasContents(): From fijal at codespeak.net Wed Nov 18 12:25:37 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 18 Nov 2009 12:25:37 +0100 (CET) Subject: [pypy-svn] r69373 - pypy/trunk/pypy/doc/config Message-ID: <20091118112537.320A916803A@codespeak.net> Author: fijal Date: Wed Nov 18 12:25:36 2009 New Revision: 69373 Added: pypy/trunk/pypy/doc/config/translation.backendopt.storesink.txt (contents, props changed) Log: Add a doc Added: pypy/trunk/pypy/doc/config/translation.backendopt.storesink.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/translation.backendopt.storesink.txt Wed Nov 18 12:25:36 2009 @@ -0,0 +1 @@ +Store sinking optimization. On by default. From cfbolz at codespeak.net Wed Nov 18 14:12:55 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Nov 2009 14:12:55 +0100 (CET) Subject: [pypy-svn] r69376 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test Message-ID: <20091118131255.9216949843F@codespeak.net> Author: cfbolz Date: Wed Nov 18 14:12:55 2009 New Revision: 69376 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): a test (written at the sprint) Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Wed Nov 18 14:12:55 2009 @@ -398,6 +398,33 @@ FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 +def test_rebuild_from_resumedata_two_guards_w_shared_virtuals(): + b1, b2, b3, b4, b5, b6 = [BoxPtr(), BoxPtr(), BoxInt(), BoxPtr(), BoxInt(), BoxInt()] + c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), + LLtypeMixin.nodebox.constbox()] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] + capture_resumedata(fs, None, storage) + + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + values = {b2: virtual_value(b2, b5, c4)} + modifier = ResumeDataVirtualAdder(storage, memo) + liveboxes = modifier.finish(values) + assert len(storage.rd_virtuals) == 1 + assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + tag(0, TAGCONST)] + + storage2 = Storage() + fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] + capture_resumedata(fs, None, storage2) + values[b4] = virtual_value(b4, b6, c4) + modifier = ResumeDataVirtualAdder(storage2, memo) + liveboxes = modifier.finish(values) + assert len(storage2.rd_virtuals) == 2 + assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums + assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] + + def test_resumedata_top_recursive_virtuals(): b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() From afa at codespeak.net Wed Nov 18 15:04:38 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 15:04:38 +0100 (CET) Subject: [pypy-svn] r69377 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118140438.EA3DD31813D@codespeak.net> Author: afa Date: Wed Nov 18 15:04:36 2009 New Revision: 69377 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: implement cursor.description Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Wed Nov 18 15:04:36 2009 @@ -351,6 +351,187 @@ self.fetchVariables = None + def getDescription(space, self): + "Return a list of 7-tuples consisting of the description of " + "the define variables" + + # make sure the cursor is open + self._checkOpen(space) + + # fixup bound cursor, if necessary + self._fixupBoundCursor() + + # if not a query, return None + if self.statementType != roci.OCI_STMT_SELECT: + return + + # determine number of items in select-list + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub1).TO, 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_STMT, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PARAM_COUNT, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_GetDescription()") + numItems = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + return space.newlist( + [space.newtuple(self._itemDescription(space, i + 1)) + for i in range(numItems)]) + + def _itemDescription(self, space, pos): + "Return a tuple describing the item at the given position" + + # acquire parameter descriptor + paramptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, + 1, flavor='raw') + try: + status = roci.OCIParamGet( + self.handle, roci.OCI_HTYPE_STMT, + self.environment.errorHandle, + rffi.cast(roci.dvoidpp, paramptr), + pos) + self.environment.checkForError( + status, + "Cursor_GetDescription(): parameter") + param = paramptr[0] + finally: + lltype.free(paramptr, flavor='raw') + + try: + # acquire usable type of item + varType = interp_variable.typeByOracleDescriptor( + param, self.environment) + + # acquire internal size of item + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_DATA_SIZE, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_ItemDescription(): internal size") + internalSize = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + # acquire name of item + nameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, 1, + flavor='raw') + lenptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, nameptr), + lenptr, + roci.OCI_ATTR_NAME, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_ItemDescription(): name") + name = rffi.charpsize2str(nameptr[0], lenptr[0]) + finally: + lltype.free(nameptr, flavor='raw') + lltype.free(lenptr, flavor='raw') + + # lookup precision and scale + if varType == interp_variable.VT_Float: + attrptr = lltype.malloc(rffi.CArrayPtr(roci.sb1).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_SCALE, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_ItemDescription(): scale") + scale = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PRECISION, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_ItemDescription(): precision") + precision = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + else: + scale = 0 + precision = 0 + + # lookup whether null is permitted for the attribute + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub1).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_IS_NULL, + self.environment.errorHandle) + self.environment.checkForError( + status, + "Cursor_ItemDescription(): nullable") + nullable = attrptr[0] != 0 + finally: + lltype.free(attrptr, flavor='raw') + + # set display size based on data type + if varType == interp_variable.VT_String: + displaySize = internalSize + elif varType == interp_variable.VT_NationalCharString: + displaySize = internalSize / 2 + elif varType == interp_variable.VT_Binary: + displaySize = internalSize + elif varType == interp_variable.VT_FixedChar: + displaySize = internalSize + elif varType == interp_variable.VT_FixedNationalChar: + displaySize = internalSize / 2 + elif varType == interp_variable.VT_Float: + if precision: + displaySize = precision + 1 + if scale > 0: + displaySize += scale + 1 + else: + displaySize = 127 + elif varType == interp_variable.VT_DateTime: + displaySize = 23 + else: + displaySize = -1 + + # return the tuple + return [space.wrap(name), space.gettypeobject(varType.typedef), + space.wrap(displaySize), space.wrap(internalSize), + space.wrap(precision), space.wrap(scale), + space.wrap(nullable)] + + finally: + roci.OCIDescriptorFree(param, roci.OCI_DTYPE_PARAM) + def _setBindVariablesByPos(self, space, w_vars, numElements, arrayPos, defer): "handle positional binds" @@ -865,5 +1046,6 @@ bindarraysize = GetSetProperty(cursor_bindarraysize_get, cursor_bindarraysize_set), rowcount = interp_attrproperty('rowCount', W_Cursor), statement = interp_attrproperty_w('w_statement', W_Cursor), - bindvars = GetSetProperty(cursor_bindvars_get) + bindvars = GetSetProperty(cursor_bindvars_get), + description = GetSetProperty(W_Cursor.getDescription), ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 15:04:36 2009 @@ -36,7 +36,7 @@ def _defineHelper(cursor, param, position, numElements): # determine data type - varType = _typeByOracleDescriptor(param, cursor.environment) + varType = typeByOracleDescriptor(param, cursor.environment) if cursor.numbersAsStrings and varType is VT_Float: varType = VT_NumberAsString @@ -656,7 +656,7 @@ ) variableTypeByTypedef[cls.typedef] = cls -def _typeByOracleDescriptor(param, environment): +def typeByOracleDescriptor(param, environment): # retrieve datatype of the parameter attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, flavor='raw') try: Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 18 15:04:36 2009 @@ -24,6 +24,7 @@ _compilation_info_ = eci ub1 = platform.SimpleType('ub1', rffi.UINT) + sb1 = platform.SimpleType('sb1', rffi.INT) ub2 = platform.SimpleType('ub2', rffi.UINT) sb2 = platform.SimpleType('sb2', rffi.INT) ub4 = platform.SimpleType('ub4', rffi.UINT) @@ -53,6 +54,7 @@ OCI_CRED_RDBMS OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT + OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Wed Nov 18 15:04:36 2009 @@ -150,3 +150,27 @@ count, = cur.fetchone() assert count == len(rows) assert var.maxlength == 100 * self.cnx.maxBytesPerCharacter + + def test_description_number(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (" + "intcol number(9) not null," + "numbercol number(9, 2) not null," + "floatcol float not null," + "unconstrainedcol number not null," + "nullablecol number(38)" + ")") + cur.execute("select * from pypy_temp_table") + got = cur.description + expected = [ + ('INTCOL', oracle.NUMBER, 10, 22, 9, 0, 0), + ('NUMBERCOL', oracle.NUMBER, 13, 22, 9, 2, 0), + ('FLOATCOL', oracle.NUMBER, 127, 22, 126, -127, 0), + ('UNCONSTRAINEDCOL', oracle.NUMBER, 127, 22, 0, -127, 0), + ('NULLABLECOL', oracle.NUMBER, 39, 22, 38, 0, 1), + ] + assert got == expected From cfbolz at codespeak.net Wed Nov 18 15:17:01 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Nov 2009 15:17:01 +0100 (CET) Subject: [pypy-svn] r69378 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091118141701.884E331813D@codespeak.net> Author: cfbolz Date: Wed Nov 18 15:17:00 2009 New Revision: 69378 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): number virtuals a bit later Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 15:17:00 2009 @@ -90,6 +90,7 @@ TAGVIRTUAL = 3 UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) @@ -205,9 +206,8 @@ self.virtuals[i] = vinfo self.vfieldboxes[i] = fieldboxes else: - tagged = tag(len(self.virtuals), TAGVIRTUAL) - self.virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) + tagged = UNASSIGNEDVIRTUAL + self.vinfos_not_env[virtualbox] = (vinfo, fieldboxes) self.liveboxes[virtualbox] = tagged self._register_boxes(fieldboxes) @@ -237,6 +237,7 @@ numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} + self.vinfos_not_env = {} storage.rd_numb = numb storage.rd_snapshot = None @@ -261,6 +262,7 @@ return liveboxes[:] def _number_virtuals(self, liveboxes): + virtuals = self.virtuals for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -269,7 +271,11 @@ liveboxes.append(box) else: assert tagbits == TAGVIRTUAL - + if tagged_eq(tagged, UNASSIGNEDVIRTUAL): + vinfo, fieldboxes = self.vinfos_not_env[box] + self.liveboxes[box] = tag(len(virtuals), TAGVIRTUAL) + virtuals.append(vinfo) + self.vfieldboxes.append(fieldboxes) storage = self.storage storage.rd_virtuals = None if len(self.virtuals) > 0: Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Wed Nov 18 15:17:00 2009 @@ -588,18 +588,20 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.virtuals = [] + modifier.vinfos_not_env = {} modifier.vfieldboxes = [] modifier.make_virtual(vbox, None, ['a', 'b'], [b1, b2]) - assert modifier.liveboxes == {vbox: tag(0, TAGVIRTUAL), b1: UNASSIGNED, + assert modifier.liveboxes == {vbox: UNASSIGNEDVIRTUAL, b1: UNASSIGNED, b2: UNASSIGNED} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2]] + assert len(modifier.virtuals) == 0 + assert modifier.vfieldboxes == [] modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} modifier.liveboxes = {} modifier.virtuals = [None] modifier.vfieldboxes = [None] + modifier.vinfos_not_env = {} modifier.make_virtual(vbox, None, ['a', 'b', 'c'], [b1, b2, vbox]) assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, vbox: tag(0, TAGVIRTUAL)} @@ -720,6 +722,7 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.virtuals = [] + modifier.vinfos_not_env = {} modifier.vfieldboxes = [] modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, @@ -749,22 +752,25 @@ metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 2 - b2t = reader._decode_box(tag(0, TAGVIRTUAL)) - b4t = reader._decode_box(tag(1, TAGVIRTUAL)) + b2t = reader._decode_box(modifier._gettagged(b2s)) + b4t = reader._decode_box(modifier._gettagged(b4s)) trace = metainterp.trace - expected = [ - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, + b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu)], - b2t, None), - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, + b2t, None) + b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu)], - b4t, None), - (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr), - ] + b4t, None) + b2set = [(rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr)] + b4set = [(rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr)] + if untag(modifier._gettagged(b2s))[0] == 0: + expected = [b2new, b4new] + b2set + b4set + else: + expected = [b4new, b2new] + b4set + b2set + for x, y in zip(expected, trace): assert x == y ptr = b2t.value._obj.container._as_ptr() @@ -785,6 +791,7 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.virtuals = [] + modifier.vinfos_not_env = {} modifier.vfieldboxes = [] modifier.make_varray(b2s, LLtypeMixin.arraydescr, @@ -831,6 +838,7 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.virtuals = [] + modifier.vinfos_not_env = {} modifier.vfieldboxes = [] modifier.make_vstruct(b2s, LLtypeMixin.ssize, From afa at codespeak.net Wed Nov 18 15:37:05 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 15:37:05 +0100 (CET) Subject: [pypy-svn] r69379 - pypy/trunk/pypy/module/oracle Message-ID: <20091118143705.2484F31813D@codespeak.net> Author: afa Date: Wed Nov 18 15:37:04 2009 New Revision: 69379 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py Log: Fix a regression in tests Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 15:37:04 2009 @@ -122,6 +122,8 @@ def __init__(self, cursor, numElements, size=0): self.environment = cursor.environment self.boundCursorHandle = lltype.nullptr(roci.OCIStmt.TO) + self.boundName = None + self.boundPos = 0 self.isArray = False self.actualElementsPtr = lltype.malloc(roci.Ptr(roci.ub4).TO, 1, zero=True, flavor='raw') @@ -184,6 +186,25 @@ self.data = lltype.malloc(rffi.CCHARP.TO, int(dataLength), flavor='raw', zero=True) + def resize(self, size): + # allocate the data for the new array + orig_data = self.data + orig_size = self.bufferSize + self.size = size + self.allocateData() + + # copy the data from the original array to the new array + for i in range(self.allocatedElements): + for j in range(orig_size): + self.data[self.bufferSize * i + j] = \ + orig_data[orig_size * i + j] + + lltype.free(orig_data, flavor='raw') + + # force rebinding + if self.boundName or self.boundPos: + self._internalBind() + def makeArray(self, space): if not self.canBeInArray: raise OperationError( From cfbolz at codespeak.net Wed Nov 18 15:59:05 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Nov 2009 15:59:05 +0100 (CET) Subject: [pypy-svn] r69380 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091118145905.3B8D4318140@codespeak.net> Author: cfbolz Date: Wed Nov 18 15:59:04 2009 New Revision: 69380 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): number virtuals and boxes that don't come from the frame with negative numbers. Slightly funny code. Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 15:59:04 2009 @@ -262,20 +262,35 @@ return liveboxes[:] def _number_virtuals(self, liveboxes): - virtuals = self.virtuals + prev_liveboxes_cache = 0 + prev_virtuals_cache = 0 + new_liveboxes = [] + new_virtuals = [] + new_vfieldboxes = [] for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: assert tagged_eq(tagged, UNASSIGNED) - self.liveboxes[box] = tag(len(liveboxes), TAGBOX) - liveboxes.append(box) + new_liveboxes.append(box) + index = -prev_liveboxes_cache - len(new_liveboxes) + self.liveboxes[box] = tag(index, TAGBOX) else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): vinfo, fieldboxes = self.vinfos_not_env[box] - self.liveboxes[box] = tag(len(virtuals), TAGVIRTUAL) - virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) + new_virtuals.append(vinfo) + new_vfieldboxes.append(fieldboxes) + index = -prev_virtuals_cache - len(new_virtuals) + self.liveboxes[box] = tag(index, TAGVIRTUAL) + new_liveboxes.reverse() + new_virtuals.reverse() + new_vfieldboxes.reverse() + + liveboxes.extend(new_liveboxes) + self.virtuals.extend(new_virtuals) + self.vfieldboxes.extend(new_vfieldboxes) + # xxx + cached stuff + storage = self.storage storage.rd_virtuals = None if len(self.virtuals) > 0: Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Wed Nov 18 15:59:04 2009 @@ -356,7 +356,7 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(0, TAGCONST)] b6 = BoxPtr() @@ -367,9 +367,9 @@ liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] # now on to resuming metainterp = MyMetaInterp() @@ -411,7 +411,7 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(0, TAGCONST)] storage2 = Storage() @@ -440,9 +440,9 @@ liveboxes = modifier.finish(values) assert liveboxes == [b3] assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), tag(0, TAGVIRTUAL)] @@ -766,7 +766,7 @@ b4set = [(rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr)] - if untag(modifier._gettagged(b2s))[0] == 0: + if untag(modifier._gettagged(b2s))[0] == -2: expected = [b2new, b4new] + b2set + b4set else: expected = [b4new, b2new] + b4set + b2set From afa at codespeak.net Wed Nov 18 16:18:20 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 16:18:20 +0100 (CET) Subject: [pypy-svn] r69381 - pypy/trunk/pypy/module/oracle Message-ID: <20091118151820.611ED49844E@codespeak.net> Author: afa Date: Wed Nov 18 16:18:19 2009 New Revision: 69381 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py Log: Fix the Decimal case, when locale settings use a comma for the decimal separator. Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 16:18:19 2009 @@ -572,11 +572,13 @@ text_buf.fill(space, w_text) format_buf = config.StringBuffer() format_buf.fill(space, w_format) + nls_params = "NLS_NUMERIC_CHARACTERS='.,'" status = roci.OCINumberFromText( self.environment.errorHandle, text_buf.ptr, text_buf.size, format_buf.ptr, format_buf.size, - None, 0, dataptr) + nls_params, len(nls_params), + dataptr) self.environment.checkForError( status, "NumberVar_SetValue(): from decimal") return From arigo at codespeak.net Wed Nov 18 16:32:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 16:32:56 +0100 (CET) Subject: [pypy-svn] r69382 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091118153256.3B3B331813D@codespeak.net> Author: arigo Date: Wed Nov 18 16:32:55 2009 New Revision: 69382 Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Log: Make sure we always reset f_forward to None, with a test. Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Wed Nov 18 16:32:55 2009 @@ -143,11 +143,12 @@ frame.f_back_some = f_back frame.f_back_forced = True if self.some_frame is frame: - self.some_frame = frame.f_back_some + f_back = frame.f_back_some + self.some_frame = f_back else: f_back = frame.f_back() - if f_back is not None: - f_back.f_forward = None + if f_back is not None: + f_back.f_forward = None self.framestackdepth -= 1 Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Wed Nov 18 16:32:55 2009 @@ -761,9 +761,22 @@ def test_unchain_with_exception(self): ec, frame, frame2 = self.enter_two_jitted_levels() + ec.jitted = False frame3 = self.Frame(ec, frame2) ec._chain(frame3) frame3.last_exception = 3 + assert ec.some_frame is frame3 ec._unchain(frame3) + ec.jitted = True assert frame3.f_back_some is frame2 assert frame3.f_back_forced + assert frame2._f_forward is None + assert not frame2.f_back_forced + assert frame2.f_back() is frame + assert frame._f_forward is frame2 + # + # test for a bug: what happens if we _unchain frame2 without an exc. + assert frame2.last_exception is None + assert ec.some_frame is frame2 + ec._unchain(frame2) + assert frame._f_forward is None From arigo at codespeak.net Wed Nov 18 16:39:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 16:39:08 +0100 (CET) Subject: [pypy-svn] r69383 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091118153908.D0FC731813D@codespeak.net> Author: arigo Date: Wed Nov 18 16:39:08 2009 New Revision: 69383 Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Log: Really fix the previous test, without breaking other tests. Argh, this is a bit of a mess. Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Wed Nov 18 16:39:08 2009 @@ -138,17 +138,17 @@ def _unchain(self, frame): #assert frame is self.gettopframe() --- slowish + if self.some_frame is frame: + self.some_frame = frame.f_back_some + else: + f_back = frame.f_back() + if f_back is not None: + f_back.f_forward = None + if frame.last_exception is not None: f_back = frame.f_back() frame.f_back_some = f_back frame.f_back_forced = True - if self.some_frame is frame: - f_back = frame.f_back_some - self.some_frame = f_back - else: - f_back = frame.f_back() - if f_back is not None: - f_back.f_forward = None self.framestackdepth -= 1 Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Wed Nov 18 16:39:08 2009 @@ -777,6 +777,5 @@ # # test for a bug: what happens if we _unchain frame2 without an exc. assert frame2.last_exception is None - assert ec.some_frame is frame2 ec._unchain(frame2) assert frame._f_forward is None From arigo at codespeak.net Wed Nov 18 16:52:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 16:52:20 +0100 (CET) Subject: [pypy-svn] r69384 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091118155220.C673B31813D@codespeak.net> Author: arigo Date: Wed Nov 18 16:52:20 2009 New Revision: 69384 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py Log: Reintroduce this, reverting r69182. Comment about why it seems to still be useful. Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Wed Nov 18 16:52:20 2009 @@ -159,6 +159,11 @@ raise if not we_are_jitted(): executioncontext.return_trace(self, w_exitvalue) + # on exit, we try to release self.last_exception -- breaks an + # obvious reference cycle, so it used to help refcounting + # implementations. Nowaways, it helps ExecutionContext._unchain() + # by saying "this frame does not throw an exception". + self.last_exception = None finally: executioncontext.leave(self) return w_exitvalue From cfbolz at codespeak.net Wed Nov 18 17:43:48 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Nov 2009 17:43:48 +0100 (CET) Subject: [pypy-svn] r69385 - in pypy/branch/compress-virtuals-resumedata/pypy/jit: backend/llgraph backend/test metainterp metainterp/test Message-ID: <20091118164348.328C749844E@codespeak.net> Author: cfbolz Date: Wed Nov 18 17:43:47 2009 New Revision: 69385 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/runner.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/compile.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/history.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): introduce "holes" into the fail_args, to make it possible to have mostly the same numbers for boxes in consecutive guards Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py Wed Nov 18 17:43:47 2009 @@ -363,7 +363,10 @@ op = loop.operations[-1] if op.fail_args is None: op.fail_args = [] - op.fail_args.append(_variables[intvar]) + if intvar == -1: + op.fail_args.append(None) + else: + op.fail_args.append(_variables[intvar]) def compile_redirect_fail(old_loop, old_index, new_loop): old_loop = _from_opaque(old_loop) @@ -382,6 +385,8 @@ self.memocast = memocast def getenv(self, v): + if v is None: + return None if isinstance(v, Constant): return v.value else: Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/runner.py Wed Nov 18 17:43:47 2009 @@ -171,7 +171,10 @@ index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: - llimpl.compile_add_fail_arg(c, var2index[box]) + if box is not None: + llimpl.compile_add_fail_arg(c, var2index[box]) + else: + llimpl.compile_add_fail_arg(c, -1) x = op.result if x is not None: if isinstance(x, history.BoxInt): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py Wed Nov 18 17:43:47 2009 @@ -123,6 +123,27 @@ res = self.cpu.get_latest_value_int(0) assert res == 10 + def test_compile_with_holes_in_fail_args(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, None, i1, None] + + self.cpu.compile_loop(inputargs, operations, looptoken) + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(2) + assert res == 10 + def test_backends_dont_keep_loops_alive(self): import weakref, gc self.cpu.dont_keepalive_stuff = True Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/compile.py Wed Nov 18 17:43:47 2009 @@ -233,7 +233,12 @@ def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes self.guard_opnum = guard_op.opnum - self.fail_arg_types = [box.type for box in boxes] + fail_arg_types = [history.HOLE] * len(boxes) + for i in range(len(boxes)): + box = boxes[i] + if box: + fail_arg_types[i] = box.type + self.fail_arg_types = fail_arg_types def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/history.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/history.py Wed Nov 18 17:43:47 2009 @@ -15,6 +15,7 @@ INT = 'i' REF = 'r' FLOAT = 'f' +HOLE = '_' FAILARGS_LIMIT = 1000 @@ -744,8 +745,9 @@ ops = op.descr._debug_suboperations TreeLoop.check_consistency_of_branch(ops, seen.copy()) for box in op.fail_args or []: - assert isinstance(box, Box) - assert box in seen + if box is not None: + assert isinstance(box, Box) + assert box in seen else: assert op.fail_args is None box = op.result Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py Wed Nov 18 17:43:47 2009 @@ -526,7 +526,7 @@ assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) - if len(newboxes) > self.metainterp_sd.options.failargs_limit: + if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) # Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py Wed Nov 18 17:43:47 2009 @@ -1645,6 +1645,8 @@ box = cpu.ts.BoxRef(cpu.get_latest_value_ref(i)) elif boxtype == history.FLOAT: box = history.BoxFloat(cpu.get_latest_value_float(i)) + elif boxtype == history.HOLE: + continue else: assert False, "bad box type: num=%d" % ord(boxtype) inputargs.append(box) Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 17:43:47 2009 @@ -102,6 +102,8 @@ self.large_ints = {} self.refs = cpu.ts.new_ref_dict_2() self.numberings = {} + self.cached_boxes = {} + self.cached_virtuals = {} def getconst(self, const): if const.type == INT: @@ -136,6 +138,8 @@ self.consts.append(const) return result + # env numbering + def number(self, values, snapshot): if snapshot is None: return None, {}, 0 @@ -174,6 +178,41 @@ def forget_numberings(self, virtualbox): # XXX ideally clear only the affected numberings self.numberings.clear() + # XXX clear cached_* + + # caching for virtuals and boxes inside them + + def num_cached_boxes(self): + return len(self.cached_boxes) + + def assign_number_to_box(self, box, boxes): + if box in self.cached_boxes: + num = self.cached_boxes[box] + boxes[-num-1] = box + else: + boxes.append(box) + num = -len(boxes) + self.cached_boxes[box] = num + return num + + def num_cached_virtuals(self): + return len(self.cached_virtuals) + + def assign_number_to_virtual(self, box, vinfo, fieldboxes, virtuals, fieldboxes_list): + if box in self.cached_virtuals: + num = self.cached_virtuals[box] + virtuals[-num-1] = vinfo + fieldboxes_list[-num-1] = fieldboxes + else: + virtuals.append(vinfo) + fieldboxes_list.append(fieldboxes) + num = -len(virtuals) + self.cached_virtuals[box] = num + return num + + def clear_box_virtual_numbers(self): + self.cached_boxes.clear() + self.cached_virtuals.clear() _frame_info_placeholder = (None, 0, 0) @@ -264,23 +303,21 @@ def _number_virtuals(self, liveboxes): prev_liveboxes_cache = 0 prev_virtuals_cache = 0 - new_liveboxes = [] - new_virtuals = [] - new_vfieldboxes = [] + new_liveboxes = [None] * self.memo.num_cached_boxes() + new_virtuals = [None] * self.memo.num_cached_virtuals() + new_vfieldboxes = [None] * self.memo.num_cached_virtuals() for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: assert tagged_eq(tagged, UNASSIGNED) - new_liveboxes.append(box) - index = -prev_liveboxes_cache - len(new_liveboxes) + index = self.memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): vinfo, fieldboxes = self.vinfos_not_env[box] - new_virtuals.append(vinfo) - new_vfieldboxes.append(fieldboxes) - index = -prev_virtuals_cache - len(new_virtuals) + index = self.memo.assign_number_to_virtual( + box, vinfo, fieldboxes, new_virtuals, new_vfieldboxes) self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() new_virtuals.reverse() @@ -297,6 +334,8 @@ storage.rd_virtuals = self.virtuals[:] for i in range(len(storage.rd_virtuals)): vinfo = storage.rd_virtuals[i] + if vinfo is None: + continue fieldboxes = self.vfieldboxes[i] vinfo.fieldnums = [self._gettagged(box) for box in fieldboxes] @@ -471,8 +510,14 @@ for const in storage.rd_consts: debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - debug_print('\tbox', box.repr_rpython()) + if box is None: + debug_print('\tbox', 'None') + else: + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - virtual.debug_prints() + if virtual is None: + debug_print('\t\t', 'None') + else: + virtual.debug_prints() debug_stop("jit-resume") Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Wed Nov 18 17:43:47 2009 @@ -363,6 +363,7 @@ v6 = virtual_value(b6, c2, None) v6.setfield(LLtypeMixin.nextdescr, v6) values = {b2: virtual_value(b2, b4, v6), b6: v6} + memo.clear_box_virtual_numbers() modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 @@ -580,6 +581,75 @@ tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 +def test_ResumeDataLoopMemo_number_boxes(): + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + b1, b2 = [BoxInt(), BoxInt()] + assert memo.num_cached_boxes() == 0 + boxes = [] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + assert memo.num_cached_boxes() == 1 + boxes = [None] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [b1, b2] + + assert memo.num_cached_boxes() == 2 + boxes = [None, None] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [None, b2] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1, b2] + + memo.clear_box_virtual_numbers() + assert memo.num_cached_boxes() == 0 + +def test_ResumeDataLoopMemo_number_virtuals(): + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + b1, b2 = [BoxInt(), BoxInt()] + vinfo1 = object() + vinfo2 = object() + fb1 = object() + fb2 = object() + assert memo.num_cached_virtuals() == 0 + virtuals = [] + vfieldboxes = [] + num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + assert num == -1 + assert virtuals == [vinfo1] + assert vfieldboxes == [fb1] + assert memo.num_cached_virtuals() == 1 + virtuals = [None] + vfieldboxes = [None] + num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + assert num == -1 + assert virtuals == [vinfo1] + assert vfieldboxes == [fb1] + num = memo.assign_number_to_virtual(b2, vinfo2, fb2, virtuals, vfieldboxes) + assert num == -2 + assert virtuals == [vinfo1, vinfo2] + assert vfieldboxes == [fb1, fb2] + + assert memo.num_cached_virtuals() == 2 + virtuals = [None, None] + vfieldboxes = [None, None] + num = memo.assign_number_to_virtual(b2, vinfo2, fb2, virtuals, vfieldboxes) + assert num == -2 + assert virtuals == [None, vinfo2] + assert vfieldboxes == [None, fb2] + num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + assert num == -1 + assert virtuals == [vinfo1, vinfo2] + assert vfieldboxes == [fb1, fb2] + + memo.clear_box_virtual_numbers() + assert memo.num_cached_virtuals() == 0 def test__make_virtual(): b1, b2 = BoxInt(), BoxInt() From afa at codespeak.net Wed Nov 18 17:59:39 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 17:59:39 +0100 (CET) Subject: [pypy-svn] r69386 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118165939.5C597318140@codespeak.net> Author: afa Date: Wed Nov 18 17:59:38 2009 New Revision: 69386 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_numbervar.py Log: Support for long integer values Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 17:59:38 2009 @@ -70,6 +70,9 @@ position == cursor.outputSizeColumn): size = cursor.outputSize + # call the procedure to set values prior to define + varType = varType.preDefine(param, cursor.environment) + # create a variable of the correct type if cursor.outputTypeHandler: var = _newByOutputTypeHandler( @@ -84,9 +87,6 @@ else: var = varType(cursor, numElements, size) - # call the procedure to set values prior to define - var.preDefine(param) - # perform the define handleptr = lltype.malloc(roci.Ptr(roci.OCIDefine).TO, 1, flavor='raw') try: @@ -216,8 +216,9 @@ def initialize(self, cursor): pass - def preDefine(self, param): - pass + @classmethod + def preDefine(cls, param, environment): + return cls def postDefine(self): pass @@ -502,6 +503,51 @@ oracleType = roci.SQLT_VNU size = rffi.sizeof(roci.OCINumber) + @classmethod + def preDefine(cls, param, environment): + # if the return type has not already been specified, check to + # see if the number can fit inside an integer by looking at + # the precision and scale + if cls is VT_Float: + attrptr = lltype.malloc(rffi.CArrayPtr(roci.sb1).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_SCALE, + environment.errorHandle) + environment.checkForError( + status, + "NumberVar_PreDefine(): scale") + scale = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, attrptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PRECISION, + environment.errorHandle) + environment.checkForError( + status, + "NumberVar_PreDefine(): precision") + precision = attrptr[0] + finally: + lltype.free(attrptr, flavor='raw') + + if scale == 0 or (scale == -127 and precision == 0): + return VT_LongInteger + elif precision > 0 and precision < 10: + return VT_Integer + + return cls + def getValueProc(self, space, pos): dataptr = rffi.ptradd( rffi.cast(roci.Ptr(roci.OCINumber), self.data), @@ -525,7 +571,32 @@ finally: lltype.free(integerValuePtr, flavor='raw') elif isinstance(self, (VT_NumberAsString, VT_LongInteger)): - XXX = NumberAsString, LongInteger + format_buf = config.StringBuffer() + format_buf.fill(space, space.wrap("TM9")) + sizeptr = lltype.malloc(rffi.CArray(roci.ub4), 1, flavor='raw') + BUFSIZE = 200 + sizeptr[0] = rffi.cast(lltype.Unsigned, BUFSIZE) + textbuf, text = rffi.alloc_buffer(BUFSIZE) + try: + status = roci.OCINumberToText( + self.environment.errorHandle, + dataptr, + format_buf.ptr, format_buf.size, + None, 0, + sizeptr, textbuf); + self.environment.checkForError( + status, "NumberVar_GetValue(): as string") + w_strvalue = space.wrap( + rffi.str_from_buffer(textbuf, text, + BUFSIZE, + rffi.cast(lltype.Signed, sizeptr[0]))) + if isinstance(self, VT_NumberAsString): + return w_strvalue + else: + return space.call_function(space.w_int, w_strvalue) + finally: + rffi.keep_buffer_alive_until_here(textbuf, text) + lltype.free(sizeptr, flavor='raw') else: return transform.OracleNumberToPythonFloat( self.environment, dataptr) @@ -551,6 +622,22 @@ finally: lltype.free(integerValuePtr, flavor='raw') return + if space.is_true(space.isinstance(w_value, space.w_long)): + text_buf = config.StringBuffer() + text_buf.fill(space, space.str(w_value)) + format_buf = config.StringBuffer() + format_buf.fill(space, space.wrap("9" * 63)) + status = roci.OCINumberFromText( + self.environment.errorHandle, + text_buf.ptr, text_buf.size, + format_buf.ptr, format_buf.size, + None, 0, + dataptr) + self.environment.checkForError( + status, "NumberVar_SetValue(): from long") + return + elif space.is_true(space.isinstance(w_value, space.w_bool)): + XXX elif space.is_true(space.isinstance(w_value, space.w_float)): doubleValuePtr = lltype.malloc(roci.Ptr(lltype.Float).TO, 1, flavor='raw') @@ -789,7 +876,8 @@ if space.is_true(space.isinstance(w_value, space.w_int)): return VT_Integer, 0, numElements - # XXX long + if space.is_true(space.isinstance(w_value, space.w_long)): + return VT_LongInteger, 0, numElements if space.is_true(space.isinstance(w_value, space.w_float)): return VT_Float, 0, numElements Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 18 17:59:38 2009 @@ -382,3 +382,15 @@ dvoidp], # rsl sword) +OCINumberToText = external( + 'OCINumberToText', + [OCIError, # err + Ptr(OCINumber), # number + oratext, # fmt + ub4, # fmt_length + oratext, # nls_params + ub4, # nls_p_length + Ptr(ub4), # buf_size + oratext], # buf + sword) + Modified: pypy/trunk/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_numbervar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_numbervar.py Wed Nov 18 17:59:38 2009 @@ -16,3 +16,18 @@ cur.execute("begin :a := :b*2.5; end;", a=var, b=decimal.Decimal("0.5")) assert var.getvalue() == 1.25 + + def test_largelong(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + var.setvalue(0, 6088343244) + cur.execute("select :x+5 from dual", x=var) + value, = cur.fetchone() + assert value == 6088343249 + + def test_smalllong(self): + cur = self.cnx.cursor() + cur.execute("select :x from dual", x=3L) + value, = cur.fetchone() + assert value == 3 + From arigo at codespeak.net Wed Nov 18 18:09:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 18:09:13 +0100 (CET) Subject: [pypy-svn] r69387 - pypy/extradoc/planning Message-ID: <20091118170913.0FB0F318140@codespeak.net> Author: arigo Date: Wed Nov 18 18:09:12 2009 New Revision: 69387 Modified: pypy/extradoc/planning/jit.txt Log: Update. Clarification. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Nov 18 18:09:12 2009 @@ -11,7 +11,7 @@ - make the assembler produced by generate_failure smaller -- jit/asmgcc + threads? +- jit/asmgcc + threads? [DONE probably. Needs a bit more testing.] - think about code memory management @@ -38,7 +38,7 @@ - improve test running, compile only once -- we would like probably enabling tracing to leave the jit instead of just being ignored +- we would like probably enabling sys.settrace() to leave the jit instead of just being ignored META ----- From cfbolz at codespeak.net Wed Nov 18 18:36:26 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 18 Nov 2009 18:36:26 +0100 (CET) Subject: [pypy-svn] r69388 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091118173626.08BD0318141@codespeak.net> Author: cfbolz Date: Wed Nov 18 18:36:26 2009 New Revision: 69388 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): WIP: steps in the direction of sharing vinfos Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py Wed Nov 18 18:36:26 2009 @@ -62,6 +62,9 @@ def get_args_for_fail(self, modifier): pass + def make_virtual_info(self, modifier, fieldnums): + raise NotImplementedError # should not be called on this level + def is_constant(self): return self.level == LEVEL_CONSTANT @@ -169,6 +172,14 @@ self._really_force() return self.box + def make_virtual_info(self, modifier, fieldnums): + vinfo = self._make_virtual(modifier) + vinfo.fieldnums = fieldnums + return vinfo + + def _make_virtual(self, modifier): + raise NotImplementedError("abstract base") + class AbstractVirtualStructValue(AbstractVirtualValue): _attrs_ = ('_fields', '_cached_sorted_fields') @@ -221,14 +232,11 @@ # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - self._make_virtual(modifier, lst, fieldboxes) + modifier.register_virtual_fields(self.keybox, fieldboxes) for ofs in lst: fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - raise NotImplementedError - class VirtualValue(AbstractVirtualStructValue): level = LEVEL_KNOWNCLASS @@ -238,10 +246,9 @@ assert isinstance(known_class, Const) self.known_class = known_class - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_virtual(self.keybox, self.known_class, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_virtual(self.known_class, fielddescrs) class VStructValue(AbstractVirtualStructValue): @@ -249,10 +256,9 @@ AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op) self.structdescr = structdescr - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_vstruct(self.keybox, self.structdescr, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_vstruct(self.structdescr, fielddescrs) class VArrayValue(AbstractVirtualValue): @@ -296,11 +302,14 @@ const = self.optimizer.new_const_item(self.arraydescr) for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) - modifier.make_varray(self.keybox, self.arraydescr, itemboxes) + modifier.register_virtual_fields(self.keybox, itemboxes) for itemvalue in self._items: if itemvalue is not self.constvalue: itemvalue.get_args_for_fail(modifier) + def _make_virtual(self, modifier): + return modifier.make_varray(self.arraydescr) + class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): raise NotImplementedError Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 18:36:26 2009 @@ -198,16 +198,11 @@ def num_cached_virtuals(self): return len(self.cached_virtuals) - def assign_number_to_virtual(self, box, vinfo, fieldboxes, virtuals, fieldboxes_list): + def assign_number_to_virtual(self, box): if box in self.cached_virtuals: num = self.cached_virtuals[box] - virtuals[-num-1] = vinfo - fieldboxes_list[-num-1] = fieldboxes else: - virtuals.append(vinfo) - fieldboxes_list.append(fieldboxes) - num = -len(virtuals) - self.cached_virtuals[box] = num + num = self.cached_virtuals[box] = -len(self.cached_virtuals) - 1 return num def clear_box_virtual_numbers(self): @@ -225,29 +220,19 @@ #self.virtuals = [] #self.vfieldboxes = [] - def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): - vinfo = VirtualInfo(known_class, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_vstruct(self, virtualbox, typedescr, fielddescrs, fieldboxes): - vinfo = VStructInfo(typedescr, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_varray(self, virtualbox, arraydescr, itemboxes): - vinfo = VArrayInfo(arraydescr) - self._make_virtual(virtualbox, vinfo, itemboxes) - - def _make_virtual(self, virtualbox, vinfo, fieldboxes): - if virtualbox in self.liveboxes_from_env: - tagged = self.liveboxes_from_env[virtualbox] - i, _ = untag(tagged) - assert self.virtuals[i] is None - self.virtuals[i] = vinfo - self.vfieldboxes[i] = fieldboxes - else: - tagged = UNASSIGNEDVIRTUAL - self.vinfos_not_env[virtualbox] = (vinfo, fieldboxes) + def make_virtual(self, known_class, fielddescrs): + return VirtualInfo(known_class, fielddescrs) + + def make_vstruct(self, typedescr, fielddescrs): + return VStructInfo(typedescr, fielddescrs) + + def make_varray(self, arraydescr): + return VArrayInfo(arraydescr) + + def register_virtual_fields(self, virtualbox, fieldboxes): + tagged = self.liveboxes_from_env.get(virtualbox, UNASSIGNEDVIRTUAL) self.liveboxes[virtualbox] = tagged + self.vfieldboxes[virtualbox] = fieldboxes self._register_boxes(fieldboxes) def register_box(self, box): @@ -283,8 +268,7 @@ # collect liveboxes and virtuals n = len(liveboxes_from_env) - v liveboxes = [None]*n - self.virtuals = [None]*v - self.vfieldboxes = [None]*v + self.vfieldboxes = {} for box, tagged in liveboxes_from_env.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -294,18 +278,16 @@ value = values[box] value.get_args_for_fail(self) - self._number_virtuals(liveboxes) + self._number_virtuals(liveboxes, values, v) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) return liveboxes[:] - def _number_virtuals(self, liveboxes): + def _number_virtuals(self, liveboxes, values, num_env_virtuals): prev_liveboxes_cache = 0 prev_virtuals_cache = 0 new_liveboxes = [None] * self.memo.num_cached_boxes() - new_virtuals = [None] * self.memo.num_cached_virtuals() - new_vfieldboxes = [None] * self.memo.num_cached_virtuals() for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -315,30 +297,25 @@ else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): - vinfo, fieldboxes = self.vinfos_not_env[box] - index = self.memo.assign_number_to_virtual( - box, vinfo, fieldboxes, new_virtuals, new_vfieldboxes) + index = self.memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() - new_virtuals.reverse() - new_vfieldboxes.reverse() liveboxes.extend(new_liveboxes) - self.virtuals.extend(new_virtuals) - self.vfieldboxes.extend(new_vfieldboxes) - # xxx + cached stuff storage = self.storage storage.rd_virtuals = None - if len(self.virtuals) > 0: - storage.rd_virtuals = self.virtuals[:] - for i in range(len(storage.rd_virtuals)): - vinfo = storage.rd_virtuals[i] - if vinfo is None: - continue - fieldboxes = self.vfieldboxes[i] - vinfo.fieldnums = [self._gettagged(box) - for box in fieldboxes] + vfieldboxes = self.vfieldboxes + if vfieldboxes: + length = num_env_virtuals + self.memo.num_cached_virtuals() + virtuals = storage.rd_virtuals = [None] * length + for virtualbox, fieldboxes in vfieldboxes.iteritems(): + num, _ = untag(self.liveboxes[virtualbox]) + value = values[virtualbox] + fieldnums = [self._gettagged(box) + for box in fieldboxes] + vinfo = value.make_virtual_info(self, fieldnums) + virtuals[num] = vinfo def _gettagged(self, box): if isinstance(box, Const): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Wed Nov 18 18:36:26 2009 @@ -613,70 +613,44 @@ def test_ResumeDataLoopMemo_number_virtuals(): memo = ResumeDataLoopMemo(LLtypeMixin.cpu) b1, b2 = [BoxInt(), BoxInt()] - vinfo1 = object() - vinfo2 = object() - fb1 = object() - fb2 = object() assert memo.num_cached_virtuals() == 0 - virtuals = [] - vfieldboxes = [] - num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + num = memo.assign_number_to_virtual(b1) assert num == -1 - assert virtuals == [vinfo1] - assert vfieldboxes == [fb1] assert memo.num_cached_virtuals() == 1 - virtuals = [None] - vfieldboxes = [None] - num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + num = memo.assign_number_to_virtual(b1) assert num == -1 - assert virtuals == [vinfo1] - assert vfieldboxes == [fb1] - num = memo.assign_number_to_virtual(b2, vinfo2, fb2, virtuals, vfieldboxes) + num = memo.assign_number_to_virtual(b2) assert num == -2 - assert virtuals == [vinfo1, vinfo2] - assert vfieldboxes == [fb1, fb2] assert memo.num_cached_virtuals() == 2 - virtuals = [None, None] - vfieldboxes = [None, None] - num = memo.assign_number_to_virtual(b2, vinfo2, fb2, virtuals, vfieldboxes) + num = memo.assign_number_to_virtual(b2) assert num == -2 - assert virtuals == [None, vinfo2] - assert vfieldboxes == [None, fb2] - num = memo.assign_number_to_virtual(b1, vinfo1, fb1, virtuals, vfieldboxes) + num = memo.assign_number_to_virtual(b1) assert num == -1 - assert virtuals == [vinfo1, vinfo2] - assert vfieldboxes == [fb1, fb2] memo.clear_box_virtual_numbers() assert memo.num_cached_virtuals() == 0 -def test__make_virtual(): +def test_register_virtual_fields(): b1, b2 = BoxInt(), BoxInt() vbox = BoxPtr() modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vinfos_not_env = {} - modifier.vfieldboxes = [] - modifier.make_virtual(vbox, None, ['a', 'b'], [b1, b2]) + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2]) assert modifier.liveboxes == {vbox: UNASSIGNEDVIRTUAL, b1: UNASSIGNED, b2: UNASSIGNED} - assert len(modifier.virtuals) == 0 - assert modifier.vfieldboxes == [] + assert modifier.vfieldboxes == {vbox: [b1, b2]} modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} modifier.liveboxes = {} - modifier.virtuals = [None] - modifier.vfieldboxes = [None] - modifier.vinfos_not_env = {} - modifier.make_virtual(vbox, None, ['a', 'b', 'c'], [b1, b2, vbox]) + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2, vbox]) assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, vbox: tag(0, TAGVIRTUAL)} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2, vbox]] + assert modifier.vfieldboxes == {vbox: [b1, b2, vbox]} def _resume_remap(liveboxes, expected, *newvalues): newboxes = [] From pedronis at codespeak.net Wed Nov 18 18:57:25 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 18:57:25 +0100 (CET) Subject: [pypy-svn] r69389 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091118175725.619DD318141@codespeak.net> Author: pedronis Date: Wed Nov 18 18:57:24 2009 New Revision: 69389 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: unused Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 18:57:24 2009 @@ -261,7 +261,6 @@ numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} - self.vinfos_not_env = {} storage.rd_numb = numb storage.rd_snapshot = None From pedronis at codespeak.net Wed Nov 18 18:59:33 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 18:59:33 +0100 (CET) Subject: [pypy-svn] r69390 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091118175933.998EA318141@codespeak.net> Author: pedronis Date: Wed Nov 18 18:59:32 2009 New Revision: 69390 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: cleanups Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 18:59:32 2009 @@ -258,7 +258,8 @@ def finish(self, values): # compute the numbering storage = self.storage - numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) + numb, liveboxes_from_env, v = self.memo.number(values, + storage.rd_snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} storage.rd_numb = numb @@ -284,19 +285,20 @@ return liveboxes[:] def _number_virtuals(self, liveboxes, values, num_env_virtuals): + memo = self.memo prev_liveboxes_cache = 0 prev_virtuals_cache = 0 - new_liveboxes = [None] * self.memo.num_cached_boxes() + new_liveboxes = [None] * memo.num_cached_boxes() for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: assert tagged_eq(tagged, UNASSIGNED) - index = self.memo.assign_number_to_box(box, new_liveboxes) + index = memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): - index = self.memo.assign_number_to_virtual(box) + index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() @@ -306,7 +308,7 @@ storage.rd_virtuals = None vfieldboxes = self.vfieldboxes if vfieldboxes: - length = num_env_virtuals + self.memo.num_cached_virtuals() + length = num_env_virtuals + memo.num_cached_virtuals() virtuals = storage.rd_virtuals = [None] * length for virtualbox, fieldboxes in vfieldboxes.iteritems(): num, _ = untag(self.liveboxes[virtualbox]) From pedronis at codespeak.net Wed Nov 18 19:04:24 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 19:04:24 +0100 (CET) Subject: [pypy-svn] r69391 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091118180424.8C3D5318141@codespeak.net> Author: pedronis Date: Wed Nov 18 19:04:24 2009 New Revision: 69391 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: unused Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 19:04:24 2009 @@ -286,8 +286,6 @@ def _number_virtuals(self, liveboxes, values, num_env_virtuals): memo = self.memo - prev_liveboxes_cache = 0 - prev_virtuals_cache = 0 new_liveboxes = [None] * memo.num_cached_boxes() for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) From pedronis at codespeak.net Wed Nov 18 19:05:19 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 18 Nov 2009 19:05:19 +0100 (CET) Subject: [pypy-svn] r69392 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091118180519.3B18A318141@codespeak.net> Author: pedronis Date: Wed Nov 18 19:05:18 2009 New Revision: 69392 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: tweak Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Wed Nov 18 19:05:18 2009 @@ -299,7 +299,6 @@ index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() - liveboxes.extend(new_liveboxes) storage = self.storage From arigo at codespeak.net Wed Nov 18 20:02:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 20:02:19 +0100 (CET) Subject: [pypy-svn] r69393 - pypy/branch/shorter-guard-path Message-ID: <20091118190219.74050318141@codespeak.net> Author: arigo Date: Wed Nov 18 20:02:18 2009 New Revision: 69393 Added: pypy/branch/shorter-guard-path/ - copied from r69392, pypy/trunk/ Log: A branch in which to shrink the assembler produced for every guard. From afa at codespeak.net Wed Nov 18 20:07:08 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 18 Nov 2009 20:07:08 +0100 (CET) Subject: [pypy-svn] r69394 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118190708.4F937318141@codespeak.net> Author: afa Date: Wed Nov 18 20:07:07 2009 New Revision: 69394 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/config.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Implement BINARY datatype Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Nov 18 20:07:07 2009 @@ -9,6 +9,7 @@ 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', 'DATETIME': 'interp_variable.VT_DateTime', + 'BINARY': 'interp_variable.VT_Binary', 'Variable': 'interp_variable.W_Variable', } Modified: pypy/trunk/pypy/module/oracle/config.py ============================================================================== --- pypy/trunk/pypy/module/oracle/config.py (original) +++ pypy/trunk/pypy/module/oracle/config.py Wed Nov 18 20:07:07 2009 @@ -3,6 +3,7 @@ WITH_UNICODE = False MAX_STRING_CHARS = 4000 +MAX_BINARY_BYTES = 4000 if WITH_UNICODE: CHARSETID = roci.OCI_UTF16ID Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 18 20:07:07 2009 @@ -149,7 +149,7 @@ # allocate the indicator for the variable self.indicator = lltype.malloc(rffi.CArrayPtr(roci.sb2).TO, self.allocatedElements, - flavor='raw', zero=True) # XXX + flavor='raw', zero=True) # ensure that all variable values start out NULL for i in range(self.allocatedElements): @@ -159,7 +159,7 @@ if self.isVariableLength: self.returnCode = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, self.allocatedElements, - flavor='raw', zero=True) # XXX + flavor='raw', zero=True) # perform extended initialization self.initialize(cursor) @@ -170,6 +170,8 @@ lltype.free(self.actualLength, flavor='raw') if self.data: lltype.free(self.data, flavor='raw') + if self.returnCode: + lltype.free(self.returnCode, flavor='raw') def getBufferSize(self): return self.size @@ -490,8 +492,9 @@ size = 18 isVariableLength = False -class VT_Binary(W_Variable): - pass +class VT_Binary(VT_String): + oracleType = roci.SQLT_BIN + size = config.MAX_BINARY_BYTES class VT_LongBinary(W_Variable): pass Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Wed Nov 18 20:07:07 2009 @@ -4,7 +4,6 @@ def test_rowid(self): cur = self.cnx.cursor() - var = cur.var(oracle.NUMBER) cur.execute("select rowid from dual") rowid, = cur.fetchone() cur.execute("select * from dual where rowid = :r", @@ -92,3 +91,18 @@ assert tablelen.getvalue() == 20 # dbms_utility.comma_to_table returns a 'NULL-terminated' table assert arrayvar.getvalue() == array + [None] + + def test_binary(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (rawcol raw(30))") + + cur.setinputsizes(p=oracle.BINARY) + cur.execute("insert into pypy_temp_table values (:p)", + p="raw string") + cur.execute("select * from pypy_temp_table") + data = cur.fetchall() + assert data == [("raw string",)] From fijal at codespeak.net Wed Nov 18 21:06:37 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 18 Nov 2009 21:06:37 +0100 (CET) Subject: [pypy-svn] r69395 - pypy/trunk/pypy/config Message-ID: <20091118200637.629E0318140@codespeak.net> Author: fijal Date: Wed Nov 18 21:06:35 2009 New Revision: 69395 Modified: pypy/trunk/pypy/config/translationoption.py Log: Store sinking seems to be safe, lib-python tests pass. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Wed Nov 18 21:06:35 2009 @@ -245,7 +245,7 @@ "Tranform graphs in SSI form into graphs tailored for " "stack based virtual machines (only for backends that support it)", default=True), - BoolOption("storesink", "Perform store sinking", default=False), + BoolOption("storesink", "Perform store sinking", default=True), BoolOption("none", "Do not run any backend optimizations", requires=[('translation.backendopt.inline', False), From arigo at codespeak.net Wed Nov 18 22:28:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 18 Nov 2009 22:28:38 +0100 (CET) Subject: [pypy-svn] r69400 - in pypy/branch/shorter-guard-path/pypy/jit: backend backend/cli backend/llgraph backend/llsupport backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20091118212838.25F7931813D@codespeak.net> Author: arigo Date: Wed Nov 18 22:28:37 2009 New Revision: 69400 Added: pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_model.py (contents, props changed) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/cli/method.py pypy/branch/shorter-guard-path/pypy/jit/backend/cli/runner.py pypy/branch/shorter-guard-path/pypy/jit/backend/llgraph/runner.py pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py pypy/branch/shorter-guard-path/pypy/jit/backend/model.py pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_random.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/runner.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/compile.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/history.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/logger.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/pyjitpl.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_compile.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_logger.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/shorter-guard-path/pypy/jit/metainterp/warmstate.py Log: (pedronis, arigo) Refactoring refactoring. Cancel a previous refactoring that moved the numbering of the guard descrs to the front-end. Instead, move that counting to AbstractCPU. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/cli/method.py Wed Nov 18 22:28:37 2009 @@ -268,9 +268,6 @@ try: return self.cpu.failing_ops.index(op) except ValueError: - descr = op.descr - assert isinstance(descr, history.AbstractFailDescr) - descr.get_index() self.cpu.failing_ops.append(op) return len(self.cpu.failing_ops)-1 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/cli/runner.py Wed Nov 18 22:28:37 2009 @@ -135,9 +135,7 @@ func = cliloop.funcbox.holder.GetFunc() func(self.get_inputargs()) op = self.failing_ops[self.inputargs.get_failed_op()] - descr = op.descr - assert isinstance(descr, AbstractFailDescr) - return descr.get_index() + return op.descr def set_future_value_int(self, index, intvalue): self.get_inputargs().set_int(index, intvalue) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/llgraph/runner.py Wed Nov 18 22:28:37 2009 @@ -88,6 +88,7 @@ def __init__(self, rtyper, stats=None, translate_support_code=False, annmixlevel=None, gcdescr=None): + model.AbstractCPU.__init__(self) self.rtyper = rtyper self.translate_support_code = translate_support_code self.stats = stats or MiniStats() @@ -167,7 +168,7 @@ if op.is_guard(): faildescr = op.descr assert isinstance(faildescr, history.AbstractFailDescr) - fail_index = faildescr.get_index() + fail_index = self.get_fail_descr_number(faildescr) index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: @@ -192,8 +193,7 @@ llimpl.compile_add_jump_target(c, compiled_version) elif op.opnum == rop.FINISH: faildescr = op.descr - assert isinstance(faildescr, history.AbstractFailDescr) - index = faildescr.get_index() + index = self.get_fail_descr_number(faildescr) llimpl.compile_add_fail(c, index) else: assert False, "unknown operation" @@ -210,7 +210,7 @@ fail_index = llimpl.frame_execute(frame) # we hit a FAIL operation. self.latest_frame = frame - return fail_index + return self.get_fail_descr_from_number(fail_index) def set_future_value_int(self, index, intvalue): llimpl.set_future_value_int(index, intvalue) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py Wed Nov 18 22:28:37 2009 @@ -20,6 +20,7 @@ def __init__(self, rtyper, stats, translate_support_code=False, gcdescr=None): from pypy.jit.backend.llsupport.gc import get_ll_description + AbstractCPU.__init__(self) self.rtyper = rtyper self.stats = stats self.translate_support_code = translate_support_code Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/model.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/model.py Wed Nov 18 22:28:37 2009 @@ -1,6 +1,25 @@ +from pypy.jit.metainterp import history + + class AbstractCPU(object): supports_floats = False + def __init__(self): + self.fail_descr_list = [] + + def get_fail_descr_number(self, descr): + assert isinstance(descr, history.AbstractFailDescr) + n = descr.index + if n < 0: + lst = self.fail_descr_list + n = len(lst) + lst.append(descr) + descr.index = n + return n + + def get_fail_descr_from_number(self, n): + return self.fail_descr_list[n] + def set_class_sizes(self, class_sizes): self.class_sizes = class_sizes Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py Wed Nov 18 22:28:37 2009 @@ -39,7 +39,7 @@ else: raise NotImplementedError(box) res = self.cpu.execute_token(looptoken) - if res == operations[-1].descr.index: + if res is operations[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -102,7 +102,7 @@ fail = self.cpu.execute_token(looptoken) res = self.cpu.get_latest_value_int(0) assert res == 3 - assert fail == 1 + assert fail.identifier == 1 def test_compile_loop(self): i0 = BoxInt() @@ -121,7 +121,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 10 @@ -179,7 +179,7 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 20 @@ -188,8 +188,6 @@ class UntouchableFailDescr(AbstractFailDescr): def __setattr__(self, name, value): py.test.fail("finish descrs should not be touched") - def get_index(self): - return 7 faildescr = UntouchableFailDescr() # to check that is not touched looptoken = LoopToken() operations = [ @@ -198,7 +196,7 @@ self.cpu.compile_loop([i0], operations, looptoken) self.cpu.set_future_value_int(0, 99) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 99 @@ -208,7 +206,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 42 @@ -218,7 +216,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr def test_execute_operations_in_env(self): cpu = self.cpu @@ -302,9 +300,9 @@ self.cpu.set_future_value_int(1, y) fail = self.cpu.execute_token(looptoken) if (z == boom) ^ reversed: - assert fail == 1 + assert fail.identifier == 1 else: - assert fail == 2 + assert fail.identifier == 2 if z != boom: assert self.cpu.get_latest_value_int(0) == z assert not self.cpu.get_exception() @@ -777,7 +775,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 15 + assert fail.identifier == 15 # dstvalues = values[:] for _ in range(11): @@ -829,7 +827,7 @@ for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_float(0) assert res == 8.5 for i in range(1, len(fboxes)): @@ -880,7 +878,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 1 + assert fail.identifier == 1 class LLtypeBackendTest(BaseBackendTest): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py Wed Nov 18 22:28:37 2009 @@ -464,7 +464,7 @@ descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -486,7 +486,7 @@ _, vtableptr = builder.get_random_structure_type_and_vtable(r) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = builder.subset_of_intvars(r) op._exc_box = None builder.should_fail_by = op @@ -509,7 +509,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -527,7 +527,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op @@ -553,7 +553,7 @@ break other_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [other_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op Added: pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_model.py ============================================================================== --- (empty file) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_model.py Wed Nov 18 22:28:37 2009 @@ -0,0 +1,21 @@ +from pypy.jit.metainterp.history import AbstractFailDescr +from pypy.jit.backend.model import AbstractCPU + + +def test_faildescr_numbering(): + cpu = AbstractCPU() + fail_descr1 = AbstractFailDescr() + fail_descr2 = AbstractFailDescr() + + n1 = cpu.get_fail_descr_number(fail_descr1) + n2 = cpu.get_fail_descr_number(fail_descr2) + assert n1 != n2 + + fail_descr = cpu.get_fail_descr_from_number(n1) + assert fail_descr is fail_descr1 + fail_descr = cpu.get_fail_descr_from_number(n2) + assert fail_descr is fail_descr2 + + # provides interning on its own + n1_1 = cpu.get_fail_descr_number(fail_descr1) + assert n1_1 == n1 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_random.py Wed Nov 18 22:28:37 2009 @@ -18,8 +18,6 @@ self.operations = subops class OperationBuilder(object): - failnumbering = 0 - def __init__(self, cpu, loop, vars): self.cpu = cpu self.loop = loop @@ -36,12 +34,6 @@ self.counter = 0 assert len(self.intvars) == len(dict.fromkeys(self.intvars)) - @classmethod - def make_fail_descr(cls): - descr = BasicFailDescr(cls.failnumbering) - cls.failnumbering += 1 - return descr - def fork(self, cpu, loop, vars): fork = self.__class__(cpu, loop, vars) fork.prebuilt_ptr_consts = self.prebuilt_ptr_consts @@ -282,7 +274,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = fail_subset builder.loop.operations.append(op) @@ -343,7 +335,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = builder.subset_of_intvars(r) if not passing: builder.should_fail_by = op @@ -559,7 +551,7 @@ endvars.append(v) r.shuffle(endvars) loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr(-2))) + descr=BasicFailDescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -605,7 +597,7 @@ else: raise NotImplementedError(box) fail = cpu.execute_token(self.loop.token) - assert fail == self.should_fail_by.descr.index + assert fail is self.should_fail_by.descr for i, v in enumerate(self.get_fail_args()): if isinstance(v, (BoxFloat, ConstFloat)): value = cpu.get_latest_value_float(i) @@ -634,7 +626,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.descr = self.builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = [] return op Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Wed Nov 18 22:28:37 2009 @@ -795,8 +795,7 @@ # don't break the following code sequence! mc = mc._mc mc.LEA(esp, addr_add(imm(0), ebp, (-RET_BP + 2) * WORD)) - assert isinstance(faildescr, AbstractFailDescr) - fail_index = faildescr.get_index() + fail_index = self.cpu.get_fail_descr_number(faildescr) mc.MOV(eax, imm(fail_index)) mc.POP(edi) mc.POP(esi) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/runner.py Wed Nov 18 22:28:37 2009 @@ -63,7 +63,7 @@ addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) fail_index = self._execute_call(func) - return fail_index + return self.get_fail_descr_from_number(fail_index) def _execute_call(self, func): # help flow objspace Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_recompilation.py Wed Nov 18 22:28:37 2009 @@ -21,7 +21,7 @@ bridge = self.attach_bridge(ops, loop, -2) self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 def test_compile_bridge_deeper(self): @@ -52,7 +52,7 @@ assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 assert self.getint(1) == 22 assert self.getint(2) == 23 @@ -78,7 +78,7 @@ bridge = self.attach_bridge(ops, other_loop, 0, looptoken=loop.token) self.cpu.set_future_value_int(0, 1) fail = self.run(other_loop) - assert fail == 1 + assert fail.identifier == 1 def test_bridge_jumps_to_self_deeper(self): loop = self.interpret(''' Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py Wed Nov 18 22:28:37 2009 @@ -260,7 +260,7 @@ self.cpu.set_future_value_int(0, b.value) else: self.cpu.set_future_value_ref(0, b.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) if guard == rop.GUARD_FALSE: assert result == execute(self.cpu, op, None, b).value @@ -306,7 +306,7 @@ self.cpu.compile_loop(inputargs, ops, looptoken) for i, box in enumerate(inputargs): self.cpu.set_future_value_int(i, box.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) expected = execute(self.cpu, op, None, a, b).value if guard == rop.GUARD_FALSE: Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/compile.py Wed Nov 18 22:28:37 2009 @@ -113,7 +113,7 @@ metainterp_sd.log("compiled new " + type) def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): - n = faildescr.get_index() + n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): show_loop(metainterp_sd) @@ -133,14 +133,7 @@ # ____________________________________________________________ class _DoneWithThisFrameDescr(AbstractFailDescr): - - def __init__(self, lst): - "NOT_RPYTHON" - self.index = len(lst) - lst.append(self) - - def get_index(self): - return self.index + pass class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): def handle_fail(self, metainterp_sd): @@ -173,18 +166,6 @@ raise metainterp_sd.ExitFrameWithExceptionRef(cpu, value) -done_with_this_frame_descrs = [] -done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid( - done_with_this_frame_descrs) -done_with_this_frame_descr_int = DoneWithThisFrameDescrInt( - done_with_this_frame_descrs) -done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef( - done_with_this_frame_descrs) -done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat( - done_with_this_frame_descrs) -exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef( - done_with_this_frame_descrs) - prebuiltNotSpecNode = NotSpecNode() class TerminatingLoopToken(LoopToken): @@ -194,22 +175,30 @@ self.specnodes = [prebuiltNotSpecNode]*nargs self.finishdescr = finishdescr -# pseudo loop tokens to make the life of optimize.py easier -loop_tokens_done_with_this_frame_int = [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ] -loop_tokens_done_with_this_frame_ref = [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ] -loop_tokens_done_with_this_frame_float = [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ] -loop_tokens_done_with_this_frame_void = [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ] -loop_tokens_exit_frame_with_exception_ref = [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ] +def make_done_loop_tokens(): + done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() + done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() + done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() + done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() + exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() + + # pseudo loop tokens to make the life of optimize.py easier + return {'loop_tokens_done_with_this_frame_int': [ + TerminatingLoopToken(1, done_with_this_frame_descr_int) + ], + 'loop_tokens_done_with_this_frame_ref': [ + TerminatingLoopToken(1, done_with_this_frame_descr_ref) + ], + 'loop_tokens_done_with_this_frame_float': [ + TerminatingLoopToken(1, done_with_this_frame_descr_float) + ], + 'loop_tokens_done_with_this_frame_void': [ + TerminatingLoopToken(0, done_with_this_frame_descr_void) + ], + 'loop_tokens_exit_frame_with_exception_ref': [ + TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) + ], + } class ResumeDescr(AbstractFailDescr): def __init__(self, original_greenkey): @@ -222,13 +211,6 @@ def __init__(self, metainterp_sd, original_greenkey): ResumeDescr.__init__(self, original_greenkey) self.metainterp_sd = metainterp_sd - self.index = -1 - - def get_index(self): - if self.index == -1: - globaldata = self.metainterp_sd.globaldata - self.index = globaldata.get_fail_descr_number(self) - return self.index def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/history.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/history.py Wed Nov 18 22:28:37 2009 @@ -122,21 +122,16 @@ return '%r' % (self,) class AbstractFailDescr(AbstractDescr): + index = -1 - def get_index(self): - raise NotImplementedError def handle_fail(self, metainterp_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError class BasicFailDescr(AbstractFailDescr): - - def __init__(self, index=-1): - self.index = index - - def get_index(self): - return self.index + def __init__(self, identifier=None): + self.identifier = identifier # for testing class AbstractMethDescr(AbstractDescr): # the base class of the result of cpu.methdescrof() Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/logger.py Wed Nov 18 22:28:37 2009 @@ -88,8 +88,8 @@ if op.descr is not None: descr = op.descr if is_guard and self.guard_number: - assert isinstance(descr, AbstractFailDescr) - r = "" % descr.get_index() + index = self.metainterp_sd.cpu.get_fail_descr_number(descr) + r = "" % index else: r = self.repr_of_descr(descr) args += ', descr=' + r Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/pyjitpl.py Wed Nov 18 22:28:37 2009 @@ -1022,6 +1022,8 @@ self._addr2name_keys = [] self._addr2name_values = [] + self.__dict__.update(compile.make_done_loop_tokens()) + def _freeze_(self): return True @@ -1044,8 +1046,7 @@ else: self.num_green_args = 0 self.state = None - self.globaldata = MetaInterpGlobalData( - self, compile.done_with_this_frame_descrs) + self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" @@ -1136,11 +1137,10 @@ # ____________________________________________________________ class MetaInterpGlobalData(object): - def __init__(self, staticdata, prebuilt_fail_descr_list): + def __init__(self, staticdata): self.initialized = False self.indirectcall_dict = None self.addr2name = None - self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # state = staticdata.state @@ -1164,16 +1164,6 @@ cell.compiled_merge_points = [] return cell.compiled_merge_points - def get_fail_descr_number(self, descr): - assert isinstance(descr, history.AbstractFailDescr) - lst = self.fail_descr_list - n = len(lst) - lst.append(descr) - return n - - def get_fail_descr_from_number(self, n): - return self.fail_descr_list[n] - # ____________________________________________________________ class MetaInterp(object): @@ -1622,16 +1612,16 @@ if sd.result_type == 'void': assert exitbox is None exits = [] - loop_tokens = compile.loop_tokens_done_with_this_frame_void + loop_tokens = sd.loop_tokens_done_with_this_frame_void elif sd.result_type == 'int': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_int + loop_tokens = sd.loop_tokens_done_with_this_frame_int elif sd.result_type == 'ref': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_ref + loop_tokens = sd.loop_tokens_done_with_this_frame_ref elif sd.result_type == 'float': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_float + loop_tokens = sd.loop_tokens_done_with_this_frame_float else: assert False self.history.record(rop.JUMP, exits, None) @@ -1643,7 +1633,8 @@ self.gen_store_back_in_virtualizable() # temporarily put a JUMP to a pseudo-loop self.history.record(rop.JUMP, [valuebox], None) - loop_tokens = compile.loop_tokens_exit_frame_with_exception_ref + sd = self.staticdata + loop_tokens = sd.loop_tokens_exit_frame_with_exception_ref target_loop_token = compile.compile_new_bridge(self, loop_tokens, self.resumekey) assert target_loop_token is loop_tokens[0] Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_compile.py Wed Nov 18 22:28:37 2009 @@ -103,22 +103,3 @@ assert loop_tokens == [loop_token] assert len(cpu.seen) == 0 assert staticdata.globaldata.loopnumbering == 2 - -def test_resumeguarddescr_get_index(): - from pypy.jit.metainterp.pyjitpl import MetaInterpGlobalData - - class FakeStaticData: - state = None - virtualizable_info = None - gd = MetaInterpGlobalData(FakeStaticData, []) - FakeStaticData.globaldata = gd - - rgd = ResumeGuardDescr(FakeStaticData, ()) - - fail_index = rgd.get_index() - fail_index1 = rgd.get_index() - - assert fail_index == fail_index1 - - assert gd.get_fail_descr_from_number(fail_index) is rgd - Modified: pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/metainterp/test/test_logger.py Wed Nov 18 22:28:37 2009 @@ -6,6 +6,8 @@ from StringIO import StringIO from pypy.jit.metainterp.test.test_optimizeopt import equaloplists from pypy.jit.metainterp.history import AbstractDescr, LoopToken, BasicFailDescr +from pypy.jit.backend.model import AbstractCPU + class Descr(AbstractDescr): pass @@ -36,8 +38,8 @@ def make_metainterp_sd(self): class FakeMetaInterpSd: - class cpu: - ts = self.ts + cpu = AbstractCPU() + cpu.ts = self.ts def get_name_from_address(self, addr): return 'Name' return FakeMetaInterpSd() @@ -113,7 +115,7 @@ pure_parse(output) def test_guard(self): - namespace = {'fdescr': BasicFailDescr(4)} + namespace = {'fdescr': BasicFailDescr()} inp = ''' [i0] guard_true(i0, descr=fdescr) [i0] @@ -121,15 +123,14 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" + assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) - def boom(): - raise Exception - namespace['fdescr'].get_index = boom logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) - assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + lastline = output.splitlines()[-1] + assert lastline.startswith("guard_true(i0, descr=<") + assert not lastline.startswith("guard_true(i0, descr= Author: arigo Date: Wed Nov 18 22:32:38 2009 New Revision: 69401 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py Log: (pedronis, arigo) Fix tests. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/runner_test.py Wed Nov 18 22:32:38 2009 @@ -187,6 +187,8 @@ i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): def __setattr__(self, name, value): + if name == 'index': + return AbstractFailDescr.__setattr__(self, name, value) py.test.fail("finish descrs should not be touched") faildescr = UntouchableFailDescr() # to check that is not touched looptoken = LoopToken() Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_ll_random.py Wed Nov 18 22:32:38 2009 @@ -4,6 +4,7 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import ConstInt, ConstPtr from pypy.jit.metainterp.history import ConstAddr, BoxPtr, BoxInt +from pypy.jit.metainterp.history import BasicFailDescr from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rarithmetic import intmask from pypy.rpython.llinterp import LLException From arigo at codespeak.net Thu Nov 19 00:04:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 19 Nov 2009 00:04:49 +0100 (CET) Subject: [pypy-svn] r69403 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091118230449.BD1E231813D@codespeak.net> Author: arigo Date: Thu Nov 19 00:04:49 2009 New Revision: 69403 Added: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py (contents, props changed) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/regalloc.py Log: (pedronis, arigo) Generate much shorter failure paths. The idea is to use generate_quick_failure() to write just a jump to control code which itself, if ever called, writes the proper failure handling code. In this way, for most guards we have only the jump to control code, and for the few guards that we have actually seen failing at least once we have the same relatively efficient code as before. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Thu Nov 19 00:04:49 2009 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs @@ -39,6 +40,17 @@ self.old_mcs = [] # keepalive self._mc = codebuf.MachineCodeBlock(self.MC_SIZE) + def bytes_free(self): + return self._mc._size - self._mc._pos + + def make_new_mc(self): + new_mc = codebuf.MachineCodeBlock(self.MC_SIZE) + self._mc.JMP(rel32(new_mc.tell())) + self._mc.done() + self.old_mcs.append(self._mc) + self._mc = new_mc + make_new_mc._dont_inline_ = True + def tell(self): return self._mc.tell() @@ -49,12 +61,8 @@ def method(self, *args): # XXX er.... pretty random number, just to be sure # not to write half-instruction - if self._mc._pos + 64 >= self._mc._size: - new_mc = codebuf.MachineCodeBlock(self.MC_SIZE) - self._mc.JMP(rel32(new_mc.tell())) - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc + if self.bytes_free() < 64: + self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method @@ -66,7 +74,6 @@ class Assembler386(object): mc = None mc2 = None - debug_markers = True def __init__(self, cpu, translate_support_code=False): self.cpu = cpu @@ -79,6 +86,7 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.setup_failure_recovery() def leave_jitted_hook(self): # XXX BIG FAT WARNING XXX @@ -115,6 +123,7 @@ # 'mc2' is for guard recovery code self.mc = MachineCodeBlockWrapper() self.mc2 = MachineCodeBlockWrapper() + self._build_failure_recovery_builder_trampoline() def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -150,9 +159,12 @@ # for the benefit of tests faildescr._x86_bridge_stack_depth = stack_depth # patch the jump from original guard + self.patch_jump(faildescr, adr_bridge) + + def patch_jump(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_bridge - adr_jump_offset - 4)) + mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) mc.valgrind_invalidated() mc.done() @@ -743,46 +755,140 @@ def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): self._no_const_locs(failargs) - addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) faildescr._x86_faillocs = fail_locs - self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) + return self.generate_quick_failure(faildescr, failargs, exc) + + def generate_quick_failure(self, faildescr, failargs, exc): + """Generate the initial code for handling a failure. We try to + keep it as compact as possible. The idea is that this code is + executed at most once (and very often, zero times); when + executed, it generates a more complete piece of code which can + really handle recovery from this particular failure. + """ + fail_index = self.cpu.get_fail_descr_number(faildescr) + bytes_needed = 20 + len(failargs) // 8 # conservative estimate + if self.mc2.bytes_free() < bytes_needed: + self.mc2.make_new_mc() + mc = self.mc2._mc + addr = mc.tell() + mc.PUSH(imm32(fail_index)) + mc.CALL(rel32(self.failure_recovery_code)) + # write a bitfield of 0's (int or float argument) or 1's (ref argument) + self.write_bitfield_for_failargs(mc, failargs, exc) + if not we_are_translated(): + faildescr._x86_exc_locs_are_ref = ( + exc, [v.type == REF for v in failargs]) return addr - def generate_failure(self, mc, faildescr, failargs, locs, exc): - pos = mc.tell() + def write_bitfield_for_failargs(self, mc, failargs, exc): + #print 'writing:', mc.tell() + #mc.writechr(0xDD) + bitfield = int(exc) # the first bit: "save exceptions" + bit = 0x02 for i in range(len(failargs)): - arg = failargs[i] + if bit == 0x100: + mc.writechr(bitfield) + bitfield = 0 + bit = 0x01 + if failargs[i].type == REF: + bitfield |= bit + bit <<= 1 + mc.writechr(bitfield) + #mc.writechr(0xCC) + + def getbit_from_bitfield(self, bitfield_ptr, i): + byte = ord(bitfield_ptr[i >> 3]) + return bool(byte & (1 << (i & 7))) + + def setup_failure_recovery(self): + + def failure_recovery_builder(fail_index, bitfield_ptr): + #assert bitfield_ptr[0] == chr(0xDD) + faildescr = self.cpu.get_fail_descr_from_number(fail_index) + locs = faildescr._x86_faillocs + #assert bitfield_ptr[1 + (1+len(locs)+7)//8] == chr(0xCC) + exc = self.getbit_from_bitfield(bitfield_ptr, 0) # the first bit + locs_are_ref = [self.getbit_from_bitfield(bitfield_ptr, 1 + i) + for i in range(len(locs))] + if not we_are_translated(): + #print 'completing:', rffi.cast(lltype.Signed, bitfield_ptr) + assert (exc, locs_are_ref) == faildescr._x86_exc_locs_are_ref + addr = self.mc2.tell() + self.generate_failure(self.mc2, fail_index, locs, exc, + locs_are_ref) + self.patch_jump(faildescr, addr) + return addr + + self.failure_recovery_builder = failure_recovery_builder + self.failure_recovery_code = 0 + + _FAILURE_RECOVERY_BUILDER = lltype.Ptr(lltype.FuncType( + [lltype.Signed, rffi.CCHARP], lltype.Signed)) + + def _build_failure_recovery_builder_trampoline(self): + failure_recovery_builder = llhelper(self._FAILURE_RECOVERY_BUILDER, + self.failure_recovery_builder) + failure_recovery_builder = rffi.cast(lltype.Signed, + failure_recovery_builder) + mc = self.mc2 + code = mc.tell() + mc.PUSH(ebp) + mc.MOV(ebp, esp) + # push all registers on the stack + mc.PUSHA() + if self.cpu.supports_floats: + mc.SUB(esp, imm(8*8)) + for i in range(8): + mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # really call the failure recovery builder code + mc.PUSH(mem(ebp, 4)) # the return address: ptr to bitfield + mc.PUSH(mem(ebp, 8)) # fail_index + mc.CALL(rel32(failure_recovery_builder)) + mc.ADD(esp, imm(8)) + # save the return value into ebp+4. This is the address of the code + # written by generate_failure. Doing so overwrites this function's + # own return address, which was just a ptr to the bitfield, so far. + mc.MOV(mem(ebp, 4), eax) + # pop all registers + if self.cpu.supports_floats: + for i in range(8): + mc.MOVSD(xmm_registers[i], mem64(esp, 8*i)) + mc.ADD(esp, imm(8*8)) + mc.POPA() + # epilogue + mc.POP(ebp) + mc.RET(imm16(4)) + self.failure_recovery_code = code + + def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): + for i in range(len(locs)): loc = locs[i] if isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), loc) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(heap(adr), loc) - for i in range(len(failargs)): - arg = failargs[i] + for i in range(len(locs)): loc = locs[i] if not isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: mc.MOVSD(xmm0, loc) adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), xmm0) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) mc.MOV(heap(adr), eax) - if self.debug_markers: - mc.MOV(eax, imm(pos)) - mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -795,7 +901,6 @@ # don't break the following code sequence! mc = mc._mc mc.LEA(esp, addr_add(imm(0), ebp, (-RET_BP + 2) * WORD)) - fail_index = self.cpu.get_fail_descr_number(faildescr) mc.MOV(eax, imm(fail_index)) mc.POP(edi) mc.POP(esi) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/regalloc.py Thu Nov 19 00:04:49 2009 @@ -389,8 +389,10 @@ def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] - self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, - locs, self.exc) + locs_are_ref = [v.type == REF for v in op.args] + fail_index = self.assembler.cpu.get_fail_descr_number(op.descr) + self.assembler.generate_failure(self.assembler.mc, fail_index, locs, + self.exc, locs_are_ref) self.possibly_free_vars(op.args) def consider_guard_no_exception(self, op, ignored): Added: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- (empty file) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Thu Nov 19 00:04:49 2009 @@ -0,0 +1,34 @@ +from pypy.jit.backend.x86.assembler import Assembler386 +from pypy.jit.metainterp.history import BoxInt, BoxPtr + + +class FakeCPU: + rtyper = None + +class FakeMC: + def __init__(self): + self.content = [] + def writechr(self, n): + self.content.append(n) + + +def test_bitfield(): + assembler = Assembler386(FakeCPU()) + mc = FakeMC() + assembler.write_bitfield_for_failargs(mc, [BoxInt(), BoxPtr()], False) + assert mc.content == [4] + bitfield = map(chr, mc.content) + assert assembler.getbit_from_bitfield(bitfield, 0) == False + assert assembler.getbit_from_bitfield(bitfield, 1) == False + assert assembler.getbit_from_bitfield(bitfield, 2) == True + +def test_larger_bitfield(): + assembler = Assembler386(FakeCPU()) + mc = FakeMC() + lst = [BoxInt(), BoxPtr(), BoxPtr()] * 6 + assembler.write_bitfield_for_failargs(mc, lst, True) + bitfield = map(chr, mc.content) + assert assembler.getbit_from_bitfield(bitfield, 0) == True + for i in range(len(lst)): + expected = (lst[i].__class__ == BoxPtr) + assert assembler.getbit_from_bitfield(bitfield, 1+i) == expected From afa at codespeak.net Thu Nov 19 00:34:07 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 00:34:07 +0100 (CET) Subject: [pypy-svn] r69405 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118233407.7DD3516803E@codespeak.net> Author: afa Date: Thu Nov 19 00:34:06 2009 New Revision: 69405 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: start supporting long strings Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 00:34:06 2009 @@ -482,7 +482,26 @@ pass class VT_LongString(W_Variable): - pass + oracleType = roci.SQLT_LVC + isVariableLength = True + size = 128 * 1024 + + def setValueProc(self, space, pos, w_value): + buf = config.StringBuffer() + buf.fill(space, w_value) + + try: + # ensure that the buffer is large enough + if buf.size + rffi.sizeof(roci.ub4) > self.bufferSize: + self.resize(buf.size + rffi.sizeof(roci.ub4)) + + # copy the string to the Oracle buffer + data = rffi.ptradd(self.data, pos * self.bufferSize) + rffi.cast(roci.Ptr(roci.ub4), data)[0] = rffi.cast(roci.ub4, buf.size) + for index in range(buf.size): + data[index + rffi.sizeof(roci.ub4)] = buf.ptr[index] + finally: + buf.clear() class VT_FixedNationalChar(W_Variable): pass @@ -496,7 +515,7 @@ oracleType = roci.SQLT_BIN size = config.MAX_BINARY_BYTES -class VT_LongBinary(W_Variable): +class VT_LongBinary(VT_LongString): pass class VT_NativeFloat(W_Variable): @@ -872,7 +891,10 @@ if space.is_true(space.isinstance(w_value, space.w_str)): size = space.int_w(space.len(w_value)) - return VT_String, size, numElements + if size > config.MAX_STRING_CHARS: + return VT_LongString, size, numElements + else: + return VT_String, size, numElements # XXX Unicode Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 19 00:34:06 2009 @@ -62,7 +62,7 @@ OCI_IND_NULL OCI_IND_NOTNULL OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE - SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI + SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Thu Nov 19 00:34:06 2009 @@ -106,3 +106,14 @@ cur.execute("select * from pypy_temp_table") data = cur.fetchall() assert data == [("raw string",)] + + def test_longstring(self): + cur = self.cnx.cursor() + cur.execute(""" + declare + t_Temp varchar2(10000); + begin + t_Temp := :bigString; + end;""", + bigString="X" * 10000) + From afa at codespeak.net Thu Nov 19 00:58:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 00:58:43 +0100 (CET) Subject: [pypy-svn] r69406 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091118235843.2674B168022@codespeak.net> Author: afa Date: Thu Nov 19 00:58:42 2009 New Revision: 69406 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Ensure that variables are created with enough size Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 19 00:58:42 2009 @@ -10,6 +10,7 @@ 'STRING': 'interp_variable.VT_String', 'DATETIME': 'interp_variable.VT_DateTime', 'BINARY': 'interp_variable.VT_Binary', + 'LONG_STRING': 'interp_variable.VT_LongString', 'Variable': 'interp_variable.W_Variable', } Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 00:58:42 2009 @@ -988,4 +988,4 @@ # everything else ought to be a Python type varType = typeByPythonType(space, cursor, w_value) - return varType(cursor, numElements) + return varType(cursor, numElements, varType.size) Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Thu Nov 19 00:58:42 2009 @@ -2,6 +2,16 @@ class AppTestStringVar(OracleTestBase): + def test_bind_inout(self): + cur = self.cnx.cursor() + vars = cur.setinputsizes(value=oracle.STRING) + cur.execute(""" + begin + :value := :value || ' output'; + end;""", + value="input") + assert vars["value"].getvalue() == "input output" + def test_rowid(self): cur = self.cnx.cursor() cur.execute("select rowid from dual") From afa at codespeak.net Thu Nov 19 01:28:50 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 01:28:50 +0100 (CET) Subject: [pypy-svn] r69407 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119002850.EF712168074@codespeak.net> Author: afa Date: Thu Nov 19 01:28:49 2009 New Revision: 69407 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: implement getvalue() for LONG_STRING variables Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 01:28:49 2009 @@ -486,6 +486,17 @@ isVariableLength = True size = 128 * 1024 + def getValueProc(self, space, pos): + ptr = rffi.ptradd(self.data, pos * self.bufferSize) + length = rffi.cast(roci.Ptr(roci.ub4), ptr)[0] + + l = [] + i = 0 + while i < length: + l.append(self.data[i + rffi.sizeof(roci.ub4)]) + i += 1 + return space.wrap(''.join(l)) + def setValueProc(self, space, pos, w_value): buf = config.StringBuffer() buf.fill(space, w_value) @@ -496,10 +507,10 @@ self.resize(buf.size + rffi.sizeof(roci.ub4)) # copy the string to the Oracle buffer - data = rffi.ptradd(self.data, pos * self.bufferSize) - rffi.cast(roci.Ptr(roci.ub4), data)[0] = rffi.cast(roci.ub4, buf.size) + ptr = rffi.ptradd(self.data, pos * self.bufferSize) + rffi.cast(roci.Ptr(roci.ub4), ptr)[0] = rffi.cast(roci.ub4, buf.size) for index in range(buf.size): - data[index + rffi.sizeof(roci.ub4)] = buf.ptr[index] + ptr[index + rffi.sizeof(roci.ub4)] = buf.ptr[index] finally: buf.clear() Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Thu Nov 19 01:28:49 2009 @@ -119,11 +119,14 @@ def test_longstring(self): cur = self.cnx.cursor() + output = cur.var(oracle.LONG_STRING) cur.execute(""" declare t_Temp varchar2(10000); begin t_Temp := :bigString; + :output := t_Temp; end;""", - bigString="X" * 10000) + bigString="X" * 10000, output=output) + assert output.getvalue() == "X" * 10000 From afa at codespeak.net Thu Nov 19 09:02:38 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 09:02:38 +0100 (CET) Subject: [pypy-svn] r69408 - pypy/trunk/pypy/module/oracle/test Message-ID: <20091119080238.BEC21168051@codespeak.net> Author: afa Date: Thu Nov 19 09:02:37 2009 New Revision: 69408 Modified: pypy/trunk/pypy/module/oracle/test/test_select.py Log: Nowadays integer columns are correcly recognised and fetched as int values Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Thu Nov 19 09:02:37 2009 @@ -6,7 +6,7 @@ cur = self.cnx.cursor() cur.execute("select 42, 'Hello' from dual") row = cur.fetchone() - assert isinstance(row[0], float) + assert isinstance(row[0], int) assert isinstance(row[1], str) assert row == (42, 'Hello') From afa at codespeak.net Thu Nov 19 10:01:06 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 10:01:06 +0100 (CET) Subject: [pypy-svn] r69409 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119090106.33D8E168075@codespeak.net> Author: afa Date: Thu Nov 19 10:01:05 2009 New Revision: 69409 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_numbervar.py Log: Fix the retrieval of a float value from an "unconstrained NUMBER" datatype. Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 10:01:05 2009 @@ -575,9 +575,10 @@ lltype.free(attrptr, flavor='raw') if scale == 0 or (scale == -127 and precision == 0): - return VT_LongInteger - elif precision > 0 and precision < 10: - return VT_Integer + if precision > 0 and precision < 10: + return VT_Integer + else: + return VT_LongInteger return cls @@ -625,8 +626,13 @@ rffi.cast(lltype.Signed, sizeptr[0]))) if isinstance(self, VT_NumberAsString): return w_strvalue - else: + + try: return space.call_function(space.w_int, w_strvalue) + except OperationError, e: + if e.match(space, space.w_ValueError): + return space.call_function(space.w_float, w_strvalue) + raise finally: rffi.keep_buffer_alive_until_here(textbuf, text) lltype.free(sizeptr, flavor='raw') Modified: pypy/trunk/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_numbervar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_numbervar.py Thu Nov 19 10:01:05 2009 @@ -2,6 +2,12 @@ class AppTestNumberVar(OracleTestBase): + def test_fetch(self): + cur = self.cnx.cursor() + cur.execute("select 1.5 from dual") + value, = cur.fetchone() + assert value == 1.5 + def test_float(self): cur = self.cnx.cursor() var = cur.var(oracle.NUMBER) From afa at codespeak.net Thu Nov 19 10:12:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 10:12:22 +0100 (CET) Subject: [pypy-svn] r69410 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119091222.50AD5168072@codespeak.net> Author: afa Date: Thu Nov 19 10:12:21 2009 New Revision: 69410 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_select.py Log: implement cursor.fetchmany() Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Thu Nov 19 10:12:21 2009 @@ -733,6 +733,18 @@ return space.w_None fetchone.unwrap_spec = ['self', ObjSpace] + def fetchmany(self, space, w_numRows=NoneNotWrapped): + if w_numRows is not None: + numRows = space.int_w(w_numRows) + else: + numRows = self.arraySize + + # verify fetch can be performed + self._verifyFetch(space) + + return self._multiFetch(space, limit=numRows) + fetchmany.unwrap_spec = ['self', ObjSpace, W_Root] + def fetchall(self, space): # verify fetch can be performed self._verifyFetch(space) @@ -811,6 +823,7 @@ # fetch as many rows as possible while limit is None or rowNum < limit: + rowNum += 1 if not self._moreRows(space): break w_row = self._createRow(space) @@ -1022,6 +1035,8 @@ unwrap_spec=W_Cursor.prepare.unwrap_spec), fetchone = interp2app(W_Cursor.fetchone, unwrap_spec=W_Cursor.fetchone.unwrap_spec), + fetchmany = interp2app(W_Cursor.fetchmany, + unwrap_spec=W_Cursor.fetchmany.unwrap_spec), fetchall = interp2app(W_Cursor.fetchall, unwrap_spec=W_Cursor.fetchall.unwrap_spec), close = interp2app(W_Cursor.close, Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Thu Nov 19 10:12:21 2009 @@ -28,9 +28,17 @@ assert rows == zip(range(42)) assert cur.rowcount == 42 + def test_fetchmany(self): + cur = self.cnx.cursor() + cur.execute("select level-1 from dual connect by level-1<442") + rows = cur.fetchmany() + assert rows == zip(range(cur.arraysize)) + rows = cur.fetchmany(3) + assert rows == zip(range(cur.arraysize, cur.arraysize+3)) + assert cur.rowcount == cur.arraysize+3 + def test_iterator(self): cur = self.cnx.cursor() - # An Oracle trick to retrieve 42 lines cur.execute("select level-1 from dual connect by level-1<42") for i, row in enumerate(cur): assert row == (i,) From afa at codespeak.net Thu Nov 19 10:44:49 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 10:44:49 +0100 (CET) Subject: [pypy-svn] r69415 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119094449.91DB216804D@codespeak.net> Author: afa Date: Thu Nov 19 10:44:48 2009 New Revision: 69415 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Log: Allow datetime.date as input type Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 19 10:44:48 2009 @@ -11,7 +11,10 @@ 'DATETIME': 'interp_variable.VT_DateTime', 'BINARY': 'interp_variable.VT_Binary', 'LONG_STRING': 'interp_variable.VT_LongString', + 'FIXED_CHAR': 'interp_variable.VT_FixedChar', 'Variable': 'interp_variable.W_Variable', + 'Timestamp': 'interp_error.get(space).w_DateTimeType', + 'Date': 'interp_error.get(space).w_DateType', } appleveldefs = { Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Thu Nov 19 10:44:48 2009 @@ -25,6 +25,7 @@ w_datetime = space.call(w_import, space.newlist( [space.wrap('datetime')])) self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) + self.w_DateType = space.getattr(w_datetime, space.wrap("date")) def get(space): Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 10:44:48 2009 @@ -747,7 +747,10 @@ minute = space.int_w(space.getattr(w_value, space.wrap('minute'))) second = space.int_w(space.getattr(w_value, space.wrap('second'))) elif space.is_true(space.isinstance(w_value, get(space).w_DateType)): - XXX + year = space.int_w(space.getattr(w_value, space.wrap('year'))) + month = space.int_w(space.getattr(w_value, space.wrap('month'))) + day = space.int_w(space.getattr(w_value, space.wrap('day'))) + hour = minute = second = 0 else: raise OperationError( space.w_TypeError, @@ -762,7 +765,7 @@ rffi.setintfield(dataptr[0], 'c_OCIDateMM', month) rffi.setintfield(dataptr[0], 'c_OCIDateDD', day) -class VT_Date(W_Variable): +class VT_Date(VT_DateTime): oracleType = roci.SQLT_ODT size = rffi.sizeof(roci.OCIDate) @@ -931,7 +934,8 @@ if space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): return VT_DateTime, 0, numElements - # XXX date + if space.is_true(space.isinstance(w_value, get(space).w_DateType)): + return VT_Date, 0, numElements # XXX Delta Modified: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_datetimevar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Thu Nov 19 10:44:48 2009 @@ -2,7 +2,7 @@ class AppTestDatetime(OracleTestBase): - def test_bind_date(self): + def test_bind_datetime(self): import datetime cur = self.cnx.cursor() cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", @@ -10,3 +10,11 @@ data = cur.fetchall() assert data == [('20021213-093615',)] + def test_bind_date(self): + import datetime + cur = self.cnx.cursor() + cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", + d=datetime.date(2002, 12, 13)) + data = cur.fetchall() + assert data == [('20021213-000000',)] + From fijal at codespeak.net Thu Nov 19 11:03:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 11:03:57 +0100 (CET) Subject: [pypy-svn] r69417 - in pypy: branch/unpackiterable-improvements unpackiterable-improvements Message-ID: <20091119100357.18C6F168072@codespeak.net> Author: fijal Date: Thu Nov 19 11:03:56 2009 New Revision: 69417 Added: pypy/branch/unpackiterable-improvements/ - copied from r69416, pypy/unpackiterable-improvements/ Removed: pypy/unpackiterable-improvements/ Log: oops oops oops, typo with location From fijal at codespeak.net Thu Nov 19 11:19:35 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 11:19:35 +0100 (CET) Subject: [pypy-svn] r69426 - pypy/branch/unpackiterable-improvements/pypy/interpreter/test Message-ID: <20091119101935.AD0C3168022@codespeak.net> Author: fijal Date: Thu Nov 19 11:19:35 2009 New Revision: 69426 Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_compiler.py Log: This test fails on 2.6. It's CPycompiler, so differences in warning module are to blame. Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_compiler.py Thu Nov 19 11:19:35 2009 @@ -651,6 +651,9 @@ elif sys.version_info < (2, 6): _unicode_error_kind = "w_UnicodeDecodeError" else: + def skip_on_2_6(self): + py.test.skip("syntax different on CPython 2.6 compiler") + test_globals_warnings = skip_on_2_6 _unicode_error_kind = "w_SyntaxError" class TestPythonAstCompiler_25_grammar(BaseTestCompiler): From fijal at codespeak.net Thu Nov 19 11:19:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 11:19:56 +0100 (CET) Subject: [pypy-svn] r69427 - pypy/branch/unpackiterable-improvements/pypy/interpreter Message-ID: <20091119101956.B26CA168022@codespeak.net> Author: fijal Date: Thu Nov 19 11:19:56 2009 New Revision: 69427 Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py Log: This two places clearly can use viewiterable Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py Thu Nov 19 11:19:56 2009 @@ -771,7 +771,7 @@ def lookup(self, w_obj, name): w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) - for w_supertype in self.unpackiterable(w_mro): + for w_supertype in self.viewiterable(w_mro): w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value @@ -880,7 +880,7 @@ if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): w_indices = self.call_method(w_index_or_slice, "indices", self.wrap(seqlength)) - w_start, w_stop, w_step = self.unpackiterable(w_indices, 3) + w_start, w_stop, w_step = self.viewiterable(w_indices, 3) start = self.int_w(w_start) stop = self.int_w(w_stop) step = self.int_w(w_step) From afa at codespeak.net Thu Nov 19 11:28:12 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 11:28:12 +0100 (CET) Subject: [pypy-svn] r69429 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119102812.1CBC7168024@codespeak.net> Author: afa Date: Thu Nov 19 11:28:11 2009 New Revision: 69429 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Log: Fix for arrays of dates Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 11:28:11 2009 @@ -732,7 +732,6 @@ dataptr = rffi.ptradd( rffi.cast(roci.Ptr(roci.OCIDate), self.data), pos) - dataptr = rffi.cast(roci.Ptr(roci.OCIDate), self.data) return transform.OracleDateToPythonDateTime(self.environment, dataptr) def setValueProc(self, space, pos, w_value): @@ -757,7 +756,8 @@ space.wrap("expecting date data")) # store a copy of the value - timePart = dataptr[0].c_OCIDateTime + value = dataptr[0] + timePart = value.c_OCIDateTime rffi.setintfield(timePart, 'c_OCITimeHH', hour) rffi.setintfield(timePart, 'c_OCITimeMI', minute) rffi.setintfield(timePart, 'c_OCITimeSS', second) Modified: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_datetimevar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Thu Nov 19 11:28:11 2009 @@ -14,7 +14,27 @@ import datetime cur = self.cnx.cursor() cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", - d=datetime.date(2002, 12, 13)) + d=oracle.Date(2002, 12, 13)) data = cur.fetchall() assert data == [('20021213-000000',)] + def test_variable(self): + import datetime + cur = self.cnx.cursor() + var = cur.var(oracle.DATETIME) + value = datetime.datetime(2002, 12, 13, 9, 36, 15) + var.setvalue(0, value) + assert var.getvalue() == value + + value = datetime.date(2002, 12, 13) + var.setvalue(0, value) + assert var.getvalue() == datetime.datetime(2002, 12, 13) + + def test_arrayvar(self): + import datetime + cur = self.cnx.cursor() + var = cur.arrayvar(oracle.DATETIME, 2) + values = [datetime.datetime(2002, 12, 13, 9, 36, 15), + datetime.datetime(2003, 7, 22, 4, 24, 32)] + var.setvalue(0, values) + assert var.getvalue() == values From afa at codespeak.net Thu Nov 19 11:45:47 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 11:45:47 +0100 (CET) Subject: [pypy-svn] r69431 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119104547.8FFA01680AF@codespeak.net> Author: afa Date: Thu Nov 19 11:45:47 2009 New Revision: 69431 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: add cursor.setoutputsize() Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Thu Nov 19 11:45:47 2009 @@ -29,6 +29,8 @@ self.bindList = None self.bindDict = None self.numbersAsStrings = False + self.outputSize = -1 + self.outputSizeColumn = -1 self.inputTypeHandler = None self.outputTypeHandler = None @@ -987,7 +989,7 @@ self.bindList = None self.bindDict = None - self.setInputSizes = 1 + self.setInputSizes = True # process each input if kw_w: @@ -1009,6 +1011,12 @@ return space.newlist(self.bindList) setinputsizes.unwrap_spec = ['self', ObjSpace, Arguments] + def setoutputsize(self, space, outputSize, outputSizeColumn=-1): + self.outputSize = outputSize + self.outputSizeColumn = outputSizeColumn + setoutputsize.unwrap_spec = ['self', ObjSpace, int, int] + + def cursor_arraysize_get(space, obj): return space.wrap(obj.arraySize) def cursor_arraysize_set(space, obj, w_value): @@ -1053,6 +1061,8 @@ unwrap_spec=W_Cursor.arrayvar.unwrap_spec), setinputsizes = interp2app(W_Cursor.setinputsizes, unwrap_spec=W_Cursor.setinputsizes.unwrap_spec), + setoutputsize = interp2app(W_Cursor.setoutputsize, + unwrap_spec=W_Cursor.setoutputsize.unwrap_spec), __iter__ = interp2app(W_Cursor.descr_iter), next = interp2app(W_Cursor.descr_next), Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Thu Nov 19 11:45:47 2009 @@ -174,3 +174,8 @@ ('NULLABLECOL', oracle.NUMBER, 39, 22, 38, 0, 1), ] assert got == expected + + def test_outputsize(self): + cur = self.cnx.cursor() + cur.setoutputsize(25000) + cur.setoutputsize(25000, 2) From fijal at codespeak.net Thu Nov 19 12:10:39 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 12:10:39 +0100 (CET) Subject: [pypy-svn] r69432 - in pypy/branch/unpackiterable-improvements/pypy: annotation interpreter interpreter/astcompiler interpreter/astcompiler/tools interpreter/test module/__builtin__ module/__builtin__/test module/_codecs module/_rawffi module/_stackless/test module/micronumpy objspace/flow objspace/std objspace/std/test rpython Message-ID: <20091119111039.0E56C16804E@codespeak.net> Author: fijal Date: Thu Nov 19 12:10:37 2009 New Revision: 69432 Modified: pypy/branch/unpackiterable-improvements/pypy/annotation/bookkeeper.py pypy/branch/unpackiterable-improvements/pypy/interpreter/argument.py pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/ast.py pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py pypy/branch/unpackiterable-improvements/pypy/interpreter/function.py pypy/branch/unpackiterable-improvements/pypy/interpreter/gateway.py pypy/branch/unpackiterable-improvements/pypy/interpreter/interactive.py pypy/branch/unpackiterable-improvements/pypy/interpreter/nestedscope.py pypy/branch/unpackiterable-improvements/pypy/interpreter/pycode.py pypy/branch/unpackiterable-improvements/pypy/interpreter/pyopcode.py pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_argument.py pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_objspace.py pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/abstractinst.py pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/interp_classobj.py pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/test/test_abstractinst.py pypy/branch/unpackiterable-improvements/pypy/module/_codecs/interp_codecs.py pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/interp_rawffi.py pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/structure.py pypy/branch/unpackiterable-improvements/pypy/module/_stackless/test/test_pickle_infrastructure.py pypy/branch/unpackiterable-improvements/pypy/module/micronumpy/numarray.py pypy/branch/unpackiterable-improvements/pypy/objspace/flow/objspace.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/formatting.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/inlinedict.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/objspace.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/strsliceobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/tupletype.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/typeobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py pypy/branch/unpackiterable-improvements/pypy/rpython/callparse.py Log: A largish checkin: Break viewiterable into two: * fixedview - returns fixed-size list. In case of W_TupleObject, no copy * listview - returns resizable list. In case of W_ListObject, no copy Modified: pypy/branch/unpackiterable-improvements/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/annotation/bookkeeper.py Thu Nov 19 12:10:37 2009 @@ -757,7 +757,8 @@ getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] raise CallPatternTooComplex, "'*' argument must be SomeTuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/argument.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/argument.py Thu Nov 19 12:10:37 2009 @@ -135,7 +135,7 @@ # unpack the * arguments if w_stararg is not None: self.arguments_w = (self.arguments_w + - self.space.viewiterable(w_stararg)) + self.space.fixedview(w_stararg)) # unpack the ** arguments if w_starstararg is not None: space = self.space Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/ast.py Thu Nov 19 12:10:37 2009 @@ -97,7 +97,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -132,7 +132,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -191,7 +191,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -244,7 +244,7 @@ self.args.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -254,7 +254,7 @@ node.sync_app_attrs(space) w_list = self.w_decorators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.decorators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -297,7 +297,7 @@ pass w_list = self.w_bases if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.bases = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -307,7 +307,7 @@ node.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -375,7 +375,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -415,7 +415,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -492,7 +492,7 @@ self.dest.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -540,7 +540,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -550,7 +550,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -595,7 +595,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -605,7 +605,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -650,7 +650,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -660,7 +660,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -707,7 +707,7 @@ self.optional_vars.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -795,7 +795,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -805,7 +805,7 @@ node.sync_app_attrs(space) w_list = self.w_handlers if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.handlers = [space.interp_w(excepthandler, w_obj) for w_obj in list_w] else: @@ -815,7 +815,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -857,7 +857,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -867,7 +867,7 @@ node.sync_app_attrs(space) w_list = self.w_finalbody if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.finalbody = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -936,7 +936,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -977,7 +977,7 @@ self.level = 0 w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -1053,7 +1053,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.str_w(w_obj) for w_obj in list_w] else: @@ -1196,7 +1196,7 @@ pass w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1359,7 +1359,7 @@ pass w_list = self.w_keys if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keys = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1369,7 +1369,7 @@ node.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1408,7 +1408,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1447,7 +1447,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1520,14 +1520,14 @@ self.left.sync_app_attrs(space) w_list = self.w_ops if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ops = [space.interp_w(cmpop, w_obj).to_simple_int(space) for w_obj in list_w] else: self.ops = None w_list = self.w_comparators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.comparators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1579,7 +1579,7 @@ self.func.sync_app_attrs(space) w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1589,7 +1589,7 @@ node.sync_app_attrs(space) w_list = self.w_keywords if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keywords = [space.interp_w(keyword, w_obj) for w_obj in list_w] else: @@ -1795,7 +1795,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1834,7 +1834,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2012,7 +2012,7 @@ pass w_list = self.w_dims if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.dims = [space.interp_w(slice, w_obj) for w_obj in list_w] else: @@ -2305,7 +2305,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_ifs if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ifs = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2344,7 +2344,7 @@ self.name.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -2379,7 +2379,7 @@ self.kwarg = None w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2389,7 +2389,7 @@ node.sync_app_attrs(space) w_list = self.w_defaults if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.defaults = [space.interp_w(expr, w_obj) for w_obj in list_w] else: Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Nov 19 12:10:37 2009 @@ -144,7 +144,7 @@ if attr.seq: self.emit("w_list = self.w_%s" % (attr.name,), 2) self.emit("if w_list is not None:", 2) - self.emit("list_w = space.viewiterable(w_list)", 3) + self.emit("list_w = space.listview(w_list)", 3) self.emit("if list_w:", 3) unwrapper = get_unwrapper(attr.type.value, "w_obj", self.data.simple_types) Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/baseobjspace.py Thu Nov 19 12:10:37 2009 @@ -668,13 +668,17 @@ (i, plural)) return items - def viewiterable(self, w_iterable, expected_length=-1): - """ More or less the same as unpackiterable, but does not return - a copy. Please don't modify the result + def fixedview(self, w_iterable, expected_length=-1): + """ A fixed list view of w_iterable. Don't modify the result """ return make_sure_not_resized(self.unpackiterable(w_iterable, expected_length)[:]) + def listview(self, w_iterable, expected_length=-1): + """ A non-fixed view of w_iterable. Don't modify the result + """ + return self.unpackiterable(w_iterable, expected_length) + def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): @@ -771,7 +775,7 @@ def lookup(self, w_obj, name): w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) - for w_supertype in self.viewiterable(w_mro): + for w_supertype in self.fixedview(w_mro): w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value @@ -880,7 +884,7 @@ if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): w_indices = self.call_method(w_index_or_slice, "indices", self.wrap(seqlength)) - w_start, w_stop, w_step = self.viewiterable(w_indices, 3) + w_start, w_stop, w_step = self.fixedview(w_indices, 3) start = self.int_w(w_start) stop = self.int_w(w_stop) step = self.int_w(w_step) Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/function.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/function.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/function.py Thu Nov 19 12:10:37 2009 @@ -198,7 +198,7 @@ else: name = None if not space.is_w(w_argdefs, space.w_None): - defs_w = space.viewiterable(w_argdefs) + defs_w = space.fixedview(w_argdefs) else: defs_w = [] nfreevars = 0 @@ -316,7 +316,7 @@ if space.is_w(w_func_dict, space.w_None): w_func_dict = None self.w_func_dict = w_func_dict - self.defs_w = space.viewiterable(w_defs_w) + self.defs_w = space.fixedview(w_defs_w) self.w_module = w_module def fget_func_defaults(space, self): @@ -331,7 +331,7 @@ return if not space.is_true(space.isinstance(w_defaults, space.w_tuple)): raise OperationError( space.w_TypeError, space.wrap("func_defaults must be set to a tuple object or None") ) - self.defs_w = space.viewiterable(w_defaults) + self.defs_w = space.fixedview(w_defaults) def fdel_func_defaults(space, self): self.defs_w = [] Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/gateway.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/gateway.py Thu Nov 19 12:10:37 2009 @@ -210,7 +210,7 @@ % (self.scopenext(), self.scopenext())) def visit_args_w(self, el): - self.run_args.append("space.viewiterable(%s)" % self.scopenext()) + self.run_args.append("space.fixedview(%s)" % self.scopenext()) def visit_w_args(self, el): self.run_args.append(self.scopenext()) @@ -416,7 +416,7 @@ # baseobjspace.W_Root is for wrapped arguments to keep wrapped # baseobjspace.Wrappable subclasses imply interp_w and a typecheck # argument.Arguments is for a final rest arguments Arguments object - # 'args_w' for viewiterable applied to rest arguments + # 'args_w' for fixedview applied to rest arguments # 'w_args' for rest arguments passed as wrapped tuple # str,int,float: unwrap argument as such type # (function, cls) use function to check/unwrap argument of type cls Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/interactive.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/interactive.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/interactive.py Thu Nov 19 12:10:37 2009 @@ -73,7 +73,7 @@ words = self.get_words(w_clz) try: w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.viewiterable(w_bases) + bases_w = s.fixedview(w_bases) except error.OperationError: return words Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/nestedscope.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/nestedscope.py Thu Nov 19 12:10:37 2009 @@ -208,7 +208,7 @@ if codeobj.magic >= 0xa0df281: # CPython 2.5 AST branch merge w_freevarstuple = f.popvalue() freevars = [f.space.interp_w(Cell, cell) - for cell in f.space.viewiterable(w_freevarstuple)] + for cell in f.space.fixedview(w_freevarstuple)] else: nfreevars = len(codeobj.co_freevars) freevars = [f.space.interp_w(Cell, f.popvalue()) Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/pycode.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/pycode.py Thu Nov 19 12:10:37 2009 @@ -349,7 +349,7 @@ if not space.is_true(space.isinstance(w_constants, space.w_tuple)): raise OperationError(space.w_TypeError, space.wrap("Expected tuple for constants")) - consts_w = space.viewiterable(w_constants) + consts_w = space.fixedview(w_constants) names = unpack_str_tuple(space, w_names) varnames = unpack_str_tuple(space, w_varnames) if w_freevars is not None: Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/pyopcode.py Thu Nov 19 12:10:37 2009 @@ -581,7 +581,7 @@ w_compile_flags, f.space.wrap(f.get_builtin()), f.space.gettypeobject(PyCode.typedef)) - w_prog, w_globals, w_locals = f.space.viewiterable(w_resulttuple, 3) + w_prog, w_globals, w_locals = f.space.fixedview(w_resulttuple, 3) plain = f.w_locals is not None and f.space.is_w(w_locals, f.w_locals) if plain: @@ -637,7 +637,7 @@ def UNPACK_SEQUENCE(f, itemcount, *ignored): w_iterable = f.popvalue() try: - items = f.space.viewiterable(w_iterable, itemcount) + items = f.space.fixedview(w_iterable, itemcount) except UnpackValueError, e: raise OperationError(f.space.w_ValueError, f.space.wrap(e.msg)) f.pushrevvalues(itemcount, items) Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_argument.py Thu Nov 19 12:10:37 2009 @@ -60,9 +60,12 @@ def is_true(self, obj): return bool(obj) - def viewiterable(self, it): + def fixedview(self, it): return list(it) + def listview(self, it): + return list(it) + def unpackiterable(self, it): return list(it) Modified: pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/interpreter/test/test_objspace.py Thu Nov 19 12:10:37 2009 @@ -58,14 +58,23 @@ raises(ValueError, self.space.unpackiterable, w_l, 3) raises(ValueError, self.space.unpackiterable, w_l, 5) - def test_viewiterable(self): + def test_fixedview(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.viewiterable(w_l) == l - assert self.space.viewiterable(w_l, 4) == l - raises(ValueError, self.space.viewiterable, w_l, 3) - raises(ValueError, self.space.viewiterable, w_l, 5) + assert self.space.fixedview(w_l) == l + assert self.space.fixedview(w_l, 4) == l + raises(ValueError, self.space.fixedview, w_l, 3) + raises(ValueError, self.space.fixedview, w_l, 5) + + def test_listview(self): + w = self.space.wrap + l = [w(1), w(2), w(3), w(4)] + w_l = self.space.newtuple(l) + assert self.space.listview(w_l) == l + assert self.space.listview(w_l, 4) == l + raises(ValueError, self.space.listview, w_l, 3) + raises(ValueError, self.space.listview, w_l, 5) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, @@ -207,7 +216,7 @@ w_res = space.call_obj_args(w_f, w_9, Arguments(space, [w_1])) - w_x, w_y = space.viewiterable(w_res, 2) + w_x, w_y = space.fixedview(w_res, 2) assert w_x is w_9 assert w_y is w_1 Modified: pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/abstractinst.py Thu Nov 19 12:10:37 2009 @@ -83,7 +83,7 @@ def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple): # -- case (anything, tuple) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_isinstance_w(space, w_obj, w_klass): return True return False @@ -109,7 +109,7 @@ return True w_bases = _get_bases(space, w_derived) if w_bases is not None: - for w_base in space.viewiterable(w_bases): + for w_base in space.fixedview(w_bases): if _issubclass_recurse(space, w_base, w_top): return True return False @@ -141,7 +141,7 @@ # -- case (class-like-object, tuple-of-classes) # XXX it might be risky that the JIT sees this if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass): return True return False Modified: pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/interp_classobj.py Thu Nov 19 12:10:37 2009 @@ -36,7 +36,7 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -79,7 +79,7 @@ raise OperationError( space.w_TypeError, space.wrap("__bases__ must be a tuple object")) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, @@ -285,7 +285,7 @@ if not e.match(space, space.w_TypeError): raise return [None, None] - return space.viewiterable(w_tup, 2) + return space.fixedview(w_tup, 2) def descr_instance_new(space, w_type, w_class, w_dict=None): # w_type is not used at all Modified: pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/test/test_abstractinst.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/test/test_abstractinst.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/__builtin__/test/test_abstractinst.py Thu Nov 19 12:10:37 2009 @@ -5,7 +5,7 @@ def test_abstract_isclass(self): space = self.space - w_B1, w_B2, w_B3, w_X, w_Y = space.viewiterable(space.appexec([], """(): + w_B1, w_B2, w_B3, w_X, w_Y = space.fixedview(space.appexec([], """(): class X(object): pass class Y: pass B1, B2, B3 = X(), X(), X() @@ -22,7 +22,7 @@ def test_abstract_getclass(self): space = self.space - w_x, w_y, w_A, w_MyInst = space.viewiterable(space.appexec([], """(): + w_x, w_y, w_A, w_MyInst = space.fixedview(space.appexec([], """(): class MyInst(object): def __init__(self, myclass): self.myclass = myclass Modified: pypy/branch/unpackiterable-improvements/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/_codecs/interp_codecs.py Thu Nov 19 12:10:37 2009 @@ -35,7 +35,7 @@ space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) - w_replace, w_newpos = space.viewiterable(w_res, 2) + w_replace, w_newpos = space.fixedview(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/interp_rawffi.py Thu Nov 19 12:10:37 2009 @@ -106,7 +106,7 @@ resshape = cache.get_array_type(letter2tp(space, letter)) else: letter = 'V' - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) from pypy.module._rawffi.structure import W_Structure resshape = space.interp_w(W_Structure, w_shapetype) ffi_type = resshape.get_ffi_type() @@ -117,7 +117,7 @@ letter = space.str_w(w_shape) return letter2tp(space, letter) else: - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) resshape = space.interp_w(W_DataShape, w_shapetype) length = space.int_w(w_length) size, alignment = resshape._size_alignment() @@ -155,7 +155,7 @@ """ ffi_restype, resshape = unpack_resshape(space, w_restype) w = space.wrap - argtypes_w = space.viewiterable(w_argtypes) + argtypes_w = space.fixedview(w_argtypes) w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w_name, w_argtypes, w(resshape)]) try: Modified: pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/structure.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/_rawffi/structure.py Thu Nov 19 12:10:37 2009 @@ -119,7 +119,7 @@ def descr_new_structure(space, w_type, w_shapeinfo): if space.is_true(space.isinstance(w_shapeinfo, space.w_tuple)): - w_size, w_alignment = space.viewiterable(w_shapeinfo, expected_length=2) + w_size, w_alignment = space.fixedview(w_shapeinfo, expected_length=2) S = W_Structure(space, None, space.int_w(w_size), space.int_w(w_alignment)) else: Modified: pypy/branch/unpackiterable-improvements/pypy/module/_stackless/test/test_pickle_infrastructure.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/_stackless/test/test_pickle_infrastructure.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/_stackless/test/test_pickle_infrastructure.py Thu Nov 19 12:10:37 2009 @@ -129,7 +129,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -188,7 +188,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -260,7 +260,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code Modified: pypy/branch/unpackiterable-improvements/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/micronumpy/numarray.py Thu Nov 19 12:10:37 2009 @@ -73,7 +73,7 @@ make_sure_not_resized(self.storage) def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.viewiterable(w_index)] + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] if len(indexes) != len(self.dim): raise OperationError(space.w_IndexError, space.wrap( 'Wrong index')) @@ -108,7 +108,7 @@ def unpack_dim(space, w_dim): if space.is_true(space.isinstance(w_dim, space.w_int)): return [space.int_w(w_dim)] - dim_w = space.viewiterable(w_dim) + dim_w = space.fixedview(w_dim) return [space.int_w(w_i) for w_i in dim_w] def unpack_dtype(space, w_dtype): Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/flow/objspace.py Thu Nov 19 12:10:37 2009 @@ -202,7 +202,7 @@ # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) # checking a tuple of classes - for w_klass in self.viewiterable(w_check_class): + for w_klass in self.fixedview(w_check_class): if ObjSpace.exception_match(self, w_exc_type, w_klass): return True return False @@ -263,8 +263,9 @@ checkgraph(graph) return graph - def viewiterable(self, w_tuple, expected_length=None): + def fixedview(self, w_tuple, expected_length=None): return self.unpackiterable(w_tuple, expected_length) + listview = fixedview def unpackiterable(self, w_iterable, expected_length=None): if not isinstance(w_iterable, Variable): Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py Thu Nov 19 12:10:37 2009 @@ -652,7 +652,7 @@ elif space.findattr(w_src, space.wrap("keys")) is None: list_of_w_pairs = space.unpackiterable(w_src) for w_pair in list_of_w_pairs: - pair = space.unpackiterable(w_pair) + pair = space.fixedview(w_pair) if len(pair)!=2: raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/formatting.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/formatting.py Thu Nov 19 12:10:37 2009 @@ -495,7 +495,7 @@ def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): - values_w = space.unpackiterable(w_values) + values_w = space.fixedview(w_values) return format(space, w_format, values_w, None, do_unicode) else: # we check directly for dict to avoid obscure checking Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/inlinedict.py Thu Nov 19 12:10:37 2009 @@ -45,7 +45,7 @@ # XXX sucky items = [] for w_item in self.impl_items(): - w_key, w_value = self.space.viewiterable(w_item) + w_key, w_value = self.space.fixedview(w_item) items.append((w_key, w_value)) return IndirectionIterImplementation(self.space, self, items) Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py Thu Nov 19 12:10:37 2009 @@ -356,7 +356,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) for w_tuple in w_dict.items(): - w_key, w_value = space.viewiterable(w_tuple, 2) + w_key, w_value = space.fixedview(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) @@ -469,14 +469,14 @@ def marshal_w_set(space, w_set, m): # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.viewiterable(w_set) + lis_w = space.unpackiterable(w_set) m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - lis_w = space.viewiterable(w_frozenset) + lis_w = space.unpackiterable(w_frozenset) m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/objspace.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/objspace.py Thu Nov 19 12:10:37 2009 @@ -383,7 +383,7 @@ space = self # too early for unpackiterable as well :-( name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) + bases = space.fixedview(space.getitem(w_args, space.wrap(1))) dic = space.unwrap(space.getitem(w_args, space.wrap(2))) dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) bases = list(bases) @@ -643,7 +643,7 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def viewiterable(self, w_obj, expected_length=-1): + def fixedview(self, w_obj, expected_length=-1): """ Fast paths """ if isinstance(w_obj, W_TupleObject): @@ -651,7 +651,18 @@ elif isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.viewiterable(self, w_obj, expected_length) + return ObjSpace.fixedview(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + return t + + def listview(self, w_obj, expected_length=-1): + if isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems + elif isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.listview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeobject.py Thu Nov 19 12:10:37 2009 @@ -537,7 +537,7 @@ (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -557,7 +557,7 @@ def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py Thu Nov 19 12:10:37 2009 @@ -504,7 +504,7 @@ def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = ropeunicode_w(space, w_prefix) if rope.startswith(unistr, prefix, start, end): return space.w_True @@ -513,7 +513,7 @@ def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = ropeunicode_w(space, w_suffix) if rope.endswith(unistr, suffix, start, end): return space.w_True Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py Thu Nov 19 12:10:37 2009 @@ -111,7 +111,7 @@ def make_setdata_from_w_iterable(space, w_iterable=None): data = r_dict(space.eq_w, space.hash_w) if w_iterable is not None: - for w_item in space.viewiterable(w_iterable): + for w_item in space.unpackiterable(w_iterable): data[w_item] = None return data Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py Thu Nov 19 12:10:37 2009 @@ -628,7 +628,7 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -647,7 +647,7 @@ def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/strsliceobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/strsliceobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/strsliceobject.py Thu Nov 19 12:10:37 2009 @@ -143,7 +143,7 @@ def str_endswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -157,7 +157,7 @@ def str_startswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_dictmultiobject.py Thu Nov 19 12:10:37 2009 @@ -607,8 +607,8 @@ StringObjectCls = FakeString w_dict = None iter = iter - viewiterable = list - + fixedview = list + listview = list class Config: class objspace: Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/tupletype.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/tupletype.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/tupletype.py Thu Nov 19 12:10:37 2009 @@ -13,7 +13,7 @@ space.is_w(space.type(w_sequence), space.w_tuple)): return w_sequence else: - tuple_w = space.viewiterable(w_sequence) + tuple_w = space.fixedview(w_sequence) w_obj = space.allocate_instance(space.TupleObjectCls, w_tupletype) space.TupleObjectCls.__init__(w_obj, tuple_w) return w_obj Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/typeobject.py Thu Nov 19 12:10:37 2009 @@ -596,7 +596,7 @@ if not space.is_w(w_where, space.w_type): w_mro_meth = space.get(w_mro_func, w_self) w_mro = space.call_function(w_mro_meth) - mro_w = space.viewiterable(w_mro) + mro_w = space.fixedview(w_mro) w_self.mro_w = validate_custom_mro(space, mro_w) return # done w_self.mro_w = w_self.compute_default_mro()[:] Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py Thu Nov 19 12:10:37 2009 @@ -11,7 +11,7 @@ w_typetype = _precheck_for_new(space, w_typetype) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) w_winner = w_typetype for base in bases_w: @@ -115,7 +115,7 @@ " to %s.__bases__, not %s"% (w_type.name, space.type(w_value).getname(space, '?')))) - newbases_w = space.viewiterable(w_value) + newbases_w = space.fixedview(w_value) if len(newbases_w) == 0: raise OperationError(space.w_TypeError, space.wrap("can only assign non-empty tuple" Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py Thu Nov 19 12:10:37 2009 @@ -489,7 +489,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.unicode_w(w_prefix) if stringstartswith(unistr, prefix, start, end): return space.w_True @@ -499,7 +499,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.unicode_w(w_suffix) if stringendswith(unistr, suffix, start, end): return space.w_True Modified: pypy/branch/unpackiterable-improvements/pypy/rpython/callparse.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/rpython/callparse.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/rpython/callparse.py Thu Nov 19 12:10:37 2009 @@ -178,7 +178,8 @@ raise ValueError return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other From fijal at codespeak.net Thu Nov 19 12:16:33 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 12:16:33 +0100 (CET) Subject: [pypy-svn] r69433 - pypy/branch/unpackiterable-improvements/pypy/objspace/std Message-ID: <20091119111633.7A82C16804E@codespeak.net> Author: fijal Date: Thu Nov 19 12:16:32 2009 New Revision: 69433 Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py Log: should be list view here, spotted by pedronis Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/setobject.py Thu Nov 19 12:16:32 2009 @@ -111,7 +111,7 @@ def make_setdata_from_w_iterable(space, w_iterable=None): data = r_dict(space.eq_w, space.hash_w) if w_iterable is not None: - for w_item in space.unpackiterable(w_iterable): + for w_item in space.listview(w_iterable): data[w_item] = None return data From fijal at codespeak.net Thu Nov 19 12:23:41 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 12:23:41 +0100 (CET) Subject: [pypy-svn] r69434 - pypy/trunk/pypy/module/Numeric Message-ID: <20091119112341.0B4D116804E@codespeak.net> Author: fijal Date: Thu Nov 19 12:23:40 2009 New Revision: 69434 Removed: pypy/trunk/pypy/module/Numeric/ Log: Remove something completely outdated. If not for anything else, for the fact that it has no tests whatsoever From fijal at codespeak.net Thu Nov 19 12:24:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 12:24:34 +0100 (CET) Subject: [pypy-svn] r69435 - pypy/trunk/pypy/doc/config Message-ID: <20091119112434.B62D216804E@codespeak.net> Author: fijal Date: Thu Nov 19 12:24:34 2009 New Revision: 69435 Removed: pypy/trunk/pypy/doc/config/objspace.usemodules.Numeric.txt Log: remove doc about module as well From fijal at codespeak.net Thu Nov 19 12:26:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 12:26:45 +0100 (CET) Subject: [pypy-svn] r69436 - pypy/branch/unpackiterable-improvements/pypy/module/posix Message-ID: <20091119112645.F31B416804E@codespeak.net> Author: fijal Date: Thu Nov 19 12:26:45 2009 New Revision: 69436 Modified: pypy/branch/unpackiterable-improvements/pypy/module/posix/interp_posix.py Log: use fixedview here as well Modified: pypy/branch/unpackiterable-improvements/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/posix/interp_posix.py Thu Nov 19 12:26:45 2009 @@ -573,7 +573,7 @@ raise wrap_oserror(space, e) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" - args_w = space.unpackiterable(w_tuple) + args_w = space.fixedview(w_tuple) if len(args_w) != 2: raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) From fijal at codespeak.net Thu Nov 19 13:12:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 13:12:32 +0100 (CET) Subject: [pypy-svn] r69437 - in pypy/branch/unpackiterable-improvements/pypy: module/rctime module/select objspace/std Message-ID: <20091119121232.CC680168024@codespeak.net> Author: fijal Date: Thu Nov 19 13:12:32 2009 New Revision: 69437 Modified: pypy/branch/unpackiterable-improvements/pypy/module/rctime/interp_time.py pypy/branch/unpackiterable-improvements/pypy/module/select/interp_select.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/listobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py Log: move a couple of places from unpackiterable to listview Modified: pypy/branch/unpackiterable-improvements/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/rctime/interp_time.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/rctime/interp_time.py Thu Nov 19 13:12:32 2009 @@ -222,7 +222,7 @@ space.wrap(_get_error_msg())) return pbuf - tup_w = space.unpackiterable(w_tup) + tup_w = space.fixedview(w_tup) if len(tup_w) != 9: raise OperationError(space.w_TypeError, space.wrap("argument must be sequence of " Modified: pypy/branch/unpackiterable-improvements/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/module/select/interp_select.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/module/select/interp_select.py Thu Nov 19 13:12:32 2009 @@ -113,9 +113,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.unpackiterable(w_iwtd) - owtd_w = space.unpackiterable(w_owtd) - ewtd_w = space.unpackiterable(w_ewtd) + iwtd_w = space.listview(w_iwtd) + owtd_w = space.listview(w_owtd) + ewtd_w = space.listview(w_ewtd) iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] owtd = [as_fd_w(space, w_f) for w_f in owtd_w] ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/dictmultiobject.py Thu Nov 19 13:12:32 2009 @@ -650,7 +650,7 @@ if w_src is None: pass elif space.findattr(w_src, space.wrap("keys")) is None: - list_of_w_pairs = space.unpackiterable(w_src) + list_of_w_pairs = space.listview(w_src) for w_pair in list_of_w_pairs: pair = space.fixedview(w_pair) if len(pair)!=2: @@ -793,7 +793,7 @@ return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): - defaults = space.unpackiterable(w_defaults) + defaults = space.listview(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/listobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/listobject.py Thu Nov 19 13:12:32 2009 @@ -260,11 +260,7 @@ _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable) def _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable): - if isinstance(w_iterable, W_ListObject): - sequence2 = w_iterable.wrappeditems - else: - sequence2 = space.unpackiterable(w_iterable) - + sequence2 = space.listview(w_iterable) assert slicelength >= 0 items = w_list.wrappeditems oldsize = len(items) @@ -357,7 +353,7 @@ return space.w_None def list_extend__List_ANY(space, w_list, w_any): - w_list.wrappeditems += space.unpackiterable(w_any) + w_list.wrappeditems += space.listview(w_any) return space.w_None # note that the default value will come back wrapped!!! Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/ropeunicodeobject.py Thu Nov 19 13:12:32 2009 @@ -233,7 +233,7 @@ return space.contains(unicode_from_string(space, w_container), w_item ) def unicode_join__RopeUnicode_ANY(space, w_self, w_list): - l_w = space.unpackiterable(w_list) + l_w = space.listview(w_list) delim = w_self._node totlen = 0 if len(l_w) == 0: Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/stringobject.py Thu Nov 19 13:12:32 2009 @@ -349,7 +349,7 @@ sliced) def str_join__String_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) str_w = space.str_w if list_w: self = w_self._value Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/typetype.py Thu Nov 19 13:12:32 2009 @@ -38,7 +38,7 @@ name = space.str_w(w_name) assert isinstance(name, str) dict_w = {} - dictkeys_w = space.unpackiterable(w_dict) + dictkeys_w = space.listview(w_dict) for w_key in dictkeys_w: key = space.str_w(w_key) dict_w[key] = space.getitem(w_dict, w_key) Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/unicodeobject.py Thu Nov 19 13:12:32 2009 @@ -173,7 +173,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.unpackiterable(w_list) + l = space.listview(w_list) delim = w_self._value totlen = 0 if len(l) == 0: From afa at codespeak.net Thu Nov 19 13:22:45 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 13:22:45 +0100 (CET) Subject: [pypy-svn] r69438 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119122245.87688168077@codespeak.net> Author: afa Date: Thu Nov 19 13:22:44 2009 New Revision: 69438 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/test/test_select.py Log: Add cursor.fetchvars() Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Thu Nov 19 13:22:44 2009 @@ -1033,6 +1033,9 @@ if obj.bindDict: return obj.bindDict +def cursor_fetchvars_get(space, obj): + return space.newlist(obj.fetchVariables) + W_Cursor.typedef = TypeDef( 'Cursor', execute = interp2app(W_Cursor.execute, @@ -1072,5 +1075,6 @@ rowcount = interp_attrproperty('rowCount', W_Cursor), statement = interp_attrproperty_w('w_statement', W_Cursor), bindvars = GetSetProperty(cursor_bindvars_get), + fetchvars = GetSetProperty(cursor_fetchvars_get), description = GetSetProperty(W_Cursor.getDescription), ) Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Thu Nov 19 13:22:44 2009 @@ -10,6 +10,9 @@ assert isinstance(row[1], str) assert row == (42, 'Hello') + assert isinstance(cur.fetchvars[0], oracle.Variable) + assert isinstance(cur.fetchvars[1], oracle.Variable) + def test_sysdate(self): import datetime cur = self.cnx.cursor() @@ -19,7 +22,7 @@ assert isinstance(sysdate, datetime.datetime) delta = abs(sysdate - datetime.datetime.now()) assert delta < datetime.timedelta(seconds=2) - + def test_fetchall(self): cur = self.cnx.cursor() # An Oracle trick to retrieve 42 lines @@ -44,4 +47,3 @@ assert row == (i,) assert i == 41 - From afa at codespeak.net Thu Nov 19 13:25:46 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 13:25:46 +0100 (CET) Subject: [pypy-svn] r69439 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119122546.77372168077@codespeak.net> Author: afa Date: Thu Nov 19 13:25:45 2009 New Revision: 69439 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_select.py Log: Handle overflow when computing allocatedElements * bufferSize Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 13:25:45 2009 @@ -144,7 +144,7 @@ self.size = size # allocate the data for the variable - self.allocateData() + self.allocateData(self.environment.space) # allocate the indicator for the variable self.indicator = lltype.malloc(rffi.CArrayPtr(roci.sb2).TO, @@ -176,24 +176,27 @@ def getBufferSize(self): return self.size - def allocateData(self): + def allocateData(self, space): # set the buffer size for the variable self.bufferSize = self.getBufferSize() # allocate the data as long as it is small enough - dataLength = ovfcheck(self.allocatedElements * self.bufferSize) - if dataLength > sys.maxint: - raise ValueError("array size too large") + try: + dataLength = ovfcheck(self.allocatedElements * self.bufferSize) + except OverflowError: + raise OperationError( + space.w_ValueError, + space.wrap("array size too large")) self.data = lltype.malloc(rffi.CCHARP.TO, int(dataLength), flavor='raw', zero=True) - def resize(self, size): + def resize(self, space, size): # allocate the data for the new array orig_data = self.data orig_size = self.bufferSize self.size = size - self.allocateData() + self.allocateData(space) # copy the data from the original array to the new array for i in range(self.allocatedElements): @@ -292,13 +295,13 @@ error = W_Error(space, self.environment, "Variable_VerifyFetch()", 0) error.code = self.returnCode[pos] - error.message = self.space.wrap( + error.message = space.wrap( "column at array pos %d fetched with error: %d" % (pos, self.returnCode[pos])) - w_error = get(self.space).w_DatabaseError + w_error = get(space).w_DatabaseError - raise OperationError(get(self.space).w_DatabaseError, - self.space.wrap(error)) + raise OperationError(get(space).w_DatabaseError, + space.wrap(error)) def getSingleValue(self, space, pos): # ensure we do not exceed the number of allocated elements @@ -464,7 +467,7 @@ # ensure that the buffer is large enough if buf.size > self.bufferSize: - self.resize(size) + self.resize(space, size) # keep a copy of the string self.actualLength[pos] = rffi.cast(roci.ub2, buf.size) @@ -504,7 +507,7 @@ try: # ensure that the buffer is large enough if buf.size + rffi.sizeof(roci.ub4) > self.bufferSize: - self.resize(buf.size + rffi.sizeof(roci.ub4)) + self.resize(space, buf.size + rffi.sizeof(roci.ub4)) # copy the string to the Oracle buffer ptr = rffi.ptradd(self.data, pos * self.bufferSize) Modified: pypy/trunk/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_select.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_select.py Thu Nov 19 13:25:45 2009 @@ -47,3 +47,9 @@ assert row == (i,) assert i == 41 + def test_arraysize_too_large(self): + cur = self.cnx.cursor() + cur.arraysize = 2 ** 20 + largevar = cur.var(oracle.STRING) + raises(ValueError, + cur.execute, "select :large from dual", large=largevar) From fijal at codespeak.net Thu Nov 19 13:57:50 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 13:57:50 +0100 (CET) Subject: [pypy-svn] r69440 - pypy/branch/unpackiterable-improvements/pypy/objspace/std Message-ID: <20091119125750.C2117168076@codespeak.net> Author: fijal Date: Thu Nov 19 13:57:50 2009 New Revision: 69440 Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py Log: fix translation Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/marshal_impl.py Thu Nov 19 13:57:50 2009 @@ -469,14 +469,14 @@ def marshal_w_set(space, w_set, m): # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.unpackiterable(w_set) + lis_w = space.fixedview(w_set) m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - lis_w = space.unpackiterable(w_frozenset) + lis_w = space.fixedview(w_frozenset) m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) From cfbolz at codespeak.net Thu Nov 19 14:07:07 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Nov 2009 14:07:07 +0100 (CET) Subject: [pypy-svn] r69441 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test Message-ID: <20091119130707.8B3D9168075@codespeak.net> Author: cfbolz Date: Thu Nov 19 14:07:07 2009 New Revision: 69441 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): adapt those tests to the new interface Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Thu Nov 19 14:07:07 2009 @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue +from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue +from pypy.jit.metainterp.optimizeopt import VStructValue from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr from pypy.jit.metainterp.history import ConstPtr, ConstFloat @@ -765,23 +766,26 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vinfos_not_env = {} - modifier.vfieldboxes = [] - modifier.make_virtual(b2s, - ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], - [b4s, c1s]) # new fields - modifier.make_virtual(b4s, - ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, - LLtypeMixin.otherdescr], - [b2s, b3s, b5s]) # new fields + modifier.vfieldboxes = {} + + v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), b2s) + v2._fields = {LLtypeMixin.nextdescr: b4s, + LLtypeMixin.valuedescr: c1s} + v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] + v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), b4s) + v4._fields = {LLtypeMixin.nextdescr: b2s, + LLtypeMixin.valuedescr: b3s, + LLtypeMixin.otherdescr: b5s} + v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, + LLtypeMixin.otherdescr] + modifier.register_virtual_fields(b2s, [b4s, c1s]) + modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) + values = {b2s: v2, b4s: v4} liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, values, 0) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -834,14 +838,17 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vinfos_not_env = {} - modifier.vfieldboxes = [] - modifier.make_varray(b2s, - LLtypeMixin.arraydescr, - [b4s, c1s]) # new fields + modifier.vfieldboxes = {} + + class FakeOptimizer(object): + def new_const_item(self, descr): + return None + v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) + v2._items = [b4s, c1s] + modifier.register_virtual_fields(b2s, [b4s, c1s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + values = {b2s: v2} + modifier._number_virtuals(liveboxes, values, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None @@ -881,15 +888,13 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vinfos_not_env = {} - modifier.vfieldboxes = [] - modifier.make_vstruct(b2s, - LLtypeMixin.ssize, - [LLtypeMixin.adescr, LLtypeMixin.bdescr], - [c1s, b4s]) # new fields + modifier.vfieldboxes = {} + v2 = VStructValue(None, LLtypeMixin.ssize, b2s) + v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} + v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] + modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, {b2s: v2}, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None From cfbolz at codespeak.net Thu Nov 19 14:50:47 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Nov 2009 14:50:47 +0100 (CET) Subject: [pypy-svn] r69442 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091119135047.A0B85168074@codespeak.net> Author: cfbolz Date: Thu Nov 19 14:50:46 2009 New Revision: 69442 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): actually pass the test that we wrote during the sprint: Share vinfo objects if the numberings didn't change. Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py Thu Nov 19 14:50:46 2009 @@ -151,9 +151,10 @@ class AbstractVirtualValue(OptValue): - _attrs_ = ('optimizer', 'keybox', 'source_op') + _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo') box = None level = LEVEL_NONNULL + _cached_vinfo = None def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer @@ -173,8 +174,13 @@ return self.box def make_virtual_info(self, modifier, fieldnums): + vinfo = self._cached_vinfo + if vinfo is not None and resume.tagged_list_eq( + vinfo.fieldnums, fieldnums): + return vinfo vinfo = self._make_virtual(modifier) vinfo.fieldnums = fieldnums + self._cached_vinfo = vinfo return vinfo def _make_virtual(self, modifier): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Thu Nov 19 14:50:46 2009 @@ -84,6 +84,14 @@ # please rpython :( return rarithmetic.widen(x) == rarithmetic.widen(y) +def tagged_list_eq(tl1, tl2): + if len(tl1) != len(tl2): + return False + for i in range(len(tl1)): + if not tagged_eq(tl1[i], tl2[i]): + return False + return True + TAGCONST = 0 TAGINT = 1 TAGBOX = 2 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_optimizeopt.py Thu Nov 19 14:50:46 2009 @@ -74,6 +74,22 @@ lst2 = virt1._get_field_descr_list() assert lst1 is lst2 +def test_reuse_vinfo(): + class FakeVInfo(object): + pass + class FakeVirtualValue(optimizeopt.AbstractVirtualValue): + def _make_virtual(self, *args): + return FakeVInfo() + v1 = FakeVirtualValue(None, None, None) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) + assert vinfo1 is vinfo2 + vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is not vinfo2 + vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is vinfo4 + + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Thu Nov 19 14:50:46 2009 @@ -34,6 +34,12 @@ assert tagged_eq(UNASSIGNED, UNASSIGNED) assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) +def test_tagged_list_eq(): + assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)], + [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)]) + assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) + assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) + class MyMetaInterp: def __init__(self, cpu=None): if cpu is None: From afa at codespeak.net Thu Nov 19 14:53:18 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 14:53:18 +0100 (CET) Subject: [pypy-svn] r69443 - pypy/trunk/pypy/module/oracle Message-ID: <20091119135318.31DE7168025@codespeak.net> Author: afa Date: Thu Nov 19 14:53:17 2009 New Revision: 69443 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: Expose and implement LONG_BINARY Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 19 14:53:17 2009 @@ -11,6 +11,7 @@ 'DATETIME': 'interp_variable.VT_DateTime', 'BINARY': 'interp_variable.VT_Binary', 'LONG_STRING': 'interp_variable.VT_LongString', + 'LONG_BINARY': 'interp_variable.VT_LongBinary', 'FIXED_CHAR': 'interp_variable.VT_FixedChar', 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 19 14:53:17 2009 @@ -389,8 +389,9 @@ setvalue = interp2app(W_Variable.setValue, unwrap_spec=W_Variable.setValue.unwrap_spec), - maxlength = interp_attrproperty('bufferSize', W_Variable), - + maxlength = interp_attrproperty('bufferSize', W_Variable), + bufferSize = interp_attrproperty('bufferSize', W_Variable), + size = interp_attrproperty('size', W_Variable), ) class VT_String(W_Variable): @@ -489,16 +490,15 @@ isVariableLength = True size = 128 * 1024 + def getBufferSize(self): + return self.size + rffi.sizeof(roci.ub4) + def getValueProc(self, space, pos): ptr = rffi.ptradd(self.data, pos * self.bufferSize) length = rffi.cast(roci.Ptr(roci.ub4), ptr)[0] - l = [] - i = 0 - while i < length: - l.append(self.data[i + rffi.sizeof(roci.ub4)]) - i += 1 - return space.wrap(''.join(l)) + ptr = rffi.ptradd(ptr, rffi.sizeof(roci.ub4)) + return space.wrap(rffi.charpsize2str(ptr, length)) def setValueProc(self, space, pos, w_value): buf = config.StringBuffer() @@ -530,7 +530,7 @@ size = config.MAX_BINARY_BYTES class VT_LongBinary(VT_LongString): - pass + oracleType = roci.SQLT_LVB class VT_NativeFloat(W_Variable): pass Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 19 14:53:17 2009 @@ -62,7 +62,7 @@ OCI_IND_NULL OCI_IND_NOTNULL OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE - SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC + SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS From arigo at codespeak.net Thu Nov 19 14:59:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 19 Nov 2009 14:59:52 +0100 (CET) Subject: [pypy-svn] r69444 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091119135952.6B1BE168075@codespeak.net> Author: arigo Date: Thu Nov 19 14:59:51 2009 New Revision: 69444 Modified: pypy/trunk/pypy/objspace/std/floatobject.py pypy/trunk/pypy/objspace/std/test/test_floatobject.py Log: Always return 0 for hash(nan). Fixes an issue with calling int(nan), on some versions of Python. Modified: pypy/trunk/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/floatobject.py Thu Nov 19 14:59:51 2009 @@ -2,7 +2,7 @@ from pypy.interpreter import gateway from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject -from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf +from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan from pypy.rlib.rarithmetic import formatd import math @@ -197,6 +197,9 @@ def _hash_float(space, v): from pypy.objspace.std.longobject import hash__Long + if isnan(v): + return 0 + # This is designed so that Python numbers of different types # that compare equal hash to the same value; otherwise comparisons # of mapping keys will turn out weird. Modified: pypy/trunk/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_floatobject.py Thu Nov 19 14:59:51 2009 @@ -59,10 +59,7 @@ inf = 1e200 * 1e200 assert hash(inf) == 314159 assert hash(-inf) == -271828 - x = hash(inf/inf) - # ^^^ assert did not crash, even though the result is a bit random - # e.g. it appears to be -32768 on Win32 and 0 on Linux - assert x == hash(inf/inf) + assert hash(inf/inf) == 0 def test_int_float(self): assert int(42.1234) == 42 From david at codespeak.net Thu Nov 19 15:26:26 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 19 Nov 2009 15:26:26 +0100 (CET) Subject: [pypy-svn] r69445 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20091119142626.3F0CF168023@codespeak.net> Author: david Date: Thu Nov 19 15:26:25 2009 New Revision: 69445 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_parse.py Log: Fix test Modified: pypy/branch/io-lang/pypy/lang/io/test/test_parse.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_parse.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_parse.py Thu Nov 19 15:26:25 2009 @@ -19,6 +19,6 @@ input = "a := b" ast = parse(input, space) a = W_Message(space, '"a"', []) - a.literal_value = space.w_immutable_sequence.clone_and_init('a') + # a.literal_value = space.w_immutable_sequence.clone_and_init('a') assert ast == W_Message(space, "setSlot", [a, W_Message(space, 'b', [])], ) \ No newline at end of file From david at codespeak.net Thu Nov 19 15:30:02 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 19 Nov 2009 15:30:02 +0100 (CET) Subject: [pypy-svn] r69446 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20091119143002.3AC14168023@codespeak.net> Author: david Date: Thu Nov 19 15:30:01 2009 New Revision: 69446 Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py pypy/branch/io-lang/pypy/lang/io/test/test_parser.py Modified: pypy/branch/io-lang/pypy/lang/io/compiler.py pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/parser.py pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Log: Parser implementation, without error messages and operator shuffling Modified: pypy/branch/io-lang/pypy/lang/io/compiler.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/compiler.py (original) +++ pypy/branch/io-lang/pypy/lang/io/compiler.py Thu Nov 19 15:30:01 2009 @@ -1,6 +1,6 @@ from pypy.lang.io.register import register_method from pypy.lang.io.model import W_Object, W_Number -from pypy.lang.io.parser import get_lexer +from pypy.lang.io.parser import get_lexer, parse @register_method('Compiler', 'tokensForString') def compiler_tokens_for_string(space, w_target, w_message, w_context): input = w_message.arguments[0].eval(space, w_context, w_target).value @@ -9,11 +9,14 @@ t = W_Object(space) if token.source in ['[', '{'] and len(io_tokens) > 0: io_tokens[-1].slots['character'].value = token.source_pos.columnno - t.slots['character'] = W_Number(space, len(token.source) + token.source_pos.columnno) t.slots['line'] = W_Number(space, token.source_pos.lineno + 1) t.slots['type'] = space.w_sequence.clone_and_init(token.name) t.slots['name'] = space.w_sequence.clone_and_init(token.source) io_tokens.append(t) - return space.w_list.clone_and_init(space, io_tokens) \ No newline at end of file + return space.w_list.clone_and_init(space, io_tokens) + + at register_method('Compiler', 'messageForString', unwrap_spec=[object, str]) +def compiler_message_for_string(space, w_target, string): + return parse(space, string) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Thu Nov 19 15:30:01 2009 @@ -292,4 +292,4 @@ except ValueError: pass if literal.startswith('"') and literal.endswith('"'): - return space.w_immutable_sequence.clone_and_init(literal[1:-1]) \ No newline at end of file + return space.w_sequence.clone_and_init(literal[1:-1]) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/parser.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parser.py (original) +++ pypy/branch/io-lang/pypy/lang/io/parser.py Thu Nov 19 15:30:01 2009 @@ -1,5 +1,10 @@ +# TODO: +# set line and character number on messages +# operator shuffling + from pypy.rlib.parsing.regexparse import parse_regex from pypy.rlib.parsing.lexer import Lexer, Token, SourcePos +from pypy.lang.io.model import W_Message, parse_literal, parse_hex # taken from rlib/parsing/test/python_lexer.py # reg exp helper methods @@ -28,7 +33,7 @@ Hexnumber = r'0[xX][0-9a-fA-F]*' # Comments and Whitespace, the ignored stuff -Whitespace = r'[ \f\t\n]*' +Whitespace = r'[ \f\t]*' slashStarComment = r'/\*[^*/]*\*/' slashSlashComment = r'//[^\n]*\n' poundComment = r'#[^\n]*\n' @@ -43,7 +48,7 @@ MonoQuote = r'"([^"]|(\\"))*"' # ; -Terminator = r'(;'+maybe(Whitespace)+')+' +Terminator = r'([\n;]'+maybe(Whitespace)+')+' # , Comma = r'\,' @@ -72,6 +77,8 @@ i = -1 while(i < len(tokens)-1): i += 1 + if tokens[i].name == 'Terminator': + tokens[i].source = ';' if tokens[i].name != 'OpenParen': continue if tokens[i].source == '(' and (i == 0 or tokens[i-1].name != 'Identifier'): @@ -89,4 +96,58 @@ tokens[i].source_pos.columnno))) i += 1 - return tokens \ No newline at end of file + return tokens + +def parse(space, string): + return IoParser(string, space).parse() + +class IoParser(object): + def __init__(self, code, space): + super(IoParser, self).__init__() + self.code = code + self.space = space + self.tokens = get_lexer().tokenize(self.code) + if len(self.tokens) > 0 and self.tokens[0].name == 'Terminator': + self.tokens.pop(0) + if len(self.tokens) > 0 and self.tokens[-1].name == 'Terminator': + self.tokens.pop() + + + def parse(self): + if len(self.tokens) == 0: + return W_Message(self.space, 'nil', []) + token = self.tokens.pop(0) + # method = getattr(self, "parse_" + token.name.lower()) + arguments = self.parse_arguments() + message = self.parse_token(token, arguments) + message.next = self.parse_next() + return message + + def parse_next(self): + if len(self.tokens) > 0 and self.tokens[0].name not in ['Comma', 'OpenParen', 'CloseParen']: + return self.parse() + else: + return None + + def parse_arguments(self): + if len(self.tokens) > 0 and self.tokens[0].name == 'OpenParen': + arguments = [] + t = self.tokens.pop(0) + assert t.name == 'OpenParen' + + while self.tokens[0].name != 'CloseParen': + if self.tokens[0].name == 'Comma': + self.tokens.pop(0) + arguments.append(self.parse()) + + t = self.tokens.pop(0) + assert t.name == 'CloseParen' + else: + arguments = [] + return arguments + + def parse_token(self, token, args=[]): + m = W_Message(self.space, token.source, args) + if token.name != 'Identifier': + m.cached_result = parse_literal(self.space, token.source) + return m Modified: pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler.py Thu Nov 19 15:30:01 2009 @@ -1,5 +1,5 @@ from pypy.lang.io.parserhack import interpret -from pypy.lang.io.model import W_Object, W_Number +from pypy.lang.io.model import W_Object, W_Number, W_Message import py def test_compiler_is_present(): @@ -8,163 +8,70 @@ assert isinstance(res, W_Object) assert res.slots['type'].value == 'Compiler' assert res.protos == [space.w_object] - -def test_compiler_token_for_number_is_number(): - inp = 'Compiler tokensForString("1")' - res, space = interpret(inp) - assert res.items[0].slots['character'].value == 1 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Number' - assert res.items[0].slots['name'].value == "1" - assert isinstance(res.items[0], W_Object) - -def test_compiler_token_HexNumber(): - inp = 'Compiler tokensForString("0x1")' - res, space = interpret(inp) - assert res.items[0].slots['character'].value == 3 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'HexNumber' - assert res.items[0].slots['name'].value == "0x1" - assert isinstance(res.items[0], W_Object) -def test_compiler_token_open_paren(): - inp = 'Compiler tokensForString("()")' +def test_parse_number(): + inp = 'Compiler messageForString("42")' res, space = interpret(inp) - assert res.items[1].slots['character'].value == 1 - assert res.items[1].slots['line'].value == 1 - assert res.items[1].slots['type'].value == 'OpenParen' - assert res.items[1].slots['name'].value == "(" - assert isinstance(res.items[1], W_Object) - -def test_compiler_token_close_paren(): - inp = 'Compiler tokensForString("()")' - res, space = interpret(inp) - assert res.items[2].slots['character'].value == 2 - assert res.items[2].slots['line'].value == 1 - assert res.items[2].slots['type'].value == 'CloseParen' - assert res.items[2].slots['name'].value == ")" - assert isinstance(res.items[2], W_Object) - -def test_compiler_parse_paren_produces_anon_message(): - inp = 'Compiler tokensForString(" ()")' - res, space = interpret(inp) - assert res.items[0].slots['character'].value == 1 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Identifier' - assert res.items[0].slots['name'].value == "" - assert isinstance(res.items[0], W_Object) - -def test_compiler_parse_paren_produces_anon_message(): - inp = 'Compiler tokensForString("()")' - res, space = interpret(inp) - assert res.items[0].slots['character'].value == 0 - assert res.items[0].slots['line'].value == 1 - assert isinstance(res.items[0], W_Object) - -# curlyBraces -def test_compiler_token_open_squareBrackets(): - inp = 'Compiler tokensForString("[]")' - res, space = interpret(inp) - assert res.items[1].slots['character'].value == 1 - assert res.items[1].slots['line'].value == 1 - assert res.items[1].slots['type'].value == 'OpenParen' - assert res.items[1].slots['name'].value == "[" - assert isinstance(res.items[1], W_Object) - -def test_compiler_token_squareBrackets(): - inp = 'Compiler tokensForString("[]")' + assert isinstance(res, W_Message) + assert res.name == '42' + assert isinstance(res.cached_result, W_Number) + assert res.cached_result.value == 42 + +def test_parse_hexnumber(): + inp = 'Compiler messageForString("0xf")' res, space = interpret(inp) - assert res.items[2].slots['character'].value == 2 - assert res.items[2].slots['line'].value == 1 - assert res.items[2].slots['type'].value == 'CloseParen' - assert res.items[2].slots['name'].value == "]" - assert isinstance(res.items[2], W_Object) - -def test_compiler_parse_paren_produces_squareBrackets_message(): - inp = 'Compiler tokensForString("[]")' + assert isinstance(res, W_Message) + assert res.name == '0xf' + assert isinstance(res.cached_result, W_Number) + assert res.cached_result.value == 15 + +def test_parse_message_with_artguments(): + inp = 'Compiler messageForString("a(1,2)")' res, space = interpret(inp) - assert res.items[0].slots['character'].value == 0 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Identifier' - assert res.items[0].slots['name'].value == "squareBrackets" - assert isinstance(res.items[0], W_Object) - -# curlyBrackets -def test_compiler_token_open_curlyBrackets(): - inp = 'Compiler tokensForString("{}")' - res, space = interpret(inp) - assert res.items[1].slots['character'].value == 1 - assert res.items[1].slots['line'].value == 1 - assert res.items[1].slots['type'].value == 'OpenParen' - assert res.items[1].slots['name'].value == "{" - assert isinstance(res.items[1], W_Object) - -def test_compiler_token_curlyBrackets(): - inp = 'Compiler tokensForString("{}")' + assert isinstance(res, W_Message) + assert res.name == 'a' + assert len(res.arguments) == 2 + assert res.arguments[0].name == '1' + assert res.arguments[1].name == '2' + +def test_parse_message_chain(): + inp = 'Compiler messageForString("1 2")' res, space = interpret(inp) - assert res.items[2].slots['character'].value == 2 - assert res.items[2].slots['line'].value == 1 - assert res.items[2].slots['type'].value == 'CloseParen' - assert res.items[2].slots['name'].value == "}" - assert isinstance(res.items[2], W_Object) - -def test_compiler_parse_paren_produces_curlyBrackets_message(): - inp = 'Compiler tokensForString("{}")' + assert isinstance(res, W_Message) + assert res.name == '1' + assert isinstance(res.next, W_Message) + assert res.next.name == '2' + assert res.next.cached_result.value == 2 + +def test_parse_longer_message_chain(): + inp = 'Compiler messageForString("1 2 3 4 5 6")' res, space = interpret(inp) - assert res.items[0].slots['character'].value == 0 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Identifier' - assert res.items[0].slots['name'].value == "curlyBrackets" - assert isinstance(res.items[0], W_Object) - -def test_compiler_identifier_token(): - inp = 'Compiler tokensForString("foo")' - res, space = interpret(inp) - assert res.items[0].slots['character'].value == 3 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Identifier' - assert res.items[0].slots['name'].value == 'foo' - assert isinstance(res.items[0], W_Object) - -def test_compiler_terminator_token(): - inp = 'Compiler tokensForString(";")' + assert isinstance(res, W_Message) + assert res.name == '1' + + n = res.next + assert isinstance(n, W_Message) + assert n.name == '2' + + n = n.next + assert isinstance(n, W_Message) + assert n.name == '3' + + n = n.next + assert isinstance(n, W_Message) + assert n.name == '4' + + n = n.next + assert isinstance(n, W_Message) + assert n.name == '5' + + n = n.next + assert isinstance(n, W_Message) + assert n.name == '6' + +def test_parse_empty_input(): + inp = 'Compiler messageForString("")' res, space = interpret(inp) - assert res.items[0].slots['character'].value == 1 - assert res.items[0].slots['line'].value == 1 - assert res.items[0].slots['type'].value == 'Terminator' - assert res.items[0].slots['name'].value == ";" - assert isinstance(res.items[0], W_Object) - -def test_compiler_comma_token(): - inp = 'Compiler tokensForString("(1,2)")' - res, space = interpret(inp) - assert res.items[3].slots['character'].value == 3 - assert res.items[3].slots['line'].value == 1 - assert res.items[3].slots['type'].value == 'Comma' - assert res.items[3].slots['name'].value == "," - assert isinstance(res.items[3], W_Object) - -def test_compiler_triquote_token(): - py.test.skip('Problem in the parserhack') - inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")' - res, space = interpret(inp) - assert res.items[0].slots['type'].value == 'TriQuote' - assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\"" - assert isinstance(res.items[0], W_Object) - -def test_compiler_monoquote_token(): - py.test.skip('Problem in the parserhack') - inp = 'Compiler tokensForString("\\\"lorem\\\"")' - res, space = interpret(inp) - assert res.items[0].slots['name'].value == "\"lorem\"" - assert res.items[0].slots['type'].value == 'MonoQuote' - assert isinstance(res.items[0], W_Object) - -def test_compiler_comment_token(): - py.test.skip("These Tokens are ignored by the lexer") - inp = 'Compiler tokensForString("xxx")' - res, space = interpret(inp) - assert res.items[0].slots['type'].value == 'Comment' - assert res.items[0].slots['name'].value == "??" - assert isinstance(res.items[0], space.x) \ No newline at end of file + assert isinstance(res, W_Message) + assert res.name == 'nil' \ No newline at end of file Added: pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_compiler_token.py Thu Nov 19 15:30:01 2009 @@ -0,0 +1,173 @@ +from pypy.lang.io.parserhack import interpret +from pypy.lang.io.model import W_Object, W_Number +import py + +def test_compiler_token_for_number_is_number(): + inp = 'Compiler tokensForString("1")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Number' + assert res.items[0].slots['name'].value == "1" + assert isinstance(res.items[0], W_Object) + +def test_compiler_token_HexNumber(): + inp = 'Compiler tokensForString("0x1")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 3 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'HexNumber' + assert res.items[0].slots['name'].value == "0x1" + assert isinstance(res.items[0], W_Object) + +def test_compiler_token_open_paren(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "(" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_close_paren(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == ")" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_anon_message(): + inp = 'Compiler tokensForString(" ()")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "" + assert isinstance(res.items[0], W_Object) + +def test_compiler_parse_paren_produces_anon_message(): + inp = 'Compiler tokensForString("()")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert isinstance(res.items[0], W_Object) + +# curlyBraces +def test_compiler_token_open_squareBrackets(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "[" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_squareBrackets(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == "]" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_squareBrackets_message(): + inp = 'Compiler tokensForString("[]")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "squareBrackets" + assert isinstance(res.items[0], W_Object) + +# curlyBrackets +def test_compiler_token_open_curlyBrackets(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[1].slots['character'].value == 1 + assert res.items[1].slots['line'].value == 1 + assert res.items[1].slots['type'].value == 'OpenParen' + assert res.items[1].slots['name'].value == "{" + assert isinstance(res.items[1], W_Object) + +def test_compiler_token_curlyBrackets(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[2].slots['character'].value == 2 + assert res.items[2].slots['line'].value == 1 + assert res.items[2].slots['type'].value == 'CloseParen' + assert res.items[2].slots['name'].value == "}" + assert isinstance(res.items[2], W_Object) + +def test_compiler_parse_paren_produces_curlyBrackets_message(): + inp = 'Compiler tokensForString("{}")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 0 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == "curlyBrackets" + assert isinstance(res.items[0], W_Object) + +def test_compiler_identifier_token(): + inp = 'Compiler tokensForString("foo")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 3 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Identifier' + assert res.items[0].slots['name'].value == 'foo' + assert isinstance(res.items[0], W_Object) + +def test_compiler_terminator_token_for_newline(): + py.test.skip("Parserhack related") + inp = """Compiler tokensForString("\n")""" + res, space = interpret(inp) + # assert res.items[0].slots['character'].value == 1 + # assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Terminator' + assert res.items[0].slots['name'].value == ";" + assert isinstance(res.items[0], W_Object) + + +def test_compiler_terminator_token(): + inp = 'Compiler tokensForString(";")' + res, space = interpret(inp) + assert res.items[0].slots['character'].value == 1 + assert res.items[0].slots['line'].value == 1 + assert res.items[0].slots['type'].value == 'Terminator' + assert res.items[0].slots['name'].value == ";" + assert isinstance(res.items[0], W_Object) + +def test_compiler_comma_token(): + inp = 'Compiler tokensForString("(1,2)")' + res, space = interpret(inp) + assert res.items[3].slots['character'].value == 3 + assert res.items[3].slots['line'].value == 1 + assert res.items[3].slots['type'].value == 'Comma' + assert res.items[3].slots['name'].value == "," + assert isinstance(res.items[3], W_Object) + +def test_compiler_triquote_token(): + py.test.skip('Problem in the parserhack') + inp = 'Compiler tokensForString("\"\"\"asdf\"\"\"")' + res, space = interpret(inp) + assert res.items[0].slots['type'].value == 'TriQuote' + assert res.items[0].slots['name'].value == "\"\"\"asdf\"\"\"" + assert isinstance(res.items[0], W_Object) + +def test_compiler_monoquote_token(): + py.test.skip('Problem in the parserhack') + inp = 'Compiler tokensForString("\\\"lorem\\\"")' + res, space = interpret(inp) + assert res.items[0].slots['name'].value == "\"lorem\"" + assert res.items[0].slots['type'].value == 'MonoQuote' + assert isinstance(res.items[0], W_Object) + +def test_compiler_comment_token(): + py.test.skip("These Tokens are ignored by the lexer") + inp = 'Compiler tokensForString("xxx")' + res, space = interpret(inp) + assert res.items[0].slots['type'].value == 'Comment' + assert res.items[0].slots['name'].value == "??" + assert isinstance(res.items[0], space.x) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_lexer.py Thu Nov 19 15:30:01 2009 @@ -85,14 +85,20 @@ def test_lex_terminator1(): inp = ';;' tokens = iolexer.tokenize(inp) - assert tokens[0] == Token('Terminator', ';;', SourcePos(0, 0, 0)) + assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0)) assert len(tokens) == 1 def test_lex_terminator2(): inp = '; ;' tokens = iolexer.tokenize(inp) - assert tokens[0] == Token('Terminator', '; ;', SourcePos(0, 0, 0)) + assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0)) assert len(tokens) == 1 + +def test_lex_newline_terminator(): + inp = '\n' + tokens = iolexer.tokenize(inp) + assert len(tokens) == 1 + assert tokens[0] == Token('Terminator', ';', SourcePos(0, 0, 0)) def test_lex_open_paren(): inp = '()' Added: pypy/branch/io-lang/pypy/lang/io/test/test_parser.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_parser.py Thu Nov 19 15:30:01 2009 @@ -0,0 +1,142 @@ +import py +from pypy.lang.io.parser import parse, IoParser +from pypy.lang.io.model import W_Message, W_Number +from pypy.lang.io.objspace import ObjSpace + +space = ObjSpace() +def test_parse_number_token(): + t = parse(space, "4") + assert isinstance(t, W_Message) + assert t.name == '4' + +def test_parse_number_sets_cached_result(): + t = parse(space, "4") + assert isinstance(t.cached_result, W_Number) + assert t.cached_result.value == 4 + +def test_parse_hexnumber_token(): + t = parse(space, "0xf") + assert isinstance(t, W_Message) + assert t.name == '0xf' + +def test_parse_hexnumber_sets_cached_result(): + t = parse(space, '0xf') + assert isinstance(t.cached_result, W_Number) + assert t.cached_result.value == 15 + +def test_parse_identifier(): + t = parse(space, 'foo') + assert isinstance(t, W_Message) + assert t.name == 'foo' + +def test_parse_string(): + t = parse(space, '"a"') + assert isinstance(t, W_Message) + assert t.name == '"a"' + +def test_parse_string_sets_cached_result(): + t = parse(space, '"a"') + assert space.w_sequence in t.cached_result.protos + assert t.cached_result.value == 'a' + +def test_parse_tripple_quoted_string(): + t = parse(space, '"""a"""') + assert isinstance(t, W_Message) + assert t.name == '"""a"""' + +def test_parse_tripple_quoted_string_sets_cached_result(): + t = parse(space, '"a"') + assert space.w_sequence in t.cached_result.protos + assert t.cached_result.value == 'a' + +def test_parse_arguments_simple(): + t = parse(space, 'a(1)') + assert len(t.arguments) == 1 + assert t.arguments[0].name == '1' + +def test_parse_argument_list(): + t = parse(space, 'a(1, "a", 0xa)') + assert len(t.arguments) == 3 + assert t.arguments[0].name == '1' + assert t.arguments[1].name == '"a"' + assert t.arguments[2].name == '0xa' + +def test_parse_message_chain(): + t = parse(space, 'a 1') + assert isinstance(t, W_Message) + assert t.name == 'a' + next = t.next + assert isinstance(next, W_Message) + assert next.name == '1' + assert next.cached_result.value == 1 + +def test_parse_message_chain_with_arguments(): + t = parse(space, 'a("foo", "bar") 1') + assert isinstance(t, W_Message) + assert t.name == 'a' + next = t.next + assert isinstance(next, W_Message) + assert next.name == '1' + assert next.cached_result.value == 1 + +def test_parse_empty_string_produces_nil_message(): + t = parse(space, '') + assert isinstance(t, W_Message) + assert t.name == 'nil' + +def test_parser_sets_line_and_char_no_on_message(): + py.test.skip("Not implemented yet") + +def test_parse_only_terminator(): + t = parse(space, ';') + assert isinstance(t, W_Message) + assert t.name == 'nil' + +def test_parse_only_terminator2(): + t = parse(space, """\n""") + assert isinstance(t, W_Message) + assert t.name == 'nil' + +def test_parse_terminator_between_messages_appends_terminator_message(): + t = parse(space, 'a ; b') + + assert isinstance(t, W_Message) + assert t.name == 'a' + t1 = t.next + + assert isinstance(t1, W_Message) + assert t1.name == ';' + t2 = t1.next + + assert isinstance(t2, W_Message) + assert t2.name == 'b' + +def test_parse_terminator_between_messages_appends_terminator_message2(): + t = parse(space, 'a ;;;;;; ; b') + + assert isinstance(t, W_Message) + assert t.name == 'a' + t1 = t.next + + assert isinstance(t1, W_Message) + assert t1.name == ';' + t2 = t1.next + + assert isinstance(t2, W_Message) + assert t2.name == 'b' + +def test_parse_terminator_at_end_is_ignored(): + t = parse(space, 'a ; b ;') + + assert isinstance(t, W_Message) + assert t.name == 'a' + t1 = t.next + + assert isinstance(t1, W_Message) + assert t1.name == ';' + t2 = t1.next + + assert isinstance(t2, W_Message) + assert t2.name == 'b' + + assert t2.next is None \ No newline at end of file From cfbolz at codespeak.net Thu Nov 19 16:07:35 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 19 Nov 2009 16:07:35 +0100 (CET) Subject: [pypy-svn] r69448 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091119150735.410CA168077@codespeak.net> Author: cfbolz Date: Thu Nov 19 16:07:34 2009 New Revision: 69448 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py Log: (pedronis, cfbolz): add counting of how much reusing of virtuals we do Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py Thu Nov 19 16:07:34 2009 @@ -20,6 +20,9 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +NVIRTUALS +NVHOLES +NVREUSED """ def _setup(): @@ -175,6 +178,9 @@ self._print_intline("forcings", cnt[OPT_FORCINGS]) self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) + self._print_intline("nvirtuals", cnt[NVIRTUALS]) + self._print_intline("nvholes", cnt[NVHOLES]) + self._print_intline("nvreused", cnt[NVREUSED]) def _print_line_time(self, string, i, tim): final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/optimizeopt.py Thu Nov 19 16:07:34 2009 @@ -2,7 +2,7 @@ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.metainterp.jitprof import OPT_OPS, OPT_GUARDS, OPT_FORCINGS +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode from pypy.jit.metainterp.specnode import AbstractVirtualStructSpecNode @@ -388,7 +388,7 @@ self.bool_boxes = {} def forget_numberings(self, virtualbox): - self.metainterp_sd.profiler.count(OPT_FORCINGS) + self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS) self.resumedata_memo.forget_numberings(virtualbox) def getinterned(self, box): @@ -514,6 +514,8 @@ else: self.optimize_default(op) self.loop.operations = self.newoperations + # accumulate counters + self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) @@ -526,9 +528,9 @@ op = op.clone() must_clone = False op.args[i] = box - self.metainterp_sd.profiler.count(OPT_OPS) + self.metainterp_sd.profiler.count(jitprof.OPT_OPS) if op.is_guard(): - self.metainterp_sd.profiler.count(OPT_GUARDS) + self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS) self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Thu Nov 19 16:07:34 2009 @@ -1,6 +1,7 @@ import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp import jitprof from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated @@ -112,6 +113,10 @@ self.numberings = {} self.cached_boxes = {} self.cached_virtuals = {} + + self.nvirtuals = 0 + self.nvholes = 0 + self.nvreused = 0 def getconst(self, const): if const.type == INT: @@ -217,6 +222,10 @@ self.cached_boxes.clear() self.cached_virtuals.clear() + def update_counters(self, profiler): + profiler.count(jitprof.NVIRTUALS, self.nvirtuals) + profiler.count(jitprof.NVHOLES, self.nvholes) + profiler.count(jitprof.NVREUSED, self.nvreused) _frame_info_placeholder = (None, 0, 0) @@ -315,12 +324,18 @@ if vfieldboxes: length = num_env_virtuals + memo.num_cached_virtuals() virtuals = storage.rd_virtuals = [None] * length + memo.nvirtuals += length + memo.nvholes += length - len(vfieldboxes) for virtualbox, fieldboxes in vfieldboxes.iteritems(): num, _ = untag(self.liveboxes[virtualbox]) value = values[virtualbox] fieldnums = [self._gettagged(box) for box in fieldboxes] vinfo = value.make_virtual_info(self, fieldnums) + # if a new vinfo instance is made, we get the fieldnums list we + # pass in as an attribute. hackish. + if vinfo.fieldnums is not fieldnums: + memo.nvreused += 1 virtuals[num] = vinfo def _gettagged(self, box): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py Thu Nov 19 16:07:34 2009 @@ -63,7 +63,8 @@ ] assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] - assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0] + assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, + 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside From arigo at codespeak.net Thu Nov 19 16:29:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 19 Nov 2009 16:29:12 +0100 (CET) Subject: [pypy-svn] r69452 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091119152912.F3C57168073@codespeak.net> Author: arigo Date: Thu Nov 19 16:29:12 2009 New Revision: 69452 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Log: Trying another approach... In-progress. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Thu Nov 19 16:29:12 2009 @@ -1,7 +1,7 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF, FLOAT +from pypy.jit.metainterp.history import Const, Box, BoxPtr, INT, REF, FLOAT from pypy.jit.metainterp.history import AbstractFailDescr from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT @@ -86,7 +86,7 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() - self.setup_failure_recovery() + self.failure_recovery_code = [0, 0] def leave_jitted_hook(self): # XXX BIG FAT WARNING XXX @@ -123,7 +123,8 @@ # 'mc2' is for guard recovery code self.mc = MachineCodeBlockWrapper() self.mc2 = MachineCodeBlockWrapper() - self._build_failure_recovery_builder_trampoline() + self._build_failure_recovery(False) + self._build_failure_recovery(True) def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -757,10 +758,9 @@ self._no_const_locs(failargs) exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = fail_locs - return self.generate_quick_failure(faildescr, failargs, exc) + return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) - def generate_quick_failure(self, faildescr, failargs, exc): + def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): """Generate the initial code for handling a failure. We try to keep it as compact as possible. The idea is that this code is executed at most once (and very often, zero times); when @@ -774,93 +774,78 @@ mc = self.mc2._mc addr = mc.tell() mc.PUSH(imm32(fail_index)) - mc.CALL(rel32(self.failure_recovery_code)) - # write a bitfield of 0's (int or float argument) or 1's (ref argument) - self.write_bitfield_for_failargs(mc, failargs, exc) - if not we_are_translated(): - faildescr._x86_exc_locs_are_ref = ( - exc, [v.type == REF for v in failargs]) + mc.CALL(rel32(self.failure_recovery_code[exc])) + # write tight data that describes the failure recovery + self.write_failure_recovery_description(mc, failargs, fail_locs) return addr - def write_bitfield_for_failargs(self, mc, failargs, exc): - #print 'writing:', mc.tell() - #mc.writechr(0xDD) - bitfield = int(exc) # the first bit: "save exceptions" - bit = 0x02 + DESCR_INT = 0x00 + DESCR_REF = 0x01 + DESCR_FLOAT = 0x02 + DESCR_FROMSTACK = 8 + DESCR_STOP = DESCR_INT + 4*esp.op + + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): - if bit == 0x100: - mc.writechr(bitfield) - bitfield = 0 - bit = 0x01 - if failargs[i].type == REF: - bitfield |= bit - bit <<= 1 - mc.writechr(bitfield) - #mc.writechr(0xCC) - - def getbit_from_bitfield(self, bitfield_ptr, i): - byte = ord(bitfield_ptr[i >> 3]) - return bool(byte & (1 << (i & 7))) - - def setup_failure_recovery(self): - - def failure_recovery_builder(fail_index, bitfield_ptr): - #assert bitfield_ptr[0] == chr(0xDD) - faildescr = self.cpu.get_fail_descr_from_number(fail_index) - locs = faildescr._x86_faillocs - #assert bitfield_ptr[1 + (1+len(locs)+7)//8] == chr(0xCC) - exc = self.getbit_from_bitfield(bitfield_ptr, 0) # the first bit - locs_are_ref = [self.getbit_from_bitfield(bitfield_ptr, 1 + i) - for i in range(len(locs))] - if not we_are_translated(): - #print 'completing:', rffi.cast(lltype.Signed, bitfield_ptr) - assert (exc, locs_are_ref) == faildescr._x86_exc_locs_are_ref - addr = self.mc2.tell() - self.generate_failure(self.mc2, fail_index, locs, exc, - locs_are_ref) - self.patch_jump(faildescr, addr) - return addr - - self.failure_recovery_builder = failure_recovery_builder - self.failure_recovery_code = 0 - - _FAILURE_RECOVERY_BUILDER = lltype.Ptr(lltype.FuncType( - [lltype.Signed, rffi.CCHARP], lltype.Signed)) - - def _build_failure_recovery_builder_trampoline(self): - failure_recovery_builder = llhelper(self._FAILURE_RECOVERY_BUILDER, - self.failure_recovery_builder) - failure_recovery_builder = rffi.cast(lltype.Signed, - failure_recovery_builder) + arg = failargs[i] + if arg.type == REF: + kind = self.DESCR_REF + elif arg.type == INT: + kind = self.DESCR_INT + elif arg.type == FLOAT: + kind = self.DESCR_FLOAT + else: + raise AssertionError("bogus kind") + loc = locs[i] + if isinstance(loc, REG): + n = loc.op + else: + n = self.DESCR_FROMSTACK + loc.position + n = kind + 4*n + while n > 0x7F: + mc.writechr((n & 0x7F) | 0x80) + n >>= 7 + mc.writechr(n) + mc.writechr(self.DESCR_STOP) + + def _build_failure_recovery(self, exc): mc = self.mc2 + esp_offset = 0 code = mc.tell() - mc.PUSH(ebp) - mc.MOV(ebp, esp) - # push all registers on the stack - mc.PUSHA() + # push the xmm registers on the stack if self.cpu.supports_floats: mc.SUB(esp, imm(8*8)) + esp_offset += 8*8 for i in range(8): mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + # push eax, edx and ecx on the stack + mc.PUSH(eax) + mc.PUSH(edx) + mc.PUSH(ecx) + esp_offset += 3*4 # really call the failure recovery builder code - mc.PUSH(mem(ebp, 4)) # the return address: ptr to bitfield - mc.PUSH(mem(ebp, 8)) # fail_index + mc.PUSH(mem(esp, esp_offset)) # the return address: ptr to bitfield + esp_offset += 4 + mc.PUSH(mem(esp, esp_offset+4)) # fail_index + esp_offset += 4 mc.CALL(rel32(failure_recovery_builder)) mc.ADD(esp, imm(8)) - # save the return value into ebp+4. This is the address of the code - # written by generate_failure. Doing so overwrites this function's - # own return address, which was just a ptr to the bitfield, so far. - mc.MOV(mem(ebp, 4), eax) + esp_offset -= 8 + # the return value in eax is the address of the code written by + # generate_failure. We write it over this function's own return + # address, which was just a ptr to the bitfield, so far. + mc.MOV(mem(esp, esp_offset), eax) # pop all registers + mc.POP(ecx) + mc.POP(edx) + mc.POP(eax) if self.cpu.supports_floats: for i in range(8): mc.MOVSD(xmm_registers[i], mem64(esp, 8*i)) mc.ADD(esp, imm(8*8)) - mc.POPA() # epilogue - mc.POP(ebp) mc.RET(imm16(4)) - self.failure_recovery_code = code + self.failure_recovery_code[exc] = code def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): for i in range(len(locs)): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Thu Nov 19 16:29:12 2009 @@ -1,5 +1,7 @@ +from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.metainterp.history import BoxInt, BoxPtr +from pypy.jit.backend.x86.regalloc import X86StackManager +from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat class FakeCPU: @@ -12,23 +14,32 @@ self.content.append(n) -def test_bitfield(): +def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) mc = FakeMC() - assembler.write_bitfield_for_failargs(mc, [BoxInt(), BoxPtr()], False) - assert mc.content == [4] - bitfield = map(chr, mc.content) - assert assembler.getbit_from_bitfield(bitfield, 0) == False - assert assembler.getbit_from_bitfield(bitfield, 1) == False - assert assembler.getbit_from_bitfield(bitfield, 2) == True - -def test_larger_bitfield(): - assembler = Assembler386(FakeCPU()) - mc = FakeMC() - lst = [BoxInt(), BoxPtr(), BoxPtr()] * 6 - assembler.write_bitfield_for_failargs(mc, lst, True) - bitfield = map(chr, mc.content) - assert assembler.getbit_from_bitfield(bitfield, 0) == True - for i in range(len(lst)): - expected = (lst[i].__class__ == BoxPtr) - assert assembler.getbit_from_bitfield(bitfield, 1+i) == expected + failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 + locs = [X86StackManager.stack_pos(0, 1), + X86StackManager.stack_pos(1, 1), + X86StackManager.stack_pos(10, 2), + X86StackManager.stack_pos(100, 1), + X86StackManager.stack_pos(101, 1), + X86StackManager.stack_pos(110, 2), + ebx, + esi, + xmm2] + assembler.write_failure_recovery_description(mc, failargs, locs) + nums = [0 + 4*(8+0), + 1 + 4*(8+1), + 2 + 4*(8+10), + 0 + 4*(8+100), + 1 + 4*(8+101), + 2 + 4*(8+110), + 0 + 4*ebx.op, + 1 + 4*esi.op, + 2 + 4*xmm2.op] + double_byte_nums = [] + for num in nums[3:6]: + double_byte_nums.append((num & 0x7F) | 0x80) + double_byte_nums.append(num >> 7) + assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + + [assembler.DESCR_STOP]) From afa at codespeak.net Thu Nov 19 17:01:46 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 19 Nov 2009 17:01:46 +0100 (CET) Subject: [pypy-svn] r69453 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091119160146.B09C7168076@codespeak.net> Author: afa Date: Thu Nov 19 17:01:45 2009 New Revision: 69453 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py Log: implement connection.encoding Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Thu Nov 19 17:01:45 2009 @@ -12,6 +12,7 @@ from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS from pypy.module.oracle.interp_environ import Environment from pypy.module.oracle.interp_cursor import W_Cursor +from pypy.module.oracle.interp_pool import W_Pool from pypy.module.oracle.interp_variable import VT_String class W_Connection(Wrappable): @@ -263,32 +264,51 @@ def _getCharacterSetName(self, space, attribute): # get character set id - status = roci.OCIAttrGet( - self.environment.handle, roci.HTYPE_ENV, - charsetId, None, - attribute, - self.environment.errorHandle) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): get charset id") + charsetIdPtr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.environment.handle, roci.OCI_HTYPE_ENV, + rffi.cast(roci.dvoidp, charsetIdPtr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + attribute, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetCharacterSetName(): get charset id") + charsetId = charsetIdPtr[0] + finally: + lltype.free(charsetIdPtr, flavor='raw') # get character set name - status = roci.OCINlsCharsetIdToName( - self.environmentHandle, - charsetNameBuf.buf, charsetNameBuf.size, - charsetIdPtr[0]) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): get Oracle charset name") + charsetname_buf, charsetname = rffi.alloc_buffer(roci.OCI_NLS_MAXBUFSZ) + try: + status = roci.OCINlsCharSetIdToName( + self.environment.handle, + charsetname_buf, roci.OCI_NLS_MAXBUFSZ, + charsetId) + self.environment.checkForError( + status, + "Connection_GetCharacterSetName(): get Oracle charset name") - # get IANA character set name - status = roci.OCINlsNameMap( - self.environmentHandle, - ianaCharsetNameBuf.buf, inaCharsetNameBuf.size, - charsetNameBuf.buf, roci.OCI_NLS_CS_ORA_TO_IANA) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): translate NLS charset") + ianacharset_buf, ianacharset = rffi.alloc_buffer( + roci.OCI_NLS_MAXBUFSZ) + + try: + # get IANA character set name + status = roci.OCINlsNameMap( + self.environment.handle, + ianacharset_buf, roci.OCI_NLS_MAXBUFSZ, + charsetname_buf, roci.OCI_NLS_CS_ORA_TO_IANA) + self.environment.checkForError( + status, + "Connection_GetCharacterSetName(): translate NLS charset") + charset = rffi.charp2str(ianacharset_buf) + finally: + rffi.keep_buffer_alive_until_here(ianacharset_buf, ianacharset) + finally: + rffi.keep_buffer_alive_until_here(charsetname_buf, charsetname) + return space.wrap(charset) - return space.wrap(ianaCharsetName) - def get_encoding(space, self): return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) def get_nationalencoding(space, self): Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 19 17:01:45 2009 @@ -30,7 +30,7 @@ ub4 = platform.SimpleType('ub4', rffi.UINT) sb4 = platform.SimpleType('sb4', rffi.INT) sword = platform.SimpleType('sword', rffi.INT) - uword = platform.SimpleType('sword', rffi.UINT) + uword = platform.SimpleType('uword', rffi.UINT) OCINumber = platform.Struct('OCINumber', []) OCITime = platform.Struct('OCITime', @@ -49,13 +49,14 @@ OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION - OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_DTYPE_PARAM - OCI_CRED_RDBMS + OCI_CRED_RDBMS OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM + OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX OCI_FETCH_NEXT @@ -69,6 +70,7 @@ SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY SQLCS_IMPLICIT SQLCS_NCHAR OCI_NUMBER_SIGNED + OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA '''.split() for c in constants: @@ -394,3 +396,21 @@ oratext], # buf sword) +# OCI Locale Functions + +OCINlsCharSetIdToName = external( + 'OCINlsCharSetIdToName', + [dvoidp, # hndl + oratext, # buf + size_t, # buflen + ub2], # id + sword) + +OCINlsNameMap = external( + 'OCINlsNameMap', + [dvoidp, # hndl + oratext, # buf + size_t, # buflen + oratext, # srcbuf + uword], # flag + sword) Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Thu Nov 19 17:01:45 2009 @@ -105,4 +105,14 @@ count, = cursor.fetchone() assert count == 0 + def test_charset(self): + self.cnx = oracle.connect(self.username, self.password, + self.tnsentry) + encoding = self.cnx.encoding + assert isinstance(encoding, str) + assert encoding != "" + encoding = self.cnx.nationalencoding + assert isinstance(encoding, str) + assert encoding != "" + From pedronis at codespeak.net Thu Nov 19 17:08:00 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 19 Nov 2009 17:08:00 +0100 (CET) Subject: [pypy-svn] r69454 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091119160800.52E8C168076@codespeak.net> Author: pedronis Date: Thu Nov 19 17:07:59 2009 New Revision: 69454 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Log: make a note comment in the code Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Thu Nov 19 17:07:59 2009 @@ -450,6 +450,7 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: + # xxx vinfo can be none, test and fix! self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] for i in range(len(virtuals)): vinfo = virtuals[i] From arigo at codespeak.net Thu Nov 19 18:21:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 19 Nov 2009 18:21:53 +0100 (CET) Subject: [pypy-svn] r69455 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091119172153.34D1B168075@codespeak.net> Author: arigo Date: Thu Nov 19 18:21:51 2009 New Revision: 69455 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386setup.py Log: Check this guy in, for reference. Will be killed next. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Thu Nov 19 18:21:51 2009 @@ -768,19 +768,23 @@ really handle recovery from this particular failure. """ fail_index = self.cpu.get_fail_descr_number(faildescr) - bytes_needed = 20 + len(failargs) // 8 # conservative estimate + bytes_needed = 20 + 5 * len(failargs) # conservative estimate if self.mc2.bytes_free() < bytes_needed: self.mc2.make_new_mc() mc = self.mc2._mc addr = mc.tell() - mc.PUSH(imm32(fail_index)) mc.CALL(rel32(self.failure_recovery_code[exc])) # write tight data that describes the failure recovery self.write_failure_recovery_description(mc, failargs, fail_locs) + # write the fail_index too + mc.write(packimm32(fail_index)) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + mc.writechr(0xCC) return addr - DESCR_INT = 0x00 - DESCR_REF = 0x01 + DESCR_REF = 0x00 + DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_FROMSTACK = 8 DESCR_STOP = DESCR_INT + 4*esp.op @@ -809,42 +813,147 @@ mc.writechr(self.DESCR_STOP) def _build_failure_recovery(self, exc): - mc = self.mc2 - esp_offset = 0 - code = mc.tell() - # push the xmm registers on the stack - if self.cpu.supports_floats: - mc.SUB(esp, imm(8*8)) - esp_offset += 8*8 - for i in range(8): - mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) - # push eax, edx and ecx on the stack - mc.PUSH(eax) - mc.PUSH(edx) - mc.PUSH(ecx) - esp_offset += 3*4 - # really call the failure recovery builder code - mc.PUSH(mem(esp, esp_offset)) # the return address: ptr to bitfield - esp_offset += 4 - mc.PUSH(mem(esp, esp_offset+4)) # fail_index - esp_offset += 4 - mc.CALL(rel32(failure_recovery_builder)) - mc.ADD(esp, imm(8)) - esp_offset -= 8 - # the return value in eax is the address of the code written by - # generate_failure. We write it over this function's own return - # address, which was just a ptr to the bitfield, so far. - mc.MOV(mem(esp, esp_offset), eax) - # pop all registers - mc.POP(ecx) - mc.POP(edx) - mc.POP(eax) - if self.cpu.supports_floats: - for i in range(8): - mc.MOVSD(xmm_registers[i], mem64(esp, 8*i)) - mc.ADD(esp, imm(8*8)) - # epilogue - mc.RET(imm16(4)) + """ + PUSH edi + PUSH esi + PUSH ebp + PUSH 0 # for ESP, not used + PUSH ebx + PUSH edx + PUSH ecx + PUSH eax + MOV esi, [esp+32] + CLD + MOV edi, -1 + + loop: + INC edi + LODSB + CMP al, 4*8 + MOVZX edx, al + JB decode_register + JL decode_multibyte + + decode_edx: + TEST edx, 3 + JZ decode_ptr + TEST edx, 2 + JNZ decode_float + + decode_int: + # (edx & 3) == 1 + NEG edx + MOV eax, [ebp + edx + 1 - 16] + + got_value_int: + MOV [fail_boxes_int + 4*edi], eax + JMP loop + + decode_ptr: + # (edx & 3) == 0 + NEG edx + MOV eax, [ebp + edx - 16] + + got_value_ptr: + MOV [fail_boxes_ptr + 4*edi], eax + JMP loop + + decode_float: + # (edx & 3) == 2 + NEG edx + MOV eax, [ebp + edx - 2 - 16] + MOV [fail_boxes_float + 8*edi], eax + MOV eax, [ebp + edx + 2 - 16] + MOV [fail_boxes_float + 8*edi + 4], eax + JMP loop + + decode_multibyte: + MOV cl, 7 + AND edx, 0x7F + JMP innerloop + + innerloop_morebytes: + AND eax, 0x7F + SHL eax, cl + OR edx, eax + ADD cl, 7 + innerloop: + LODSB + CMP al, 0 + MOVZX eax, al + JL innerloop_morebytes + + SHL eax, cl + OR edx, eax + JMP decode_edx + + decode_register: + TEST al, 2 + JNZ decode_register_float + CMP al, DESCR_STOP + JE stop + AND edx, 0x3C + TEST al, 1 + MOV eax, [esp+edx] + JNZ got_value_int + MOV [fail_boxes_ptr + 4*edi], eax + JMP loop + + decode_register_float: + CMP al, 0x10 + JB case_0123 + CMP al, 0x18 + JB case_45 + CMP al, 0x1C + JB case_6 + case_7: + MOVSD [fail_boxes_float + 8*edi], xmm7 + JMP loop + case_6: + MOVSD [fail_boxes_float + 8*edi], xmm6 + JMP loop + case_45: + CMP al, 0x14 + JB case_4 + case_5: + MOVSD [fail_boxes_float + 8*edi], xmm5 + JMP loop + case_4: + MOVSD [fail_boxes_float + 8*edi], xmm4 + JMP loop + case_0123: + CMP al, 0x08 + JB case_01 + CMP al, 0x0C + JB case_2 + case_3: + MOVSD [fail_boxes_float + 8*edi], xmm3 + JMP loop + case_2: + MOVSD [fail_boxes_float + 8*edi], xmm2 + JMP loop + case_01: + CMP al, 0x04 + JB case_0 + case_1: + MOVSD [fail_boxes_float + 8*edi], xmm1 + JMP loop + case_0: + MOVSD [fail_boxes_float + 8*edi], xmm0 + JMP loop + + stop: + CALL on_leave_jitted + MOV eax, [esp+36] # fail_index + LEA esp, [ebp-12] + POP edi + POP esi + POP ebx + POP ebp + RET + """ + ... + ... self.failure_recovery_code[exc] = code def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386setup.py Thu Nov 19 18:21:51 2009 @@ -459,6 +459,13 @@ SAHF = Instruction() SAHF.mode0(['\x9E']) +LODSB = Instruction() +LODSB.mode0(['\xAC']) + +LODSD = Instruction() +LODSD.mode0(['\xAD']) +LODSD.as_alias = "LODSL" + # ------------------------- floating point instructions ------------------ FLDL = Instruction() From fijal at codespeak.net Thu Nov 19 22:38:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 22:38:07 +0100 (CET) Subject: [pypy-svn] r69456 - pypy/branch/unpackiterable-improvements/pypy/objspace/std/test Message-ID: <20091119213807.6F95F16806F@codespeak.net> Author: fijal Date: Thu Nov 19 22:38:06 2009 New Revision: 69456 Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_userobject.py Log: Simply skip this test when run appdirect, makes no sense Modified: pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_userobject.py ============================================================================== --- pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_userobject.py (original) +++ pypy/branch/unpackiterable-improvements/pypy/objspace/std/test/test_userobject.py Thu Nov 19 22:38:06 2009 @@ -283,6 +283,8 @@ cls.prev_installer = multimethod.Installer multimethod.Installer = multimethod.InstallerVersion2 + if conftest.option.runappdirect: + py.test.skip("Cannot run different installers when runappdirect") config = conftest.make_config(conftest.option, **cls.OPTIONS) cls.space = conftest.maketestobjspace(config) From fijal at codespeak.net Thu Nov 19 22:41:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 22:41:03 +0100 (CET) Subject: [pypy-svn] r69457 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091119214103.1C8A5168023@codespeak.net> Author: fijal Date: Thu Nov 19 22:41:02 2009 New Revision: 69457 Modified: pypy/trunk/pypy/objspace/std/test/test_userobject.py Log: Argh. Previous checkin belonged to trunk, fix Modified: pypy/trunk/pypy/objspace/std/test/test_userobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_userobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_userobject.py Thu Nov 19 22:41:02 2009 @@ -283,6 +283,8 @@ cls.prev_installer = multimethod.Installer multimethod.Installer = multimethod.InstallerVersion2 + if conftest.option.runappdirect: + py.test.skip("Cannot run different installers when runappdirect") config = conftest.make_config(conftest.option, **cls.OPTIONS) cls.space = conftest.maketestobjspace(config) From fijal at codespeak.net Thu Nov 19 22:43:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 22:43:55 +0100 (CET) Subject: [pypy-svn] r69458 - in pypy/trunk/pypy: annotation interpreter interpreter/astcompiler interpreter/astcompiler/tools interpreter/test module/__builtin__ module/__builtin__/test module/_codecs module/_rawffi module/_stackless/test module/micronumpy module/posix module/rctime module/select objspace/flow objspace/std objspace/std/test rpython Message-ID: <20091119214355.9623A31813D@codespeak.net> Author: fijal Date: Thu Nov 19 22:43:52 2009 New Revision: 69458 Modified: pypy/trunk/pypy/annotation/bookkeeper.py pypy/trunk/pypy/interpreter/argument.py pypy/trunk/pypy/interpreter/astcompiler/ast.py pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/gateway.py pypy/trunk/pypy/interpreter/interactive.py pypy/trunk/pypy/interpreter/nestedscope.py pypy/trunk/pypy/interpreter/pycode.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/interpreter/test/test_argument.py pypy/trunk/pypy/interpreter/test/test_compiler.py pypy/trunk/pypy/interpreter/test/test_objspace.py pypy/trunk/pypy/module/__builtin__/abstractinst.py pypy/trunk/pypy/module/__builtin__/interp_classobj.py pypy/trunk/pypy/module/__builtin__/test/test_abstractinst.py pypy/trunk/pypy/module/_codecs/interp_codecs.py pypy/trunk/pypy/module/_rawffi/interp_rawffi.py pypy/trunk/pypy/module/_rawffi/structure.py pypy/trunk/pypy/module/_stackless/test/test_pickle_infrastructure.py pypy/trunk/pypy/module/micronumpy/numarray.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/rctime/interp_time.py pypy/trunk/pypy/module/select/interp_select.py pypy/trunk/pypy/objspace/flow/objspace.py pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/formatting.py pypy/trunk/pypy/objspace/std/inlinedict.py pypy/trunk/pypy/objspace/std/listobject.py pypy/trunk/pypy/objspace/std/marshal_impl.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/ropeobject.py pypy/trunk/pypy/objspace/std/ropeunicodeobject.py pypy/trunk/pypy/objspace/std/setobject.py pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/objspace/std/strsliceobject.py pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py pypy/trunk/pypy/objspace/std/tupletype.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/objspace/std/typetype.py pypy/trunk/pypy/objspace/std/unicodeobject.py pypy/trunk/pypy/rpython/callparse.py Log: Merge unpackiterable-improvements branch. This branch splits viewiterable into two parts: * fixedview, returns a non-resizable list, won't copy if w_obj is a W_TupleObj * listview, returns a resizable list, won't copy if w_obj is a W_ListObj Modified: pypy/trunk/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/trunk/pypy/annotation/bookkeeper.py (original) +++ pypy/trunk/pypy/annotation/bookkeeper.py Thu Nov 19 22:43:52 2009 @@ -757,7 +757,8 @@ getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] raise CallPatternTooComplex, "'*' argument must be SomeTuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/trunk/pypy/interpreter/argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/argument.py (original) +++ pypy/trunk/pypy/interpreter/argument.py Thu Nov 19 22:43:52 2009 @@ -135,7 +135,7 @@ # unpack the * arguments if w_stararg is not None: self.arguments_w = (self.arguments_w + - self.space.viewiterable(w_stararg)) + self.space.fixedview(w_stararg)) # unpack the ** arguments if w_starstararg is not None: space = self.space Modified: pypy/trunk/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/ast.py Thu Nov 19 22:43:52 2009 @@ -97,7 +97,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -132,7 +132,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -191,7 +191,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -244,7 +244,7 @@ self.args.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -254,7 +254,7 @@ node.sync_app_attrs(space) w_list = self.w_decorators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.decorators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -297,7 +297,7 @@ pass w_list = self.w_bases if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.bases = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -307,7 +307,7 @@ node.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -375,7 +375,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -415,7 +415,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -492,7 +492,7 @@ self.dest.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -540,7 +540,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -550,7 +550,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -595,7 +595,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -605,7 +605,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -650,7 +650,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -660,7 +660,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -707,7 +707,7 @@ self.optional_vars.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -795,7 +795,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -805,7 +805,7 @@ node.sync_app_attrs(space) w_list = self.w_handlers if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.handlers = [space.interp_w(excepthandler, w_obj) for w_obj in list_w] else: @@ -815,7 +815,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -857,7 +857,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -867,7 +867,7 @@ node.sync_app_attrs(space) w_list = self.w_finalbody if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.finalbody = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -936,7 +936,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -977,7 +977,7 @@ self.level = 0 w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -1053,7 +1053,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.str_w(w_obj) for w_obj in list_w] else: @@ -1196,7 +1196,7 @@ pass w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1359,7 +1359,7 @@ pass w_list = self.w_keys if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keys = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1369,7 +1369,7 @@ node.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1408,7 +1408,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1447,7 +1447,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1520,14 +1520,14 @@ self.left.sync_app_attrs(space) w_list = self.w_ops if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ops = [space.interp_w(cmpop, w_obj).to_simple_int(space) for w_obj in list_w] else: self.ops = None w_list = self.w_comparators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.comparators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1579,7 +1579,7 @@ self.func.sync_app_attrs(space) w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1589,7 +1589,7 @@ node.sync_app_attrs(space) w_list = self.w_keywords if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keywords = [space.interp_w(keyword, w_obj) for w_obj in list_w] else: @@ -1795,7 +1795,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1834,7 +1834,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2012,7 +2012,7 @@ pass w_list = self.w_dims if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.dims = [space.interp_w(slice, w_obj) for w_obj in list_w] else: @@ -2305,7 +2305,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_ifs if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ifs = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2344,7 +2344,7 @@ self.name.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -2379,7 +2379,7 @@ self.kwarg = None w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2389,7 +2389,7 @@ node.sync_app_attrs(space) w_list = self.w_defaults if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.defaults = [space.interp_w(expr, w_obj) for w_obj in list_w] else: Modified: pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/trunk/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Nov 19 22:43:52 2009 @@ -144,7 +144,7 @@ if attr.seq: self.emit("w_list = self.w_%s" % (attr.name,), 2) self.emit("if w_list is not None:", 2) - self.emit("list_w = space.viewiterable(w_list)", 3) + self.emit("list_w = space.listview(w_list)", 3) self.emit("if list_w:", 3) unwrapper = get_unwrapper(attr.type.value, "w_obj", self.data.simple_types) Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Thu Nov 19 22:43:52 2009 @@ -668,13 +668,17 @@ (i, plural)) return items - def viewiterable(self, w_iterable, expected_length=-1): - """ More or less the same as unpackiterable, but does not return - a copy. Please don't modify the result + def fixedview(self, w_iterable, expected_length=-1): + """ A fixed list view of w_iterable. Don't modify the result """ return make_sure_not_resized(self.unpackiterable(w_iterable, expected_length)[:]) + def listview(self, w_iterable, expected_length=-1): + """ A non-fixed view of w_iterable. Don't modify the result + """ + return self.unpackiterable(w_iterable, expected_length) + def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): @@ -771,7 +775,7 @@ def lookup(self, w_obj, name): w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) - for w_supertype in self.unpackiterable(w_mro): + for w_supertype in self.fixedview(w_mro): w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value @@ -880,7 +884,7 @@ if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): w_indices = self.call_method(w_index_or_slice, "indices", self.wrap(seqlength)) - w_start, w_stop, w_step = self.unpackiterable(w_indices, 3) + w_start, w_stop, w_step = self.fixedview(w_indices, 3) start = self.int_w(w_start) stop = self.int_w(w_stop) step = self.int_w(w_step) Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Thu Nov 19 22:43:52 2009 @@ -198,7 +198,7 @@ else: name = None if not space.is_w(w_argdefs, space.w_None): - defs_w = space.viewiterable(w_argdefs) + defs_w = space.fixedview(w_argdefs) else: defs_w = [] nfreevars = 0 @@ -316,7 +316,7 @@ if space.is_w(w_func_dict, space.w_None): w_func_dict = None self.w_func_dict = w_func_dict - self.defs_w = space.viewiterable(w_defs_w) + self.defs_w = space.fixedview(w_defs_w) self.w_module = w_module def fget_func_defaults(space, self): @@ -331,7 +331,7 @@ return if not space.is_true(space.isinstance(w_defaults, space.w_tuple)): raise OperationError( space.w_TypeError, space.wrap("func_defaults must be set to a tuple object or None") ) - self.defs_w = space.viewiterable(w_defaults) + self.defs_w = space.fixedview(w_defaults) def fdel_func_defaults(space, self): self.defs_w = [] Modified: pypy/trunk/pypy/interpreter/gateway.py ============================================================================== --- pypy/trunk/pypy/interpreter/gateway.py (original) +++ pypy/trunk/pypy/interpreter/gateway.py Thu Nov 19 22:43:52 2009 @@ -210,7 +210,7 @@ % (self.scopenext(), self.scopenext())) def visit_args_w(self, el): - self.run_args.append("space.viewiterable(%s)" % self.scopenext()) + self.run_args.append("space.fixedview(%s)" % self.scopenext()) def visit_w_args(self, el): self.run_args.append(self.scopenext()) @@ -416,7 +416,7 @@ # baseobjspace.W_Root is for wrapped arguments to keep wrapped # baseobjspace.Wrappable subclasses imply interp_w and a typecheck # argument.Arguments is for a final rest arguments Arguments object - # 'args_w' for viewiterable applied to rest arguments + # 'args_w' for fixedview applied to rest arguments # 'w_args' for rest arguments passed as wrapped tuple # str,int,float: unwrap argument as such type # (function, cls) use function to check/unwrap argument of type cls Modified: pypy/trunk/pypy/interpreter/interactive.py ============================================================================== --- pypy/trunk/pypy/interpreter/interactive.py (original) +++ pypy/trunk/pypy/interpreter/interactive.py Thu Nov 19 22:43:52 2009 @@ -73,7 +73,7 @@ words = self.get_words(w_clz) try: w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.viewiterable(w_bases) + bases_w = s.fixedview(w_bases) except error.OperationError: return words Modified: pypy/trunk/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/trunk/pypy/interpreter/nestedscope.py (original) +++ pypy/trunk/pypy/interpreter/nestedscope.py Thu Nov 19 22:43:52 2009 @@ -208,7 +208,7 @@ if codeobj.magic >= 0xa0df281: # CPython 2.5 AST branch merge w_freevarstuple = f.popvalue() freevars = [f.space.interp_w(Cell, cell) - for cell in f.space.viewiterable(w_freevarstuple)] + for cell in f.space.fixedview(w_freevarstuple)] else: nfreevars = len(codeobj.co_freevars) freevars = [f.space.interp_w(Cell, f.popvalue()) Modified: pypy/trunk/pypy/interpreter/pycode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pycode.py (original) +++ pypy/trunk/pypy/interpreter/pycode.py Thu Nov 19 22:43:52 2009 @@ -349,7 +349,7 @@ if not space.is_true(space.isinstance(w_constants, space.w_tuple)): raise OperationError(space.w_TypeError, space.wrap("Expected tuple for constants")) - consts_w = space.viewiterable(w_constants) + consts_w = space.fixedview(w_constants) names = unpack_str_tuple(space, w_names) varnames = unpack_str_tuple(space, w_varnames) if w_freevars is not None: Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Thu Nov 19 22:43:52 2009 @@ -581,7 +581,7 @@ w_compile_flags, f.space.wrap(f.get_builtin()), f.space.gettypeobject(PyCode.typedef)) - w_prog, w_globals, w_locals = f.space.viewiterable(w_resulttuple, 3) + w_prog, w_globals, w_locals = f.space.fixedview(w_resulttuple, 3) plain = f.w_locals is not None and f.space.is_w(w_locals, f.w_locals) if plain: @@ -637,7 +637,7 @@ def UNPACK_SEQUENCE(f, itemcount, *ignored): w_iterable = f.popvalue() try: - items = f.space.viewiterable(w_iterable, itemcount) + items = f.space.fixedview(w_iterable, itemcount) except UnpackValueError, e: raise OperationError(f.space.w_ValueError, f.space.wrap(e.msg)) f.pushrevvalues(itemcount, items) Modified: pypy/trunk/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_argument.py (original) +++ pypy/trunk/pypy/interpreter/test/test_argument.py Thu Nov 19 22:43:52 2009 @@ -60,9 +60,12 @@ def is_true(self, obj): return bool(obj) - def viewiterable(self, it): + def fixedview(self, it): return list(it) + def listview(self, it): + return list(it) + def unpackiterable(self, it): return list(it) Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Thu Nov 19 22:43:52 2009 @@ -651,6 +651,9 @@ elif sys.version_info < (2, 6): _unicode_error_kind = "w_UnicodeDecodeError" else: + def skip_on_2_6(self): + py.test.skip("syntax different on CPython 2.6 compiler") + test_globals_warnings = skip_on_2_6 _unicode_error_kind = "w_SyntaxError" class TestPythonAstCompiler_25_grammar(BaseTestCompiler): Modified: pypy/trunk/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_objspace.py (original) +++ pypy/trunk/pypy/interpreter/test/test_objspace.py Thu Nov 19 22:43:52 2009 @@ -58,14 +58,23 @@ raises(ValueError, self.space.unpackiterable, w_l, 3) raises(ValueError, self.space.unpackiterable, w_l, 5) - def test_viewiterable(self): + def test_fixedview(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.viewiterable(w_l) == l - assert self.space.viewiterable(w_l, 4) == l - raises(ValueError, self.space.viewiterable, w_l, 3) - raises(ValueError, self.space.viewiterable, w_l, 5) + assert self.space.fixedview(w_l) == l + assert self.space.fixedview(w_l, 4) == l + raises(ValueError, self.space.fixedview, w_l, 3) + raises(ValueError, self.space.fixedview, w_l, 5) + + def test_listview(self): + w = self.space.wrap + l = [w(1), w(2), w(3), w(4)] + w_l = self.space.newtuple(l) + assert self.space.listview(w_l) == l + assert self.space.listview(w_l, 4) == l + raises(ValueError, self.space.listview, w_l, 3) + raises(ValueError, self.space.listview, w_l, 5) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, @@ -207,7 +216,7 @@ w_res = space.call_obj_args(w_f, w_9, Arguments(space, [w_1])) - w_x, w_y = space.viewiterable(w_res, 2) + w_x, w_y = space.fixedview(w_res, 2) assert w_x is w_9 assert w_y is w_1 Modified: pypy/trunk/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/trunk/pypy/module/__builtin__/abstractinst.py Thu Nov 19 22:43:52 2009 @@ -83,7 +83,7 @@ def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple): # -- case (anything, tuple) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_isinstance_w(space, w_obj, w_klass): return True return False @@ -109,7 +109,7 @@ return True w_bases = _get_bases(space, w_derived) if w_bases is not None: - for w_base in space.viewiterable(w_bases): + for w_base in space.fixedview(w_bases): if _issubclass_recurse(space, w_base, w_top): return True return False @@ -141,7 +141,7 @@ # -- case (class-like-object, tuple-of-classes) # XXX it might be risky that the JIT sees this if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass): return True return False Modified: pypy/trunk/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/trunk/pypy/module/__builtin__/interp_classobj.py Thu Nov 19 22:43:52 2009 @@ -36,7 +36,7 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -79,7 +79,7 @@ raise OperationError( space.w_TypeError, space.wrap("__bases__ must be a tuple object")) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, @@ -285,7 +285,7 @@ if not e.match(space, space.w_TypeError): raise return [None, None] - return space.viewiterable(w_tup, 2) + return space.fixedview(w_tup, 2) def descr_instance_new(space, w_type, w_class, w_dict=None): # w_type is not used at all Modified: pypy/trunk/pypy/module/__builtin__/test/test_abstractinst.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_abstractinst.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_abstractinst.py Thu Nov 19 22:43:52 2009 @@ -5,7 +5,7 @@ def test_abstract_isclass(self): space = self.space - w_B1, w_B2, w_B3, w_X, w_Y = space.viewiterable(space.appexec([], """(): + w_B1, w_B2, w_B3, w_X, w_Y = space.fixedview(space.appexec([], """(): class X(object): pass class Y: pass B1, B2, B3 = X(), X(), X() @@ -22,7 +22,7 @@ def test_abstract_getclass(self): space = self.space - w_x, w_y, w_A, w_MyInst = space.viewiterable(space.appexec([], """(): + w_x, w_y, w_A, w_MyInst = space.fixedview(space.appexec([], """(): class MyInst(object): def __init__(self, myclass): self.myclass = myclass Modified: pypy/trunk/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/trunk/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/trunk/pypy/module/_codecs/interp_codecs.py Thu Nov 19 22:43:52 2009 @@ -35,7 +35,7 @@ space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) - w_replace, w_newpos = space.viewiterable(w_res, 2) + w_replace, w_newpos = space.fixedview(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/trunk/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/trunk/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/trunk/pypy/module/_rawffi/interp_rawffi.py Thu Nov 19 22:43:52 2009 @@ -106,7 +106,7 @@ resshape = cache.get_array_type(letter2tp(space, letter)) else: letter = 'V' - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) from pypy.module._rawffi.structure import W_Structure resshape = space.interp_w(W_Structure, w_shapetype) ffi_type = resshape.get_ffi_type() @@ -117,7 +117,7 @@ letter = space.str_w(w_shape) return letter2tp(space, letter) else: - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) resshape = space.interp_w(W_DataShape, w_shapetype) length = space.int_w(w_length) size, alignment = resshape._size_alignment() @@ -155,7 +155,7 @@ """ ffi_restype, resshape = unpack_resshape(space, w_restype) w = space.wrap - argtypes_w = space.viewiterable(w_argtypes) + argtypes_w = space.fixedview(w_argtypes) w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w_name, w_argtypes, w(resshape)]) try: Modified: pypy/trunk/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/trunk/pypy/module/_rawffi/structure.py (original) +++ pypy/trunk/pypy/module/_rawffi/structure.py Thu Nov 19 22:43:52 2009 @@ -119,7 +119,7 @@ def descr_new_structure(space, w_type, w_shapeinfo): if space.is_true(space.isinstance(w_shapeinfo, space.w_tuple)): - w_size, w_alignment = space.viewiterable(w_shapeinfo, expected_length=2) + w_size, w_alignment = space.fixedview(w_shapeinfo, expected_length=2) S = W_Structure(space, None, space.int_w(w_size), space.int_w(w_alignment)) else: Modified: pypy/trunk/pypy/module/_stackless/test/test_pickle_infrastructure.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/test/test_pickle_infrastructure.py (original) +++ pypy/trunk/pypy/module/_stackless/test/test_pickle_infrastructure.py Thu Nov 19 22:43:52 2009 @@ -129,7 +129,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -188,7 +188,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -260,7 +260,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code Modified: pypy/trunk/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/trunk/pypy/module/micronumpy/numarray.py (original) +++ pypy/trunk/pypy/module/micronumpy/numarray.py Thu Nov 19 22:43:52 2009 @@ -73,7 +73,7 @@ make_sure_not_resized(self.storage) def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.viewiterable(w_index)] + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] if len(indexes) != len(self.dim): raise OperationError(space.w_IndexError, space.wrap( 'Wrong index')) @@ -108,7 +108,7 @@ def unpack_dim(space, w_dim): if space.is_true(space.isinstance(w_dim, space.w_int)): return [space.int_w(w_dim)] - dim_w = space.viewiterable(w_dim) + dim_w = space.fixedview(w_dim) return [space.int_w(w_i) for w_i in dim_w] def unpack_dtype(space, w_dtype): Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Nov 19 22:43:52 2009 @@ -573,7 +573,7 @@ raise wrap_oserror(space, e) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" - args_w = space.unpackiterable(w_tuple) + args_w = space.fixedview(w_tuple) if len(args_w) != 2: raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) Modified: pypy/trunk/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/trunk/pypy/module/rctime/interp_time.py (original) +++ pypy/trunk/pypy/module/rctime/interp_time.py Thu Nov 19 22:43:52 2009 @@ -222,7 +222,7 @@ space.wrap(_get_error_msg())) return pbuf - tup_w = space.unpackiterable(w_tup) + tup_w = space.fixedview(w_tup) if len(tup_w) != 9: raise OperationError(space.w_TypeError, space.wrap("argument must be sequence of " Modified: pypy/trunk/pypy/module/select/interp_select.py ============================================================================== --- pypy/trunk/pypy/module/select/interp_select.py (original) +++ pypy/trunk/pypy/module/select/interp_select.py Thu Nov 19 22:43:52 2009 @@ -113,9 +113,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.unpackiterable(w_iwtd) - owtd_w = space.unpackiterable(w_owtd) - ewtd_w = space.unpackiterable(w_ewtd) + iwtd_w = space.listview(w_iwtd) + owtd_w = space.listview(w_owtd) + ewtd_w = space.listview(w_ewtd) iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] owtd = [as_fd_w(space, w_f) for w_f in owtd_w] ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] Modified: pypy/trunk/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/objspace.py Thu Nov 19 22:43:52 2009 @@ -202,7 +202,7 @@ # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) # checking a tuple of classes - for w_klass in self.viewiterable(w_check_class): + for w_klass in self.fixedview(w_check_class): if ObjSpace.exception_match(self, w_exc_type, w_klass): return True return False @@ -263,8 +263,9 @@ checkgraph(graph) return graph - def viewiterable(self, w_tuple, expected_length=None): + def fixedview(self, w_tuple, expected_length=None): return self.unpackiterable(w_tuple, expected_length) + listview = fixedview def unpackiterable(self, w_iterable, expected_length=None): if not isinstance(w_iterable, Variable): Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Thu Nov 19 22:43:52 2009 @@ -650,9 +650,9 @@ if w_src is None: pass elif space.findattr(w_src, space.wrap("keys")) is None: - list_of_w_pairs = space.unpackiterable(w_src) + list_of_w_pairs = space.listview(w_src) for w_pair in list_of_w_pairs: - pair = space.unpackiterable(w_pair) + pair = space.fixedview(w_pair) if len(pair)!=2: raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) @@ -793,7 +793,7 @@ return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): - defaults = space.unpackiterable(w_defaults) + defaults = space.listview(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) Modified: pypy/trunk/pypy/objspace/std/formatting.py ============================================================================== --- pypy/trunk/pypy/objspace/std/formatting.py (original) +++ pypy/trunk/pypy/objspace/std/formatting.py Thu Nov 19 22:43:52 2009 @@ -495,7 +495,7 @@ def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): - values_w = space.unpackiterable(w_values) + values_w = space.fixedview(w_values) return format(space, w_format, values_w, None, do_unicode) else: # we check directly for dict to avoid obscure checking Modified: pypy/trunk/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/inlinedict.py Thu Nov 19 22:43:52 2009 @@ -45,7 +45,7 @@ # XXX sucky items = [] for w_item in self.impl_items(): - w_key, w_value = self.space.viewiterable(w_item) + w_key, w_value = self.space.fixedview(w_item) items.append((w_key, w_value)) return IndirectionIterImplementation(self.space, self, items) Modified: pypy/trunk/pypy/objspace/std/listobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/listobject.py (original) +++ pypy/trunk/pypy/objspace/std/listobject.py Thu Nov 19 22:43:52 2009 @@ -260,11 +260,7 @@ _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable) def _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable): - if isinstance(w_iterable, W_ListObject): - sequence2 = w_iterable.wrappeditems - else: - sequence2 = space.unpackiterable(w_iterable) - + sequence2 = space.listview(w_iterable) assert slicelength >= 0 items = w_list.wrappeditems oldsize = len(items) @@ -357,7 +353,7 @@ return space.w_None def list_extend__List_ANY(space, w_list, w_any): - w_list.wrappeditems += space.unpackiterable(w_any) + w_list.wrappeditems += space.listview(w_any) return space.w_None # note that the default value will come back wrapped!!! Modified: pypy/trunk/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/trunk/pypy/objspace/std/marshal_impl.py (original) +++ pypy/trunk/pypy/objspace/std/marshal_impl.py Thu Nov 19 22:43:52 2009 @@ -356,7 +356,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) for w_tuple in w_dict.items(): - w_key, w_value = space.viewiterable(w_tuple, 2) + w_key, w_value = space.fixedview(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) @@ -469,14 +469,14 @@ def marshal_w_set(space, w_set, m): # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.viewiterable(w_set) + lis_w = space.fixedview(w_set) m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - lis_w = space.viewiterable(w_frozenset) + lis_w = space.fixedview(w_frozenset) m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) Modified: pypy/trunk/pypy/objspace/std/objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/std/objspace.py (original) +++ pypy/trunk/pypy/objspace/std/objspace.py Thu Nov 19 22:43:52 2009 @@ -383,7 +383,7 @@ space = self # too early for unpackiterable as well :-( name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) + bases = space.fixedview(space.getitem(w_args, space.wrap(1))) dic = space.unwrap(space.getitem(w_args, space.wrap(2))) dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) bases = list(bases) @@ -643,7 +643,7 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def viewiterable(self, w_obj, expected_length=-1): + def fixedview(self, w_obj, expected_length=-1): """ Fast paths """ if isinstance(w_obj, W_TupleObject): @@ -651,7 +651,18 @@ elif isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.viewiterable(self, w_obj, expected_length) + return ObjSpace.fixedview(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + return t + + def listview(self, w_obj, expected_length=-1): + if isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems + elif isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.listview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t Modified: pypy/trunk/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/ropeobject.py (original) +++ pypy/trunk/pypy/objspace/std/ropeobject.py Thu Nov 19 22:43:52 2009 @@ -537,7 +537,7 @@ (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -557,7 +557,7 @@ def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/trunk/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/ropeunicodeobject.py Thu Nov 19 22:43:52 2009 @@ -233,7 +233,7 @@ return space.contains(unicode_from_string(space, w_container), w_item ) def unicode_join__RopeUnicode_ANY(space, w_self, w_list): - l_w = space.unpackiterable(w_list) + l_w = space.listview(w_list) delim = w_self._node totlen = 0 if len(l_w) == 0: @@ -504,7 +504,7 @@ def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = ropeunicode_w(space, w_prefix) if rope.startswith(unistr, prefix, start, end): return space.w_True @@ -513,7 +513,7 @@ def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = ropeunicode_w(space, w_suffix) if rope.endswith(unistr, suffix, start, end): return space.w_True Modified: pypy/trunk/pypy/objspace/std/setobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/setobject.py (original) +++ pypy/trunk/pypy/objspace/std/setobject.py Thu Nov 19 22:43:52 2009 @@ -111,7 +111,7 @@ def make_setdata_from_w_iterable(space, w_iterable=None): data = r_dict(space.eq_w, space.hash_w) if w_iterable is not None: - for w_item in space.viewiterable(w_iterable): + for w_item in space.listview(w_iterable): data[w_item] = None return data Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Thu Nov 19 22:43:52 2009 @@ -349,7 +349,7 @@ sliced) def str_join__String_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) str_w = space.str_w if list_w: self = w_self._value @@ -628,7 +628,7 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -647,7 +647,7 @@ def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/trunk/pypy/objspace/std/strsliceobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/strsliceobject.py (original) +++ pypy/trunk/pypy/objspace/std/strsliceobject.py Thu Nov 19 22:43:52 2009 @@ -143,7 +143,7 @@ def str_endswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -157,7 +157,7 @@ def str_startswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py Thu Nov 19 22:43:52 2009 @@ -607,8 +607,8 @@ StringObjectCls = FakeString w_dict = None iter = iter - viewiterable = list - + fixedview = list + listview = list class Config: class objspace: Modified: pypy/trunk/pypy/objspace/std/tupletype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/tupletype.py (original) +++ pypy/trunk/pypy/objspace/std/tupletype.py Thu Nov 19 22:43:52 2009 @@ -13,7 +13,7 @@ space.is_w(space.type(w_sequence), space.w_tuple)): return w_sequence else: - tuple_w = space.viewiterable(w_sequence) + tuple_w = space.fixedview(w_sequence) w_obj = space.allocate_instance(space.TupleObjectCls, w_tupletype) space.TupleObjectCls.__init__(w_obj, tuple_w) return w_obj Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Thu Nov 19 22:43:52 2009 @@ -596,7 +596,7 @@ if not space.is_w(w_where, space.w_type): w_mro_meth = space.get(w_mro_func, w_self) w_mro = space.call_function(w_mro_meth) - mro_w = space.viewiterable(w_mro) + mro_w = space.fixedview(w_mro) w_self.mro_w = validate_custom_mro(space, mro_w) return # done w_self.mro_w = w_self.compute_default_mro()[:] Modified: pypy/trunk/pypy/objspace/std/typetype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typetype.py (original) +++ pypy/trunk/pypy/objspace/std/typetype.py Thu Nov 19 22:43:52 2009 @@ -11,7 +11,7 @@ w_typetype = _precheck_for_new(space, w_typetype) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) w_winner = w_typetype for base in bases_w: @@ -38,7 +38,7 @@ name = space.str_w(w_name) assert isinstance(name, str) dict_w = {} - dictkeys_w = space.unpackiterable(w_dict) + dictkeys_w = space.listview(w_dict) for w_key in dictkeys_w: key = space.str_w(w_key) dict_w[key] = space.getitem(w_dict, w_key) @@ -115,7 +115,7 @@ " to %s.__bases__, not %s"% (w_type.name, space.type(w_value).getname(space, '?')))) - newbases_w = space.viewiterable(w_value) + newbases_w = space.fixedview(w_value) if len(newbases_w) == 0: raise OperationError(space.w_TypeError, space.wrap("can only assign non-empty tuple" Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Thu Nov 19 22:43:52 2009 @@ -173,7 +173,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.unpackiterable(w_list) + l = space.listview(w_list) delim = w_self._value totlen = 0 if len(l) == 0: @@ -489,7 +489,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.unicode_w(w_prefix) if stringstartswith(unistr, prefix, start, end): return space.w_True @@ -499,7 +499,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.unicode_w(w_suffix) if stringendswith(unistr, suffix, start, end): return space.w_True Modified: pypy/trunk/pypy/rpython/callparse.py ============================================================================== --- pypy/trunk/pypy/rpython/callparse.py (original) +++ pypy/trunk/pypy/rpython/callparse.py Thu Nov 19 22:43:52 2009 @@ -178,7 +178,8 @@ raise ValueError return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other From fijal at codespeak.net Thu Nov 19 22:44:40 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 22:44:40 +0100 (CET) Subject: [pypy-svn] r69459 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091119214440.E5E9216806F@codespeak.net> Author: fijal Date: Thu Nov 19 22:44:40 2009 New Revision: 69459 Modified: pypy/trunk/pypy/objspace/std/test/test_userobject.py Log: Fix import Modified: pypy/trunk/pypy/objspace/std/test/test_userobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_userobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_userobject.py Thu Nov 19 22:44:40 2009 @@ -1,3 +1,4 @@ +import py from pypy.interpreter import gateway From fijal at codespeak.net Thu Nov 19 22:50:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 19 Nov 2009 22:50:03 +0100 (CET) Subject: [pypy-svn] r69460 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091119215003.E562016806F@codespeak.net> Author: fijal Date: Thu Nov 19 22:50:03 2009 New Revision: 69460 Modified: pypy/trunk/pypy/objspace/std/test/test_celldict.py Log: I don't think this can work with apptests Modified: pypy/trunk/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_celldict.py Thu Nov 19 22:50:03 2009 @@ -1,4 +1,5 @@ -from pypy.conftest import gettestobjspace +import py +from pypy.conftest import gettestobjspace, option from pypy.objspace.std.celldict import get_global_cache, ModuleCell, ModuleDictImplementation from pypy.interpreter import gateway @@ -8,6 +9,8 @@ class AppTestCellDict(object): def setup_class(cls): + if option.runappdirect: + py.test.skip("not appdirect tests") cls.space = gettestobjspace(**{"objspace.std.withcelldict": True}) cls.w_impl_used = cls.space.appexec([], """(): import __pypy__ From afa at codespeak.net Fri Nov 20 10:28:55 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 20 Nov 2009 10:28:55 +0100 (CET) Subject: [pypy-svn] r69461 - pypy/trunk/pypy/module/oracle Message-ID: <20091120092855.0046F168051@codespeak.net> Author: afa Date: Fri Nov 20 10:28:54 2009 New Revision: 69461 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/roci.py Log: Some fixes found when trying to translate. Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 20 10:28:54 2009 @@ -43,7 +43,7 @@ W_Connection.__init__(self) # set up the environment - if w_pool: + if 0 and w_pool: # XXX pool = space.instance_w(W_Pool, w_pool) self.environment = pool.environment.clone() else: @@ -150,6 +150,8 @@ finally: lltype.free(handleptr, flavor='raw') + credentialType = roci.OCI_CRED_EXT + # set user name in session handle stringBuffer.fill(space, self.w_username) try: @@ -322,7 +324,7 @@ return self.w_version # allocate a cursor to retrieve the version - cursor = self.newCursor(space) + cursor = W_Cursor(space, self) # allocate version and compatibility variables versionVar = VT_String(cursor, cursor.arraySize, MAX_STRING_CHARS) Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Fri Nov 20 10:28:54 2009 @@ -128,7 +128,7 @@ # queries are not supported as the result is undefined if self.statementType == roci.OCI_STMT_SELECT: raise OperationError( - w_NotSupportedErrorException, + get(space).w_NotSupportedErrorException, space.wrap("queries not supported: results undefined")) # perform binds @@ -207,8 +207,8 @@ w_vars = w_args # build up the statement - args = ', '.join(':%d' % (i + offset + 1,) - for i in range(numArguments)) + args = ', '.join([':%d' % (i + offset + 1,) + for i in range(numArguments)]) if retvar: stmt = "begin :1 := %s(%s); end;" % (name, args) else: @@ -226,7 +226,7 @@ if not self.handle: return if self.isOwned: - roci.OciHandleFree(self.handle, OCI_HTYPE_STMT) + roci.OCIHandleFree(self.handle, roci.OCI_HTYPE_STMT) elif self.connection.handle: tagBuffer = StringBuffer() tagBuffer.fill(space, self.w_statementTag) @@ -958,7 +958,7 @@ numElements = space.int_w(w_value) else: raise OperationError( - w_NotSupportedErrorException, + get(space).w_NotSupportedErrorException, space.wrap("expecting integer or list of values")) # create the variable @@ -1017,24 +1017,24 @@ setoutputsize.unwrap_spec = ['self', ObjSpace, int, int] -def cursor_arraysize_get(space, obj): - return space.wrap(obj.arraySize) -def cursor_arraysize_set(space, obj, w_value): - obj.arraySize = space.int_w(w_value) - -def cursor_bindarraysize_get(space, obj): - return space.wrap(obj.bindArraySize) -def cursor_bindarraysize_set(space, obj, w_value): - obj.bindArraySize = space.int_w(w_value) - -def cursor_bindvars_get(space, obj): - if obj.bindList: - return space.newlist(obj.bindList) - if obj.bindDict: - return obj.bindDict + def arraysize_get(space, self): + return space.wrap(self.arraySize) + def arraysize_set(space, self, w_value): + self.arraySize = space.int_w(w_value) + + def bindarraysize_get(space, self): + return space.wrap(self.bindArraySize) + def bindarraysize_set(space, self, w_value): + self.bindArraySize = space.int_w(w_value) -def cursor_fetchvars_get(space, obj): - return space.newlist(obj.fetchVariables) + def bindvars_get(space, self): + if self.bindList: + return space.newlist(self.bindList) + if self.bindDict: + return self.bindDict + + def fetchvars_get(space, self): + return space.newlist(self.fetchVariables) W_Cursor.typedef = TypeDef( 'Cursor', @@ -1070,11 +1070,13 @@ __iter__ = interp2app(W_Cursor.descr_iter), next = interp2app(W_Cursor.descr_next), - arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), - bindarraysize = GetSetProperty(cursor_bindarraysize_get, cursor_bindarraysize_set), + arraysize = GetSetProperty(W_Cursor.arraysize_get, + W_Cursor.arraysize_set), + bindarraysize = GetSetProperty(W_Cursor.bindarraysize_get, + W_Cursor.bindarraysize_set), rowcount = interp_attrproperty('rowCount', W_Cursor), statement = interp_attrproperty_w('w_statement', W_Cursor), - bindvars = GetSetProperty(cursor_bindvars_get), - fetchvars = GetSetProperty(cursor_fetchvars_get), + bindvars = GetSetProperty(W_Cursor.bindvars_get), + fetchvars = GetSetProperty(W_Cursor.fetchvars_get), description = GetSetProperty(W_Cursor.getDescription), ) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Fri Nov 20 10:28:54 2009 @@ -51,7 +51,7 @@ OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_DTYPE_PARAM - OCI_CRED_RDBMS OCI_SPOOL_ATTRVAL_NOWAIT + OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL From arigo at codespeak.net Fri Nov 20 11:49:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 11:49:03 +0100 (CET) Subject: [pypy-svn] r69462 - pypy/branch/faster-raise/pypy/doc/config Message-ID: <20091120104903.EC2FF168072@codespeak.net> Author: arigo Date: Fri Nov 20 11:49:02 2009 New Revision: 69462 Added: pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules.exceptions.txt (contents, props changed) Log: Add missing file. Added: pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules.exceptions.txt ============================================================================== --- (empty file) +++ pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules.exceptions.txt Fri Nov 20 11:49:02 2009 @@ -0,0 +1,2 @@ +Use the 'exceptions' module. +This module is essential, included by default and should not be removed. From afa at codespeak.net Fri Nov 20 11:53:47 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 20 Nov 2009 11:53:47 +0100 (CET) Subject: [pypy-svn] r69463 - pypy/trunk/pypy/module/oracle Message-ID: <20091120105347.AD239168072@codespeak.net> Author: afa Date: Fri Nov 20 11:53:47 2009 New Revision: 69463 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/transform.py Log: Use the new space.fixedview and space.listview operations Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 20 11:53:47 2009 @@ -56,13 +56,13 @@ # perform some parsing, if necessary if (self.w_username and not self.w_password and space.is_true(space.contains(self.w_username, space.wrap('/')))): - (self.w_username, self.w_password) = space.unpackiterable( + (self.w_username, self.w_password) = space.listview( space.call_method(self.w_username, 'split', space.wrap('/'), space.wrap(1))) - + if (self.w_password and not self.w_tnsentry and space.is_true(space.contains(self.w_password, space.wrap('@')))): - (self.w_password, self.w_tnsentry) = space.unpackiterable( + (self.w_password, self.w_tnsentry) = space.listview( space.call_method(self.w_password, 'split', space.wrap('@'), space.wrap(1))) Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Fri Nov 20 11:53:47 2009 @@ -132,15 +132,16 @@ space.wrap("queries not supported: results undefined")) # perform binds - numrows = space.int_w(space.len(w_list_of_args)) - for i, arguments in enumerate(space.viewiterable(w_list_of_args)): + args_w = space.listview(w_list_of_args) + numrows = len(args_w) + for i, w_arguments in enumerate(args_w): deferred = i < numrows - 1 - if space.is_true(space.isinstance(arguments, space.w_dict)): + if space.is_true(space.isinstance(w_arguments, space.w_dict)): self._setBindVariablesByName( - space, arguments, numrows, i, deferred) + space, w_arguments, numrows, i, deferred) else: self._setBindVariablesByPos( - space, arguments, numrows, i, deferred) + space, w_arguments, numrows, i, deferred) self._performBind(space) # execute the statement, but only if the number of rows is greater than @@ -546,7 +547,7 @@ if self.bindList is None: self.bindList = [] - for i, w_value in enumerate(space.viewiterable(w_vars)): + for i, w_value in enumerate(space.fixedview(w_vars)): if i < len(self.bindList): origVar = self.bindList[i] if space.is_w(origVar, space.w_None): @@ -574,9 +575,9 @@ if self.bindDict is None: self.bindDict = space.newdict() - items = space.viewiterable(space.call_method(w_vars, "iteritems")) + items = space.fixedview(space.call_method(w_vars, "iteritems")) for item in items: - w_key, w_value = space.viewiterable(item, 2) + w_key, w_value = space.fixedview(item, 2) origVar = space.finditem(self.bindDict, w_key) newVar = self._setBindVariableHelper(space, w_value, origVar, numElements, arrayPos, defer) @@ -643,10 +644,10 @@ for i, var in enumerate(self.bindList): var.bind(space, self, None, i + 1) if self.bindDict: - items = space.viewiterable( + items_w = space.fixedview( space.call_method(self.bindDict, "iteritems")) - for item in items: - w_key, var = space.viewiterable(item, 2) + for w_item in items_w: + w_key, var = space.fixedview(w_item, 2) var.bind(space, self, w_key, 0) # ensure that input sizes are reset Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Fri Nov 20 11:53:47 2009 @@ -361,7 +361,7 @@ space.w_TypeError, space.wrap("expecting array data")) - elements_w = space.viewiterable(w_value) + elements_w = space.listview(w_value) # ensure we haven't exceeded the number of allocated elements if len(elements_w) > self.allocatedElements: @@ -949,7 +949,7 @@ # handle arrays if space.is_true(space.isinstance(w_value, space.w_list)): - elements_w = space.viewiterable(w_value) + elements_w = space.listview(w_value) for w_element in elements_w: if not space.is_w(w_element, space.w_None): break @@ -982,7 +982,7 @@ def newArrayVariableByType(space, cursor, w_value): "Allocate a new PL/SQL array by looking at the Python data type." - w_type, w_numElements = space.viewiterable(w_value, 2) + w_type, w_numElements = space.fixedview(w_value, 2) numElements = space.int_w(w_numElements) varType = typeByPythonType(space, cursor, w_type) Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Fri Nov 20 11:53:47 2009 @@ -63,12 +63,12 @@ w_tuple_value = space.call_method(w_value, "as_tuple") # acquire basic information from the value tuple - w_sign, w_digits, w_scale = space.viewiterable(w_tuple_value) + w_sign, w_digits, w_scale = space.fixedview(w_tuple_value, 3) text = '' format = '' - digits_w = space.viewiterable(w_digits) + digits_w = space.listview(w_digits) num_digits = len(digits_w) scale = space.int_w(w_scale) From arigo at codespeak.net Fri Nov 20 12:43:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 12:43:49 +0100 (CET) Subject: [pypy-svn] r69464 - pypy/branch/faster-raise/pypy/jit/metainterp/test Message-ID: <20091120114349.30E25168072@codespeak.net> Author: arigo Date: Fri Nov 20 12:43:48 2009 New Revision: 69464 Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py Log: Fix test. Modified: pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/faster-raise/pypy/jit/metainterp/test/test_pyjitpl.py Fri Nov 20 12:43:48 2009 @@ -149,7 +149,7 @@ pass metainterp_sd = FakeMetaInterpSd() metainterp_sd.info_from_codewriter(None, None, None, - [(123, "a"), (456, "b")]) + [(123, "a"), (456, "b")], None) assert metainterp_sd.get_name_from_address(123) == 'a' assert metainterp_sd.get_name_from_address(456) == 'b' assert metainterp_sd.get_name_from_address(789) == '' From cfbolz at codespeak.net Fri Nov 20 14:41:54 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Nov 2009 14:41:54 +0100 (CET) Subject: [pypy-svn] r69465 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091120134154.9886249844E@codespeak.net> Author: cfbolz Date: Fri Nov 20 14:41:53 2009 New Revision: 69465 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Log: (pedronis, cfbolz): fix the reader to deal with holes in the virtuals list Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Fri Nov 20 14:41:53 2009 @@ -450,11 +450,15 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - # xxx vinfo can be none, test and fix! - self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(metainterp) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): numb = self.cur_numb Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_resume.py Fri Nov 20 14:41:53 2009 @@ -112,6 +112,22 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] + +def test_prepare_virtuals(): + class FakeVinfo(object): + def allocate(self, metainterp): + return "allocated" + def setfields(self, metainterp, virtual, func): + assert virtual == "allocated" + class FakeStorage(object): + rd_virtuals = [FakeVinfo(), None] + rd_numb = [] + rd_consts = [] + class FakeMetainterp(object): + cpu = None + reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) + assert reader.virtuals == ["allocated", None] + # ____________________________________________________________ From afa at codespeak.net Fri Nov 20 15:01:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 20 Nov 2009 15:01:11 +0100 (CET) Subject: [pypy-svn] r69467 - pypy/trunk/pypy/translator/c/gcc Message-ID: <20091120140111.01292168071@codespeak.net> Author: afa Date: Fri Nov 20 15:01:11 2009 New Revision: 69467 Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Log: "__eprintf" prints an error message and abort the program. Add it to the list of non-returning functions. Modified: pypy/trunk/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/trunk/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/trunk/pypy/translator/c/gcc/trackgcroot.py Fri Nov 20 15:01:11 2009 @@ -754,7 +754,8 @@ '_exit': None, '__assert_fail': None, '___assert_rtn': None, - 'L___assert_rtn$stub': None + 'L___assert_rtn$stub': None, + 'L___eprintf$stub': None, } def __init__(self, lines, filetag=0): From cfbolz at codespeak.net Fri Nov 20 15:14:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Nov 2009 15:14:06 +0100 (CET) Subject: [pypy-svn] r69468 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test Message-ID: <20091120141406.A4322168076@codespeak.net> Author: cfbolz Date: Fri Nov 20 15:14:04 2009 New Revision: 69468 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Log: (pedronis, cfbolz): trying to write a failing test. this one passes, unfortunately. Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Fri Nov 20 15:14:04 2009 @@ -329,6 +329,32 @@ return 0 self.meta_interp(f, [50]) + def test_guards_around_forcing_of_virtual_in_virtual(self): + class A(object): + def __init__(self, x): + self.x = x + mydriver = JitDriver(reds = ['n'], greens = []) + global_a = A(0) + + def g(b): + n = b.x + if n < 10: + n += 1 + global_a.forced = b.next + if n < 20: + assert global_a.forced is b.next + + def f(n): + while n > 0: + mydriver.can_enter_jit(n=n) + mydriver.jit_merge_point(n=n) + a = A(n) + b = A(n) + b.next = a + g(b) + n -= 1 + return 0 + self.meta_interp(f, [50]) # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class From cfbolz at codespeak.net Fri Nov 20 15:29:47 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Nov 2009 15:29:47 +0100 (CET) Subject: [pypy-svn] r69469 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091120142947.581E1168076@codespeak.net> Author: cfbolz Date: Fri Nov 20 15:29:46 2009 New Revision: 69469 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Log: (pedronis, cfbolz): kill the test again, it was not helping. conservatively clear caches when we force. Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Fri Nov 20 15:29:46 2009 @@ -191,7 +191,7 @@ def forget_numberings(self, virtualbox): # XXX ideally clear only the affected numberings self.numberings.clear() - # XXX clear cached_* + self.clear_box_virtual_numbers() # caching for virtuals and boxes inside them Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Fri Nov 20 15:29:46 2009 @@ -329,33 +329,6 @@ return 0 self.meta_interp(f, [50]) - def test_guards_around_forcing_of_virtual_in_virtual(self): - class A(object): - def __init__(self, x): - self.x = x - mydriver = JitDriver(reds = ['n'], greens = []) - global_a = A(0) - - def g(b): - n = b.x - if n < 10: - n += 1 - global_a.forced = b.next - if n < 20: - assert global_a.forced is b.next - - def f(n): - while n > 0: - mydriver.can_enter_jit(n=n) - mydriver.jit_merge_point(n=n) - a = A(n) - b = A(n) - b.next = a - g(b) - n -= 1 - return 0 - self.meta_interp(f, [50]) - # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class From fijal at codespeak.net Fri Nov 20 16:22:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 20 Nov 2009 16:22:55 +0100 (CET) Subject: [pypy-svn] r69470 - pypy/trunk/pypy/objspace/std Message-ID: <20091120152255.D1F5949844E@codespeak.net> Author: fijal Date: Fri Nov 20 16:22:55 2009 New Revision: 69470 Modified: pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/objspace/std/unicodeobject.py Log: Preallocate a correct list upfront. It still can see some improvements in terms of copying, but that's already something. Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Fri Nov 20 16:22:55 2009 @@ -355,7 +355,7 @@ self = w_self._value listlen = 0 reslen = 0 - l = [] + l = [None] * len(list_w) for i in range(len(list_w)): w_s = list_w[i] if not space.is_true(space.isinstance(w_s, space.w_str)): @@ -367,7 +367,7 @@ space.wrap("sequence item %d: expected string, %s " "found" % (i, space.type(w_s).getname(space, '?')))) - l.append(space.str_w(w_s)) + l[i] = space.str_w(w_s) return space.wrap(self.join(l)) else: return W_StringObject.EMPTY Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Fri Nov 20 16:22:55 2009 @@ -182,7 +182,7 @@ space.is_w(space.type(l[0]), space.w_unicode)): return l[0] - values_list = [] + values_list = [None] * len(l) for i in range(len(l)): item = l[i] if isinstance(item, W_UnicodeObject): @@ -194,7 +194,7 @@ w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - values_list.append(item) + values_list[i] = item return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): From arigo at codespeak.net Fri Nov 20 16:25:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 16:25:34 +0100 (CET) Subject: [pypy-svn] r69471 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091120152534.8A12849844E@codespeak.net> Author: arigo Date: Fri Nov 20 16:25:34 2009 New Revision: 69471 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Log: Current status: wrote a test, painfully. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 16:25:34 2009 @@ -86,7 +86,7 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() - self.failure_recovery_code = [0, 0] + self.setup_failure_recovery() def leave_jitted_hook(self): # XXX BIG FAT WARNING XXX @@ -812,149 +812,58 @@ mc.writechr(n) mc.writechr(self.DESCR_STOP) + def setup_failure_recovery(self): + + def failure_recovery_func(registers): + pass #... + + self.failure_recovery_func = failure_recovery_func + self.failure_recovery_code = [0, 0] + + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], + lltype.Signed)) + def _build_failure_recovery(self, exc): - """ - PUSH edi - PUSH esi - PUSH ebp - PUSH 0 # for ESP, not used - PUSH ebx - PUSH edx - PUSH ecx - PUSH eax - MOV esi, [esp+32] - CLD - MOV edi, -1 - - loop: - INC edi - LODSB - CMP al, 4*8 - MOVZX edx, al - JB decode_register - JL decode_multibyte - - decode_edx: - TEST edx, 3 - JZ decode_ptr - TEST edx, 2 - JNZ decode_float - - decode_int: - # (edx & 3) == 1 - NEG edx - MOV eax, [ebp + edx + 1 - 16] - - got_value_int: - MOV [fail_boxes_int + 4*edi], eax - JMP loop - - decode_ptr: - # (edx & 3) == 0 - NEG edx - MOV eax, [ebp + edx - 16] - - got_value_ptr: - MOV [fail_boxes_ptr + 4*edi], eax - JMP loop - - decode_float: - # (edx & 3) == 2 - NEG edx - MOV eax, [ebp + edx - 2 - 16] - MOV [fail_boxes_float + 8*edi], eax - MOV eax, [ebp + edx + 2 - 16] - MOV [fail_boxes_float + 8*edi + 4], eax - JMP loop - - decode_multibyte: - MOV cl, 7 - AND edx, 0x7F - JMP innerloop - - innerloop_morebytes: - AND eax, 0x7F - SHL eax, cl - OR edx, eax - ADD cl, 7 - innerloop: - LODSB - CMP al, 0 - MOVZX eax, al - JL innerloop_morebytes - - SHL eax, cl - OR edx, eax - JMP decode_edx - - decode_register: - TEST al, 2 - JNZ decode_register_float - CMP al, DESCR_STOP - JE stop - AND edx, 0x3C - TEST al, 1 - MOV eax, [esp+edx] - JNZ got_value_int - MOV [fail_boxes_ptr + 4*edi], eax - JMP loop - - decode_register_float: - CMP al, 0x10 - JB case_0123 - CMP al, 0x18 - JB case_45 - CMP al, 0x1C - JB case_6 - case_7: - MOVSD [fail_boxes_float + 8*edi], xmm7 - JMP loop - case_6: - MOVSD [fail_boxes_float + 8*edi], xmm6 - JMP loop - case_45: - CMP al, 0x14 - JB case_4 - case_5: - MOVSD [fail_boxes_float + 8*edi], xmm5 - JMP loop - case_4: - MOVSD [fail_boxes_float + 8*edi], xmm4 - JMP loop - case_0123: - CMP al, 0x08 - JB case_01 - CMP al, 0x0C - JB case_2 - case_3: - MOVSD [fail_boxes_float + 8*edi], xmm3 - JMP loop - case_2: - MOVSD [fail_boxes_float + 8*edi], xmm2 - JMP loop - case_01: - CMP al, 0x04 - JB case_0 - case_1: - MOVSD [fail_boxes_float + 8*edi], xmm1 - JMP loop - case_0: - MOVSD [fail_boxes_float + 8*edi], xmm0 - JMP loop - - stop: - CALL on_leave_jitted - MOV eax, [esp+36] # fail_index - LEA esp, [ebp-12] - POP edi - POP esi - POP ebx - POP ebp - RET - """ - ... - ... - self.failure_recovery_code[exc] = code + failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, + self.failure_recovery_func) + failure_recovery_func = rffi.cast(lltype.Signed, + failure_recovery_func) + mc = self.mc2._mc + # Assume that we are called at the beginning, when there is no risk + # that 'mc' runs out of space. Checked by asserts in mc.write(). + addr = mc.tell() + mc.PUSH(edi) + mc.PUSH(esi) + mc.PUSH(ebp) + mc.PUSH(esp) # not really used, but needed to take up the space + mc.PUSH(ebx) + mc.PUSH(edx) + mc.PUSH(ecx) + mc.PUSH(eax) + mc.MOV(eax, esp) + mc.PUSH(eax) + mc.CALL(rel32(failure_recovery_func)) + + # we call a provided function that will + # - call our on_leave_jitted_hook which will mark + # the fail_boxes_ptr array as pointing to young objects to + # avoid unwarranted freeing + # - optionally save exception depending on the flag + addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) + mc.PUSH(eax) + mc.CALL(rel32(addr)) + mc.POP(eax) + + # now we return from the complete frame, which starts from + # _assemble_bootstrap_code(). The LEA below throws away most + # of the frame, including all the PUSHes that we did just above. + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) + mc.POP(edi) + mc.POP(esi) + mc.POP(ebx) + mc.POP(ebp) + mc.RET() + self.failure_recovery_code[exc] = addr def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): for i in range(len(locs)): @@ -994,7 +903,7 @@ # don't break the following code sequence! mc = mc._mc - mc.LEA(esp, addr_add(imm(0), ebp, (-RET_BP + 2) * WORD)) + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) mc.MOV(eax, imm(fail_index)) mc.POP(edi) mc.POP(esi) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Fri Nov 20 16:25:34 2009 @@ -2,6 +2,8 @@ from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.x86.regalloc import X86StackManager from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat +from pypy.rlib.rarithmetic import intmask +from pypy.rpython.lltypesystem import lltype, llmemory, rffi class FakeCPU: @@ -28,18 +30,126 @@ esi, xmm2] assembler.write_failure_recovery_description(mc, failargs, locs) - nums = [0 + 4*(8+0), - 1 + 4*(8+1), - 2 + 4*(8+10), - 0 + 4*(8+100), - 1 + 4*(8+101), - 2 + 4*(8+110), - 0 + 4*ebx.op, - 1 + 4*esi.op, - 2 + 4*xmm2.op] + nums = [Assembler386.DESCR_INT + 4*(8+0), + Assembler386.DESCR_REF + 4*(8+1), + Assembler386.DESCR_FLOAT + 4*(8+10), + Assembler386.DESCR_INT + 4*(8+100), + Assembler386.DESCR_REF + 4*(8+101), + Assembler386.DESCR_FLOAT + 4*(8+110), + Assembler386.DESCR_INT + 4*ebx.op, + Assembler386.DESCR_REF + 4*esi.op, + Assembler386.DESCR_FLOAT + 4*xmm2.op] double_byte_nums = [] for num in nums[3:6]: double_byte_nums.append((num & 0x7F) | 0x80) double_byte_nums.append(num >> 7) assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + [assembler.DESCR_STOP]) + +def test_failure_recovery_func(): + import random + S = lltype.GcStruct('S') + + def get_random_int(): + return random.randrange(-10000, 10000) + + def get_random_ptr(): + return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) + + def get_random_float(): + return random.random() - 0.5 + + # memory locations: 30 integers, 30 pointers, 26 floats + # main registers: half of them as signed and the other half as ptrs + # xmm registers: all floats, from xmm0 to xmm7 + locations = [] + baseloc = 4 + for i in range(30+30+26): + if baseloc < 128: + baseloc += random.randrange(2, 20) + else: + baseloc += random.randrange(2, 1000) + locations.append(baseloc) + random.shuffle(locations) + content = ([('int', locations.pop()) for _ in range(30)] + + [('ptr', locations.pop()) for _ in range(30)] + + [('float', locations.pop()) for _ in range(26)] + + [(['int', 'ptr'][random.randrange(0, 2)], reg) + for reg in [eax, ecx, edx, ebx, esi, edi]] + + [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7]]) + assert len(content) == 100 + random.shuffle(content) + + # prepare the expected target arrays, the descr_bytecode, + # the 'registers' and the 'stack' arrays according to 'content' + registers = lltype.malloc(rffi.LONGP.TO, 9, flavor='raw') + xmmregisters = [0.0] * 8 + stacklen = baseloc + 3 + stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') + expected_ints = [0] * 100 + expected_ptrs = [lltype.nullptr(llmemory.GCREF.TO)] * 100 + expected_floats = [0.0] * 100 + + descr_bytecode = [] + for i, (kind, loc) in enumerate(content): + if kind == 'float': + value = get_random_float() + expected_floats[i] = value + kind = Assembler386.DESCR_FLOAT + if isinstance(loc, REG): + xmmregisters[loc.op] = value + else: + tmp = lltype.malloc(rffi.LONGP.TO, 2, flavor='raw') + rffi.cast(rffi.DOUBLEP, tmp)[0] = value + stack[stacklen - loc] = tmp[1] + stack[stacklen - (loc+1)] = tmp[0] + else: + if kind == 'int': + value = get_random_int() + expected_ints[i] = value + kind = Assembler386.DESCR_INT + elif kind == 'ptr': + value = get_random_ptr() + expected_ptrs[i] = value + kind = Assembler386.DESCR_REF + value = rffi.cast(rffi.LONG, value) + else: + assert 0, kind + if isinstance(loc, REG): + registers[loc.op] = value + else: + stack[stacklen - loc] = value + + if isinstance(loc, REG): + num = kind + 4*loc.op + else: + num = kind + 4*(8+loc) + while num >= 0x80: + descr_bytecode.append((num & 0x7F) | 0x80) + num >>= 7 + descr_bytecode.append(num) + + descr_bytecode.append(Assembler386.DESCR_STOP) + descr_bytecode.append(0xC3) # fail_index = 0x1C3 + descr_bytecode.append(0x01) + descr_bytecode.append(0x00) + descr_bytecode.append(0x00) + descr_bytecode.append(0xCC) # end marker + descr_bytes = lltype.malloc(rffi.UCHARP.TO, len(descr_bytecode), + flavor='raw') + for i in range(len(descr_bytecode)): + assert 0 <= descr_bytecode[i] <= 255 + descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) + registers[8] = rffi.cast(rffi.LONG, descr_bytes) + + # run! + assembler = Assembler386(FakeCPU()) + res = assembler.failure_recovery_func(registers) + assert res == 0x1C3 + + # check the fail_boxes + for i in range(100): + assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] + assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] + assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] From cfbolz at codespeak.net Fri Nov 20 17:22:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Nov 2009 17:22:39 +0100 (CET) Subject: [pypy-svn] r69472 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp Message-ID: <20091120162239.13C47168075@codespeak.net> Author: cfbolz Date: Fri Nov 20 17:22:38 2009 New Revision: 69472 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py Log: (cfbolz, pedronis) xxx about current issue, needs testing Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py Fri Nov 20 17:22:38 2009 @@ -1622,6 +1622,7 @@ def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around + # xxx now that we have holes all this is delicate xxx inputargs = self.load_values_from_failure(resumedescr) warmrunnerstate = self.staticdata.state if must_compile: From arigo at codespeak.net Fri Nov 20 17:23:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 17:23:52 +0100 (CET) Subject: [pypy-svn] r69473 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091120162352.08D44168075@codespeak.net> Author: arigo Date: Fri Nov 20 17:23:51 2009 New Revision: 69473 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/support.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Log: Phew. The failure_recovery code. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 17:23:51 2009 @@ -125,6 +125,9 @@ self.mc2 = MachineCodeBlockWrapper() self._build_failure_recovery(False) self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -773,7 +776,12 @@ self.mc2.make_new_mc() mc = self.mc2._mc addr = mc.tell() - mc.CALL(rel32(self.failure_recovery_code[exc])) + withfloats = False + for box in failargs: + if box.type == FLOAT: + withfloats = True + break + mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery self.write_failure_recovery_description(mc, failargs, fail_locs) # write the fail_index too @@ -811,19 +819,84 @@ n >>= 7 mc.writechr(n) mc.writechr(self.DESCR_STOP) + # preallocate the fail_boxes + i = len(failargs) - 1 + if i >= 0: + self.fail_boxes_int.get_addr_for_num(i) + self.fail_boxes_ptr.get_addr_for_num(i) + if self.cpu.supports_floats: + self.fail_boxes_float.get_addr_for_num(i) def setup_failure_recovery(self): def failure_recovery_func(registers): - pass #... + # no malloc allowed here!! + stack_at_ebp = registers[ebp.op] + bytecode = rffi.cast(rffi.UCHARP, registers[8]) + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + stackloc = stack_at_ebp + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + stackloc -= 4 + value = rffi.cast(rffi.LONGP, stackloc)[0] + elif code == self.DESCR_STOP: + break + else: + # 'code' identifies a register: load its value + kind = code & 3 + code >>= 2 + if kind == self.DESCR_FLOAT: + xmmregisters = rffi.ptradd(registers, -16) + value = xmmregisters[2*code] + value_hi = xmmregisters[2*code + 1] + else: + value = registers[code] + + # store the loaded value into fail_boxes_ + if kind == self.DESCR_INT: + tgt = self.fail_boxes_int.get_addr_for_num(num) + elif kind == self.DESCR_REF: + tgt = self.fail_boxes_ptr.get_addr_for_num(num) + elif kind == self.DESCR_FLOAT: + tgt = self.fail_boxes_float.get_addr_for_num(num) + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + else: + assert 0, "bogus kind" + rffi.cast(rffi.LONGP, tgt)[0] = value + num += 1 + # + if not we_are_translated(): + assert bytecode[4] == 0xCC + fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + return fail_index self.failure_recovery_func = failure_recovery_func - self.failure_recovery_code = [0, 0] + self.failure_recovery_code = [0, 0, 0, 0] _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], lltype.Signed)) - def _build_failure_recovery(self, exc): + def _build_failure_recovery(self, exc, withfloats=False): failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, @@ -841,8 +914,13 @@ mc.PUSH(ecx) mc.PUSH(eax) mc.MOV(eax, esp) + if withfloats: + mc.SUB(esp, imm(8*8)) + for i in range(8): + mc.MOVSD(mem(esp, 8*i), xmm_registers[i]) mc.PUSH(eax) mc.CALL(rel32(failure_recovery_func)) + # returns in eax the fail_index # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -850,9 +928,9 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.PUSH(eax) + mc.MOV(edi, eax) mc.CALL(rel32(addr)) - mc.POP(eax) + mc.MOV(eax, edi) # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most @@ -863,7 +941,7 @@ mc.POP(ebx) mc.POP(ebp) mc.RET() - self.failure_recovery_code[exc] = addr + self.failure_recovery_code[exc + 2 * withfloats] = addr def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): for i in range(len(locs)): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/support.py Fri Nov 20 17:23:51 2009 @@ -3,7 +3,8 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated -CHUNK_SIZE = 1000 +CHUNK_SIZE_BITS = 8 +CHUNK_SIZE = 1 << CHUNK_SIZE_BITS def new_nonmovable_growable_array(TP): ATP = lltype.GcArray(TP) @@ -34,7 +35,7 @@ def _no_of(self, i): while i >= len(self.chunks) * CHUNK_SIZE: self._grow() - return i / CHUNK_SIZE, i % CHUNK_SIZE + return i >> CHUNK_SIZE_BITS, i & (CHUNK_SIZE-1) _no_of._always_inline_ = True def setitem(self, i, v): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Fri Nov 20 17:23:51 2009 @@ -1,6 +1,6 @@ from pypy.jit.backend.x86.ri386 import * from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import X86StackManager +from pypy.jit.backend.x86.regalloc import X86StackManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat from pypy.rlib.rarithmetic import intmask from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -8,6 +8,7 @@ class FakeCPU: rtyper = None + supports_floats = True class FakeMC: def __init__(self): @@ -46,7 +47,13 @@ assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + [assembler.DESCR_STOP]) -def test_failure_recovery_func(): +def test_failure_recovery_func_no_floats(): + do_failure_recovery_func(withfloats=False) + +def test_failure_recovery_func_with_floats(): + do_failure_recovery_func(withfloats=True) + +def do_failure_recovery_func(withfloats): import random S = lltype.GcStruct('S') @@ -57,7 +64,12 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) def get_random_float(): - return random.random() - 0.5 + assert withfloats + value = random.random() - 0.5 + # make sure it fits into 64 bits + tmp = lltype.malloc(rffi.LONGP.TO, 2, flavor='raw') + rffi.cast(rffi.DOUBLEP, tmp)[0] = value + return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] # memory locations: 30 integers, 30 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs @@ -73,37 +85,43 @@ random.shuffle(locations) content = ([('int', locations.pop()) for _ in range(30)] + [('ptr', locations.pop()) for _ in range(30)] + - [('float', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in [eax, ecx, edx, ebx, esi, edi]] + - [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7]]) - assert len(content) == 100 + for reg in [eax, ecx, edx, ebx, esi, edi]]) + if withfloats: + content += ([('float', locations.pop()) for _ in range(26)] + + [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, + xmm4, xmm5, xmm6, xmm7]]) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, # the 'registers' and the 'stack' arrays according to 'content' - registers = lltype.malloc(rffi.LONGP.TO, 9, flavor='raw') - xmmregisters = [0.0] * 8 - stacklen = baseloc + 3 + xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+9, flavor='raw') + registers = rffi.ptradd(xmmregisters, 16) + stacklen = baseloc + 10 stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw') - expected_ints = [0] * 100 - expected_ptrs = [lltype.nullptr(llmemory.GCREF.TO)] * 100 - expected_floats = [0.0] * 100 + expected_ints = [0] * len(content) + expected_ptrs = [lltype.nullptr(llmemory.GCREF.TO)] * len(content) + expected_floats = [0.0] * len(content) + + def write_in_stack(loc, value): + assert loc >= 0 + ofs = get_ebp_ofs(loc) + assert ofs < 0 + assert (ofs % 4) == 0 + stack[stacklen + ofs//4] = value descr_bytecode = [] for i, (kind, loc) in enumerate(content): if kind == 'float': - value = get_random_float() + value, lo, hi = get_random_float() expected_floats[i] = value kind = Assembler386.DESCR_FLOAT if isinstance(loc, REG): - xmmregisters[loc.op] = value + xmmregisters[2*loc.op] = lo + xmmregisters[2*loc.op+1] = hi else: - tmp = lltype.malloc(rffi.LONGP.TO, 2, flavor='raw') - rffi.cast(rffi.DOUBLEP, tmp)[0] = value - stack[stacklen - loc] = tmp[1] - stack[stacklen - (loc+1)] = tmp[0] + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) else: if kind == 'int': value = get_random_int() @@ -119,7 +137,7 @@ if isinstance(loc, REG): registers[loc.op] = value else: - stack[stacklen - loc] = value + write_in_stack(loc, value) if isinstance(loc, REG): num = kind + 4*loc.op @@ -142,14 +160,21 @@ assert 0 <= descr_bytecode[i] <= 255 descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) registers[8] = rffi.cast(rffi.LONG, descr_bytes) + registers[ebp.op] = rffi.cast(rffi.LONG, stack) + 4*stacklen # run! assembler = Assembler386(FakeCPU()) + assembler.fail_boxes_int.get_addr_for_num(len(content)-1) # preallocate + assembler.fail_boxes_ptr.get_addr_for_num(len(content)-1) + assembler.fail_boxes_float.get_addr_for_num(len(content)-1) res = assembler.failure_recovery_func(registers) assert res == 0x1C3 # check the fail_boxes - for i in range(100): + for i in range(len(content)): assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] + # note: we expect *exact* results below. If you have only + # an approximate result, it might mean that only the first 32 + # bits of the float were correctly saved and restored. assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] From arigo at codespeak.net Fri Nov 20 17:49:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 17:49:21 +0100 (CET) Subject: [pypy-svn] r69474 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091120164921.D09C4168075@codespeak.net> Author: arigo Date: Fri Nov 20 17:49:21 2009 New Revision: 69474 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Log: Minor fixes. test_basic passes again :-) Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 17:49:21 2009 @@ -151,6 +151,7 @@ def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() regalloc = RegAlloc(self, self.cpu.translate_support_code) + # XXXXXXXXXXXXXXXXXXXXXX remove usage of _x86_faillocs arglocs = faildescr._x86_faillocs fail_stack_depth = faildescr._x86_current_stack_depth regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, @@ -783,12 +784,14 @@ break mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery + faildescr._x86_failure_recovery_bytecode = mc.tell() self.write_failure_recovery_description(mc, failargs, fail_locs) # write the fail_index too mc.write(packimm32(fail_index)) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) + faildescr._x86_faillocs = fail_locs return addr DESCR_REF = 0x00 @@ -904,7 +907,7 @@ mc = self.mc2._mc # Assume that we are called at the beginning, when there is no risk # that 'mc' runs out of space. Checked by asserts in mc.write(). - addr = mc.tell() + recovery_addr = mc.tell() mc.PUSH(edi) mc.PUSH(esi) mc.PUSH(ebp) @@ -917,7 +920,7 @@ if withfloats: mc.SUB(esp, imm(8*8)) for i in range(8): - mc.MOVSD(mem(esp, 8*i), xmm_registers[i]) + mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) mc.PUSH(eax) mc.CALL(rel32(failure_recovery_func)) # returns in eax the fail_index @@ -941,7 +944,8 @@ mc.POP(ebx) mc.POP(ebp) mc.RET() - self.failure_recovery_code[exc + 2 * withfloats] = addr + self.mc2.done() + self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): for i in range(len(locs)): From cfbolz at codespeak.net Fri Nov 20 17:51:38 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 20 Nov 2009 17:51:38 +0100 (CET) Subject: [pypy-svn] r69475 - pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86 Message-ID: <20091120165138.C302B168075@codespeak.net> Author: cfbolz Date: Fri Nov 20 17:51:38 2009 New Revision: 69475 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/assembler.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py Log: (pedronis, cfbolz) fixes for x86, not enough though, the whole holes business needs more care Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/assembler.py Fri Nov 20 17:51:38 2009 @@ -738,7 +738,7 @@ """ assert that all args are actually Boxes """ for arg in args: - assert isinstance(arg, Box) + assert arg is None or isinstance(arg, Box) def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): @@ -754,6 +754,8 @@ pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] + if arg is None: + continue loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: @@ -767,6 +769,8 @@ mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] + if arg is None: + continue loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py Fri Nov 20 17:51:38 2009 @@ -203,7 +203,8 @@ def possibly_free_vars(self, vars): for var in vars: - self.possibly_free_var(var) + if var is not None: + self.possibly_free_var(var) def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, imm_fine=True, @@ -361,6 +362,8 @@ longevity[arg] = (start_live[arg], i) if op.is_guard(): for arg in op.fail_args: + if arg is None: # hole + continue assert isinstance(arg, Box) if arg not in start_live: print "Bogus arg in guard %d at %d" % (op.opnum, i) @@ -374,6 +377,8 @@ return longevity def loc(self, v): + if v is None: + return None if v.type == FLOAT: return self.xrm.loc(v) return self.rm.loc(v) From arigo at codespeak.net Fri Nov 20 18:03:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 18:03:37 +0100 (CET) Subject: [pypy-svn] r69476 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091120170337.9AA24168076@codespeak.net> Author: arigo Date: Fri Nov 20 18:03:37 2009 New Revision: 69476 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Log: Test and write rebuild_faillocs_from_descr. Now guards don't need to store the attribute _x86_faillocs any more. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 18:03:37 2009 @@ -150,9 +150,12 @@ def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() + arglocs = self.rebuild_faillocs_from_descr( + faildescr._x86_failure_recovery_bytecode) + if not we_are_translated(): + assert ([loc.assembler() for loc in arglocs] == + [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - # XXXXXXXXXXXXXXXXXXXXXX remove usage of _x86_faillocs - arglocs = faildescr._x86_faillocs fail_stack_depth = faildescr._x86_current_stack_depth regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, operations) @@ -791,7 +794,7 @@ # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) - faildescr._x86_faillocs = fail_locs + faildescr._x86_debug_faillocs = fail_locs return addr DESCR_REF = 0x00 @@ -830,6 +833,46 @@ if self.cpu.supports_floats: self.fail_boxes_float.get_addr_for_num(i) + def rebuild_faillocs_from_descr(self, bytecode): + from pypy.jit.backend.x86.regalloc import X86StackManager + bytecode = rffi.cast(rffi.UCHARP, bytecode) + arglocs = [] + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + # 'code' identifies a stack location + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + if kind == self.DESCR_FLOAT: + size = 2 + else: + size = 1 + loc = X86StackManager.stack_pos(code, size) + elif code == self.DESCR_STOP: + break + else: + # 'code' identifies a register + kind = code & 3 + code >>= 2 + if kind == self.DESCR_FLOAT: + loc = xmm_registers[code] + else: + loc = registers[code] + arglocs.append(loc) + return arglocs[:] + def setup_failure_recovery(self): def failure_recovery_func(registers): Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Fri Nov 20 18:03:37 2009 @@ -47,6 +47,18 @@ assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + [assembler.DESCR_STOP]) + # also test rebuild_faillocs_from_descr() + bytecode = lltype.malloc(rffi.UCHARP.TO, len(mc.content), flavor='raw') + for i in range(len(mc.content)): + assert 0 <= mc.content[i] <= 255 + bytecode[i] = rffi.cast(rffi.UCHAR, mc.content[i]) + bytecode_addr = rffi.cast(lltype.Signed, bytecode) + newlocs = assembler.rebuild_faillocs_from_descr(bytecode_addr) + assert ([loc.assembler() for loc in newlocs] == + [loc.assembler() for loc in locs]) + +# ____________________________________________________________ + def test_failure_recovery_func_no_floats(): do_failure_recovery_func(withfloats=False) From arigo at codespeak.net Fri Nov 20 18:31:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 18:31:57 +0100 (CET) Subject: [pypy-svn] r69477 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test Message-ID: <20091120173157.A64CD16807A@codespeak.net> Author: arigo Date: Fri Nov 20 18:31:57 2009 New Revision: 69477 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py Log: Improve the testing of "machine code block full". Although it is still a bit random, now it runs all runner tests a second time with a 1KB limit. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_runner.py Fri Nov 20 18:31:57 2009 @@ -314,31 +314,27 @@ else: assert result != expected - def test_overflow_mc(self): - from pypy.jit.backend.x86.assembler import MachineCodeBlockWrapper - orig_size = MachineCodeBlockWrapper.MC_SIZE - MachineCodeBlockWrapper.MC_SIZE = 1024 - old_mc = self.cpu.assembler.mc - old_mc2 = self.cpu.assembler.mc2 - self.cpu.assembler.mc = None - try: - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc != old_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - finally: - MachineCodeBlockWrapper.MC_SIZE = orig_size - self.cpu.assembler.mc = old_mc - self.cpu.assembler.mc2 = old_mc2 +class TestX86OverflowMC(TestX86): + + def setup_class(cls): + cls.cpu = CPU(rtyper=None, stats=FakeStats()) + cls.cpu.assembler.mc_size = 1024 + + def test_overflow_mc(self): + ops = [] + base_v = BoxInt() + v = base_v + for i in range(1024): + next_v = BoxInt() + ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) + v = next_v + ops.append(ResOperation(rop.FINISH, [v], None, + descr=BasicFailDescr())) + looptoken = LoopToken() + old_mc_mc = self.cpu.assembler.mc._mc + self.cpu.compile_loop([base_v], ops, looptoken) + assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed + self.cpu.set_future_value_int(0, base_v.value) + self.cpu.execute_token(looptoken) + assert self.cpu.get_latest_value_int(0) == 1024 From arigo at codespeak.net Fri Nov 20 18:32:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 18:32:42 +0100 (CET) Subject: [pypy-svn] r69478 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091120173242.73B8B16807A@codespeak.net> Author: arigo Date: Fri Nov 20 18:32:42 2009 New Revision: 69478 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Log: This belongs to the previous checkin. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 18:32:42 2009 @@ -17,6 +17,7 @@ from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ CHUNK_SIZE +from pypy.rlib.debug import debug_print # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -34,17 +35,19 @@ return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) class MachineCodeBlockWrapper(object): - MC_SIZE = 1024*1024 + MC_DEFAULT_SIZE = 1024*1024 - def __init__(self): + def __init__(self, bigsize): self.old_mcs = [] # keepalive - self._mc = codebuf.MachineCodeBlock(self.MC_SIZE) + self.bigsize = bigsize + self._mc = codebuf.MachineCodeBlock(bigsize) def bytes_free(self): return self._mc._size - self._mc._pos def make_new_mc(self): - new_mc = codebuf.MachineCodeBlock(self.MC_SIZE) + new_mc = codebuf.MachineCodeBlock(self.bigsize) + debug_print('[new machine code block at 0x%x]' % fixid(new_mc.tell())) self._mc.JMP(rel32(new_mc.tell())) self._mc.done() self.old_mcs.append(self._mc) @@ -74,6 +77,7 @@ class Assembler386(object): mc = None mc2 = None + mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE def __init__(self, cpu, translate_support_code=False): self.cpu = cpu @@ -121,8 +125,8 @@ # done # we generate the loop body in 'mc' # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper() - self.mc2 = MachineCodeBlockWrapper() + self.mc = MachineCodeBlockWrapper(self.mc_size) + self.mc2 = MachineCodeBlockWrapper(self.mc_size) self._build_failure_recovery(False) self._build_failure_recovery(True) if self.cpu.supports_floats: From arigo at codespeak.net Fri Nov 20 18:52:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 18:52:38 +0100 (CET) Subject: [pypy-svn] r69479 - in pypy/branch/shorter-guard-path/pypy/jit/backend/x86: . test Message-ID: <20091120175238.E97FC168078@codespeak.net> Author: arigo Date: Fri Nov 20 18:52:38 2009 New Revision: 69479 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Log: Grrr. Test and fix for mutating global lists. Bad idea. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py Fri Nov 20 18:52:38 2009 @@ -259,9 +259,16 @@ xmm6 = XMM6() xmm7 = XMM7() -registers = [eax, ecx, edx, ebx, esp, ebp, esi, edi] -registers8 = [al, cl, dl, bl, ah, ch, dh, bh] -xmm_registers = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] +class immutablelist(list): + def _dont_modify(self, *args): + raise Exception("don't modify this list!") + __delitem__ = __delslice__ = __iadd__ = __imul__ = _dont_modify + __setitem__ = __setslice__ = _dont_modify + append = extend = insert = pop = remove = reverse = sort = _dont_modify + +registers = immutablelist([eax, ecx, edx, ebx, esp, ebp, esi, edi]) +registers8 = immutablelist([al, cl, dl, bl, ah, ch, dh, bh]) +xmm_registers = immutablelist([xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7]) for r in registers + registers8: r.bitmask = 1 << r.op Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Fri Nov 20 18:52:38 2009 @@ -92,7 +92,7 @@ return [pick1(i386.memSIB64) for i in range(COUNT2)] def xmm_tests(): - return i386.xmm_registers + return i386.xmm_registers[:] def modrm8_tests(): return i386.registers8 + [pick1(i386.memSIB8) for i in range(COUNT2)] From arigo at codespeak.net Fri Nov 20 19:30:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 19:30:20 +0100 (CET) Subject: [pypy-svn] r69480 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091120183020.D173C168071@codespeak.net> Author: arigo Date: Fri Nov 20 19:30:20 2009 New Revision: 69480 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Log: Can't use fixid() in RPython. :-( Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 19:30:20 2009 @@ -47,7 +47,7 @@ def make_new_mc(self): new_mc = codebuf.MachineCodeBlock(self.bigsize) - debug_print('[new machine code block at 0x%x]' % fixid(new_mc.tell())) + debug_print('[new machine code block at', new_mc.tell(), ']') self._mc.JMP(rel32(new_mc.tell())) self._mc.done() self.old_mcs.append(self._mc) From arigo at codespeak.net Fri Nov 20 19:30:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 19:30:46 +0100 (CET) Subject: [pypy-svn] r69481 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091120183046.8B022168071@codespeak.net> Author: arigo Date: Fri Nov 20 19:30:46 2009 New Revision: 69481 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py Log: Revert the test for r69479 (but not the fix). I cannot find an RPythonic way to say that :-( Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/ri386.py Fri Nov 20 19:30:46 2009 @@ -259,16 +259,9 @@ xmm6 = XMM6() xmm7 = XMM7() -class immutablelist(list): - def _dont_modify(self, *args): - raise Exception("don't modify this list!") - __delitem__ = __delslice__ = __iadd__ = __imul__ = _dont_modify - __setitem__ = __setslice__ = _dont_modify - append = extend = insert = pop = remove = reverse = sort = _dont_modify - -registers = immutablelist([eax, ecx, edx, ebx, esp, ebp, esi, edi]) -registers8 = immutablelist([al, cl, dl, bl, ah, ch, dh, bh]) -xmm_registers = immutablelist([xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7]) +registers = [eax, ecx, edx, ebx, esp, ebp, esi, edi] +registers8 = [al, cl, dl, bl, ah, ch, dh, bh] +xmm_registers = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7] for r in registers + registers8: r.bitmask = 1 << r.op From arigo at codespeak.net Fri Nov 20 19:37:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 19:37:59 +0100 (CET) Subject: [pypy-svn] r69482 - pypy/branch/shorter-guard-path/pypy/jit/backend/x86 Message-ID: <20091120183759.8F765168071@codespeak.net> Author: arigo Date: Fri Nov 20 19:37:59 2009 New Revision: 69482 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Log: Translation fix. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 19:37:59 2009 @@ -819,10 +819,11 @@ else: raise AssertionError("bogus kind") loc = locs[i] - if isinstance(loc, REG): - n = loc.op - else: + if isinstance(loc, MODRM): n = self.DESCR_FROMSTACK + loc.position + else: + assert isinstance(loc, REG) + n = loc.op n = kind + 4*n while n > 0x7F: mc.writechr((n & 0x7F) | 0x80) From arigo at codespeak.net Fri Nov 20 20:17:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 20:17:13 +0100 (CET) Subject: [pypy-svn] r69483 - in pypy/branch/shorter-guard-path/pypy/jit/backend: llsupport x86 Message-ID: <20091120191713.7DAB6168073@codespeak.net> Author: arigo Date: Fri Nov 20 20:17:12 2009 New Revision: 69483 Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Log: Small fixes, and documentation. Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/llsupport/llmodel.py Fri Nov 20 20:17:12 2009 @@ -150,8 +150,8 @@ on_leave_jitted_hook() def on_leave_jitted_save_exc(): - on_leave_jitted_hook() save_exception() + on_leave_jitted_hook() self.on_leave_jitted_noexc = on_leave_jitted_noexc self.on_leave_jitted_save_exc = on_leave_jitted_save_exc Modified: pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path/pypy/jit/backend/x86/assembler.py Fri Nov 20 20:17:12 2009 @@ -93,9 +93,6 @@ self.setup_failure_recovery() def leave_jitted_hook(self): - # XXX BIG FAT WARNING XXX - # At this point, we should not call anyone here, because - # RPython-level exception might be set. Here be dragons i = 0 while i < self.fail_boxes_ptr.lgt: chunk = self.fail_boxes_ptr.chunks[i] @@ -882,6 +879,10 @@ def failure_recovery_func(registers): # no malloc allowed here!! + # 'registers' is a pointer to a structure containing the + # original value of the registers, optionally the original + # value of XMM registers, and finally a reference to the + # recovery bytecode. See _build_failure_recovery() for details. stack_at_ebp = registers[ebp.op] bytecode = rffi.cast(rffi.UCHARP, registers[8]) num = 0 @@ -908,8 +909,7 @@ value = rffi.cast(rffi.LONGP, stackloc)[0] if kind == self.DESCR_FLOAT: value_hi = value - stackloc -= 4 - value = rffi.cast(rffi.LONGP, stackloc)[0] + value = rffi.cast(rffi.LONGP, stackloc - 4)[0] elif code == self.DESCR_STOP: break else: @@ -959,19 +959,16 @@ mc.PUSH(edi) mc.PUSH(esi) mc.PUSH(ebp) - mc.PUSH(esp) # not really used, but needed to take up the space + mc.PUSH(esp) # <-- not really used, but needed to take up the space mc.PUSH(ebx) mc.PUSH(edx) mc.PUSH(ecx) mc.PUSH(eax) - mc.MOV(eax, esp) + mc.MOV(esi, esp) if withfloats: mc.SUB(esp, imm(8*8)) for i in range(8): mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) - mc.PUSH(eax) - mc.CALL(rel32(failure_recovery_func)) - # returns in eax the fail_index # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -979,9 +976,18 @@ # avoid unwarranted freeing # - optionally save exception depending on the flag addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) - mc.MOV(edi, eax) mc.CALL(rel32(addr)) - mc.MOV(eax, edi) + + # the following call saves all values from the stack and from + # registers to the right 'fail_boxes_' location. + # Note that the registers are saved so far in esi[0] to esi[7], + # as pushed above, plus optionally in esi[-16] to esi[-1] for + # the XMM registers. Moreover, esi[8] is a pointer to the recovery + # bytecode, pushed just before by the CALL instruction written by + # generate_quick_failure(). + mc.PUSH(esi) + mc.CALL(rel32(failure_recovery_func)) + # returns in eax the fail_index # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most From arigo at codespeak.net Fri Nov 20 21:35:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 21:35:08 +0100 (CET) Subject: [pypy-svn] r69484 - in pypy/branch/shorter-guard-path/pypy: annotation config doc/config interpreter interpreter/astcompiler interpreter/astcompiler/tools interpreter/test module/Numeric module/__builtin__ module/__builtin__/test module/_codecs module/_rawffi module/_stackless/test module/micronumpy module/oracle module/oracle/test module/posix module/rctime module/select objspace/flow objspace/std objspace/std/test rpython translator/c/gcc Message-ID: <20091120203508.BD9CB16807B@codespeak.net> Author: arigo Date: Fri Nov 20 21:35:04 2009 New Revision: 69484 Removed: pypy/branch/shorter-guard-path/pypy/doc/config/objspace.usemodules.Numeric.txt pypy/branch/shorter-guard-path/pypy/module/Numeric/ Modified: pypy/branch/shorter-guard-path/pypy/annotation/bookkeeper.py pypy/branch/shorter-guard-path/pypy/config/translationoption.py pypy/branch/shorter-guard-path/pypy/interpreter/argument.py pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/ast.py pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/shorter-guard-path/pypy/interpreter/baseobjspace.py pypy/branch/shorter-guard-path/pypy/interpreter/function.py pypy/branch/shorter-guard-path/pypy/interpreter/gateway.py pypy/branch/shorter-guard-path/pypy/interpreter/interactive.py pypy/branch/shorter-guard-path/pypy/interpreter/nestedscope.py pypy/branch/shorter-guard-path/pypy/interpreter/pycode.py pypy/branch/shorter-guard-path/pypy/interpreter/pyopcode.py pypy/branch/shorter-guard-path/pypy/interpreter/test/test_argument.py pypy/branch/shorter-guard-path/pypy/interpreter/test/test_compiler.py pypy/branch/shorter-guard-path/pypy/interpreter/test/test_objspace.py pypy/branch/shorter-guard-path/pypy/module/__builtin__/abstractinst.py pypy/branch/shorter-guard-path/pypy/module/__builtin__/interp_classobj.py pypy/branch/shorter-guard-path/pypy/module/__builtin__/test/test_abstractinst.py pypy/branch/shorter-guard-path/pypy/module/_codecs/interp_codecs.py pypy/branch/shorter-guard-path/pypy/module/_rawffi/interp_rawffi.py pypy/branch/shorter-guard-path/pypy/module/_rawffi/structure.py pypy/branch/shorter-guard-path/pypy/module/_stackless/test/test_pickle_infrastructure.py pypy/branch/shorter-guard-path/pypy/module/micronumpy/numarray.py pypy/branch/shorter-guard-path/pypy/module/oracle/__init__.py pypy/branch/shorter-guard-path/pypy/module/oracle/config.py pypy/branch/shorter-guard-path/pypy/module/oracle/interp_connect.py pypy/branch/shorter-guard-path/pypy/module/oracle/interp_cursor.py pypy/branch/shorter-guard-path/pypy/module/oracle/interp_error.py pypy/branch/shorter-guard-path/pypy/module/oracle/interp_variable.py pypy/branch/shorter-guard-path/pypy/module/oracle/roci.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_connect.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_cursor.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_datetimevar.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_numbervar.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_select.py pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_stringvar.py pypy/branch/shorter-guard-path/pypy/module/oracle/transform.py pypy/branch/shorter-guard-path/pypy/module/posix/interp_posix.py pypy/branch/shorter-guard-path/pypy/module/rctime/interp_time.py pypy/branch/shorter-guard-path/pypy/module/select/interp_select.py pypy/branch/shorter-guard-path/pypy/objspace/flow/objspace.py pypy/branch/shorter-guard-path/pypy/objspace/std/dictmultiobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/floatobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/formatting.py pypy/branch/shorter-guard-path/pypy/objspace/std/inlinedict.py pypy/branch/shorter-guard-path/pypy/objspace/std/listobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/marshal_impl.py pypy/branch/shorter-guard-path/pypy/objspace/std/objspace.py pypy/branch/shorter-guard-path/pypy/objspace/std/ropeobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/ropeunicodeobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/setobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/stringobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/strsliceobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_celldict.py pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_floatobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_userobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/tupletype.py pypy/branch/shorter-guard-path/pypy/objspace/std/typeobject.py pypy/branch/shorter-guard-path/pypy/objspace/std/typetype.py pypy/branch/shorter-guard-path/pypy/objspace/std/unicodeobject.py pypy/branch/shorter-guard-path/pypy/rpython/callparse.py pypy/branch/shorter-guard-path/pypy/translator/c/gcc/trackgcroot.py Log: Merge the trunk. Fixes a failure. svn merge -r69393:69483 svn+ssh://codespeak.net/svn/pypy/trunk Modified: pypy/branch/shorter-guard-path/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/shorter-guard-path/pypy/annotation/bookkeeper.py Fri Nov 20 21:35:04 2009 @@ -757,7 +757,8 @@ getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] raise CallPatternTooComplex, "'*' argument must be SomeTuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/branch/shorter-guard-path/pypy/config/translationoption.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/config/translationoption.py (original) +++ pypy/branch/shorter-guard-path/pypy/config/translationoption.py Fri Nov 20 21:35:04 2009 @@ -245,7 +245,7 @@ "Tranform graphs in SSI form into graphs tailored for " "stack based virtual machines (only for backends that support it)", default=True), - BoolOption("storesink", "Perform store sinking", default=False), + BoolOption("storesink", "Perform store sinking", default=True), BoolOption("none", "Do not run any backend optimizations", requires=[('translation.backendopt.inline', False), Modified: pypy/branch/shorter-guard-path/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/argument.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/argument.py Fri Nov 20 21:35:04 2009 @@ -135,7 +135,7 @@ # unpack the * arguments if w_stararg is not None: self.arguments_w = (self.arguments_w + - self.space.viewiterable(w_stararg)) + self.space.fixedview(w_stararg)) # unpack the ** arguments if w_starstararg is not None: space = self.space Modified: pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/ast.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/ast.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/ast.py Fri Nov 20 21:35:04 2009 @@ -97,7 +97,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -132,7 +132,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -191,7 +191,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -244,7 +244,7 @@ self.args.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -254,7 +254,7 @@ node.sync_app_attrs(space) w_list = self.w_decorators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.decorators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -297,7 +297,7 @@ pass w_list = self.w_bases if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.bases = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -307,7 +307,7 @@ node.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -375,7 +375,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -415,7 +415,7 @@ pass w_list = self.w_targets if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.targets = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -492,7 +492,7 @@ self.dest.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -540,7 +540,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -550,7 +550,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -595,7 +595,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -605,7 +605,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -650,7 +650,7 @@ self.test.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -660,7 +660,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -707,7 +707,7 @@ self.optional_vars.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -795,7 +795,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -805,7 +805,7 @@ node.sync_app_attrs(space) w_list = self.w_handlers if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.handlers = [space.interp_w(excepthandler, w_obj) for w_obj in list_w] else: @@ -815,7 +815,7 @@ node.sync_app_attrs(space) w_list = self.w_orelse if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.orelse = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -857,7 +857,7 @@ pass w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -867,7 +867,7 @@ node.sync_app_attrs(space) w_list = self.w_finalbody if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.finalbody = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -936,7 +936,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -977,7 +977,7 @@ self.level = 0 w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.interp_w(alias, w_obj) for w_obj in list_w] else: @@ -1053,7 +1053,7 @@ pass w_list = self.w_names if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.names = [space.str_w(w_obj) for w_obj in list_w] else: @@ -1196,7 +1196,7 @@ pass w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1359,7 +1359,7 @@ pass w_list = self.w_keys if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keys = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1369,7 +1369,7 @@ node.sync_app_attrs(space) w_list = self.w_values if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.values = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1408,7 +1408,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1447,7 +1447,7 @@ self.elt.sync_app_attrs(space) w_list = self.w_generators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.generators = [space.interp_w(comprehension, w_obj) for w_obj in list_w] else: @@ -1520,14 +1520,14 @@ self.left.sync_app_attrs(space) w_list = self.w_ops if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ops = [space.interp_w(cmpop, w_obj).to_simple_int(space) for w_obj in list_w] else: self.ops = None w_list = self.w_comparators if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.comparators = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1579,7 +1579,7 @@ self.func.sync_app_attrs(space) w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1589,7 +1589,7 @@ node.sync_app_attrs(space) w_list = self.w_keywords if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.keywords = [space.interp_w(keyword, w_obj) for w_obj in list_w] else: @@ -1795,7 +1795,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -1834,7 +1834,7 @@ pass w_list = self.w_elts if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.elts = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2012,7 +2012,7 @@ pass w_list = self.w_dims if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.dims = [space.interp_w(slice, w_obj) for w_obj in list_w] else: @@ -2305,7 +2305,7 @@ self.iter.sync_app_attrs(space) w_list = self.w_ifs if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.ifs = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2344,7 +2344,7 @@ self.name.sync_app_attrs(space) w_list = self.w_body if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.body = [space.interp_w(stmt, w_obj) for w_obj in list_w] else: @@ -2379,7 +2379,7 @@ self.kwarg = None w_list = self.w_args if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.args = [space.interp_w(expr, w_obj) for w_obj in list_w] else: @@ -2389,7 +2389,7 @@ node.sync_app_attrs(space) w_list = self.w_defaults if w_list is not None: - list_w = space.viewiterable(w_list) + list_w = space.listview(w_list) if list_w: self.defaults = [space.interp_w(expr, w_obj) for w_obj in list_w] else: Modified: pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/astcompiler/tools/asdl_py.py Fri Nov 20 21:35:04 2009 @@ -144,7 +144,7 @@ if attr.seq: self.emit("w_list = self.w_%s" % (attr.name,), 2) self.emit("if w_list is not None:", 2) - self.emit("list_w = space.viewiterable(w_list)", 3) + self.emit("list_w = space.listview(w_list)", 3) self.emit("if list_w:", 3) unwrapper = get_unwrapper(attr.type.value, "w_obj", self.data.simple_types) Modified: pypy/branch/shorter-guard-path/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/baseobjspace.py Fri Nov 20 21:35:04 2009 @@ -668,13 +668,17 @@ (i, plural)) return items - def viewiterable(self, w_iterable, expected_length=-1): - """ More or less the same as unpackiterable, but does not return - a copy. Please don't modify the result + def fixedview(self, w_iterable, expected_length=-1): + """ A fixed list view of w_iterable. Don't modify the result """ return make_sure_not_resized(self.unpackiterable(w_iterable, expected_length)[:]) + def listview(self, w_iterable, expected_length=-1): + """ A non-fixed view of w_iterable. Don't modify the result + """ + return self.unpackiterable(w_iterable, expected_length) + def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): @@ -771,7 +775,7 @@ def lookup(self, w_obj, name): w_type = self.type(w_obj) w_mro = self.getattr(w_type, self.wrap("__mro__")) - for w_supertype in self.unpackiterable(w_mro): + for w_supertype in self.fixedview(w_mro): w_value = w_supertype.getdictvalue(self, name) if w_value is not None: return w_value @@ -880,7 +884,7 @@ if self.is_true(self.isinstance(w_index_or_slice, self.w_slice)): w_indices = self.call_method(w_index_or_slice, "indices", self.wrap(seqlength)) - w_start, w_stop, w_step = self.unpackiterable(w_indices, 3) + w_start, w_stop, w_step = self.fixedview(w_indices, 3) start = self.int_w(w_start) stop = self.int_w(w_stop) step = self.int_w(w_step) Modified: pypy/branch/shorter-guard-path/pypy/interpreter/function.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/function.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/function.py Fri Nov 20 21:35:04 2009 @@ -198,7 +198,7 @@ else: name = None if not space.is_w(w_argdefs, space.w_None): - defs_w = space.viewiterable(w_argdefs) + defs_w = space.fixedview(w_argdefs) else: defs_w = [] nfreevars = 0 @@ -316,7 +316,7 @@ if space.is_w(w_func_dict, space.w_None): w_func_dict = None self.w_func_dict = w_func_dict - self.defs_w = space.viewiterable(w_defs_w) + self.defs_w = space.fixedview(w_defs_w) self.w_module = w_module def fget_func_defaults(space, self): @@ -331,7 +331,7 @@ return if not space.is_true(space.isinstance(w_defaults, space.w_tuple)): raise OperationError( space.w_TypeError, space.wrap("func_defaults must be set to a tuple object or None") ) - self.defs_w = space.viewiterable(w_defaults) + self.defs_w = space.fixedview(w_defaults) def fdel_func_defaults(space, self): self.defs_w = [] Modified: pypy/branch/shorter-guard-path/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/gateway.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/gateway.py Fri Nov 20 21:35:04 2009 @@ -210,7 +210,7 @@ % (self.scopenext(), self.scopenext())) def visit_args_w(self, el): - self.run_args.append("space.viewiterable(%s)" % self.scopenext()) + self.run_args.append("space.fixedview(%s)" % self.scopenext()) def visit_w_args(self, el): self.run_args.append(self.scopenext()) @@ -416,7 +416,7 @@ # baseobjspace.W_Root is for wrapped arguments to keep wrapped # baseobjspace.Wrappable subclasses imply interp_w and a typecheck # argument.Arguments is for a final rest arguments Arguments object - # 'args_w' for viewiterable applied to rest arguments + # 'args_w' for fixedview applied to rest arguments # 'w_args' for rest arguments passed as wrapped tuple # str,int,float: unwrap argument as such type # (function, cls) use function to check/unwrap argument of type cls Modified: pypy/branch/shorter-guard-path/pypy/interpreter/interactive.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/interactive.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/interactive.py Fri Nov 20 21:35:04 2009 @@ -73,7 +73,7 @@ words = self.get_words(w_clz) try: w_bases = s.getattr(w_clz, s.wrap("__bases__")) - bases_w = s.viewiterable(w_bases) + bases_w = s.fixedview(w_bases) except error.OperationError: return words Modified: pypy/branch/shorter-guard-path/pypy/interpreter/nestedscope.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/nestedscope.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/nestedscope.py Fri Nov 20 21:35:04 2009 @@ -208,7 +208,7 @@ if codeobj.magic >= 0xa0df281: # CPython 2.5 AST branch merge w_freevarstuple = f.popvalue() freevars = [f.space.interp_w(Cell, cell) - for cell in f.space.viewiterable(w_freevarstuple)] + for cell in f.space.fixedview(w_freevarstuple)] else: nfreevars = len(codeobj.co_freevars) freevars = [f.space.interp_w(Cell, f.popvalue()) Modified: pypy/branch/shorter-guard-path/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/pycode.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/pycode.py Fri Nov 20 21:35:04 2009 @@ -349,7 +349,7 @@ if not space.is_true(space.isinstance(w_constants, space.w_tuple)): raise OperationError(space.w_TypeError, space.wrap("Expected tuple for constants")) - consts_w = space.viewiterable(w_constants) + consts_w = space.fixedview(w_constants) names = unpack_str_tuple(space, w_names) varnames = unpack_str_tuple(space, w_varnames) if w_freevars is not None: Modified: pypy/branch/shorter-guard-path/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/pyopcode.py Fri Nov 20 21:35:04 2009 @@ -581,7 +581,7 @@ w_compile_flags, f.space.wrap(f.get_builtin()), f.space.gettypeobject(PyCode.typedef)) - w_prog, w_globals, w_locals = f.space.viewiterable(w_resulttuple, 3) + w_prog, w_globals, w_locals = f.space.fixedview(w_resulttuple, 3) plain = f.w_locals is not None and f.space.is_w(w_locals, f.w_locals) if plain: @@ -637,7 +637,7 @@ def UNPACK_SEQUENCE(f, itemcount, *ignored): w_iterable = f.popvalue() try: - items = f.space.viewiterable(w_iterable, itemcount) + items = f.space.fixedview(w_iterable, itemcount) except UnpackValueError, e: raise OperationError(f.space.w_ValueError, f.space.wrap(e.msg)) f.pushrevvalues(itemcount, items) Modified: pypy/branch/shorter-guard-path/pypy/interpreter/test/test_argument.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/test/test_argument.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/test/test_argument.py Fri Nov 20 21:35:04 2009 @@ -60,9 +60,12 @@ def is_true(self, obj): return bool(obj) - def viewiterable(self, it): + def fixedview(self, it): return list(it) + def listview(self, it): + return list(it) + def unpackiterable(self, it): return list(it) Modified: pypy/branch/shorter-guard-path/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/test/test_compiler.py Fri Nov 20 21:35:04 2009 @@ -651,6 +651,9 @@ elif sys.version_info < (2, 6): _unicode_error_kind = "w_UnicodeDecodeError" else: + def skip_on_2_6(self): + py.test.skip("syntax different on CPython 2.6 compiler") + test_globals_warnings = skip_on_2_6 _unicode_error_kind = "w_SyntaxError" class TestPythonAstCompiler_25_grammar(BaseTestCompiler): Modified: pypy/branch/shorter-guard-path/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/shorter-guard-path/pypy/interpreter/test/test_objspace.py Fri Nov 20 21:35:04 2009 @@ -58,14 +58,23 @@ raises(ValueError, self.space.unpackiterable, w_l, 3) raises(ValueError, self.space.unpackiterable, w_l, 5) - def test_viewiterable(self): + def test_fixedview(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.viewiterable(w_l) == l - assert self.space.viewiterable(w_l, 4) == l - raises(ValueError, self.space.viewiterable, w_l, 3) - raises(ValueError, self.space.viewiterable, w_l, 5) + assert self.space.fixedview(w_l) == l + assert self.space.fixedview(w_l, 4) == l + raises(ValueError, self.space.fixedview, w_l, 3) + raises(ValueError, self.space.fixedview, w_l, 5) + + def test_listview(self): + w = self.space.wrap + l = [w(1), w(2), w(3), w(4)] + w_l = self.space.newtuple(l) + assert self.space.listview(w_l) == l + assert self.space.listview(w_l, 4) == l + raises(ValueError, self.space.listview, w_l, 3) + raises(ValueError, self.space.listview, w_l, 5) def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, @@ -207,7 +216,7 @@ w_res = space.call_obj_args(w_f, w_9, Arguments(space, [w_1])) - w_x, w_y = space.viewiterable(w_res, 2) + w_x, w_y = space.fixedview(w_res, 2) assert w_x is w_9 assert w_y is w_1 Modified: pypy/branch/shorter-guard-path/pypy/module/__builtin__/abstractinst.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/__builtin__/abstractinst.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/__builtin__/abstractinst.py Fri Nov 20 21:35:04 2009 @@ -83,7 +83,7 @@ def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple): # -- case (anything, tuple) if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_isinstance_w(space, w_obj, w_klass): return True return False @@ -109,7 +109,7 @@ return True w_bases = _get_bases(space, w_derived) if w_bases is not None: - for w_base in space.viewiterable(w_bases): + for w_base in space.fixedview(w_bases): if _issubclass_recurse(space, w_base, w_top): return True return False @@ -141,7 +141,7 @@ # -- case (class-like-object, tuple-of-classes) # XXX it might be risky that the JIT sees this if space.is_true(space.isinstance(w_klass_or_tuple, space.w_tuple)): - for w_klass in space.viewiterable(w_klass_or_tuple): + for w_klass in space.fixedview(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass): return True return False Modified: pypy/branch/shorter-guard-path/pypy/module/__builtin__/interp_classobj.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/__builtin__/interp_classobj.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/__builtin__/interp_classobj.py Fri Nov 20 21:35:04 2009 @@ -36,7 +36,7 @@ # XXX missing: lengthy and obscure logic about "__module__" - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): w_metaclass = space.type(w_base) @@ -79,7 +79,7 @@ raise OperationError( space.w_TypeError, space.wrap("__bases__ must be a tuple object")) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) for w_base in bases_w: if not isinstance(w_base, W_ClassObject): raise OperationError(space.w_TypeError, @@ -285,7 +285,7 @@ if not e.match(space, space.w_TypeError): raise return [None, None] - return space.viewiterable(w_tup, 2) + return space.fixedview(w_tup, 2) def descr_instance_new(space, w_type, w_class, w_dict=None): # w_type is not used at all Modified: pypy/branch/shorter-guard-path/pypy/module/__builtin__/test/test_abstractinst.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/__builtin__/test/test_abstractinst.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/__builtin__/test/test_abstractinst.py Fri Nov 20 21:35:04 2009 @@ -5,7 +5,7 @@ def test_abstract_isclass(self): space = self.space - w_B1, w_B2, w_B3, w_X, w_Y = space.viewiterable(space.appexec([], """(): + w_B1, w_B2, w_B3, w_X, w_Y = space.fixedview(space.appexec([], """(): class X(object): pass class Y: pass B1, B2, B3 = X(), X(), X() @@ -22,7 +22,7 @@ def test_abstract_getclass(self): space = self.space - w_x, w_y, w_A, w_MyInst = space.viewiterable(space.appexec([], """(): + w_x, w_y, w_A, w_MyInst = space.fixedview(space.appexec([], """(): class MyInst(object): def __init__(self, myclass): self.myclass = myclass Modified: pypy/branch/shorter-guard-path/pypy/module/_codecs/interp_codecs.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/_codecs/interp_codecs.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/_codecs/interp_codecs.py Fri Nov 20 21:35:04 2009 @@ -35,7 +35,7 @@ space.wrap("encoding error handler must return " "(unicode, int) tuple, not %s" % ( space.str_w(space.repr(w_res))))) - w_replace, w_newpos = space.viewiterable(w_res, 2) + w_replace, w_newpos = space.fixedview(w_res, 2) newpos = space.int_w(w_newpos) if (newpos < 0): newpos = len(input) + newpos Modified: pypy/branch/shorter-guard-path/pypy/module/_rawffi/interp_rawffi.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/_rawffi/interp_rawffi.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/_rawffi/interp_rawffi.py Fri Nov 20 21:35:04 2009 @@ -106,7 +106,7 @@ resshape = cache.get_array_type(letter2tp(space, letter)) else: letter = 'V' - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) from pypy.module._rawffi.structure import W_Structure resshape = space.interp_w(W_Structure, w_shapetype) ffi_type = resshape.get_ffi_type() @@ -117,7 +117,7 @@ letter = space.str_w(w_shape) return letter2tp(space, letter) else: - w_shapetype, w_length = space.viewiterable(w_shape, expected_length=2) + w_shapetype, w_length = space.fixedview(w_shape, expected_length=2) resshape = space.interp_w(W_DataShape, w_shapetype) length = space.int_w(w_length) size, alignment = resshape._size_alignment() @@ -155,7 +155,7 @@ """ ffi_restype, resshape = unpack_resshape(space, w_restype) w = space.wrap - argtypes_w = space.viewiterable(w_argtypes) + argtypes_w = space.fixedview(w_argtypes) w_argtypes = space.newtuple(argtypes_w) w_key = space.newtuple([w_name, w_argtypes, w(resshape)]) try: Modified: pypy/branch/shorter-guard-path/pypy/module/_rawffi/structure.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/_rawffi/structure.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/_rawffi/structure.py Fri Nov 20 21:35:04 2009 @@ -119,7 +119,7 @@ def descr_new_structure(space, w_type, w_shapeinfo): if space.is_true(space.isinstance(w_shapeinfo, space.w_tuple)): - w_size, w_alignment = space.viewiterable(w_shapeinfo, expected_length=2) + w_size, w_alignment = space.fixedview(w_shapeinfo, expected_length=2) S = W_Structure(space, None, space.int_w(w_size), space.int_w(w_alignment)) else: Modified: pypy/branch/shorter-guard-path/pypy/module/_stackless/test/test_pickle_infrastructure.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/_stackless/test/test_pickle_infrastructure.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/_stackless/test/test_pickle_infrastructure.py Fri Nov 20 21:35:04 2009 @@ -129,7 +129,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -188,7 +188,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code @@ -260,7 +260,7 @@ return co, f, g """) - w_co, w_f, w_g = space.viewiterable(w_res) + w_co, w_f, w_g = space.fixedview(w_res) ec = space.getexecutioncontext() fcode = w_f.code.co_code Modified: pypy/branch/shorter-guard-path/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/micronumpy/numarray.py Fri Nov 20 21:35:04 2009 @@ -73,7 +73,7 @@ make_sure_not_resized(self.storage) def _unpack_indexes(self, space, w_index): - indexes = [space.int_w(w_i) for w_i in space.viewiterable(w_index)] + indexes = [space.int_w(w_i) for w_i in space.fixedview(w_index)] if len(indexes) != len(self.dim): raise OperationError(space.w_IndexError, space.wrap( 'Wrong index')) @@ -108,7 +108,7 @@ def unpack_dim(space, w_dim): if space.is_true(space.isinstance(w_dim, space.w_int)): return [space.int_w(w_dim)] - dim_w = space.viewiterable(w_dim) + dim_w = space.fixedview(w_dim) return [space.int_w(w_i) for w_i in dim_w] def unpack_dtype(space, w_dtype): Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/__init__.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/__init__.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/__init__.py Fri Nov 20 21:35:04 2009 @@ -9,7 +9,13 @@ 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', 'DATETIME': 'interp_variable.VT_DateTime', + 'BINARY': 'interp_variable.VT_Binary', + 'LONG_STRING': 'interp_variable.VT_LongString', + 'LONG_BINARY': 'interp_variable.VT_LongBinary', + 'FIXED_CHAR': 'interp_variable.VT_FixedChar', 'Variable': 'interp_variable.W_Variable', + 'Timestamp': 'interp_error.get(space).w_DateTimeType', + 'Date': 'interp_error.get(space).w_DateType', } appleveldefs = { Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/config.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/config.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/config.py Fri Nov 20 21:35:04 2009 @@ -3,6 +3,7 @@ WITH_UNICODE = False MAX_STRING_CHARS = 4000 +MAX_BINARY_BYTES = 4000 if WITH_UNICODE: CHARSETID = roci.OCI_UTF16ID Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/interp_connect.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/interp_connect.py Fri Nov 20 21:35:04 2009 @@ -12,6 +12,7 @@ from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS from pypy.module.oracle.interp_environ import Environment from pypy.module.oracle.interp_cursor import W_Cursor +from pypy.module.oracle.interp_pool import W_Pool from pypy.module.oracle.interp_variable import VT_String class W_Connection(Wrappable): @@ -42,7 +43,7 @@ W_Connection.__init__(self) # set up the environment - if w_pool: + if 0 and w_pool: # XXX pool = space.instance_w(W_Pool, w_pool) self.environment = pool.environment.clone() else: @@ -55,13 +56,13 @@ # perform some parsing, if necessary if (self.w_username and not self.w_password and space.is_true(space.contains(self.w_username, space.wrap('/')))): - (self.w_username, self.w_password) = space.unpackiterable( + (self.w_username, self.w_password) = space.listview( space.call_method(self.w_username, 'split', space.wrap('/'), space.wrap(1))) - + if (self.w_password and not self.w_tnsentry and space.is_true(space.contains(self.w_password, space.wrap('@')))): - (self.w_password, self.w_tnsentry) = space.unpackiterable( + (self.w_password, self.w_tnsentry) = space.listview( space.call_method(self.w_password, 'split', space.wrap('@'), space.wrap(1))) @@ -149,6 +150,8 @@ finally: lltype.free(handleptr, flavor='raw') + credentialType = roci.OCI_CRED_EXT + # set user name in session handle stringBuffer.fill(space, self.w_username) try: @@ -263,32 +266,51 @@ def _getCharacterSetName(self, space, attribute): # get character set id - status = roci.OCIAttrGet( - self.environment.handle, roci.HTYPE_ENV, - charsetId, None, - attribute, - self.environment.errorHandle) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): get charset id") + charsetIdPtr = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.environment.handle, roci.OCI_HTYPE_ENV, + rffi.cast(roci.dvoidp, charsetIdPtr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + attribute, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetCharacterSetName(): get charset id") + charsetId = charsetIdPtr[0] + finally: + lltype.free(charsetIdPtr, flavor='raw') # get character set name - status = roci.OCINlsCharsetIdToName( - self.environmentHandle, - charsetNameBuf.buf, charsetNameBuf.size, - charsetIdPtr[0]) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): get Oracle charset name") + charsetname_buf, charsetname = rffi.alloc_buffer(roci.OCI_NLS_MAXBUFSZ) + try: + status = roci.OCINlsCharSetIdToName( + self.environment.handle, + charsetname_buf, roci.OCI_NLS_MAXBUFSZ, + charsetId) + self.environment.checkForError( + status, + "Connection_GetCharacterSetName(): get Oracle charset name") - # get IANA character set name - status = roci.OCINlsNameMap( - self.environmentHandle, - ianaCharsetNameBuf.buf, inaCharsetNameBuf.size, - charsetNameBuf.buf, roci.OCI_NLS_CS_ORA_TO_IANA) - self.environment.checkForError( - status, "Connection_GetCharacterSetName(): translate NLS charset") + ianacharset_buf, ianacharset = rffi.alloc_buffer( + roci.OCI_NLS_MAXBUFSZ) + + try: + # get IANA character set name + status = roci.OCINlsNameMap( + self.environment.handle, + ianacharset_buf, roci.OCI_NLS_MAXBUFSZ, + charsetname_buf, roci.OCI_NLS_CS_ORA_TO_IANA) + self.environment.checkForError( + status, + "Connection_GetCharacterSetName(): translate NLS charset") + charset = rffi.charp2str(ianacharset_buf) + finally: + rffi.keep_buffer_alive_until_here(ianacharset_buf, ianacharset) + finally: + rffi.keep_buffer_alive_until_here(charsetname_buf, charsetname) + return space.wrap(charset) - return space.wrap(ianaCharsetName) - def get_encoding(space, self): return self._getCharacterSetName(space, roci.OCI_ATTR_ENV_CHARSET_ID) def get_nationalencoding(space, self): @@ -302,7 +324,7 @@ return self.w_version # allocate a cursor to retrieve the version - cursor = self.newCursor(space) + cursor = W_Cursor(space, self) # allocate version and compatibility variables versionVar = VT_String(cursor, cursor.arraySize, MAX_STRING_CHARS) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/interp_cursor.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/interp_cursor.py Fri Nov 20 21:35:04 2009 @@ -29,6 +29,8 @@ self.bindList = None self.bindDict = None self.numbersAsStrings = False + self.outputSize = -1 + self.outputSizeColumn = -1 self.inputTypeHandler = None self.outputTypeHandler = None @@ -126,19 +128,20 @@ # queries are not supported as the result is undefined if self.statementType == roci.OCI_STMT_SELECT: raise OperationError( - w_NotSupportedErrorException, + get(space).w_NotSupportedErrorException, space.wrap("queries not supported: results undefined")) # perform binds - numrows = space.int_w(space.len(w_list_of_args)) - for i, arguments in enumerate(space.viewiterable(w_list_of_args)): + args_w = space.listview(w_list_of_args) + numrows = len(args_w) + for i, w_arguments in enumerate(args_w): deferred = i < numrows - 1 - if space.is_true(space.isinstance(arguments, space.w_dict)): + if space.is_true(space.isinstance(w_arguments, space.w_dict)): self._setBindVariablesByName( - space, arguments, numrows, i, deferred) + space, w_arguments, numrows, i, deferred) else: self._setBindVariablesByPos( - space, arguments, numrows, i, deferred) + space, w_arguments, numrows, i, deferred) self._performBind(space) # execute the statement, but only if the number of rows is greater than @@ -205,8 +208,8 @@ w_vars = w_args # build up the statement - args = ', '.join(':%d' % (i + offset + 1,) - for i in range(numArguments)) + args = ', '.join([':%d' % (i + offset + 1,) + for i in range(numArguments)]) if retvar: stmt = "begin :1 := %s(%s); end;" % (name, args) else: @@ -224,7 +227,7 @@ if not self.handle: return if self.isOwned: - roci.OciHandleFree(self.handle, OCI_HTYPE_STMT) + roci.OCIHandleFree(self.handle, roci.OCI_HTYPE_STMT) elif self.connection.handle: tagBuffer = StringBuffer() tagBuffer.fill(space, self.w_statementTag) @@ -544,7 +547,7 @@ if self.bindList is None: self.bindList = [] - for i, w_value in enumerate(space.viewiterable(w_vars)): + for i, w_value in enumerate(space.fixedview(w_vars)): if i < len(self.bindList): origVar = self.bindList[i] if space.is_w(origVar, space.w_None): @@ -572,9 +575,9 @@ if self.bindDict is None: self.bindDict = space.newdict() - items = space.viewiterable(space.call_method(w_vars, "iteritems")) + items = space.fixedview(space.call_method(w_vars, "iteritems")) for item in items: - w_key, w_value = space.viewiterable(item, 2) + w_key, w_value = space.fixedview(item, 2) origVar = space.finditem(self.bindDict, w_key) newVar = self._setBindVariableHelper(space, w_value, origVar, numElements, arrayPos, defer) @@ -641,10 +644,10 @@ for i, var in enumerate(self.bindList): var.bind(space, self, None, i + 1) if self.bindDict: - items = space.viewiterable( + items_w = space.fixedview( space.call_method(self.bindDict, "iteritems")) - for item in items: - w_key, var = space.viewiterable(item, 2) + for w_item in items_w: + w_key, var = space.fixedview(w_item, 2) var.bind(space, self, w_key, 0) # ensure that input sizes are reset @@ -733,6 +736,18 @@ return space.w_None fetchone.unwrap_spec = ['self', ObjSpace] + def fetchmany(self, space, w_numRows=NoneNotWrapped): + if w_numRows is not None: + numRows = space.int_w(w_numRows) + else: + numRows = self.arraySize + + # verify fetch can be performed + self._verifyFetch(space) + + return self._multiFetch(space, limit=numRows) + fetchmany.unwrap_spec = ['self', ObjSpace, W_Root] + def fetchall(self, space): # verify fetch can be performed self._verifyFetch(space) @@ -811,6 +826,7 @@ # fetch as many rows as possible while limit is None or rowNum < limit: + rowNum += 1 if not self._moreRows(space): break w_row = self._createRow(space) @@ -943,7 +959,7 @@ numElements = space.int_w(w_value) else: raise OperationError( - w_NotSupportedErrorException, + get(space).w_NotSupportedErrorException, space.wrap("expecting integer or list of values")) # create the variable @@ -974,7 +990,7 @@ self.bindList = None self.bindDict = None - self.setInputSizes = 1 + self.setInputSizes = True # process each input if kw_w: @@ -996,21 +1012,30 @@ return space.newlist(self.bindList) setinputsizes.unwrap_spec = ['self', ObjSpace, Arguments] -def cursor_arraysize_get(space, obj): - return space.wrap(obj.arraySize) -def cursor_arraysize_set(space, obj, w_value): - obj.arraySize = space.int_w(w_value) - -def cursor_bindarraysize_get(space, obj): - return space.wrap(obj.bindArraySize) -def cursor_bindarraysize_set(space, obj, w_value): - obj.bindArraySize = space.int_w(w_value) - -def cursor_bindvars_get(space, obj): - if obj.bindList: - return space.newlist(obj.bindList) - if obj.bindDict: - return obj.bindDict + def setoutputsize(self, space, outputSize, outputSizeColumn=-1): + self.outputSize = outputSize + self.outputSizeColumn = outputSizeColumn + setoutputsize.unwrap_spec = ['self', ObjSpace, int, int] + + + def arraysize_get(space, self): + return space.wrap(self.arraySize) + def arraysize_set(space, self, w_value): + self.arraySize = space.int_w(w_value) + + def bindarraysize_get(space, self): + return space.wrap(self.bindArraySize) + def bindarraysize_set(space, self, w_value): + self.bindArraySize = space.int_w(w_value) + + def bindvars_get(space, self): + if self.bindList: + return space.newlist(self.bindList) + if self.bindDict: + return self.bindDict + + def fetchvars_get(space, self): + return space.newlist(self.fetchVariables) W_Cursor.typedef = TypeDef( 'Cursor', @@ -1022,6 +1047,8 @@ unwrap_spec=W_Cursor.prepare.unwrap_spec), fetchone = interp2app(W_Cursor.fetchone, unwrap_spec=W_Cursor.fetchone.unwrap_spec), + fetchmany = interp2app(W_Cursor.fetchmany, + unwrap_spec=W_Cursor.fetchmany.unwrap_spec), fetchall = interp2app(W_Cursor.fetchall, unwrap_spec=W_Cursor.fetchall.unwrap_spec), close = interp2app(W_Cursor.close, @@ -1038,14 +1065,19 @@ unwrap_spec=W_Cursor.arrayvar.unwrap_spec), setinputsizes = interp2app(W_Cursor.setinputsizes, unwrap_spec=W_Cursor.setinputsizes.unwrap_spec), + setoutputsize = interp2app(W_Cursor.setoutputsize, + unwrap_spec=W_Cursor.setoutputsize.unwrap_spec), __iter__ = interp2app(W_Cursor.descr_iter), next = interp2app(W_Cursor.descr_next), - arraysize = GetSetProperty(cursor_arraysize_get, cursor_arraysize_set), - bindarraysize = GetSetProperty(cursor_bindarraysize_get, cursor_bindarraysize_set), + arraysize = GetSetProperty(W_Cursor.arraysize_get, + W_Cursor.arraysize_set), + bindarraysize = GetSetProperty(W_Cursor.bindarraysize_get, + W_Cursor.bindarraysize_set), rowcount = interp_attrproperty('rowCount', W_Cursor), statement = interp_attrproperty_w('w_statement', W_Cursor), - bindvars = GetSetProperty(cursor_bindvars_get), + bindvars = GetSetProperty(W_Cursor.bindvars_get), + fetchvars = GetSetProperty(W_Cursor.fetchvars_get), description = GetSetProperty(W_Cursor.getDescription), ) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/interp_error.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/interp_error.py Fri Nov 20 21:35:04 2009 @@ -25,6 +25,7 @@ w_datetime = space.call(w_import, space.newlist( [space.wrap('datetime')])) self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) + self.w_DateType = space.getattr(w_datetime, space.wrap("date")) def get(space): Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/interp_variable.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/interp_variable.py Fri Nov 20 21:35:04 2009 @@ -144,12 +144,12 @@ self.size = size # allocate the data for the variable - self.allocateData() + self.allocateData(self.environment.space) # allocate the indicator for the variable self.indicator = lltype.malloc(rffi.CArrayPtr(roci.sb2).TO, self.allocatedElements, - flavor='raw', zero=True) # XXX + flavor='raw', zero=True) # ensure that all variable values start out NULL for i in range(self.allocatedElements): @@ -159,7 +159,7 @@ if self.isVariableLength: self.returnCode = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, self.allocatedElements, - flavor='raw', zero=True) # XXX + flavor='raw', zero=True) # perform extended initialization self.initialize(cursor) @@ -170,28 +170,33 @@ lltype.free(self.actualLength, flavor='raw') if self.data: lltype.free(self.data, flavor='raw') + if self.returnCode: + lltype.free(self.returnCode, flavor='raw') def getBufferSize(self): return self.size - def allocateData(self): + def allocateData(self, space): # set the buffer size for the variable self.bufferSize = self.getBufferSize() # allocate the data as long as it is small enough - dataLength = ovfcheck(self.allocatedElements * self.bufferSize) - if dataLength > sys.maxint: - raise ValueError("array size too large") + try: + dataLength = ovfcheck(self.allocatedElements * self.bufferSize) + except OverflowError: + raise OperationError( + space.w_ValueError, + space.wrap("array size too large")) self.data = lltype.malloc(rffi.CCHARP.TO, int(dataLength), flavor='raw', zero=True) - def resize(self, size): + def resize(self, space, size): # allocate the data for the new array orig_data = self.data orig_size = self.bufferSize self.size = size - self.allocateData() + self.allocateData(space) # copy the data from the original array to the new array for i in range(self.allocatedElements): @@ -290,13 +295,13 @@ error = W_Error(space, self.environment, "Variable_VerifyFetch()", 0) error.code = self.returnCode[pos] - error.message = self.space.wrap( + error.message = space.wrap( "column at array pos %d fetched with error: %d" % (pos, self.returnCode[pos])) - w_error = get(self.space).w_DatabaseError + w_error = get(space).w_DatabaseError - raise OperationError(get(self.space).w_DatabaseError, - self.space.wrap(error)) + raise OperationError(get(space).w_DatabaseError, + space.wrap(error)) def getSingleValue(self, space, pos): # ensure we do not exceed the number of allocated elements @@ -356,7 +361,7 @@ space.w_TypeError, space.wrap("expecting array data")) - elements_w = space.viewiterable(w_value) + elements_w = space.listview(w_value) # ensure we haven't exceeded the number of allocated elements if len(elements_w) > self.allocatedElements: @@ -384,8 +389,9 @@ setvalue = interp2app(W_Variable.setValue, unwrap_spec=W_Variable.setValue.unwrap_spec), - maxlength = interp_attrproperty('bufferSize', W_Variable), - + maxlength = interp_attrproperty('bufferSize', W_Variable), + bufferSize = interp_attrproperty('bufferSize', W_Variable), + size = interp_attrproperty('size', W_Variable), ) class VT_String(W_Variable): @@ -462,7 +468,7 @@ # ensure that the buffer is large enough if buf.size > self.bufferSize: - self.resize(size) + self.resize(space, size) # keep a copy of the string self.actualLength[pos] = rffi.cast(roci.ub2, buf.size) @@ -480,7 +486,36 @@ pass class VT_LongString(W_Variable): - pass + oracleType = roci.SQLT_LVC + isVariableLength = True + size = 128 * 1024 + + def getBufferSize(self): + return self.size + rffi.sizeof(roci.ub4) + + def getValueProc(self, space, pos): + ptr = rffi.ptradd(self.data, pos * self.bufferSize) + length = rffi.cast(roci.Ptr(roci.ub4), ptr)[0] + + ptr = rffi.ptradd(ptr, rffi.sizeof(roci.ub4)) + return space.wrap(rffi.charpsize2str(ptr, length)) + + def setValueProc(self, space, pos, w_value): + buf = config.StringBuffer() + buf.fill(space, w_value) + + try: + # ensure that the buffer is large enough + if buf.size + rffi.sizeof(roci.ub4) > self.bufferSize: + self.resize(space, buf.size + rffi.sizeof(roci.ub4)) + + # copy the string to the Oracle buffer + ptr = rffi.ptradd(self.data, pos * self.bufferSize) + rffi.cast(roci.Ptr(roci.ub4), ptr)[0] = rffi.cast(roci.ub4, buf.size) + for index in range(buf.size): + ptr[index + rffi.sizeof(roci.ub4)] = buf.ptr[index] + finally: + buf.clear() class VT_FixedNationalChar(W_Variable): pass @@ -490,11 +525,12 @@ size = 18 isVariableLength = False -class VT_Binary(W_Variable): - pass +class VT_Binary(VT_String): + oracleType = roci.SQLT_BIN + size = config.MAX_BINARY_BYTES -class VT_LongBinary(W_Variable): - pass +class VT_LongBinary(VT_LongString): + oracleType = roci.SQLT_LVB class VT_NativeFloat(W_Variable): pass @@ -542,9 +578,10 @@ lltype.free(attrptr, flavor='raw') if scale == 0 or (scale == -127 and precision == 0): - return VT_LongInteger - elif precision > 0 and precision < 10: - return VT_Integer + if precision > 0 and precision < 10: + return VT_Integer + else: + return VT_LongInteger return cls @@ -592,8 +629,13 @@ rffi.cast(lltype.Signed, sizeptr[0]))) if isinstance(self, VT_NumberAsString): return w_strvalue - else: + + try: return space.call_function(space.w_int, w_strvalue) + except OperationError, e: + if e.match(space, space.w_ValueError): + return space.call_function(space.w_float, w_strvalue) + raise finally: rffi.keep_buffer_alive_until_here(textbuf, text) lltype.free(sizeptr, flavor='raw') @@ -693,7 +735,6 @@ dataptr = rffi.ptradd( rffi.cast(roci.Ptr(roci.OCIDate), self.data), pos) - dataptr = rffi.cast(roci.Ptr(roci.OCIDate), self.data) return transform.OracleDateToPythonDateTime(self.environment, dataptr) def setValueProc(self, space, pos, w_value): @@ -708,14 +749,18 @@ minute = space.int_w(space.getattr(w_value, space.wrap('minute'))) second = space.int_w(space.getattr(w_value, space.wrap('second'))) elif space.is_true(space.isinstance(w_value, get(space).w_DateType)): - XXX + year = space.int_w(space.getattr(w_value, space.wrap('year'))) + month = space.int_w(space.getattr(w_value, space.wrap('month'))) + day = space.int_w(space.getattr(w_value, space.wrap('day'))) + hour = minute = second = 0 else: raise OperationError( space.w_TypeError, space.wrap("expecting date data")) # store a copy of the value - timePart = dataptr[0].c_OCIDateTime + value = dataptr[0] + timePart = value.c_OCIDateTime rffi.setintfield(timePart, 'c_OCITimeHH', hour) rffi.setintfield(timePart, 'c_OCITimeMI', minute) rffi.setintfield(timePart, 'c_OCITimeSS', second) @@ -723,7 +768,7 @@ rffi.setintfield(dataptr[0], 'c_OCIDateMM', month) rffi.setintfield(dataptr[0], 'c_OCIDateDD', day) -class VT_Date(W_Variable): +class VT_Date(VT_DateTime): oracleType = roci.SQLT_ODT size = rffi.sizeof(roci.OCIDate) @@ -869,7 +914,10 @@ if space.is_true(space.isinstance(w_value, space.w_str)): size = space.int_w(space.len(w_value)) - return VT_String, size, numElements + if size > config.MAX_STRING_CHARS: + return VT_LongString, size, numElements + else: + return VT_String, size, numElements # XXX Unicode @@ -889,7 +937,8 @@ if space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): return VT_DateTime, 0, numElements - # XXX date + if space.is_true(space.isinstance(w_value, get(space).w_DateType)): + return VT_Date, 0, numElements # XXX Delta @@ -900,7 +949,7 @@ # handle arrays if space.is_true(space.isinstance(w_value, space.w_list)): - elements_w = space.viewiterable(w_value) + elements_w = space.listview(w_value) for w_element in elements_w: if not space.is_w(w_element, space.w_None): break @@ -933,7 +982,7 @@ def newArrayVariableByType(space, cursor, w_value): "Allocate a new PL/SQL array by looking at the Python data type." - w_type, w_numElements = space.viewiterable(w_value, 2) + w_type, w_numElements = space.fixedview(w_value, 2) numElements = space.int_w(w_numElements) varType = typeByPythonType(space, cursor, w_type) @@ -963,4 +1012,4 @@ # everything else ought to be a Python type varType = typeByPythonType(space, cursor, w_value) - return varType(cursor, numElements) + return varType(cursor, numElements, varType.size) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/roci.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/roci.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/roci.py Fri Nov 20 21:35:04 2009 @@ -30,7 +30,7 @@ ub4 = platform.SimpleType('ub4', rffi.UINT) sb4 = platform.SimpleType('sb4', rffi.INT) sword = platform.SimpleType('sword', rffi.INT) - uword = platform.SimpleType('sword', rffi.UINT) + uword = platform.SimpleType('uword', rffi.UINT) OCINumber = platform.Struct('OCINumber', []) OCITime = platform.Struct('OCITime', @@ -49,26 +49,28 @@ OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION - OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_DTYPE_PARAM - OCI_CRED_RDBMS + OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM + OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE - SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI + SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB SQLT_BFLOAT SQLT_IBFLOAT SQLT_BDOUBLE SQLT_IBDOUBLE SQLT_NUM SQLT_VNU SQLT_DAT SQLT_ODT SQLT_DATE SQLT_TIMESTAMP SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY SQLCS_IMPLICIT SQLCS_NCHAR OCI_NUMBER_SIGNED + OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA '''.split() for c in constants: @@ -394,3 +396,21 @@ oratext], # buf sword) +# OCI Locale Functions + +OCINlsCharSetIdToName = external( + 'OCINlsCharSetIdToName', + [dvoidp, # hndl + oratext, # buf + size_t, # buflen + ub2], # id + sword) + +OCINlsNameMap = external( + 'OCINlsNameMap', + [dvoidp, # hndl + oratext, # buf + size_t, # buflen + oratext, # srcbuf + uword], # flag + sword) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_connect.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_connect.py Fri Nov 20 21:35:04 2009 @@ -105,4 +105,14 @@ count, = cursor.fetchone() assert count == 0 + def test_charset(self): + self.cnx = oracle.connect(self.username, self.password, + self.tnsentry) + encoding = self.cnx.encoding + assert isinstance(encoding, str) + assert encoding != "" + encoding = self.cnx.nationalencoding + assert isinstance(encoding, str) + assert encoding != "" + Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_cursor.py Fri Nov 20 21:35:04 2009 @@ -174,3 +174,8 @@ ('NULLABLECOL', oracle.NUMBER, 39, 22, 38, 0, 1), ] assert got == expected + + def test_outputsize(self): + cur = self.cnx.cursor() + cur.setoutputsize(25000) + cur.setoutputsize(25000, 2) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_datetimevar.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_datetimevar.py Fri Nov 20 21:35:04 2009 @@ -2,7 +2,7 @@ class AppTestDatetime(OracleTestBase): - def test_bind_date(self): + def test_bind_datetime(self): import datetime cur = self.cnx.cursor() cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", @@ -10,3 +10,31 @@ data = cur.fetchall() assert data == [('20021213-093615',)] + def test_bind_date(self): + import datetime + cur = self.cnx.cursor() + cur.execute("select to_char(:d, 'YYYYMMDD-HH24MISS') from dual", + d=oracle.Date(2002, 12, 13)) + data = cur.fetchall() + assert data == [('20021213-000000',)] + + def test_variable(self): + import datetime + cur = self.cnx.cursor() + var = cur.var(oracle.DATETIME) + value = datetime.datetime(2002, 12, 13, 9, 36, 15) + var.setvalue(0, value) + assert var.getvalue() == value + + value = datetime.date(2002, 12, 13) + var.setvalue(0, value) + assert var.getvalue() == datetime.datetime(2002, 12, 13) + + def test_arrayvar(self): + import datetime + cur = self.cnx.cursor() + var = cur.arrayvar(oracle.DATETIME, 2) + values = [datetime.datetime(2002, 12, 13, 9, 36, 15), + datetime.datetime(2003, 7, 22, 4, 24, 32)] + var.setvalue(0, values) + assert var.getvalue() == values Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_numbervar.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_numbervar.py Fri Nov 20 21:35:04 2009 @@ -2,6 +2,12 @@ class AppTestNumberVar(OracleTestBase): + def test_fetch(self): + cur = self.cnx.cursor() + cur.execute("select 1.5 from dual") + value, = cur.fetchone() + assert value == 1.5 + def test_float(self): cur = self.cnx.cursor() var = cur.var(oracle.NUMBER) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_select.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_select.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_select.py Fri Nov 20 21:35:04 2009 @@ -6,10 +6,13 @@ cur = self.cnx.cursor() cur.execute("select 42, 'Hello' from dual") row = cur.fetchone() - assert isinstance(row[0], float) + assert isinstance(row[0], int) assert isinstance(row[1], str) assert row == (42, 'Hello') + assert isinstance(cur.fetchvars[0], oracle.Variable) + assert isinstance(cur.fetchvars[1], oracle.Variable) + def test_sysdate(self): import datetime cur = self.cnx.cursor() @@ -19,7 +22,7 @@ assert isinstance(sysdate, datetime.datetime) delta = abs(sysdate - datetime.datetime.now()) assert delta < datetime.timedelta(seconds=2) - + def test_fetchall(self): cur = self.cnx.cursor() # An Oracle trick to retrieve 42 lines @@ -28,12 +31,25 @@ assert rows == zip(range(42)) assert cur.rowcount == 42 + def test_fetchmany(self): + cur = self.cnx.cursor() + cur.execute("select level-1 from dual connect by level-1<442") + rows = cur.fetchmany() + assert rows == zip(range(cur.arraysize)) + rows = cur.fetchmany(3) + assert rows == zip(range(cur.arraysize, cur.arraysize+3)) + assert cur.rowcount == cur.arraysize+3 + def test_iterator(self): cur = self.cnx.cursor() - # An Oracle trick to retrieve 42 lines cur.execute("select level-1 from dual connect by level-1<42") for i, row in enumerate(cur): assert row == (i,) assert i == 41 - + def test_arraysize_too_large(self): + cur = self.cnx.cursor() + cur.arraysize = 2 ** 20 + largevar = cur.var(oracle.STRING) + raises(ValueError, + cur.execute, "select :large from dual", large=largevar) Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/test/test_stringvar.py Fri Nov 20 21:35:04 2009 @@ -2,9 +2,18 @@ class AppTestStringVar(OracleTestBase): + def test_bind_inout(self): + cur = self.cnx.cursor() + vars = cur.setinputsizes(value=oracle.STRING) + cur.execute(""" + begin + :value := :value || ' output'; + end;""", + value="input") + assert vars["value"].getvalue() == "input output" + def test_rowid(self): cur = self.cnx.cursor() - var = cur.var(oracle.NUMBER) cur.execute("select rowid from dual") rowid, = cur.fetchone() cur.execute("select * from dual where rowid = :r", @@ -92,3 +101,32 @@ assert tablelen.getvalue() == 20 # dbms_utility.comma_to_table returns a 'NULL-terminated' table assert arrayvar.getvalue() == array + [None] + + def test_binary(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (rawcol raw(30))") + + cur.setinputsizes(p=oracle.BINARY) + cur.execute("insert into pypy_temp_table values (:p)", + p="raw string") + cur.execute("select * from pypy_temp_table") + data = cur.fetchall() + assert data == [("raw string",)] + + def test_longstring(self): + cur = self.cnx.cursor() + output = cur.var(oracle.LONG_STRING) + cur.execute(""" + declare + t_Temp varchar2(10000); + begin + t_Temp := :bigString; + :output := t_Temp; + end;""", + bigString="X" * 10000, output=output) + assert output.getvalue() == "X" * 10000 + Modified: pypy/branch/shorter-guard-path/pypy/module/oracle/transform.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/oracle/transform.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/oracle/transform.py Fri Nov 20 21:35:04 2009 @@ -63,12 +63,12 @@ w_tuple_value = space.call_method(w_value, "as_tuple") # acquire basic information from the value tuple - w_sign, w_digits, w_scale = space.viewiterable(w_tuple_value) + w_sign, w_digits, w_scale = space.fixedview(w_tuple_value, 3) text = '' format = '' - digits_w = space.viewiterable(w_digits) + digits_w = space.listview(w_digits) num_digits = len(digits_w) scale = space.int_w(w_scale) Modified: pypy/branch/shorter-guard-path/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/posix/interp_posix.py Fri Nov 20 21:35:04 2009 @@ -573,7 +573,7 @@ raise wrap_oserror(space, e) try: msg = "utime() arg 2 must be a tuple (atime, mtime) or None" - args_w = space.unpackiterable(w_tuple) + args_w = space.fixedview(w_tuple) if len(args_w) != 2: raise OperationError(space.w_TypeError, space.wrap(msg)) actime = space.float_w(args_w[0]) Modified: pypy/branch/shorter-guard-path/pypy/module/rctime/interp_time.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/rctime/interp_time.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/rctime/interp_time.py Fri Nov 20 21:35:04 2009 @@ -222,7 +222,7 @@ space.wrap(_get_error_msg())) return pbuf - tup_w = space.unpackiterable(w_tup) + tup_w = space.fixedview(w_tup) if len(tup_w) != 9: raise OperationError(space.w_TypeError, space.wrap("argument must be sequence of " Modified: pypy/branch/shorter-guard-path/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/module/select/interp_select.py (original) +++ pypy/branch/shorter-guard-path/pypy/module/select/interp_select.py Fri Nov 20 21:35:04 2009 @@ -113,9 +113,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.unpackiterable(w_iwtd) - owtd_w = space.unpackiterable(w_owtd) - ewtd_w = space.unpackiterable(w_ewtd) + iwtd_w = space.listview(w_iwtd) + owtd_w = space.listview(w_owtd) + ewtd_w = space.listview(w_ewtd) iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] owtd = [as_fd_w(space, w_f) for w_f in owtd_w] ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] Modified: pypy/branch/shorter-guard-path/pypy/objspace/flow/objspace.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/flow/objspace.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/flow/objspace.py Fri Nov 20 21:35:04 2009 @@ -202,7 +202,7 @@ # the simple case return ObjSpace.exception_match(self, w_exc_type, w_check_class) # checking a tuple of classes - for w_klass in self.viewiterable(w_check_class): + for w_klass in self.fixedview(w_check_class): if ObjSpace.exception_match(self, w_exc_type, w_klass): return True return False @@ -263,8 +263,9 @@ checkgraph(graph) return graph - def viewiterable(self, w_tuple, expected_length=None): + def fixedview(self, w_tuple, expected_length=None): return self.unpackiterable(w_tuple, expected_length) + listview = fixedview def unpackiterable(self, w_iterable, expected_length=None): if not isinstance(w_iterable, Variable): Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/dictmultiobject.py Fri Nov 20 21:35:04 2009 @@ -650,9 +650,9 @@ if w_src is None: pass elif space.findattr(w_src, space.wrap("keys")) is None: - list_of_w_pairs = space.unpackiterable(w_src) + list_of_w_pairs = space.listview(w_src) for w_pair in list_of_w_pairs: - pair = space.unpackiterable(w_pair) + pair = space.fixedview(w_pair) if len(pair)!=2: raise OperationError(space.w_ValueError, space.wrap("dict() takes a sequence of pairs")) @@ -793,7 +793,7 @@ return w_default def dict_pop__DictMulti_ANY(space, w_dict, w_key, w_defaults): - defaults = space.unpackiterable(w_defaults) + defaults = space.listview(w_defaults) len_defaults = len(defaults) if len_defaults > 1: raise OperationError(space.w_TypeError, space.wrap("pop expected at most 2 arguments, got %d" % (1 + len_defaults, ))) Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/floatobject.py Fri Nov 20 21:35:04 2009 @@ -2,7 +2,7 @@ from pypy.interpreter import gateway from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject -from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf +from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan from pypy.rlib.rarithmetic import formatd import math @@ -197,6 +197,9 @@ def _hash_float(space, v): from pypy.objspace.std.longobject import hash__Long + if isnan(v): + return 0 + # This is designed so that Python numbers of different types # that compare equal hash to the same value; otherwise comparisons # of mapping keys will turn out weird. Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/formatting.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/formatting.py Fri Nov 20 21:35:04 2009 @@ -495,7 +495,7 @@ def mod_format(space, w_format, w_values, do_unicode=False): if space.is_true(space.isinstance(w_values, space.w_tuple)): - values_w = space.unpackiterable(w_values) + values_w = space.fixedview(w_values) return format(space, w_format, values_w, None, do_unicode) else: # we check directly for dict to avoid obscure checking Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/inlinedict.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/inlinedict.py Fri Nov 20 21:35:04 2009 @@ -45,7 +45,7 @@ # XXX sucky items = [] for w_item in self.impl_items(): - w_key, w_value = self.space.viewiterable(w_item) + w_key, w_value = self.space.fixedview(w_item) items.append((w_key, w_value)) return IndirectionIterImplementation(self.space, self, items) Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/listobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/listobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/listobject.py Fri Nov 20 21:35:04 2009 @@ -260,11 +260,7 @@ _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable) def _setitem_slice_helper(space, w_list, start, step, slicelength, w_iterable): - if isinstance(w_iterable, W_ListObject): - sequence2 = w_iterable.wrappeditems - else: - sequence2 = space.unpackiterable(w_iterable) - + sequence2 = space.listview(w_iterable) assert slicelength >= 0 items = w_list.wrappeditems oldsize = len(items) @@ -357,7 +353,7 @@ return space.w_None def list_extend__List_ANY(space, w_list, w_any): - w_list.wrappeditems += space.unpackiterable(w_any) + w_list.wrappeditems += space.listview(w_any) return space.w_None # note that the default value will come back wrapped!!! Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/marshal_impl.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/marshal_impl.py Fri Nov 20 21:35:04 2009 @@ -356,7 +356,7 @@ def marshal_w__DictMulti(space, w_dict, m): m.start(TYPE_DICT) for w_tuple in w_dict.items(): - w_key, w_value = space.viewiterable(w_tuple, 2) + w_key, w_value = space.fixedview(w_tuple, 2) m.put_w_obj(w_key) m.put_w_obj(w_value) m.atom(TYPE_NULL) @@ -469,14 +469,14 @@ def marshal_w_set(space, w_set, m): # cannot access this list directly, because it's # type is not exactly known through applevel. - lis_w = space.viewiterable(w_set) + lis_w = space.fixedview(w_set) m.put_tuple_w(TYPE_SET, lis_w) handled_by_any.append( ('set', marshal_w_set) ) # not directly supported: def marshal_w_frozenset(space, w_frozenset, m): - lis_w = space.viewiterable(w_frozenset) + lis_w = space.fixedview(w_frozenset) m.put_tuple_w(TYPE_FROZENSET, lis_w) handled_by_any.append( ('frozenset', marshal_w_frozenset) ) Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/objspace.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/objspace.py Fri Nov 20 21:35:04 2009 @@ -383,7 +383,7 @@ space = self # too early for unpackiterable as well :-( name = space.unwrap(space.getitem(w_args, space.wrap(0))) - bases = space.viewiterable(space.getitem(w_args, space.wrap(1))) + bases = space.fixedview(space.getitem(w_args, space.wrap(1))) dic = space.unwrap(space.getitem(w_args, space.wrap(2))) dic = dict([(key,space.wrap(value)) for (key, value) in dic.items()]) bases = list(bases) @@ -643,7 +643,7 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def viewiterable(self, w_obj, expected_length=-1): + def fixedview(self, w_obj, expected_length=-1): """ Fast paths """ if isinstance(w_obj, W_TupleObject): @@ -651,7 +651,18 @@ elif isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.viewiterable(self, w_obj, expected_length) + return ObjSpace.fixedview(self, w_obj, expected_length) + if expected_length != -1 and len(t) != expected_length: + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + return t + + def listview(self, w_obj, expected_length=-1): + if isinstance(w_obj, W_ListObject): + t = w_obj.wrappeditems + elif isinstance(w_obj, W_TupleObject): + t = w_obj.wrappeditems[:] + else: + return ObjSpace.listview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/ropeobject.py Fri Nov 20 21:35:04 2009 @@ -537,7 +537,7 @@ (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -557,7 +557,7 @@ def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/ropeunicodeobject.py Fri Nov 20 21:35:04 2009 @@ -233,7 +233,7 @@ return space.contains(unicode_from_string(space, w_container), w_item ) def unicode_join__RopeUnicode_ANY(space, w_self, w_list): - l_w = space.unpackiterable(w_list) + l_w = space.listview(w_list) delim = w_self._node totlen = 0 if len(l_w) == 0: @@ -504,7 +504,7 @@ def unicode_startswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_prefixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = ropeunicode_w(space, w_prefix) if rope.startswith(unistr, prefix, start, end): return space.w_True @@ -513,7 +513,7 @@ def unicode_endswith__RopeUnicode_Tuple_ANY_ANY(space, w_unistr, w_suffixes, w_start, w_end): unistr, start, end = _convert_idx_params(space, w_unistr, w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = ropeunicode_w(space, w_suffix) if rope.endswith(unistr, suffix, start, end): return space.w_True Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/setobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/setobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/setobject.py Fri Nov 20 21:35:04 2009 @@ -111,7 +111,7 @@ def make_setdata_from_w_iterable(space, w_iterable=None): data = r_dict(space.eq_w, space.hash_w) if w_iterable is not None: - for w_item in space.viewiterable(w_iterable): + for w_item in space.listview(w_iterable): data[w_item] = None return data Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/stringobject.py Fri Nov 20 21:35:04 2009 @@ -349,13 +349,13 @@ sliced) def str_join__String_ANY(space, w_self, w_list): - list_w = space.unpackiterable(w_list) + list_w = space.listview(w_list) str_w = space.str_w if list_w: self = w_self._value listlen = 0 reslen = 0 - l = [] + l = [None] * len(list_w) for i in range(len(list_w)): w_s = list_w[i] if not space.is_true(space.isinstance(w_s, space.w_str)): @@ -367,7 +367,7 @@ space.wrap("sequence item %d: expected string, %s " "found" % (i, space.type(w_s).getname(space, '?')))) - l.append(space.str_w(w_s)) + l[i] = space.str_w(w_s) return space.wrap(self.join(l)) else: return W_StringObject.EMPTY @@ -628,7 +628,7 @@ (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): if space.is_true(space.isinstance(w_suffix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, @@ -647,7 +647,7 @@ def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): if space.is_true(space.isinstance(w_prefix, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "startswith", w_prefixes, w_start, Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/strsliceobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/strsliceobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/strsliceobject.py Fri Nov 20 21:35:04 2009 @@ -143,7 +143,7 @@ def str_endswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True @@ -157,7 +157,7 @@ def str_startswith__StringSlice_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''), w_start, w_end) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.str_w(w_prefix) if stringstartswith(u_self, prefix, start, end): return space.w_True Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_celldict.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_celldict.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_celldict.py Fri Nov 20 21:35:04 2009 @@ -1,4 +1,5 @@ -from pypy.conftest import gettestobjspace +import py +from pypy.conftest import gettestobjspace, option from pypy.objspace.std.celldict import get_global_cache, ModuleCell, ModuleDictImplementation from pypy.interpreter import gateway @@ -8,6 +9,8 @@ class AppTestCellDict(object): def setup_class(cls): + if option.runappdirect: + py.test.skip("not appdirect tests") cls.space = gettestobjspace(**{"objspace.std.withcelldict": True}) cls.w_impl_used = cls.space.appexec([], """(): import __pypy__ Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_dictmultiobject.py Fri Nov 20 21:35:04 2009 @@ -607,8 +607,8 @@ StringObjectCls = FakeString w_dict = None iter = iter - viewiterable = list - + fixedview = list + listview = list class Config: class objspace: Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_floatobject.py Fri Nov 20 21:35:04 2009 @@ -59,10 +59,7 @@ inf = 1e200 * 1e200 assert hash(inf) == 314159 assert hash(-inf) == -271828 - x = hash(inf/inf) - # ^^^ assert did not crash, even though the result is a bit random - # e.g. it appears to be -32768 on Win32 and 0 on Linux - assert x == hash(inf/inf) + assert hash(inf/inf) == 0 def test_int_float(self): assert int(42.1234) == 42 Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_userobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_userobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/test/test_userobject.py Fri Nov 20 21:35:04 2009 @@ -1,3 +1,4 @@ +import py from pypy.interpreter import gateway @@ -283,6 +284,8 @@ cls.prev_installer = multimethod.Installer multimethod.Installer = multimethod.InstallerVersion2 + if conftest.option.runappdirect: + py.test.skip("Cannot run different installers when runappdirect") config = conftest.make_config(conftest.option, **cls.OPTIONS) cls.space = conftest.maketestobjspace(config) Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/tupletype.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/tupletype.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/tupletype.py Fri Nov 20 21:35:04 2009 @@ -13,7 +13,7 @@ space.is_w(space.type(w_sequence), space.w_tuple)): return w_sequence else: - tuple_w = space.viewiterable(w_sequence) + tuple_w = space.fixedview(w_sequence) w_obj = space.allocate_instance(space.TupleObjectCls, w_tupletype) space.TupleObjectCls.__init__(w_obj, tuple_w) return w_obj Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/typeobject.py Fri Nov 20 21:35:04 2009 @@ -596,7 +596,7 @@ if not space.is_w(w_where, space.w_type): w_mro_meth = space.get(w_mro_func, w_self) w_mro = space.call_function(w_mro_meth) - mro_w = space.viewiterable(w_mro) + mro_w = space.fixedview(w_mro) w_self.mro_w = validate_custom_mro(space, mro_w) return # done w_self.mro_w = w_self.compute_default_mro()[:] Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/typetype.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/typetype.py Fri Nov 20 21:35:04 2009 @@ -11,7 +11,7 @@ w_typetype = _precheck_for_new(space, w_typetype) - bases_w = space.viewiterable(w_bases) + bases_w = space.fixedview(w_bases) w_winner = w_typetype for base in bases_w: @@ -38,7 +38,7 @@ name = space.str_w(w_name) assert isinstance(name, str) dict_w = {} - dictkeys_w = space.unpackiterable(w_dict) + dictkeys_w = space.listview(w_dict) for w_key in dictkeys_w: key = space.str_w(w_key) dict_w[key] = space.getitem(w_dict, w_key) @@ -115,7 +115,7 @@ " to %s.__bases__, not %s"% (w_type.name, space.type(w_value).getname(space, '?')))) - newbases_w = space.viewiterable(w_value) + newbases_w = space.fixedview(w_value) if len(newbases_w) == 0: raise OperationError(space.w_TypeError, space.wrap("can only assign non-empty tuple" Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/unicodeobject.py Fri Nov 20 21:35:04 2009 @@ -173,7 +173,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.unpackiterable(w_list) + l = space.listview(w_list) delim = w_self._value totlen = 0 if len(l) == 0: @@ -182,7 +182,7 @@ space.is_w(space.type(l[0]), space.w_unicode)): return l[0] - values_list = [] + values_list = [None] * len(l) for i in range(len(l)): item = l[i] if isinstance(item, W_UnicodeObject): @@ -194,7 +194,7 @@ w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - values_list.append(item) + values_list[i] = item return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): @@ -489,7 +489,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_prefix in space.viewiterable(w_prefixes): + for w_prefix in space.fixedview(w_prefixes): prefix = space.unicode_w(w_prefix) if stringstartswith(unistr, prefix, start, end): return space.w_True @@ -499,7 +499,7 @@ w_start, w_end): unistr, _, start, end = _convert_idx_params(space, w_unistr, space.wrap(u''), w_start, w_end, True) - for w_suffix in space.viewiterable(w_suffixes): + for w_suffix in space.fixedview(w_suffixes): suffix = space.unicode_w(w_suffix) if stringendswith(unistr, suffix, start, end): return space.w_True Modified: pypy/branch/shorter-guard-path/pypy/rpython/callparse.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/rpython/callparse.py (original) +++ pypy/branch/shorter-guard-path/pypy/rpython/callparse.py Fri Nov 20 21:35:04 2009 @@ -178,7 +178,8 @@ raise ValueError return list(items) raise CallPatternTooComplex, "'*' argument must be a tuple" - viewiterable = unpackiterable + fixedview = unpackiterable + listview = unpackiterable def is_w(self, one, other): return one is other Modified: pypy/branch/shorter-guard-path/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/shorter-guard-path/pypy/translator/c/gcc/trackgcroot.py Fri Nov 20 21:35:04 2009 @@ -754,7 +754,8 @@ '_exit': None, '__assert_fail': None, '___assert_rtn': None, - 'L___assert_rtn$stub': None + 'L___assert_rtn$stub': None, + 'L___eprintf$stub': None, } def __init__(self, lines, filetag=0): From arigo at codespeak.net Fri Nov 20 21:37:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 20 Nov 2009 21:37:45 +0100 (CET) Subject: [pypy-svn] r69485 - pypy/branch/shorter-guard-path/pypy/objspace/std Message-ID: <20091120203745.822AF16807B@codespeak.net> Author: arigo Date: Fri Nov 20 21:37:44 2009 New Revision: 69485 Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/multimethod.py Log: Don't crash when printing a FailedToImplement error. Modified: pypy/branch/shorter-guard-path/pypy/objspace/std/multimethod.py ============================================================================== --- pypy/branch/shorter-guard-path/pypy/objspace/std/multimethod.py (original) +++ pypy/branch/shorter-guard-path/pypy/objspace/std/multimethod.py Fri Nov 20 21:37:44 2009 @@ -29,6 +29,9 @@ self.w_type = w_type self.w_value = w_value + def __str__(self): + return '' % (self.w_type, self.w_value) + def raiseFailedToImplement(): raise FailedToImplement From benjamin at codespeak.net Sat Nov 21 02:55:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 21 Nov 2009 02:55:59 +0100 (CET) Subject: [pypy-svn] r69486 - pypy/trunk/pypy/interpreter/pyparser Message-ID: <20091121015559.58DAE168074@codespeak.net> Author: benjamin Date: Sat Nov 21 02:55:58 2009 New Revision: 69486 Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py Log: fix typo Modified: pypy/trunk/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/metaparser.py Sat Nov 21 02:55:58 2009 @@ -110,7 +110,7 @@ del dfa[j] for sub_state in dfa: sub_state.unify_state(other_state, state) - changes = True + changed = True break From arigo at codespeak.net Sat Nov 21 12:11:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 12:11:27 +0100 (CET) Subject: [pypy-svn] r69487 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20091121111127.CF2CC168051@codespeak.net> Author: arigo Date: Sat Nov 21 12:11:26 2009 New Revision: 69487 Modified: pypy/trunk/pypy/interpreter/executioncontext.py pypy/trunk/pypy/interpreter/test/test_executioncontext.py Log: Improve the test, and see it failing. For now, no fix; I am trying to fix this in the branch/faster-raise, because it requires a bit deeper changes. Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Sat Nov 21 12:11:26 2009 @@ -71,6 +71,8 @@ def _init_frame_chain(self): # 'some_frame' points to any frame from this thread's frame stack # (although in general it should point to the top one). + # XXX not true: some_frame must point to a frame from which we can + # reach the top frame by following the chain of f_forward self.some_frame = None self.framestackdepth = 0 Modified: pypy/trunk/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/test/test_executioncontext.py Sat Nov 21 12:11:26 2009 @@ -575,7 +575,7 @@ assert frame.f_back_some is None assert ec.gettopframe() is None - + @py.test.mark.xfail def test_frame_chain_jitted_forced(self): ec = self.EC() @@ -593,8 +593,10 @@ # recursive enter/leave seen by the jit frame3 = self.Frame(ec) ec._chain(frame3) + assert ec.gettopframe() is frame3 res = frame3.force_f_back() assert res is frame2 + assert ec.gettopframe() is frame3 assert frame3.f_back() is frame2 ec._unchain(frame3) From arigo at codespeak.net Sat Nov 21 13:34:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 13:34:55 +0100 (CET) Subject: [pypy-svn] r69488 - in pypy/branch/faster-raise/pypy: interpreter interpreter/test module/_rawffi module/_stackless module/sys objspace/flow Message-ID: <20091121123455.7F6F6168074@codespeak.net> Author: arigo Date: Sat Nov 21 13:34:54 2009 New Revision: 69488 Modified: pypy/branch/faster-raise/pypy/interpreter/error.py pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/pyframe.py pypy/branch/faster-raise/pypy/interpreter/pyopcode.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py pypy/branch/faster-raise/pypy/interpreter/test/test_zzpickle_and_slow.py pypy/branch/faster-raise/pypy/module/_rawffi/callback.py pypy/branch/faster-raise/pypy/module/_stackless/interp_coroutine.py pypy/branch/faster-raise/pypy/module/sys/__init__.py pypy/branch/faster-raise/pypy/module/sys/vm.py pypy/branch/faster-raise/pypy/objspace/flow/flowcontext.py Log: Current status of fighting with f_back_some and f_forward. I think that this has a better chance to pass serious tests. Check-in to move to a faster machine to run more tests... Modified: pypy/branch/faster-raise/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/error.py Sat Nov 21 13:34:54 2009 @@ -47,6 +47,11 @@ "NOT_RPYTHON: Convenience for tracebacks." return '[%s: %s]' % (self.w_type, self.w_value) + def wrap_application_traceback(self, space): + ec = space.getexecutioncontext() + ec.force_frame_chain() + return space.wrap(self.application_traceback) + def errorstr(self, space): "The exception class and value, as a string." if space is None: Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Sat Nov 21 13:34:54 2009 @@ -5,6 +5,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import we_are_jitted from pypy.rlib import jit +from pypy.rlib.objectmodel import we_are_translated def app_profile_call(space, w_callable, frame, event, w_arg): space.call_function(w_callable, @@ -62,16 +63,17 @@ @jit.unroll_safe def gettopframe(self): - frame = self.some_frame + frame = self.top_real_frame if frame is not None: while frame.f_forward is not None: frame = frame.f_forward return frame def _init_frame_chain(self): - # 'some_frame' points to any frame from this thread's frame stack - # (although in general it should point to the top one). - self.some_frame = None + # 'top_real_frame' points to the topmost "real" frame from this + # thread's frame stack. See gettopframe() for how to go from + # there to the actual top of frame. + self.top_real_frame = None self.framestackdepth = 0 @staticmethod @@ -113,43 +115,44 @@ ... This ensures that the virtual frames don't escape via the f_back of the - real frames. For the same reason, the executioncontext's some_frame - attribute should only point to real frames. + real frames. For the same reason, the executioncontext's top_real_frame + attribute should only point to real frames (as the name implies). All places where a frame can become accessed from applevel-code (like - sys._getframe and traceback catching) need to call force_f_back to ensure - that the intermediate virtual frames are forced to be real ones. - + sys._getframe, sys.exc_info) need to call force_frame_chain to ensure + that the intermediate virtual frames are forced to be real ones. For + sys.exc_info() and similar places that expose the traceback to app- + level, this is done by OperationError.wrap_application_traceback(). + + The 'f_no_forward_from_there' flag on PyFrame is an optimization: + it is set to True when we are sure that f_forward is None on frame and + on any of its f_backs. """ frame.f_back_some = None frame.f_forward = None - frame.f_back_forced = False + frame.f_no_forward_from_there = False def _chain(self, frame): self.framestackdepth += 1 # - frame.f_back_some = self.some_frame + frame.f_back_some = self.top_real_frame if self._we_are_jitted(): curtopframe = self.gettopframe() assert curtopframe is not None curtopframe.f_forward = frame else: - self.some_frame = frame + self.top_real_frame = frame def _unchain(self, frame): - #assert frame is self.gettopframe() --- slowish - if self.some_frame is frame: - self.some_frame = frame.f_back_some + if not we_are_translated(): + assert frame is self.gettopframe() # slowish + if self.top_real_frame is frame: + self.top_real_frame = frame.f_back_some else: f_back = frame.f_back() - if f_back is not None: - f_back.f_forward = None - - if frame.last_exception is not None: - f_back = frame.f_back() - frame.f_back_some = f_back - frame.f_back_forced = True - + assert f_back is not None + f_back.f_forward = None + # self.framestackdepth -= 1 @staticmethod @@ -160,13 +163,13 @@ # executed outside of the jit. Note that this makes it important that # _unchain does not call we_are_jitted frame.f_back().f_forward = None - ec.some_frame = frame + ec.top_real_frame = frame @staticmethod @jit.unroll_safe def _extract_back_from_frame(frame): back_some = frame.f_back_some - if frame.f_back_forced: + if frame.f_no_forward_from_there: # don't check back_some.f_forward in this case return back_some if back_some is None: @@ -177,17 +180,33 @@ return back_some back_some = f_forward - @staticmethod - def _force_back_of_frame(frame): - orig_frame = frame - while frame is not None and not frame.f_back_forced: - frame.f_back_some = f_back = ExecutionContext._extract_back_from_frame(frame) - frame.f_back_forced = True + def force_frame_chain(self): + frame = self.gettopframe() + self.top_real_frame = frame + while frame is not None and not frame.f_no_forward_from_there: + f_back = ExecutionContext._extract_back_from_frame(frame) + frame.f_back_some = f_back # now that we force the whole chain, we also have to set the # forward links to None frame.f_forward = None + frame.f_no_forward_from_there = True frame = f_back - return orig_frame.f_back_some + + def escape_frame_via_traceback(self, topframe): + # The topframe escapes via a traceback. There are two cases: either + # this topframe is a virtual frame, or not. + if not we_are_translated(): + assert topframe is self.gettopframe() + if self.top_real_frame is not topframe: + # Case of a virtual frame: set the f_back_some pointer on the + # frame to the real back, and set f_no_forward_from_there as a + # way to ensure that the correct thing will happen in + # _extract_back_from_frame(), should it ever be called. + topframe.f_back_some = topframe.f_back() + topframe.f_no_forward_from_there = True + else: + # Normal case + self.force_frame_chain() _we_are_jitted = staticmethod(we_are_jitted) # indirection for testing @@ -207,7 +226,7 @@ self.is_tracing = 0 def enter(self, ec): - ec.some_frame = self.topframe + ec.top_real_frame = self.topframe ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc @@ -216,6 +235,7 @@ ec.space.frame_trace_action.fire() def leave(self, ec): + ec.force_frame_chain() # for now self.topframe = ec.gettopframe() self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc @@ -392,7 +412,7 @@ if w_callback is not None and event != "leaveframe": if operr is not None: w_arg = space.newtuple([operr.w_type, operr.w_value, - space.wrap(operr.application_traceback)]) + operr.wrap_application_traceback(space)]) frame.fast2locals() self.is_tracing += 1 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Sat Nov 21 13:34:54 2009 @@ -310,7 +310,7 @@ w_tb = space.w_None else: w_exc_value = self.last_exception.w_value - w_tb = w(self.last_exception.application_traceback) + w_tb = self.last_exception.wrap_application_traceback(space) tup_state = [ w(self.f_back()), @@ -366,7 +366,6 @@ # everything like cells from here PyFrame.__init__(self, space, pycode, w_globals, closure) new_frame.f_back_some = space.interp_w(PyFrame, w_f_back, can_be_None=True) - new_frame.f_back_forced = True new_frame.builtin = space.interp_w(Module, w_builtin) new_frame.set_blocklist([unpickle_block(space, w_blk) @@ -439,9 +438,6 @@ def f_back(self): return ExecutionContext._extract_back_from_frame(self) - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - ### line numbers ### # for f*_f_* unwrapping through unwrap_spec in typedef.py @@ -632,7 +628,7 @@ while f is not None and f.last_exception is None: f = f.f_back() if f is not None: - return space.wrap(f.last_exception.application_traceback) + return f.last_exception.wrap_application_traceback(space) return space.w_None def fget_f_restricted(space, self): Modified: pypy/branch/faster-raise/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyopcode.py Sat Nov 21 13:34:54 2009 @@ -159,6 +159,7 @@ block = self.unrollstack(SApplicationException.kind) if block is None: # no handler found for the OperationError + ec.escape_frame_via_traceback(self) if we_are_translated(): raise operr else: @@ -874,10 +875,11 @@ unroller = f.space.interpclass_w(w_unroller) if isinstance(unroller, SApplicationException): operr = unroller.operr + w_traceback = operr.wrap_application_traceback(f.space) w_result = f.space.call_function(w_exitfunc, operr.w_type, operr.w_value, - operr.application_traceback) + w_traceback) if f.space.is_true(w_result): # __exit__() returned True -> Swallow the exception. f.settopvalue(f.space.w_None, 2) Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Sat Nov 21 13:34:54 2009 @@ -225,7 +225,7 @@ class TestFrameChaining(object): class EC(ExecutionContext): - _some_frame = None + _top_real_frame = None def __init__(self, jitted=False): self.jitted = jitted self.virtualizable = None @@ -235,20 +235,19 @@ def _we_are_jitted(self): return self.jitted - def _get_some_frame(self): - if self._some_frame: - self._some_frame.look_at() - return self._some_frame - def _set_some_frame(self, frame): + def _get_top_real_frame(self): + if self._top_real_frame: + self._top_real_frame.look_at() + return self._top_real_frame + def _set_top_real_frame(self, frame): if frame is not None: frame.force() - self._some_frame = frame - some_frame = property(_get_some_frame, _set_some_frame) + self._top_real_frame = frame + top_real_frame = property(_get_top_real_frame, _set_top_real_frame) class Frame(object): _f_back_some = None _f_forward = None - last_exception = None def __init__(self, ec, virtual_with_base_frame=None): self.ec = ec @@ -259,9 +258,6 @@ def f_back(self): return ExecutionContext._extract_back_from_frame(self) - def force_f_back(self): - return ExecutionContext._force_back_of_frame(self) - def force(self): if not self.escaped: self.virtual_with_base_frame = None @@ -338,13 +334,13 @@ def test_gettopframe_no_jit(self): ec = self.EC() frame = self.Frame(ec) - ec.some_frame = frame + ec.top_real_frame = frame assert ec.gettopframe() is frame def test_gettopframe_jit(self): ec = self.EC() frame = self.Frame(ec) # real frame - ec.some_frame = frame + ec.top_real_frame = frame assert ec.gettopframe() is frame frame2 = self.Frame(ec) # virtual frame @@ -359,19 +355,19 @@ frame4 = self.Frame(ec) # real frame again frame4.f_back_some = frame - ec.some_frame = frame4 + ec.top_real_frame = frame4 assert ec.gettopframe() is frame4 def test_frame_chain(self): ec = self.EC() - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 frame = self.Frame(ec) ec._chain(frame) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 1 assert frame.f_back_some is None assert frame.f_forward is None @@ -380,7 +376,7 @@ frame2 = self.Frame(ec) ec._chain(frame2) - assert ec.some_frame is frame2 + assert ec.top_real_frame is frame2 assert ec.framestackdepth == 2 assert frame2.f_back_some is frame assert frame.f_forward is None @@ -391,7 +387,7 @@ frame3 = self.Frame(ec) ec._chain(frame3) - assert ec.some_frame is frame3 + assert ec.top_real_frame is frame3 assert frame3.f_back_some is frame2 assert frame2.f_forward is None assert ec.gettopframe() is frame3 @@ -401,7 +397,7 @@ assert frame3.f_back() is frame2 ec._unchain(frame3) - assert ec.some_frame is frame2 + assert ec.top_real_frame is frame2 assert ec.framestackdepth == 2 assert frame2.f_forward is None assert frame3.f_back_some is frame2 @@ -411,7 +407,7 @@ assert frame2.f_back() is frame ec._unchain(frame2) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 1 assert frame.f_forward is None assert frame2.f_back_some is frame @@ -420,7 +416,7 @@ assert frame.f_back() is None ec._unchain(frame) - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 assert frame.f_back_some is None assert ec.gettopframe() is None @@ -436,14 +432,13 @@ frame2 = self.Frame(ec) ec._chain(frame2) - assert ec.some_frame is frame2 + assert ec.top_real_frame is frame2 assert ec.framestackdepth == 2 assert frame2.f_back_some is frame assert frame.f_forward is None assert frame2.f_forward is None - res = frame2.force_f_back() - assert res is frame - assert frame.f_back_forced + ec.force_frame_chain() + assert frame.f_no_forward_from_there assert ec.gettopframe() is frame2 assert ec._extract_back_from_frame(frame2) is frame assert ec._extract_back_from_frame(frame) is None @@ -457,7 +452,7 @@ assert frame3.f_back() is frame2 ec._unchain(frame3) - assert ec.some_frame is frame2 + assert ec.top_real_frame is frame2 assert frame3.f_back_some is frame2 assert ec.gettopframe() is frame2 assert ec._extract_back_from_frame(frame2) is frame @@ -471,7 +466,7 @@ assert frame.f_back() is None ec._unchain(frame) - assert ec.some_frame is None + assert ec.top_real_frame is None assert frame.f_back_some is None assert frame2.f_back() is frame @@ -483,13 +478,13 @@ ec = self.EC() - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 assert ec.gettopframe() is None frame = self.Frame(ec) ec._chain(frame) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 1 assert frame.f_back_some is None assert frame.f_forward is None @@ -500,7 +495,7 @@ ec.virtualizable = frame frame2 = self.Frame(ec, frame) ec._chain(frame2) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 2 assert frame2.f_back_some is frame assert frame.f_forward is frame2 @@ -512,7 +507,7 @@ # recursive enter/leave seen by the jit frame3 = self.Frame(ec, frame) ec._chain(frame3) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert frame3.f_back_some is frame assert frame2.f_forward is frame3 assert ec.gettopframe() is frame3 @@ -522,7 +517,7 @@ assert frame3.f_back() is frame2 ec._unchain(frame3) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 2 assert frame2.f_forward is None assert frame3.f_back_some is frame @@ -537,7 +532,7 @@ ec.virtualizable = None ec._chain(frame3) assert not frame2.escaped - assert ec.some_frame is frame3 + assert ec.top_real_frame is frame3 assert frame3.f_back_some is frame assert frame2.f_forward is None assert frame3.escaped @@ -548,7 +543,7 @@ assert frame3.f_back() is frame2 ec._unchain(frame3) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 2 assert frame2.f_forward is None assert frame3.f_back_some is frame @@ -561,7 +556,7 @@ assert frame2.f_back() is frame ec._unchain(frame2) - assert ec.some_frame is frame + assert ec.top_real_frame is frame assert ec.framestackdepth == 1 assert frame.f_forward is None assert frame2.f_back_some is frame @@ -571,7 +566,7 @@ ec.jitted = False assert frame.f_back() is None ec._unchain(frame) - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 assert frame.f_back_some is None assert ec.gettopframe() is None @@ -581,7 +576,7 @@ ec = self.EC() - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 frame = self.Frame(ec) @@ -594,8 +589,9 @@ # recursive enter/leave seen by the jit frame3 = self.Frame(ec) ec._chain(frame3) - res = frame3.force_f_back() - assert res is frame2 + assert frame3 is ec.gettopframe() + ec.force_frame_chain() + assert frame3 is ec.gettopframe() assert frame3.f_back() is frame2 ec._unchain(frame3) @@ -613,7 +609,7 @@ def enter_two_jitted_levels(self): ec = self.EC() - assert ec.some_frame is None + assert ec.top_real_frame is None assert ec.framestackdepth == 0 frame = self.Frame(ec) @@ -747,8 +743,8 @@ frame3 = self.Frame(ec, frame2) ec.jitted = False ec._chain(frame3) - ec.gettopframe() - frame3.force_f_back() + assert ec.gettopframe() is frame3 + ec.force_frame_chain() ec._unchain(frame3) assert not frame2.f_forward assert ec.gettopframe() is frame2 @@ -759,23 +755,84 @@ ec._unchain(frame) assert ec.gettopframe() is None - def test_unchain_with_exception(self): + def test_unchain_virtual_with_exception(self): ec, frame, frame2 = self.enter_two_jitted_levels() - ec.jitted = False frame3 = self.Frame(ec, frame2) ec._chain(frame3) - frame3.last_exception = 3 - assert ec.some_frame is frame3 + frame4 = self.Frame(ec, frame3) + ec._chain(frame4) + ec.escape_frame_via_traceback(frame4) + ec._unchain(frame4) + assert ec._top_real_frame is frame + assert ec.gettopframe() is frame3 + assert frame._f_forward is frame2 + assert frame2._f_forward is frame3 + assert frame3._f_forward is None + assert frame4._f_back_some is frame3 + assert frame4.f_back() is frame3 + # + # exit frame3 without an exception set: + # the traceback was not forced, so it is forgotten ec._unchain(frame3) - ec.jitted = True - assert frame3.f_back_some is frame2 - assert frame3.f_back_forced + assert ec._top_real_frame is frame + assert ec.gettopframe() is frame2 + assert frame._f_forward is frame2 + + def test_unchain_virtual_with_exception_forced(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + frame3 = self.Frame(ec, frame2) + ec._chain(frame3) + ec.escape_frame_via_traceback(frame3) + ec._unchain(frame3) + # + ec.force_frame_chain() + assert ec._top_real_frame is frame2 + assert ec.gettopframe() is frame2 + assert frame._f_forward is None + assert frame2._f_forward is None + assert frame3._f_back_some is frame2 + assert frame2._f_back_some is frame + + def test_unchain_virtual_with_exception_forced_later(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + frame3 = self.Frame(ec, frame2) + ec._chain(frame3) + ec.escape_frame_via_traceback(frame3) + ec._unchain(frame3) + # + frame4 = self.Frame(ec, frame2) + ec._chain(frame4) + ec.force_frame_chain() + assert ec._top_real_frame is frame4 + assert ec.gettopframe() is frame4 + assert frame._f_forward is None assert frame2._f_forward is None - assert not frame2.f_back_forced + assert frame3._f_back_some is frame2 + assert frame2._f_back_some is frame + assert frame4.f_back() is frame2 + assert frame3.f_back() is frame2 assert frame2.f_back() is frame + + def test_unchain_virtual_then_real_with_exception(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + # + ec.jitted = False + frame3 = self.Frame(ec, frame2) + ec._chain(frame3) + # + ec.jitted = True + frame4 = self.Frame(ec, frame3) + ec._chain(frame4) + # + ec.escape_frame_via_traceback(frame4) # not forced + ec._unchain(frame4) assert frame._f_forward is frame2 + assert not frame.f_no_forward_from_there + assert not frame2.f_no_forward_from_there # - # test for a bug: what happens if we _unchain frame2 without an exc. - assert frame2.last_exception is None - ec._unchain(frame2) + ec.escape_frame_via_traceback(frame3) # calls force_frame_chain() + ec._unchain(frame3) + assert frame2.f_no_forward_from_there + assert frame2._f_forward is None + assert frame.f_no_forward_from_there assert frame._f_forward is None Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_zzpickle_and_slow.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_zzpickle_and_slow.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_zzpickle_and_slow.py Sat Nov 21 13:34:54 2009 @@ -32,7 +32,7 @@ w_last = None while w_frame.f_back(): # should have been forced by traceback capturing - assert w_frame.f_back_forced + assert w_frame.f_no_forward_from_there w_last = w_frame w_frame = w_frame.f_back() assert w_last @@ -42,7 +42,7 @@ def restore_top_frame(space, w_frame, w_saved): while w_frame.f_back(): - assert w_frame.f_back_forced + assert w_frame.f_no_forward_from_there w_frame = w_frame.f_back() w_frame.f_back_some = w_saved Modified: pypy/branch/faster-raise/pypy/module/_rawffi/callback.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_rawffi/callback.py (original) +++ pypy/branch/faster-raise/pypy/module/_rawffi/callback.py Sat Nov 21 13:34:54 2009 @@ -36,7 +36,7 @@ unwrap_value(space, push_elem, ll_res, 0, callback_ptr.result, w_res) except OperationError, e: - tbprint(space, space.wrap(e.application_traceback), + tbprint(space, e.wrap_application_traceback(space), space.wrap(e.errorstr(space))) # force the result to be zero if callback_ptr.result != 'O': Modified: pypy/branch/faster-raise/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/faster-raise/pypy/module/_stackless/interp_coroutine.py Sat Nov 21 13:34:54 2009 @@ -275,7 +275,6 @@ return space.newtuple([]) items = [None] * index f = self.subctx.topframe - f.force_f_back() while index > 0: index -= 1 items[index] = space.wrap(f) Modified: pypy/branch/faster-raise/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/sys/__init__.py (original) +++ pypy/branch/faster-raise/pypy/module/sys/__init__.py Sat Nov 21 13:34:54 2009 @@ -122,7 +122,7 @@ if operror is None: return space.w_None else: - return space.wrap(operror.application_traceback) + return operror.wrap_application_traceback(space) return None def get_w_default_encoder(self): Modified: pypy/branch/faster-raise/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/sys/vm.py (original) +++ pypy/branch/faster-raise/pypy/module/sys/vm.py Sat Nov 21 13:34:54 2009 @@ -30,8 +30,8 @@ raise OperationError(space.w_ValueError, space.wrap("frame index must not be negative")) ec = space.getexecutioncontext() + ec.force_frame_chain() f = ec.gettopframe_nohidden() - f.force_f_back() while True: if f is None: raise OperationError(space.w_ValueError, @@ -92,7 +92,7 @@ return space.newtuple([space.w_None,space.w_None,space.w_None]) else: return space.newtuple([operror.w_type, operror.w_value, - space.wrap(operror.application_traceback)]) + operror.wrap_application_traceback(space)]) def exc_clear(space): """Clear global information on the current exception. Subsequent calls Modified: pypy/branch/faster-raise/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/objspace/flow/flowcontext.py (original) +++ pypy/branch/faster-raise/pypy/objspace/flow/flowcontext.py Sat Nov 21 13:34:54 2009 @@ -260,8 +260,8 @@ except StopFlowing: continue # restarting a dead SpamBlock try: - old_frame = self.some_frame - self.some_frame = frame + old_frame = self.top_real_frame + self.top_real_frame = frame self.crnt_frame = frame try: w_result = frame.dispatch(frame.pycode, @@ -269,7 +269,7 @@ self) finally: self.crnt_frame = None - self.some_frame = old_frame + self.top_real_frame = old_frame except OperationThatShouldNotBePropagatedError, e: raise Exception( From arigo at codespeak.net Sat Nov 21 13:39:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 13:39:09 +0100 (CET) Subject: [pypy-svn] r69489 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091121123909.69BDA168074@codespeak.net> Author: arigo Date: Sat Nov 21 13:39:08 2009 New Revision: 69489 Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py Log: Re-kill this :-) Modified: pypy/branch/faster-raise/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/pyframe.py Sat Nov 21 13:39:08 2009 @@ -159,11 +159,6 @@ raise if not we_are_jitted(): executioncontext.return_trace(self, w_exitvalue) - # on exit, we try to release self.last_exception -- breaks an - # obvious reference cycle, so it used to help refcounting - # implementations. Nowaways, it helps ExecutionContext._unchain() - # by saying "this frame does not throw an exception". - self.last_exception = None finally: executioncontext.leave(self) return w_exitvalue From arigo at codespeak.net Sat Nov 21 16:29:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 16:29:12 +0100 (CET) Subject: [pypy-svn] r69490 - pypy/branch/faster-raise/pypy/lib/app_test Message-ID: <20091121152912.7AA121680C1@codespeak.net> Author: arigo Date: Sat Nov 21 16:29:11 2009 New Revision: 69490 Modified: pypy/branch/faster-raise/pypy/lib/app_test/test_exception_extra.py Log: Fix test: this is also what we get on Python 2.4 or 2.5. Modified: pypy/branch/faster-raise/pypy/lib/app_test/test_exception_extra.py ============================================================================== --- pypy/branch/faster-raise/pypy/lib/app_test/test_exception_extra.py (original) +++ pypy/branch/faster-raise/pypy/lib/app_test/test_exception_extra.py Sat Nov 21 16:29:11 2009 @@ -6,7 +6,7 @@ e = ex.EnvironmentError(1, "hello") assert str(e) == "[Errno 1] hello" e = ex.EnvironmentError(1, "hello", "world") - assert str(e) == "[Errno 1] hello: world" + assert str(e) == "[Errno 1] hello: 'world'" def app_test_import(): import exceptions From fijal at codespeak.net Sat Nov 21 16:35:51 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 21 Nov 2009 16:35:51 +0100 (CET) Subject: [pypy-svn] r69491 - pypy/branch/llop-copy-list Message-ID: <20091121153551.2F84C1680F4@codespeak.net> Author: fijal Date: Sat Nov 21 16:35:50 2009 New Revision: 69491 Added: pypy/branch/llop-copy-list/ - copied from r69490, pypy/trunk/ Log: A branch to try to implement more general copy-list operation, so list copying would not need to perform write barrier From arigo at codespeak.net Sat Nov 21 16:43:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 16:43:21 +0100 (CET) Subject: [pypy-svn] r69492 - in pypy/branch/faster-raise/pypy/module/exceptions: . test Message-ID: <20091121154321.B4FFE1680F3@codespeak.net> Author: arigo Date: Sat Nov 21 16:43:21 2009 New Revision: 69492 Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Log: Test and fix. Modified: pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/interp_exceptions.py Sat Nov 21 16:43:21 2009 @@ -111,7 +111,7 @@ if lgt == 0: return space.wrap('') elif lgt == 1: - return space.str(self.w_message) + return space.str(self.args_w[0]) else: return space.str(space.newtuple(self.args_w)) descr_str.unwrap_spec = ['self', ObjSpace] Modified: pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise/pypy/module/exceptions/test/test_exc.py Sat Nov 21 16:43:21 2009 @@ -25,6 +25,7 @@ assert x.xyz == 3 x.args = [42] assert x.args == (42,) + assert str(x) == '42' assert x[0] == 42 x.args = (1, 2, 3) assert x[1:2] == (2,) From arigo at codespeak.net Sat Nov 21 16:49:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 16:49:46 +0100 (CET) Subject: [pypy-svn] r69493 - pypy/branch/faster-raise/pypy/doc Message-ID: <20091121154946.9E2421680F3@codespeak.net> Author: arigo Date: Sat Nov 21 16:49:45 2009 New Revision: 69493 Modified: pypy/branch/faster-raise/pypy/doc/geninterp.txt Log: Remove the references to pypy/lib/_exceptions.py from geninterp.txt. Modified: pypy/branch/faster-raise/pypy/doc/geninterp.txt ============================================================================== --- pypy/branch/faster-raise/pypy/doc/geninterp.txt (original) +++ pypy/branch/faster-raise/pypy/doc/geninterp.txt Sat Nov 21 16:49:45 2009 @@ -18,20 +18,15 @@ any longer to execute this code. .. _`application-level`: coding-guide.html#app-preferable -.. _exceptions: ../../pypy/lib/_exceptions.py -An example is exceptions_. They are -needed in a very early phase of bootstrapping StdObjspace, but -for simplicity, they are written as RPythonic application -level code. This implies that the interpreter must be quite -completely initialized to execute this code, which is -impossible in the early phase, where we have neither -exceptions implemented nor classes available. +Bootstrap issue ++++++++++++++++ -Solution -++++++++ +One issue we had so far was of bootstrapping: some pieces of the +interpreter (e.g. exceptions) were written in geninterped code. +It is unclear how much of it is left, thought. -This bootstrap issue is solved by invoking a new bytecode interpreter +That bootstrap issue is (was?) solved by invoking a new bytecode interpreter which runs on FlowObjspace. FlowObjspace is complete without complicated initialization. It is able to do abstract interpretation of any Rpythonic code, without actually implementing anything. It just @@ -176,10 +171,8 @@ Interplevel Snippets in the Sources +++++++++++++++++++++++++++++++++++ -.. _`_exceptions.py`: ../../pypy/lib/_exceptions.py - Code written in application space can consist of complete files -to be translated (e.g. `_exceptions.py`_), or they +to be translated, or they can be tiny snippets scattered all over a source file, similar to our example from above. From arigo at codespeak.net Sat Nov 21 18:32:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 18:32:14 +0100 (CET) Subject: [pypy-svn] r69494 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091121173214.E0722168025@codespeak.net> Author: arigo Date: Sat Nov 21 18:32:13 2009 New Revision: 69494 Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Log: (pedronis, arigo) Rename yet again the flag, and change subtle its meaning yet again. Fixes an issue, described by the new test. Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Sat Nov 21 18:32:13 2009 @@ -124,13 +124,15 @@ sys.exc_info() and similar places that expose the traceback to app- level, this is done by OperationError.wrap_application_traceback(). - The 'f_no_forward_from_there' flag on PyFrame is an optimization: - it is set to True when we are sure that f_forward is None on frame and - on any of its f_backs. + The 'f_back_is_correct' flag on PyFrame means 'the f_back_some is + actually really my f_back'. If not set, we must check + 'f_back_some.f_forward' to access the real f_back. Note that we + guarantee that if f_back_is_correct is set on a frame in the frame + chain, then it is also set on all older frames. """ frame.f_back_some = None frame.f_forward = None - frame.f_no_forward_from_there = False + frame.f_back_is_correct = False def _chain(self, frame): self.framestackdepth += 1 @@ -144,8 +146,7 @@ self.top_real_frame = frame def _unchain(self, frame): - if not we_are_translated(): - assert frame is self.gettopframe() # slowish + assert frame is self.gettopframe() # slowish, XXX disable me again if self.top_real_frame is frame: self.top_real_frame = frame.f_back_some else: @@ -169,7 +170,7 @@ @jit.unroll_safe def _extract_back_from_frame(frame): back_some = frame.f_back_some - if frame.f_no_forward_from_there: + if frame.f_back_is_correct: # don't check back_some.f_forward in this case return back_some if back_some is None: @@ -183,27 +184,30 @@ def force_frame_chain(self): frame = self.gettopframe() self.top_real_frame = frame - while frame is not None and not frame.f_no_forward_from_there: + if frame is None: + return + while not frame.f_back_is_correct: f_back = ExecutionContext._extract_back_from_frame(frame) frame.f_back_some = f_back + frame.f_back_is_correct = True # now that we force the whole chain, we also have to set the # forward links to None - frame.f_forward = None - frame.f_no_forward_from_there = True + if f_back is None: + break + f_back.f_forward = None frame = f_back def escape_frame_via_traceback(self, topframe): # The topframe escapes via a traceback. There are two cases: either # this topframe is a virtual frame, or not. - if not we_are_translated(): - assert topframe is self.gettopframe() + assert topframe is self.gettopframe() if self.top_real_frame is not topframe: # Case of a virtual frame: set the f_back_some pointer on the - # frame to the real back, and set f_no_forward_from_there as a + # frame to the real back, and set f_back_is_correct as a # way to ensure that the correct thing will happen in # _extract_back_from_frame(), should it ever be called. topframe.f_back_some = topframe.f_back() - topframe.f_no_forward_from_there = True + topframe.f_back_is_correct = True else: # Normal case self.force_frame_chain() Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Sat Nov 21 18:32:13 2009 @@ -438,7 +438,7 @@ assert frame.f_forward is None assert frame2.f_forward is None ec.force_frame_chain() - assert frame.f_no_forward_from_there + assert frame.f_back_is_correct assert ec.gettopframe() is frame2 assert ec._extract_back_from_frame(frame2) is frame assert ec._extract_back_from_frame(frame) is None @@ -827,12 +827,21 @@ ec.escape_frame_via_traceback(frame4) # not forced ec._unchain(frame4) assert frame._f_forward is frame2 - assert not frame.f_no_forward_from_there - assert not frame2.f_no_forward_from_there + assert not frame.f_back_is_correct + assert not frame2.f_back_is_correct # ec.escape_frame_via_traceback(frame3) # calls force_frame_chain() ec._unchain(frame3) - assert frame2.f_no_forward_from_there + assert frame2.f_back_is_correct assert frame2._f_forward is None - assert frame.f_no_forward_from_there + assert frame.f_back_is_correct + assert frame._f_forward is None + + def test_force_frame_chain_bug(self): + ec, frame, frame2 = self.enter_two_jitted_levels() + ec.force_frame_chain() + frame3 = self.Frame(ec, frame2) + ec._chain(frame3) + ec.force_frame_chain() + assert frame2._f_forward is None # because we just forced assert frame._f_forward is None From arigo at codespeak.net Sat Nov 21 18:36:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 18:36:23 +0100 (CET) Subject: [pypy-svn] r69495 - in pypy/branch/faster-raise/pypy/interpreter: . test Message-ID: <20091121173623.31F4C168027@codespeak.net> Author: arigo Date: Sat Nov 21 18:36:22 2009 New Revision: 69495 Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Log: (pedronis, arigo) Yet Another Fix For The Day. With test. Modified: pypy/branch/faster-raise/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/executioncontext.py Sat Nov 21 18:36:22 2009 @@ -142,6 +142,7 @@ curtopframe = self.gettopframe() assert curtopframe is not None curtopframe.f_forward = frame + frame.f_back_is_correct = False else: self.top_real_frame = frame Modified: pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/test/test_executioncontext.py Sat Nov 21 18:36:22 2009 @@ -845,3 +845,12 @@ ec.force_frame_chain() assert frame2._f_forward is None # because we just forced assert frame._f_forward is None + + def test_rechain_frame(self): + # test rechaining a frame, as is done with generators + ec, frame, frame2 = self.enter_two_jitted_levels() + ec.force_frame_chain() + ec._unchain(frame2) + ec._chain(frame2) + assert frame._f_forward is frame2 + assert not frame2.f_back_is_correct From arigo at codespeak.net Sat Nov 21 18:53:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 18:53:50 +0100 (CET) Subject: [pypy-svn] r69496 - pypy/trunk/pypy/objspace/std Message-ID: <20091121175350.7C7A216802A@codespeak.net> Author: arigo Date: Sat Nov 21 18:53:49 2009 New Revision: 69496 Modified: pypy/trunk/pypy/objspace/std/complexobject.py pypy/trunk/pypy/objspace/std/floatobject.py pypy/trunk/pypy/objspace/std/longobject.py pypy/trunk/pypy/objspace/std/noneobject.py pypy/trunk/pypy/objspace/std/ropeobject.py pypy/trunk/pypy/objspace/std/ropeunicodeobject.py pypy/trunk/pypy/objspace/std/sliceobject.py pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/objspace/std/tupleobject.py pypy/trunk/pypy/objspace/std/unicodeobject.py Log: A generous sprinkle of "_immutable_=True" in the std object space. Modified: pypy/trunk/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/complexobject.py (original) +++ pypy/trunk/pypy/objspace/std/complexobject.py Sat Nov 21 18:53:49 2009 @@ -8,8 +8,8 @@ class W_ComplexObject(W_Object): """This is a reimplementation of the CPython "PyComplexObject" """ - from pypy.objspace.std.complextype import complex_typedef as typedef + _immutable_ = True def __init__(w_self, realval=0.0, imgval=0.0): w_self.realval = float(realval) Modified: pypy/trunk/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/floatobject.py Sat Nov 21 18:53:49 2009 @@ -13,7 +13,8 @@ it is assumed that the constructor takes a real Python float as an argument""" from pypy.objspace.std.floattype import float_typedef as typedef - + _immutable_ = True + def __init__(w_self, floatval): w_self.floatval = floatval Modified: pypy/trunk/pypy/objspace/std/longobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/longobject.py (original) +++ pypy/trunk/pypy/objspace/std/longobject.py Sat Nov 21 18:53:49 2009 @@ -7,7 +7,8 @@ class W_LongObject(W_Object): """This is a wrapper of rbigint.""" from pypy.objspace.std.longtype import long_typedef as typedef - + _immutable_ = True + def __init__(w_self, l): w_self.num = l # instance of rbigint Modified: pypy/trunk/pypy/objspace/std/noneobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/noneobject.py (original) +++ pypy/trunk/pypy/objspace/std/noneobject.py Sat Nov 21 18:53:49 2009 @@ -8,6 +8,7 @@ class W_NoneObject(W_Object): from pypy.objspace.std.nonetype import none_typedef as typedef + _immutable_ = True def unwrap(w_self, space): return None Modified: pypy/trunk/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/ropeobject.py (original) +++ pypy/trunk/pypy/objspace/std/ropeobject.py Sat Nov 21 18:53:49 2009 @@ -16,6 +16,7 @@ class W_RopeObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef + _immutable_ = True def __init__(w_self, node): if not we_are_translated(): Modified: pypy/trunk/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/ropeunicodeobject.py Sat Nov 21 18:53:49 2009 @@ -73,6 +73,7 @@ class W_RopeUnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef + _immutable_ = True def __init__(w_self, node): w_self._node = node Modified: pypy/trunk/pypy/objspace/std/sliceobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/sliceobject.py (original) +++ pypy/trunk/pypy/objspace/std/sliceobject.py Sat Nov 21 18:53:49 2009 @@ -12,7 +12,8 @@ class W_SliceObject(W_Object): from pypy.objspace.std.slicetype import slice_typedef as typedef - + _immutable_ = True + def __init__(w_self, w_start, w_stop, w_step): assert w_start is not None assert w_stop is not None Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Sat Nov 21 18:53:49 2009 @@ -20,8 +20,8 @@ class W_StringObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef - _immutable_ = True + def __init__(w_self, str): w_self._value = str Modified: pypy/trunk/pypy/objspace/std/tupleobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/tupleobject.py (original) +++ pypy/trunk/pypy/objspace/std/tupleobject.py Sat Nov 21 18:53:49 2009 @@ -7,6 +7,7 @@ class W_TupleObject(W_Object): from pypy.objspace.std.tupletype import tuple_typedef as typedef + _immutable_ = True def __init__(w_self, wrappeditems): make_sure_not_resized(wrappeditems) Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Sat Nov 21 18:53:49 2009 @@ -16,6 +16,7 @@ class W_UnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef + _immutable_ = True def __init__(w_self, unistr): assert isinstance(unistr, unicode) From arigo at codespeak.net Sat Nov 21 18:59:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 18:59:53 +0100 (CET) Subject: [pypy-svn] r69497 - pypy/branch/faster-raise/pypy/interpreter Message-ID: <20091121175953.1E1A216802D@codespeak.net> Author: arigo Date: Sat Nov 21 18:59:52 2009 New Revision: 69497 Modified: pypy/branch/faster-raise/pypy/interpreter/generator.py Log: Probable fix for the genrator's throw() method: around simulating a call to handle_operation_error, we must also simulate ec.enter() and ec.leave(). Modified: pypy/branch/faster-raise/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/faster-raise/pypy/interpreter/generator.py (original) +++ pypy/branch/faster-raise/pypy/interpreter/generator.py Sat Nov 21 18:59:52 2009 @@ -87,7 +87,12 @@ operr.normalize_exception(space) ec = space.getexecutioncontext() - next_instr = self.frame.handle_operation_error(ec, operr) + frame = self.frame + ec.enter(frame) + try: + next_instr = frame.handle_operation_error(ec, operr) + finally: + ec.leave(frame) self.frame.last_instr = intmask(next_instr - 1) return self.send_ex(space.w_None, True) From arigo at codespeak.net Sat Nov 21 19:13:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 19:13:49 +0100 (CET) Subject: [pypy-svn] r69498 - pypy/branch/faster-raise/pypy/module/pypyjit/test Message-ID: <20091121181349.338B816802F@codespeak.net> Author: arigo Date: Sat Nov 21 19:13:47 2009 New Revision: 69498 Modified: pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py Log: Two tests about raising and catching exceptions in a loop. Modified: pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/faster-raise/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 21 19:13:47 2009 @@ -290,6 +290,49 @@ assert len(bytecode.get_opnames("call")) == 1 # the call to append assert len(bytecode.get_opnames("guard")) == 1 # guard_no_exception after the call + def test_exception_inside_loop_1(self): + self.run_source(''' + def main(n): + while n: + try: + raise ValueError + except ValueError: + pass + n -= 1 + return n + ''', + ([30], 0)) + + bytecode, = self.get_by_bytecode("SETUP_EXCEPT") + #assert not bytecode.get_opnames("new") -- currently, we have + # new_with_vtable(pypy.interpreter.pyopcode.ExceptBlock) + bytecode, = self.get_by_bytecode("RAISE_VARARGS") + assert not bytecode.get_opnames("new") + bytecode, = self.get_by_bytecode("COMPARE_OP") + assert not bytecode.get_opnames() + + def test_exception_inside_loop_2(self): + self.run_source(''' + def g(n): + raise ValueError(n) + def f(n): + g(n) + def main(n): + while n: + try: + f(n) + except ValueError: + pass + n -= 1 + return n + ''', + ([30], 0)) + + bytecode, = self.get_by_bytecode("RAISE_VARARGS") + assert not bytecode.get_opnames("new") + bytecode, = self.get_by_bytecode("COMPARE_OP") + assert len(bytecode.get_opnames()) <= 2 # oois, guard_true + class AppTestJIT(PyPyCJITTests): def setup_class(cls): From arigo at codespeak.net Sat Nov 21 19:47:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 21 Nov 2009 19:47:16 +0100 (CET) Subject: [pypy-svn] r69499 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091121184716.CCBF216803A@codespeak.net> Author: arigo Date: Sat Nov 21 19:47:16 2009 New Revision: 69499 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: (pedronis, arigo) - Enable looking inside bridges too, as long as they come ultimately from a loop and not from an entry bridge. - Put counters on the total number of expected operations, too, to check random regressions. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Sat Nov 21 19:47:16 2009 @@ -2,7 +2,7 @@ from pypy.tool.udir import udir import py from py.test import skip -import sys, os +import sys, os, re class BytecodeTrace(list): def get_opnames(self, prefix=""): @@ -27,8 +27,60 @@ 'STORE_FAST', ] + +r_bridge = re.compile(r"bridge out of Guard (\d+)") + +def from_entry_bridge(text, allparts): + firstline = text.splitlines()[0] + if 'entry bridge' in firstline: + return True + match = r_bridge.search(firstline) + if match: + search = '' + for part in allparts: + if search in part: + break + else: + raise AssertionError, "%s not found??" % (search,) + return from_entry_bridge(part, allparts) + return False + +def test_from_entry_bridge(): + assert from_entry_bridge( + "# Loop 4 : entry bridge with 31 ops\n[p0, etc", []) + assert not from_entry_bridge( + "# Loop 1 : loop with 31 ops\n[p0, p1, etc", []) + assert not from_entry_bridge( + "# bridge out of Guard 5 with 24 ops\n[p0, p1, etc", + ["# Loop 1 : loop with 31 ops\n" + "[p0, p1]\n" + "guard_stuff(descr=)\n"]) + assert from_entry_bridge( + "# bridge out of Guard 5 with 24 ops\n[p0, p1, etc", + ["# Loop 1 : entry bridge with 31 ops\n" + "[p0, p1]\n" + "guard_stuff(descr=)\n"]) + assert not from_entry_bridge( + "# bridge out of Guard 51 with 24 ops\n[p0, p1, etc", + ["# Loop 1 : loop with 31 ops\n" + "[p0, p1]\n" + "guard_stuff(descr=)\n", + "# bridge out of Guard 5 with 13 ops\n" + "[p0, p1]\n" + "guard_other(p1, descr=)\n"]) + assert from_entry_bridge( + "# bridge out of Guard 51 with 24 ops\n[p0, p1, etc", + ["# Loop 1 : entry bridge with 31 ops\n" + "[p0, p1]\n" + "guard_stuff(descr=)\n", + "# bridge out of Guard 5 with 13 ops\n" + "[p0, p1]\n" + "guard_other(p1, descr=)\n"]) + + class PyPyCJITTests(object): - def run_source(self, source, *testcases): + def run_source(self, source, expected_max_ops, *testcases): + assert isinstance(expected_max_ops, int) source = py.code.Source(source) filepath = self.tmpdir.join('case%d.py' % self.counter) logfilepath = filepath.new(ext='.log') @@ -63,6 +115,7 @@ assert result assert result.splitlines()[-1].strip() == 'OK :-)' self.parse_loops(logfilepath) + assert self.total_ops <= expected_max_ops def parse_loops(self, opslogfile): from pypy.jit.metainterp.test.oparser import parse @@ -72,9 +125,11 @@ parts = logparser.extract_category(log, 'jit-log-opt-') # skip entry bridges, they can contain random things self.loops = [parse(part, no_namespace=True) for part in parts - if "bridge" not in part.lower()] + if not from_entry_bridge(part, parts)] self.sliced_loops = [] # contains all bytecodes of all loops + self.total_ops = 0 for loop in self.loops: + self.total_ops += len(loop.operations) for op in loop.operations: if op.getopname() == "debug_merge_point": sliced_loop = BytecodeTrace() @@ -106,7 +161,7 @@ x = x + (i&j) i = i + 1 return x - ''', + ''', 194, ([2117], 1083876708)) def test_factorial(self): @@ -117,7 +172,7 @@ r *= n n -= 1 return r - ''', + ''', 26, ([5], 120), ([20], 2432902008176640000L)) @@ -128,7 +183,7 @@ return n * main(n-1) else: return 1 - ''', + ''', 0, ([5], 120), ([20], 2432902008176640000L)) @@ -139,7 +194,7 @@ def main(): return richards.main(iterations = 1) - ''' % (sys.path,), + ''' % (sys.path,), 7000, ([], 42)) def test_simple_call(self): @@ -151,7 +206,7 @@ while i < n: i = f(f(i)) return i - ''', + ''', 76, ([20], 20), ([31], 32)) ops = self.get_by_bytecode("LOAD_GLOBAL") @@ -180,7 +235,7 @@ x = a.f(i) i = a.f(x) return i - ''', + ''', 92, ([20], 20), ([31], 32)) ops = self.get_by_bytecode("LOOKUP_METHOD") @@ -213,7 +268,7 @@ while i < n: i = f(f(i), j=1) return i - ''', + ''', 98, ([20], 20), ([31], 32)) ops = self.get_by_bytecode("CALL_FUNCTION") @@ -237,7 +292,7 @@ a.x = 2 i = i + a.x return i - ''', + ''', 63, ([20], 20), ([31], 32)) @@ -268,7 +323,7 @@ while i < n: i = i + a.x return i - ''', + ''', 39, ([20], 20), ([31], 32)) @@ -289,7 +344,7 @@ while i < n: i = j + i return i, type(i) is float - ''', + ''', 35, ([20], (20, True)), ([31], (32, True))) @@ -309,7 +364,7 @@ i += 1 l.append(i) return i, len(l) - ''', + ''', 37, ([20], (20, 18)), ([31], (31, 29))) From pedronis at codespeak.net Sun Nov 22 16:33:38 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 22 Nov 2009 16:33:38 +0100 (CET) Subject: [pypy-svn] r69501 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091122153338.26A97168024@codespeak.net> Author: pedronis Date: Sun Nov 22 16:33:37 2009 New Revision: 69501 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Log: failing test about inputargs/holes interaction Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Sun Nov 22 16:33:37 2009 @@ -307,12 +307,14 @@ for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: + assert box not in self.liveboxes_from_env assert tagged_eq(tagged, UNASSIGNED) index = memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): + assert box not in self.liveboxes_from_env index = memo.assign_number_to_virtual(box) self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Sun Nov 22 16:33:37 2009 @@ -304,6 +304,8 @@ # ENTER - compile the leaving path self.check_enter_count(4) +class VirtualMiscTests: + def test_guards_around_forcing(self): class A(object): def __init__(self, x): @@ -329,6 +331,28 @@ return 0 self.meta_interp(f, [50]) + def test_guards_and_holes(self): + class A(object): + def __init__(self, x): + self.x = x + mydriver = JitDriver(reds = ['n', 'tot'], greens = []) + + def f(n): + tot = 0 + while n > 0: + mydriver.can_enter_jit(n=n, tot=tot) + mydriver.jit_merge_point(n=n, tot=tot) + a = A(n) + b = A(n+1) + if n % 9 == 0: + tot += (a.x + b.x) % 3 + c = A(n+1) + if n % 10 == 0: + tot -= (c.x + a.x) % 3 + n -= 1 + return tot + self.meta_interp(f, [70]) + # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class @@ -434,3 +458,11 @@ p = lltype.malloc(NODE2) p.parent.typeptr = vtable2 return p + +# misc + +class TestOOTypeMisc(VirtualMiscTests, OOJitMixin): + pass + +class TestLLTypeMisc(VirtualMiscTests, LLJitMixin): + pass From arigo at codespeak.net Sun Nov 22 16:46:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 16:46:06 +0100 (CET) Subject: [pypy-svn] r69502 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091122154606.C7C99168024@codespeak.net> Author: arigo Date: Sun Nov 22 16:46:06 2009 New Revision: 69502 Modified: pypy/trunk/pypy/objspace/std/test/test_longobject.py Log: Bah. Modified: pypy/trunk/pypy/objspace/std/test/test_longobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_longobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_longobject.py Sun Nov 22 16:46:06 2009 @@ -169,13 +169,13 @@ -1895067127, # with 32-bit platforms 1234567890123456789) # with 64-bit platforms - def math_log(self): + def test_math_log(self): import math raises(ValueError, math.log, 0L) raises(ValueError, math.log, -1L) raises(ValueError, math.log, -2L) raises(ValueError, math.log, -(1L << 10000)) - raises(ValueError, math.log, 0) + #raises(ValueError, math.log, 0) raises(ValueError, math.log, -1) raises(ValueError, math.log, -2) From arigo at codespeak.net Sun Nov 22 16:53:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 16:53:03 +0100 (CET) Subject: [pypy-svn] r69503 - pypy/branch/float-comparison Message-ID: <20091122155303.08B6D168024@codespeak.net> Author: arigo Date: Sun Nov 22 16:53:03 2009 New Revision: 69503 Added: pypy/branch/float-comparison/ - copied from r69502, pypy/trunk/ Log: Trying to fix issue384. From pedronis at codespeak.net Sun Nov 22 17:09:47 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 22 Nov 2009 17:09:47 +0100 (CET) Subject: [pypy-svn] r69504 - pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test Message-ID: <20091122160947.7E22C168023@codespeak.net> Author: pedronis Date: Sun Nov 22 17:09:46 2009 New Revision: 69504 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Log: check the result also Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_virtual.py Sun Nov 22 17:09:46 2009 @@ -351,7 +351,9 @@ tot -= (c.x + a.x) % 3 n -= 1 return tot - self.meta_interp(f, [70]) + r = self.meta_interp(f, [70]) + expected = f(70) + assert r == expected # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class From arigo at codespeak.net Sun Nov 22 17:44:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 17:44:30 +0100 (CET) Subject: [pypy-svn] r69505 - in pypy/branch/float-comparison/pypy/rlib: . test Message-ID: <20091122164430.4A830168024@codespeak.net> Author: arigo Date: Sun Nov 22 17:44:29 2009 New Revision: 69505 Modified: pypy/branch/float-comparison/pypy/rlib/rbigint.py pypy/branch/float-comparison/pypy/rlib/test/test_rbigint.py Log: Add the argument 'rounding_towards' to rbigint.fromfloat(). Add tests. Also make constants ZERO and ONE global constants. Modified: pypy/branch/float-comparison/pypy/rlib/rbigint.py ============================================================================== --- pypy/branch/float-comparison/pypy/rlib/rbigint.py (original) +++ pypy/branch/float-comparison/pypy/rlib/rbigint.py Sun Nov 22 17:44:29 2009 @@ -75,7 +75,7 @@ sign = 1 ival = r_uint(intval) else: - return rbigint() + return ZERO # Count the number of Python digits. # We used to pick 5 ("big enough for anything"), but that's a # waste of time and space given that 5*15 = 75 bits are rarely @@ -97,35 +97,43 @@ def frombool(b): if b: - return rbigint([1], 1) - return rbigint() + return ONE + return ZERO frombool = staticmethod(frombool) def fromlong(l): return rbigint(*args_from_long(l)) fromlong = staticmethod(fromlong) - def fromfloat(dval): + def fromfloat(dval, rounding_towards=0): """ Create a new bigint object from a float """ - neg = 0 + neg = False if isinf(dval): raise OverflowError if dval < 0.0: - neg = 1 + neg = True dval = -dval frac, expo = math.frexp(dval) # dval = frac*2**expo; 0.0 <= frac < 1.0 if expo <= 0: - return rbigint() - ndig = (expo-1) // SHIFT + 1 # Number of 'digits' in result - v = rbigint([0] * ndig, 1) - frac = math.ldexp(frac, (expo-1) % SHIFT + 1) - for i in range(ndig-1, -1, -1): - bits = mask_digit(int(frac)) - v._setdigit(i, bits) - frac -= float(bits) - frac = math.ldexp(frac, SHIFT) - if neg: - v.sign = -1 + v = ZERO + else: + ndig = (expo-1) // SHIFT + 1 # Number of 'digits' in result + v = rbigint([0] * ndig, 1) + frac = math.ldexp(frac, (expo-1) % SHIFT + 1) + for i in range(ndig-1, -1, -1): + bits = mask_digit(int(frac)) + v._setdigit(i, bits) + frac -= float(bits) + frac = math.ldexp(frac, SHIFT) + if neg: + v.sign = -1 + if rounding_towards != 0 and frac != 0.0: + if not neg: + if rounding_towards > 0: + v = v.add(ONE) + else: + if rounding_towards < 0: + v = v.sub(ONE) return v fromfloat = staticmethod(fromfloat) @@ -352,7 +360,7 @@ div, mod = _divrem(v, w) if mod.sign * w.sign == -1: mod = mod.add(w) - div = div.sub(rbigint([1], 1)) + div = div.sub(ONE) return div, mod def pow(a, b, c=None): @@ -384,7 +392,7 @@ # if modulus == 1: # return 0 if c._numdigits() == 1 and c.digits[0] == 1: - return rbigint() + return ZERO # if base < 0: # base = base % modulus @@ -396,7 +404,7 @@ # At this point a, b, and c are guaranteed non-negative UNLESS # c is NULL, in which case a may be negative. */ - z = rbigint([1], 1) + z = ONE # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result) # into helper function result = _help_mult(x, y, c) @@ -416,8 +424,8 @@ else: # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82) # z still holds 1L - table = [z] * 32 - table[0] = z; + table = [None] * 32 + table[0] = z for i in range(1, 32): table[i] = _help_mult(table[i-1], a, c) i = b._numdigits() - 1 @@ -444,7 +452,7 @@ return rbigint(self.digits, abs(self.sign)) def invert(self): #Implement ~x as -(x + 1) - return self.add(rbigint([1], 1)).neg() + return self.add(ONE).neg() def lshift(self, int_other): if int_other < 0: @@ -490,7 +498,7 @@ wordshift = int_other // SHIFT newsize = self._numdigits() - wordshift if newsize <= 0: - return rbigint() + return ZERO loshift = int_other % SHIFT hishift = SHIFT - loshift @@ -565,6 +573,9 @@ # Helper Functions +ZERO = rbigint() +ONE = rbigint([1], 1) + def _help_mult(x, y, c): """ Modified: pypy/branch/float-comparison/pypy/rlib/test/test_rbigint.py ============================================================================== --- pypy/branch/float-comparison/pypy/rlib/test/test_rbigint.py (original) +++ pypy/branch/float-comparison/pypy/rlib/test/test_rbigint.py Sun Nov 22 17:44:29 2009 @@ -172,6 +172,32 @@ x = 12345.6789e200 x *= x assert raises(OverflowError, rbigint.fromfloat, x) + # + f1 = rbigint.fromfloat(12345.0, rounding_towards=-1) + assert f1.toint() == 12345 + f1 = rbigint.fromfloat(12345.0, rounding_towards=0) + assert f1.toint() == 12345 + f1 = rbigint.fromfloat(12345.0, rounding_towards=+1) + assert f1.toint() == 12345 + f1 = rbigint.fromfloat(-12345.0, rounding_towards=-1) + assert f1.toint() == -12345 + f1 = rbigint.fromfloat(-12345.0, rounding_towards=0) + assert f1.toint() == -12345 + f1 = rbigint.fromfloat(-12345.0, rounding_towards=+1) + assert f1.toint() == -12345 + # + f1 = rbigint.fromfloat(12345.6, rounding_towards=-1) + assert f1.toint() == 12345 + f1 = rbigint.fromfloat(12345.6, rounding_towards=0) + assert f1.toint() == 12345 + f1 = rbigint.fromfloat(12345.6, rounding_towards=+1) + assert f1.toint() == 12346 # here! + f1 = rbigint.fromfloat(-12345.6, rounding_towards=-1) + assert f1.toint() == -12346 # here! + f1 = rbigint.fromfloat(-12345.6, rounding_towards=0) + assert f1.toint() == -12345 + f1 = rbigint.fromfloat(-12345.6, rounding_towards=+1) + assert f1.toint() == -12345 def test_eq(self): x = 5858393919192332223L From pedronis at codespeak.net Sun Nov 22 17:47:35 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 22 Nov 2009 17:47:35 +0100 (CET) Subject: [pypy-svn] r69506 - in pypy/branch/compress-virtuals-resumedata/pypy/jit: backend/llgraph backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20091122164735.1FACE168024@codespeak.net> Author: pedronis Date: Sun Nov 22 17:47:34 2009 New Revision: 69506 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/test/test_virtual.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_pyjitpl.py Log: repair frontend/backend wrt holes with tests, we could do slightly saner things in backends but I want first to see if the branch helps Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/llgraph/llimpl.py Sun Nov 22 17:47:34 2009 @@ -385,8 +385,6 @@ self.memocast = memocast def getenv(self, v): - if v is None: - return None if isinstance(v, Constant): return v.value else: @@ -411,23 +409,27 @@ except GuardFailed: assert op.is_guard() _stats.exec_conditional_jumps += 1 - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args] - else: - args = [] if op.jump_target is not None: # a patched guard, pointing to further code + args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) operations = op.jump_target.operations opindex = 0 continue else: + fail_args = [] + if op.fail_args: + for fail_arg in op.fail_args: + if fail_arg is None: + fail_args.append(None) + else: + fail_args.append(self.getenv(fail_arg)) # a non-patched guard if self.verbose: log.trace('failed: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args + ', '.join(map(str, fail_args)),)) + self.fail_args = fail_args return op.fail_index #verbose = self.verbose assert (result is None) == (op.result is None) Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/test/runner_test.py Sun Nov 22 17:47:34 2009 @@ -202,6 +202,40 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_compile_bridge_with_holes(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, i1, None] + self.cpu.compile_loop(inputargs, operations, looptoken) + + i1b = BoxInt() + i3 = BoxInt() + bridge = [ + ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3), + ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2), + ResOperation(rop.JUMP, [i1b], None, descr=looptoken), + ] + bridge[1].fail_args = [i1b] + + self.cpu.compile_bridge(faildescr1, [i1b], bridge) + + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(0) + assert res == 20 + def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/regalloc.py Sun Nov 22 17:47:34 2009 @@ -240,9 +240,12 @@ def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py used = {} - for i in range(len(inputargs)): + i = 0 + for loc in locs: + if loc is None: # xxx bit kludgy + continue arg = inputargs[i] - loc = locs[i] + i += 1 if arg.type == FLOAT: if isinstance(loc, REG): self.xrm.reg_bindings[arg] = loc Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/backend/x86/test/test_virtual.py Sun Nov 22 17:47:34 2009 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.test.test_virtual import VirtualTests +from pypy.jit.metainterp.test.test_virtual import VirtualTests, VirtualMiscTests from pypy.jit.backend.x86.test.test_basic import Jit386Mixin class MyClass: @@ -13,3 +13,8 @@ @staticmethod def _new(): return MyClass() + +class TestsVirtualMisc(Jit386Mixin, VirtualMiscTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_virtual.py + pass Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/pyjitpl.py Sun Nov 22 17:47:34 2009 @@ -1622,22 +1622,20 @@ def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around - # xxx now that we have holes all this is delicate xxx - inputargs = self.load_values_from_failure(resumedescr) - warmrunnerstate = self.staticdata.state + inputargs_and_holes = self.load_values_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) - self.history.inputargs = inputargs + self.history.inputargs = [box for box in inputargs_and_holes if box] self.staticdata.profiler.start_tracing() else: self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true - self.rebuild_state_after_failure(resumedescr, inputargs) + self.rebuild_state_after_failure(resumedescr, inputargs_and_holes) def load_values_from_failure(self, resumedescr): cpu = self.cpu fail_arg_types = resumedescr.fail_arg_types - inputargs = [] + inputargs_and_holes = [] for i in range(len(fail_arg_types)): boxtype = fail_arg_types[i] if boxtype == history.INT: @@ -1647,11 +1645,11 @@ elif boxtype == history.FLOAT: box = history.BoxFloat(cpu.get_latest_value_float(i)) elif boxtype == history.HOLE: - continue + box = None else: assert False, "bad box type: num=%d" % ord(boxtype) - inputargs.append(box) - return inputargs + inputargs_and_holes.append(box) + return inputargs_and_holes def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_pyjitpl.py Sun Nov 22 17:47:34 2009 @@ -1,7 +1,8 @@ # some unit tests for the bytecode decoding -from pypy.jit.metainterp import pyjitpl, codewriter, resoperation +from pypy.jit.metainterp import pyjitpl, codewriter, resoperation, history +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt from pypy.jit.metainterp.history import History from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -142,3 +143,41 @@ assert equaloplists(metainterp.history.operations, [ ResOperation(rop.SAME_AS, [b2], boxes[1]), ]) + + +def test_initialize_state_from_guard_failure(): + from pypy.jit.metainterp.typesystem import llhelper + calls = [] + + class FakeCPU: + ts = llhelper + + def get_latest_value_int(self, index): + return index + + class FakeStaticData: + cpu = FakeCPU() + profiler = jitprof.EmptyProfiler() + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + + def rebuild_state_after_failure(descr, newboxes): + calls.append(newboxes) + metainterp.rebuild_state_after_failure = rebuild_state_after_failure + + class FakeResumeDescr: + pass + resumedescr = FakeResumeDescr() + resumedescr.fail_arg_types = [history.INT, history.HOLE, + history.INT, history.HOLE, + history.INT] + + metainterp.initialize_state_from_guard_failure(resumedescr, True) + + inp = metainterp.history.inputargs + assert len(inp) == 3 + assert [box.value for box in inp] == [0, 2, 4] + b0, b2, b4 = inp + assert len(calls) == 1 + assert calls[0] == [b0, None, b2, None, b4] + From pedronis at codespeak.net Sun Nov 22 17:59:27 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 22 Nov 2009 17:59:27 +0100 (CET) Subject: [pypy-svn] r69507 - in pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp: . test Message-ID: <20091122165927.22739168024@codespeak.net> Author: pedronis Date: Sun Nov 22 17:59:26 2009 New Revision: 69507 Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py Log: count right holes Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/jitprof.py Sun Nov 22 17:59:26 2009 @@ -23,6 +23,7 @@ NVIRTUALS NVHOLES NVREUSED +NVRIGHTHOLES """ def _setup(): @@ -180,6 +181,7 @@ self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) + self._print_intline("nvrightholes", cnt[NVRIGHTHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) def _print_line_time(self, string, i, tim): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/resume.py Sun Nov 22 17:59:26 2009 @@ -116,6 +116,7 @@ self.nvirtuals = 0 self.nvholes = 0 + self.nvrightholes = 0 self.nvreused = 0 def getconst(self, const): @@ -226,6 +227,7 @@ profiler.count(jitprof.NVIRTUALS, self.nvirtuals) profiler.count(jitprof.NVHOLES, self.nvholes) profiler.count(jitprof.NVREUSED, self.nvreused) + profiler.count(jitprof.NVRIGHTHOLES, self.nvrightholes) _frame_info_placeholder = (None, 0, 0) @@ -339,6 +341,13 @@ if vinfo.fieldnums is not fieldnums: memo.nvreused += 1 virtuals[num] = vinfo + # count right holes + r = 0 + while r < length: + if virtuals[-1-r]: + break + r += 1 + memo.nvrightholes += r def _gettagged(self, box): if isinstance(box, Const): Modified: pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata/pypy/jit/metainterp/test/test_jitprof.py Sun Nov 22 17:59:26 2009 @@ -64,7 +64,7 @@ assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, - 0, 0, 0] + 0, 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside From arigo at codespeak.net Sun Nov 22 18:41:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 18:41:19 +0100 (CET) Subject: [pypy-svn] r69508 - pypy/branch/float-comparison Message-ID: <20091122174119.2C0AA168024@codespeak.net> Author: arigo Date: Sun Nov 22 18:41:18 2009 New Revision: 69508 Removed: pypy/branch/float-comparison/ Log: Kill. From arigo at codespeak.net Sun Nov 22 18:41:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 18:41:33 +0100 (CET) Subject: [pypy-svn] r69509 - pypy/branch/float-comparison Message-ID: <20091122174133.EBF1C168024@codespeak.net> Author: arigo Date: Sun Nov 22 18:41:32 2009 New Revision: 69509 Added: pypy/branch/float-comparison/ - copied from r69508, pypy/trunk/ Log: Try again. From arigo at codespeak.net Sun Nov 22 18:44:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 18:44:44 +0100 (CET) Subject: [pypy-svn] r69510 - in pypy/branch/float-comparison/pypy/objspace/std: . test Message-ID: <20091122174444.C3733168024@codespeak.net> Author: arigo Date: Sun Nov 22 18:44:44 2009 New Revision: 69510 Modified: pypy/branch/float-comparison/pypy/objspace/std/floatobject.py pypy/branch/float-comparison/pypy/objspace/std/model.py pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py Log: Attempt to fix the issue by swapping the order: now int-to-float delegation comes before int-to-long. Added tests that try to cover int-float comparisons. I expect one of these tests to fail on 64-bit machines. Modified: pypy/branch/float-comparison/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/float-comparison/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/float-comparison/pypy/objspace/std/floatobject.py Sun Nov 22 18:44:44 2009 @@ -100,36 +100,9 @@ name = opname + "__Float_Float" return func_with_new_name(f, name), name -def declare_new_int_float_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_int1, w_float2): - i = w_int1.intval - j = w_float2.floatval - return space.newbool(op(float(i), j)) - name = opname + "__Int_Float" - return func_with_new_name(f, name), name - -def declare_new_float_int_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_float1, w_int2): - i = w_float1.floatval - j = w_int2.intval - return space.newbool(op(i, float(j))) - name = opname + "__Float_Int" - return func_with_new_name(f, name), name - for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: func, name = declare_new_float_comparison(op) globals()[name] = func - # XXX shortcuts disabled: see r54171 and issue #384. - #func, name = declare_new_int_float_comparison(op) - #globals()[name] = func - #func, name = declare_new_float_int_comparison(op) - #globals()[name] = func # for overflowing comparisons between longs and floats # XXX we might have to worry (later) about eq__Float_Int, for the case Modified: pypy/branch/float-comparison/pypy/objspace/std/model.py ============================================================================== --- pypy/branch/float-comparison/pypy/objspace/std/model.py (original) +++ pypy/branch/float-comparison/pypy/objspace/std/model.py Sun Nov 22 18:44:44 2009 @@ -153,20 +153,20 @@ ] self.typeorder[smallintobject.W_SmallIntObject] += [ (intobject.W_IntObject, smallintobject.delegate_SmallInt2Int), - (longobject.W_LongObject, smallintobject.delegate_SmallInt2Long), (floatobject.W_FloatObject, smallintobject.delegate_SmallInt2Float), + (longobject.W_LongObject, smallintobject.delegate_SmallInt2Long), (complexobject.W_ComplexObject, smallintobject.delegate_SmallInt2Complex), ] self.typeorder[boolobject.W_BoolObject] += [ (intobject.W_IntObject, boolobject.delegate_Bool2IntObject), - (longobject.W_LongObject, longobject.delegate_Bool2Long), (floatobject.W_FloatObject, floatobject.delegate_Bool2Float), + (longobject.W_LongObject, longobject.delegate_Bool2Long), (complexobject.W_ComplexObject, complexobject.delegate_Bool2Complex), ] self.typeorder[intobject.W_IntObject] += [ - (longobject.W_LongObject, longobject.delegate_Int2Long), (floatobject.W_FloatObject, floatobject.delegate_Int2Float), + (longobject.W_LongObject, longobject.delegate_Int2Long), (complexobject.W_ComplexObject, complexobject.delegate_Int2Complex), ] self.typeorder[longobject.W_LongObject] += [ Modified: pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py Sun Nov 22 18:44:44 2009 @@ -35,6 +35,30 @@ v = fobj.pow__Float_Float_ANY(self.space, f1, f2, self.space.w_None) assert v.floatval == x**y + def test_dont_use_long_impl(self): + from pypy.objspace.std.longobject import W_LongObject + space = self.space + saved = W_LongObject.__dict__['fromfloat'] + W_LongObject.fromfloat = lambda x: disabled + try: + w_i = space.wrap(12) + w_f = space.wrap(12.3) + assert space.unwrap(space.eq(w_f, w_i)) is False + assert space.unwrap(space.eq(w_i, w_f)) is False + assert space.unwrap(space.ne(w_f, w_i)) is True + assert space.unwrap(space.ne(w_i, w_f)) is True + assert space.unwrap(space.lt(w_f, w_i)) is False + assert space.unwrap(space.lt(w_i, w_f)) is True + assert space.unwrap(space.le(w_f, w_i)) is False + assert space.unwrap(space.le(w_i, w_f)) is True + assert space.unwrap(space.gt(w_f, w_i)) is True + assert space.unwrap(space.gt(w_i, w_f)) is False + assert space.unwrap(space.ge(w_f, w_i)) is True + assert space.unwrap(space.ge(w_i, w_f)) is False + finally: + W_LongObject.fromfloat = saved + + class AppTestAppFloatTest: def test_negatives(self): assert -1.1 < 0 @@ -187,9 +211,75 @@ assert 13 <= 13.0 assert 13 <= 13.01 + def test_comparison_more(self): + infinity = 1e200*1e200 + nan = infinity/infinity + for x in (123, 1 << 30, + (1 << 33) - 1, 1 << 33, (1 << 33) + 1, + 1 << 63, 1 << 70): + # + assert (x == float(x)) + assert (x >= float(x)) + assert (x <= float(x)) + assert not (x != float(x)) + assert not (x > float(x)) + assert not (x < float(x)) + # + assert not ((x - 1) == float(x)) + assert not ((x - 1) >= float(x)) + assert ((x - 1) <= float(x)) + assert ((x - 1) != float(x)) + assert not ((x - 1) > float(x)) + assert ((x - 1) < float(x)) + # + assert not ((x + 1) == float(x)) + assert ((x + 1) >= float(x)) + assert not ((x + 1) <= float(x)) + assert ((x + 1) != float(x)) + assert ((x + 1) > float(x)) + assert not ((x + 1) < float(x)) + # + #assert not (x == nan) + #assert not (x >= nan) + #assert not (x <= nan) + #assert (x != nan) + #assert not (x > nan) + #assert not (x < nan) + # + assert (float(x) == x) + assert (float(x) <= x) + assert (float(x) >= x) + assert not (float(x) != x) + assert not (float(x) < x) + assert not (float(x) > x) + # + assert not (float(x) == (x - 1)) + assert not (float(x) <= (x - 1)) + assert (float(x) >= (x - 1)) + assert (float(x) != (x - 1)) + assert not (float(x) < (x - 1)) + assert (float(x) > (x - 1)) + # + assert not (float(x) == (x + 1)) + assert (float(x) <= (x + 1)) + assert not (float(x) >= (x + 1)) + assert (float(x) != (x + 1)) + assert (float(x) < (x + 1)) + assert not (float(x) > (x + 1)) + # + #assert not (nan == x) + #assert not (nan <= x) + #assert not (nan >= x) + #assert (nan != x) + #assert not (nan < x) + #assert not (nan > x) + def test_multimethod_slice(self): assert 5 .__add__(3.14) is NotImplemented assert 3.25 .__add__(5) == 8.25 if hasattr(int, '__eq__'): # for py.test -A: CPython is inconsistent assert 5 .__eq__(3.14) is NotImplemented assert 3.14 .__eq__(5) is False + #if hasattr(long, '__eq__'): # for py.test -A: CPython is inconsistent + # assert 5L .__eq__(3.14) is NotImplemented + # assert 3.14 .__eq__(5L) is False From arigo at codespeak.net Sun Nov 22 18:52:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 18:52:26 +0100 (CET) Subject: [pypy-svn] r69511 - pypy/branch/float-comparison/pypy/objspace/std/test Message-ID: <20091122175226.1DF41168024@codespeak.net> Author: arigo Date: Sun Nov 22 18:52:25 2009 New Revision: 69511 Modified: pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py Log: This is what I intended. Modified: pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/float-comparison/pypy/objspace/std/test/test_floatobject.py Sun Nov 22 18:52:25 2009 @@ -216,7 +216,7 @@ nan = infinity/infinity for x in (123, 1 << 30, (1 << 33) - 1, 1 << 33, (1 << 33) + 1, - 1 << 63, 1 << 70): + 1 << 62, 1 << 70): # assert (x == float(x)) assert (x >= float(x)) From arigo at codespeak.net Sun Nov 22 20:12:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 20:12:41 +0100 (CET) Subject: [pypy-svn] r69512 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20091122191241.2B94F168023@codespeak.net> Author: arigo Date: Sun Nov 22 20:12:39 2009 New Revision: 69512 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_subprocess.py Log: This "except ImportError" means that the code here catches and ignores failed attempts at importing 'resource'. However, it does not work, because in that case it gets an UnboundLocalError on 'resource.error'. Must be factored in two distinct try:excepts. Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_subprocess.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_subprocess.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_subprocess.py Sun Nov 22 20:12:39 2009 @@ -485,10 +485,13 @@ """ try: import resource + except ImportError: + return None + try: old_limit = resource.getrlimit(resource.RLIMIT_CORE) resource.setrlimit(resource.RLIMIT_CORE, (0,0)) return old_limit - except (ImportError, ValueError, resource.error): + except (ValueError, resource.error): return None def _unsuppress_core_files(self, old_limit): @@ -497,8 +500,11 @@ return try: import resource + except ImportError: + return + try: resource.setrlimit(resource.RLIMIT_CORE, old_limit) - except (ImportError, ValueError, resource.error): + except (ValueError, resource.error): return def test_run_abort(self): From arigo at codespeak.net Sun Nov 22 20:38:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 20:38:10 +0100 (CET) Subject: [pypy-svn] r69513 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091122193810.A88F3168023@codespeak.net> Author: arigo Date: Sun Nov 22 20:38:10 2009 New Revision: 69513 Modified: pypy/trunk/pypy/objspace/std/floatobject.py pypy/trunk/pypy/objspace/std/model.py pypy/trunk/pypy/objspace/std/test/test_floatobject.py Log: issue384 testing Merge the branch/float-comparison: we found out that we can get the "expected" order of comparison 'float > int' delegates to 'float > float' rather than to 'float > long' simply by tweaking the order of delegations in model.py. No other test seems to fail because of that, neither our own nor from the stdlib, so let's merge. Modified: pypy/trunk/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/floatobject.py Sun Nov 22 20:38:10 2009 @@ -100,36 +100,9 @@ name = opname + "__Float_Float" return func_with_new_name(f, name), name -def declare_new_int_float_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_int1, w_float2): - i = w_int1.intval - j = w_float2.floatval - return space.newbool(op(float(i), j)) - name = opname + "__Int_Float" - return func_with_new_name(f, name), name - -def declare_new_float_int_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_float1, w_int2): - i = w_float1.floatval - j = w_int2.intval - return space.newbool(op(i, float(j))) - name = opname + "__Float_Int" - return func_with_new_name(f, name), name - for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: func, name = declare_new_float_comparison(op) globals()[name] = func - # XXX shortcuts disabled: see r54171 and issue #384. - #func, name = declare_new_int_float_comparison(op) - #globals()[name] = func - #func, name = declare_new_float_int_comparison(op) - #globals()[name] = func # for overflowing comparisons between longs and floats # XXX we might have to worry (later) about eq__Float_Int, for the case Modified: pypy/trunk/pypy/objspace/std/model.py ============================================================================== --- pypy/trunk/pypy/objspace/std/model.py (original) +++ pypy/trunk/pypy/objspace/std/model.py Sun Nov 22 20:38:10 2009 @@ -153,20 +153,20 @@ ] self.typeorder[smallintobject.W_SmallIntObject] += [ (intobject.W_IntObject, smallintobject.delegate_SmallInt2Int), - (longobject.W_LongObject, smallintobject.delegate_SmallInt2Long), (floatobject.W_FloatObject, smallintobject.delegate_SmallInt2Float), + (longobject.W_LongObject, smallintobject.delegate_SmallInt2Long), (complexobject.W_ComplexObject, smallintobject.delegate_SmallInt2Complex), ] self.typeorder[boolobject.W_BoolObject] += [ (intobject.W_IntObject, boolobject.delegate_Bool2IntObject), - (longobject.W_LongObject, longobject.delegate_Bool2Long), (floatobject.W_FloatObject, floatobject.delegate_Bool2Float), + (longobject.W_LongObject, longobject.delegate_Bool2Long), (complexobject.W_ComplexObject, complexobject.delegate_Bool2Complex), ] self.typeorder[intobject.W_IntObject] += [ - (longobject.W_LongObject, longobject.delegate_Int2Long), (floatobject.W_FloatObject, floatobject.delegate_Int2Float), + (longobject.W_LongObject, longobject.delegate_Int2Long), (complexobject.W_ComplexObject, complexobject.delegate_Int2Complex), ] self.typeorder[longobject.W_LongObject] += [ Modified: pypy/trunk/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_floatobject.py Sun Nov 22 20:38:10 2009 @@ -35,6 +35,30 @@ v = fobj.pow__Float_Float_ANY(self.space, f1, f2, self.space.w_None) assert v.floatval == x**y + def test_dont_use_long_impl(self): + from pypy.objspace.std.longobject import W_LongObject + space = self.space + saved = W_LongObject.__dict__['fromfloat'] + W_LongObject.fromfloat = lambda x: disabled + try: + w_i = space.wrap(12) + w_f = space.wrap(12.3) + assert space.unwrap(space.eq(w_f, w_i)) is False + assert space.unwrap(space.eq(w_i, w_f)) is False + assert space.unwrap(space.ne(w_f, w_i)) is True + assert space.unwrap(space.ne(w_i, w_f)) is True + assert space.unwrap(space.lt(w_f, w_i)) is False + assert space.unwrap(space.lt(w_i, w_f)) is True + assert space.unwrap(space.le(w_f, w_i)) is False + assert space.unwrap(space.le(w_i, w_f)) is True + assert space.unwrap(space.gt(w_f, w_i)) is True + assert space.unwrap(space.gt(w_i, w_f)) is False + assert space.unwrap(space.ge(w_f, w_i)) is True + assert space.unwrap(space.ge(w_i, w_f)) is False + finally: + W_LongObject.fromfloat = saved + + class AppTestAppFloatTest: def test_negatives(self): assert -1.1 < 0 @@ -187,9 +211,75 @@ assert 13 <= 13.0 assert 13 <= 13.01 + def test_comparison_more(self): + infinity = 1e200*1e200 + nan = infinity/infinity + for x in (123, 1 << 30, + (1 << 33) - 1, 1 << 33, (1 << 33) + 1, + 1 << 62, 1 << 70): + # + assert (x == float(x)) + assert (x >= float(x)) + assert (x <= float(x)) + assert not (x != float(x)) + assert not (x > float(x)) + assert not (x < float(x)) + # + assert not ((x - 1) == float(x)) + assert not ((x - 1) >= float(x)) + assert ((x - 1) <= float(x)) + assert ((x - 1) != float(x)) + assert not ((x - 1) > float(x)) + assert ((x - 1) < float(x)) + # + assert not ((x + 1) == float(x)) + assert ((x + 1) >= float(x)) + assert not ((x + 1) <= float(x)) + assert ((x + 1) != float(x)) + assert ((x + 1) > float(x)) + assert not ((x + 1) < float(x)) + # + #assert not (x == nan) + #assert not (x >= nan) + #assert not (x <= nan) + #assert (x != nan) + #assert not (x > nan) + #assert not (x < nan) + # + assert (float(x) == x) + assert (float(x) <= x) + assert (float(x) >= x) + assert not (float(x) != x) + assert not (float(x) < x) + assert not (float(x) > x) + # + assert not (float(x) == (x - 1)) + assert not (float(x) <= (x - 1)) + assert (float(x) >= (x - 1)) + assert (float(x) != (x - 1)) + assert not (float(x) < (x - 1)) + assert (float(x) > (x - 1)) + # + assert not (float(x) == (x + 1)) + assert (float(x) <= (x + 1)) + assert not (float(x) >= (x + 1)) + assert (float(x) != (x + 1)) + assert (float(x) < (x + 1)) + assert not (float(x) > (x + 1)) + # + #assert not (nan == x) + #assert not (nan <= x) + #assert not (nan >= x) + #assert (nan != x) + #assert not (nan < x) + #assert not (nan > x) + def test_multimethod_slice(self): assert 5 .__add__(3.14) is NotImplemented assert 3.25 .__add__(5) == 8.25 if hasattr(int, '__eq__'): # for py.test -A: CPython is inconsistent assert 5 .__eq__(3.14) is NotImplemented assert 3.14 .__eq__(5) is False + #if hasattr(long, '__eq__'): # for py.test -A: CPython is inconsistent + # assert 5L .__eq__(3.14) is NotImplemented + # assert 3.14 .__eq__(5L) is False From arigo at codespeak.net Sun Nov 22 20:38:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 22 Nov 2009 20:38:31 +0100 (CET) Subject: [pypy-svn] r69514 - pypy/branch/float-comparison Message-ID: <20091122193831.C2207168015@codespeak.net> Author: arigo Date: Sun Nov 22 20:38:31 2009 New Revision: 69514 Removed: pypy/branch/float-comparison/ Log: Remove merged branch. From fijal at codespeak.net Sun Nov 22 21:52:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 22 Nov 2009 21:52:57 +0100 (CET) Subject: [pypy-svn] r69515 - pypy/trunk/pypy/objspace/std Message-ID: <20091122205257.DFFB2168023@codespeak.net> Author: fijal Date: Sun Nov 22 21:52:54 2009 New Revision: 69515 Modified: pypy/trunk/pypy/objspace/std/stringobject.py Log: Implement str.join in a slightly faster manner. Use strinbuilder and precompute necessary length. Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Sun Nov 22 21:52:54 2009 @@ -351,6 +351,7 @@ def str_join__String_ANY(space, w_self, w_list): list_w = space.listview(w_list) str_w = space.str_w + s = space.str_w(w_self) if list_w: self = w_self._value listlen = 0 @@ -367,8 +368,17 @@ space.wrap("sequence item %d: expected string, %s " "found" % (i, space.type(w_s).getname(space, '?')))) - l[i] = space.str_w(w_s) - return space.wrap(self.join(l)) + # We can do a shortcut here "if isinstance(w_s, W_StringObject)", + # but this should be rendered pointless by having multilist + # which will simply always contain strings + reslen += len(space.str_w(w_s)) + reslen += len(s) * (len(list_w) - 1) + res = StringBuilder(reslen) + for i in range(len(list_w)): + res.append(space.str_w(list_w[i])) + if i != len(list_w) - 1: + res.append(s) + return space.wrap(res.build()) else: return W_StringObject.EMPTY From fijal at codespeak.net Sun Nov 22 23:37:22 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 22 Nov 2009 23:37:22 +0100 (CET) Subject: [pypy-svn] r69516 - pypy/trunk/pypy/objspace/std Message-ID: <20091122223722.A7B71168023@codespeak.net> Author: fijal Date: Sun Nov 22 23:37:20 2009 New Revision: 69516 Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py Log: improve join on unicode Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Sun Nov 22 23:37:20 2009 @@ -13,6 +13,7 @@ from pypy.objspace.std.formatting import mod_format from pypy.objspace.std.stringtype import stringstartswith, stringendswith +from pypy.rlib.rstring import UnicodeBuilder class W_UnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef @@ -174,29 +175,38 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.listview(w_list) + l_w = space.listview(w_list) delim = w_self._value - totlen = 0 - if len(l) == 0: + if len(l_w) == 0: return W_UnicodeObject.EMPTY - if (len(l) == 1 and - space.is_w(space.type(l[0]), space.w_unicode)): - return l[0] - - values_list = [None] * len(l) - for i in range(len(l)): - item = l[i] - if isinstance(item, W_UnicodeObject): + if (len(l_w) == 1 and + space.is_w(space.type(l_w[0]), space.w_unicode)): + return l_w[0] + lgt = 0 + for i in range(len(l_w)): + w_item = l_w[i] + if isinstance(w_item, W_UnicodeObject): # shortcut for performane - item = item._value - elif space.is_true(space.isinstance(item, space.w_str)): - item = space.unicode_w(item) + lgt += len(w_item._value) + elif space.is_true(space.isinstance(w_item, space.w_str)): + lgt += len(space.str_w(w_item)) + # some estimate, does not need to be perfect else: w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - values_list[i] = item - return W_UnicodeObject(w_self._value.join(values_list)) + # now we know it's a list of unicode or string + lgt += len(delim) * (len(l_w) - 1) + builder = UnicodeBuilder(lgt) + for i in range(len(l_w)): + w_item = l_w[i] + if isinstance(w_item, W_UnicodeObject): + builder.append(w_item._value) + else: + builder.append(space.unicode_w(w_item)) + if i != len(l_w) - 1: + builder.append(delim) + return W_UnicodeObject(builder.build()) def hash__Unicode(space, w_uni): s = w_uni._value From fijal at codespeak.net Sun Nov 22 23:42:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 22 Nov 2009 23:42:01 +0100 (CET) Subject: [pypy-svn] r69517 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20091122224201.943A3168023@codespeak.net> Author: fijal Date: Sun Nov 22 23:42:00 2009 New Revision: 69517 Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py Log: I'm confused. Either hybrid gc can realloc and we should enable it (which was the original idea), or we should simply kill the whole reallocing logic Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Sun Nov 22 23:42:00 2009 @@ -82,7 +82,7 @@ """ first_unused_gcflag = _gcflag_next_bit prebuilt_gc_objects_are_static_roots = True - can_realloc = False + can_realloc = True # the following values override the default arguments of __init__ when # translating to a real backend. From fijal at codespeak.net Sun Nov 22 23:42:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 22 Nov 2009 23:42:31 +0100 (CET) Subject: [pypy-svn] r69518 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20091122224231.1A063168023@codespeak.net> Author: fijal Date: Sun Nov 22 23:42:30 2009 New Revision: 69518 Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py Log: In case size is identical, don't realloc (missing test probably) Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Sun Nov 22 23:42:30 2009 @@ -240,8 +240,11 @@ result = llop.raw_realloc_grow(llmemory.Address, source_addr, old_tot_size, tot_size) else: - result = llop.raw_realloc_shrink(llmemory.Address, source_addr, - old_tot_size, tot_size) + if old_tot_size == tot_size: + result = source_addr + else: + result = llop.raw_realloc_shrink(llmemory.Address, source_addr, + old_tot_size, tot_size) if not result: self.gen2_resizable_objects.append(addr) raise MemoryError() From fijal at codespeak.net Sun Nov 22 23:44:39 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 22 Nov 2009 23:44:39 +0100 (CET) Subject: [pypy-svn] r69519 - pypy/trunk/pypy/rpython/memory/test Message-ID: <20091122224439.CFD12168023@codespeak.net> Author: fijal Date: Sun Nov 22 23:44:39 2009 New Revision: 69519 Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py Log: A test for previous checkin Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gc.py Sun Nov 22 23:44:39 2009 @@ -660,6 +660,21 @@ res = self.interpret(f, [15]) assert res == 16 + def test_resizable_buffer_no_realloc(self): + from pypy.rpython.lltypesystem.rstr import STR + from pypy.rpython.annlowlevel import hlstr + + def f(): + ptr = rgc.resizable_buffer_of_shape(STR, 1) + ptr.chars[0] = 'a' + ptr = rgc.resize_buffer(ptr, 1, 2) + ptr.chars[1] = 'b' + newptr = rgc.finish_building_buffer(ptr, 2) + return ptr == newptr + + assert self.interpret(f, []) == 1 + + def test_malloc_nonmovable_fixsize(self): py.test.skip("Not supported") From afa at codespeak.net Mon Nov 23 00:11:56 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 23 Nov 2009 00:11:56 +0100 (CET) Subject: [pypy-svn] r69520 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091122231156.602A1168023@codespeak.net> Author: afa Date: Mon Nov 23 00:11:55 2009 New Revision: 69520 Added: pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: Start implementing cursor bind variables Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Mon Nov 23 00:11:55 2009 @@ -12,7 +12,7 @@ from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS from pypy.module.oracle.interp_environ import Environment from pypy.module.oracle.interp_cursor import W_Cursor -from pypy.module.oracle.interp_pool import W_Pool +#from pypy.module.oracle.interp_pool import W_Pool from pypy.module.oracle.interp_variable import VT_String class W_Connection(Wrappable): Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 23 00:11:55 2009 @@ -21,6 +21,7 @@ self.statementType = -1 self.handle = None self.isOpen = True + self.isOwned = False self.setInputSizes = False self.arraySize = 50 @@ -155,7 +156,7 @@ self._checkOpen(space) # close the cursor - self._freeHandle(space, raiseError=True) + self.freeHandle(space, raiseError=True) self.isOpen = False self.handle = None @@ -223,7 +224,22 @@ interp_error.get(space).w_InterfaceError, space.wrap("not open")) - def _freeHandle(self, space, raiseError=True): + def allocateHandle(self): + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIStmt).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_STMT, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "Cursor_New()") + self.handle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + self.isOwned = True + + def freeHandle(self, space, raiseError=True): if not self.handle: return if self.isOwned: @@ -262,7 +278,7 @@ # release existing statement, if necessary self.w_statementTag = w_tag - self._freeHandle(space) + self.freeHandle(space) # prepare statement self.isOwned = False Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 23 00:11:55 2009 @@ -162,9 +162,10 @@ flavor='raw', zero=True) # perform extended initialization - self.initialize(cursor) + self.initialize(self.environment.space, cursor) def __del__(self): + self.finalize() lltype.free(self.actualElementsPtr, flavor='raw') if self.actualLength: lltype.free(self.actualLength, flavor='raw') @@ -172,6 +173,8 @@ lltype.free(self.data, flavor='raw') if self.returnCode: lltype.free(self.returnCode, flavor='raw') + if self.indicator: + lltype.free(self.indicator, flavor='raw') def getBufferSize(self): return self.size @@ -218,7 +221,10 @@ "Variable_MakeArray(): type does not support arrays")) self.isArray = True - def initialize(self, cursor): + def initialize(self, space, cursor): + pass + + def finalize(self): pass @classmethod @@ -408,7 +414,7 @@ else: return self.size * 2 - def initialize(self, cursor): + def initialize(self, space, cursor): self.actualLength = lltype.malloc(rffi.CArrayPtr(roci.ub2).TO, self.allocatedElements, zero=True, flavor='raw') @@ -797,8 +803,47 @@ pass class VT_Cursor(W_Variable): + oracleType = roci.SQLT_RSET + size = rffi.sizeof(roci.OCIStmt) canBeInArray = False + def initialize(self, space, cursor): + from pypy.module.oracle import interp_cursor + self.connection = cursor.connection + self.cursors_w = [None] * self.allocatedElements + for i in range(self.allocatedElements): + tempCursor = interp_cursor.W_Cursor(space, self.connection) + tempCursor.allocateHandle() + self.cursors_w[i] = space.wrap(tempCursor) + + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIStmt), self.data), + i) + dataptr[0] = tempCursor.handle + + def setValueProc(self, space, pos, w_value): + from pypy.module.oracle import interp_cursor + w_CursorType = space.gettypeobject(interp_cursor.W_Cursor.typedef) + if not space.is_true(space.isinstance(w_value, w_CursorType)): + raise OperationError( + space.w_TypeError, + space.wrap("expecting cursor")) + + self.cursors_w[pos] = w_value + + cursor = space.interp_w(interp_cursor.W_Cursor, w_value) + if not cursor.isOwned: + cursor.freeHandle(space, raiseError=True) + cursor.isOwned = True + cursor.allocateHandle() + + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIStmt), self.data), + pos) + dataptr[0] = cursor.handle + cursor.statementType = -1 + + class VT_Object(W_Variable): canBeInArray = False @@ -942,7 +987,11 @@ # XXX Delta - # XXX cursorType + from pypy.module.oracle import interp_cursor + if space.is_true(space.isinstance( # XXX is there an easier way? + w_value, + space.gettypeobject(interp_cursor.W_Cursor.typedef))): + return VT_Cursor, 0, numElements if space.is_true(space.isinstance(w_value, get(space).w_DecimalType)): return VT_NumberAsString, 0, numElements Added: pypy/trunk/pypy/module/oracle/test/test_cursorvar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Mon Nov 23 00:11:55 2009 @@ -0,0 +1,17 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class AppTestCursorVar(OracleTestBase): + + def test_bind_inout(self): + cur = self.cnx.cursor() + cursor = self.cnx.cursor() + assert cursor.description is None + cur.execute(""" + begin + open :cursor for select 1 numbercol from dual; + end;""", + cursor=cursor) + assert cursor.description == [ + ('NUMBERCOL', oracle.NUMBER, 127, 2, 0, 0, 1)] + data = cursor.fetchall() + assert data == [(1,)] From afa at codespeak.net Mon Nov 23 00:26:33 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 23 Nov 2009 00:26:33 +0100 (CET) Subject: [pypy-svn] r69521 - pypy/trunk/pypy/module/oracle Message-ID: <20091122232633.54D9E168023@codespeak.net> Author: afa Date: Mon Nov 23 00:26:32 2009 New Revision: 69521 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py Log: free the buffer sooner, and un-nest a few lines Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 23 00:26:32 2009 @@ -633,18 +633,19 @@ rffi.str_from_buffer(textbuf, text, BUFSIZE, rffi.cast(lltype.Signed, sizeptr[0]))) - if isinstance(self, VT_NumberAsString): - return w_strvalue - - try: - return space.call_function(space.w_int, w_strvalue) - except OperationError, e: - if e.match(space, space.w_ValueError): - return space.call_function(space.w_float, w_strvalue) - raise finally: rffi.keep_buffer_alive_until_here(textbuf, text) lltype.free(sizeptr, flavor='raw') + + if isinstance(self, VT_NumberAsString): + return w_strvalue + + try: + return space.call_function(space.w_int, w_strvalue) + except OperationError, e: + if e.match(space, space.w_ValueError): + return space.call_function(space.w_float, w_strvalue) + raise else: return transform.OracleNumberToPythonFloat( self.environment, dataptr) From afa at codespeak.net Mon Nov 23 09:23:01 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 23 Nov 2009 09:23:01 +0100 (CET) Subject: [pypy-svn] r69522 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091123082301.3BFFE168021@codespeak.net> Author: afa Date: Mon Nov 23 09:22:58 2009 New Revision: 69522 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_cursor.py Log: Add a check, and a test for it. Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 23 09:22:58 2009 @@ -942,6 +942,10 @@ def typeByPythonType(space, cursor, w_type): """Return a variable type given a Python type object""" moduledict = get(space) + if not space.is_true(space.isinstance(w_type, space.w_type)): + raise OperationError( + space.w_TypeError, + space.wrap("Variable_TypeByPythonType(): type expected")) if w_type.instancetypedef in variableTypeByTypedef: return variableTypeByTypedef[w_type.instancetypedef] if space.is_w(w_type, space.w_int): Modified: pypy/trunk/pypy/module/oracle/test/test_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursor.py Mon Nov 23 09:22:58 2009 @@ -25,6 +25,11 @@ assert cur.bindnames() == ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L"] + def test_var(self): + cur = self.cnx.cursor() + cur.var(oracle.NUMBER) + raises(TypeError, cur.var, 0) + def test_bind_out(self): cur = self.cnx.cursor() var = cur.var(oracle.NUMBER) From afa at codespeak.net Mon Nov 23 09:51:53 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 23 Nov 2009 09:51:53 +0100 (CET) Subject: [pypy-svn] r69523 - pypy/trunk/pypy/module/oracle/test Message-ID: <20091123085153.4F649168021@codespeak.net> Author: afa Date: Mon Nov 23 09:51:51 2009 New Revision: 69523 Modified: pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Log: Oracle 11 seems to describe columns differently Modified: pypy/trunk/pypy/module/oracle/test/test_cursorvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursorvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Mon Nov 23 09:51:51 2009 @@ -11,7 +11,9 @@ open :cursor for select 1 numbercol from dual; end;""", cursor=cursor) - assert cursor.description == [ - ('NUMBERCOL', oracle.NUMBER, 127, 2, 0, 0, 1)] + assert (cursor.description == + [('NUMBERCOL', oracle.NUMBER, 127, 2, 0, 0, 1)] + or cursor.description == + [('NUMBERCOL', oracle.NUMBER, 127, 2, 0, -127, 1)]) data = cursor.fetchall() assert data == [(1,)] From afa at codespeak.net Mon Nov 23 09:52:19 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 23 Nov 2009 09:52:19 +0100 (CET) Subject: [pypy-svn] r69524 - pypy/trunk/pypy/module/oracle Message-ID: <20091123085219.7E7EF168021@codespeak.net> Author: afa Date: Mon Nov 23 09:52:19 2009 New Revision: 69524 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: enumerate() is not RPython :-( Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Mon Nov 23 09:52:19 2009 @@ -135,7 +135,8 @@ # perform binds args_w = space.listview(w_list_of_args) numrows = len(args_w) - for i, w_arguments in enumerate(args_w): + for i in range(numrows): + w_arguments = args_w[i] deferred = i < numrows - 1 if space.is_true(space.isinstance(w_arguments, space.w_dict)): self._setBindVariablesByName( @@ -563,7 +564,9 @@ if self.bindList is None: self.bindList = [] - for i, w_value in enumerate(space.fixedview(w_vars)): + vars_w = space.fixedview(w_vars) + for i in range(len(vars_w)): + w_value = vars_w[i] if i < len(self.bindList): origVar = self.bindList[i] if space.is_w(origVar, space.w_None): @@ -657,7 +660,8 @@ def _performBind(self, space): # set values and perform binds for all bind variables if self.bindList: - for i, var in enumerate(self.bindList): + for i in range(len(self.bindList)): + var = self.bindList[i] var.bind(space, self, None, i + 1) if self.bindDict: items_w = space.fixedview( @@ -1018,7 +1022,8 @@ return self.bindDict else: self.bindList = [None] * len(args_w) - for i, w_value in enumerate(args_w): + for i in range(len(args_w)): + w_value = args_w[i] if space.is_w(w_value, space.w_None): var = space.w_None else: Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Mon Nov 23 09:52:19 2009 @@ -377,8 +377,8 @@ # set all of the values self.actualElementsPtr[0] = rffi.cast(lltype.Unsigned, len(elements_w)) - for i, w_element in enumerate(elements_w): - self.setSingleValue(space, i, w_element) + for i in range(len(elements_w)): + self.setSingleValue(space, i, elements_w[i]) def setValue(self, space, pos, w_value): if self.isArray: From fijal at codespeak.net Mon Nov 23 10:42:23 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 10:42:23 +0100 (CET) Subject: [pypy-svn] r69525 - pypy/branch/stringbuilder Message-ID: <20091123094223.8F17B168021@codespeak.net> Author: fijal Date: Mon Nov 23 10:42:22 2009 New Revision: 69525 Added: pypy/branch/stringbuilder/ - copied from r69524, pypy/trunk/ Log: Copy current trunk to a branch, since it broke stuff From fijal at codespeak.net Mon Nov 23 10:43:57 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 10:43:57 +0100 (CET) Subject: [pypy-svn] r69526 - in pypy/trunk/pypy: objspace/std rpython/memory/gc rpython/memory/test Message-ID: <20091123094357.59C46168021@codespeak.net> Author: fijal Date: Mon Nov 23 10:43:56 2009 New Revision: 69526 Modified: pypy/trunk/pypy/objspace/std/stringobject.py pypy/trunk/pypy/objspace/std/unicodeobject.py pypy/trunk/pypy/rpython/memory/gc/hybrid.py pypy/trunk/pypy/rpython/memory/test/test_gc.py Log: Revert a couple of last changes, will be tried and debugged on a branch Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Mon Nov 23 10:43:56 2009 @@ -351,7 +351,6 @@ def str_join__String_ANY(space, w_self, w_list): list_w = space.listview(w_list) str_w = space.str_w - s = space.str_w(w_self) if list_w: self = w_self._value listlen = 0 @@ -368,17 +367,8 @@ space.wrap("sequence item %d: expected string, %s " "found" % (i, space.type(w_s).getname(space, '?')))) - # We can do a shortcut here "if isinstance(w_s, W_StringObject)", - # but this should be rendered pointless by having multilist - # which will simply always contain strings - reslen += len(space.str_w(w_s)) - reslen += len(s) * (len(list_w) - 1) - res = StringBuilder(reslen) - for i in range(len(list_w)): - res.append(space.str_w(list_w[i])) - if i != len(list_w) - 1: - res.append(s) - return space.wrap(res.build()) + l[i] = space.str_w(w_s) + return space.wrap(self.join(l)) else: return W_StringObject.EMPTY Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Mon Nov 23 10:43:56 2009 @@ -13,7 +13,6 @@ from pypy.objspace.std.formatting import mod_format from pypy.objspace.std.stringtype import stringstartswith, stringendswith -from pypy.rlib.rstring import UnicodeBuilder class W_UnicodeObject(W_Object): from pypy.objspace.std.unicodetype import unicode_typedef as typedef @@ -175,38 +174,29 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l_w = space.listview(w_list) + l = space.listview(w_list) delim = w_self._value - if len(l_w) == 0: + totlen = 0 + if len(l) == 0: return W_UnicodeObject.EMPTY - if (len(l_w) == 1 and - space.is_w(space.type(l_w[0]), space.w_unicode)): - return l_w[0] - lgt = 0 - for i in range(len(l_w)): - w_item = l_w[i] - if isinstance(w_item, W_UnicodeObject): + if (len(l) == 1 and + space.is_w(space.type(l[0]), space.w_unicode)): + return l[0] + + values_list = [None] * len(l) + for i in range(len(l)): + item = l[i] + if isinstance(item, W_UnicodeObject): # shortcut for performane - lgt += len(w_item._value) - elif space.is_true(space.isinstance(w_item, space.w_str)): - lgt += len(space.str_w(w_item)) - # some estimate, does not need to be perfect + item = item._value + elif space.is_true(space.isinstance(item, space.w_str)): + item = space.unicode_w(item) else: w_msg = space.mod(space.wrap('sequence item %d: expected string or Unicode'), space.wrap(i)) raise OperationError(space.w_TypeError, w_msg) - # now we know it's a list of unicode or string - lgt += len(delim) * (len(l_w) - 1) - builder = UnicodeBuilder(lgt) - for i in range(len(l_w)): - w_item = l_w[i] - if isinstance(w_item, W_UnicodeObject): - builder.append(w_item._value) - else: - builder.append(space.unicode_w(w_item)) - if i != len(l_w) - 1: - builder.append(delim) - return W_UnicodeObject(builder.build()) + values_list[i] = item + return W_UnicodeObject(w_self._value.join(values_list)) def hash__Unicode(space, w_uni): s = w_uni._value Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Mon Nov 23 10:43:56 2009 @@ -82,7 +82,7 @@ """ first_unused_gcflag = _gcflag_next_bit prebuilt_gc_objects_are_static_roots = True - can_realloc = True + can_realloc = False # the following values override the default arguments of __init__ when # translating to a real backend. @@ -240,11 +240,8 @@ result = llop.raw_realloc_grow(llmemory.Address, source_addr, old_tot_size, tot_size) else: - if old_tot_size == tot_size: - result = source_addr - else: - result = llop.raw_realloc_shrink(llmemory.Address, source_addr, - old_tot_size, tot_size) + result = llop.raw_realloc_shrink(llmemory.Address, source_addr, + old_tot_size, tot_size) if not result: self.gen2_resizable_objects.append(addr) raise MemoryError() Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/trunk/pypy/rpython/memory/test/test_gc.py Mon Nov 23 10:43:56 2009 @@ -660,21 +660,6 @@ res = self.interpret(f, [15]) assert res == 16 - def test_resizable_buffer_no_realloc(self): - from pypy.rpython.lltypesystem.rstr import STR - from pypy.rpython.annlowlevel import hlstr - - def f(): - ptr = rgc.resizable_buffer_of_shape(STR, 1) - ptr.chars[0] = 'a' - ptr = rgc.resize_buffer(ptr, 1, 2) - ptr.chars[1] = 'b' - newptr = rgc.finish_building_buffer(ptr, 2) - return ptr == newptr - - assert self.interpret(f, []) == 1 - - def test_malloc_nonmovable_fixsize(self): py.test.skip("Not supported") From pedronis at codespeak.net Mon Nov 23 10:46:26 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 23 Nov 2009 10:46:26 +0100 (CET) Subject: [pypy-svn] r69527 - pypy/branch/compress-virtuals-resumedata2 Message-ID: <20091123094626.D3004168021@codespeak.net> Author: pedronis Date: Mon Nov 23 10:46:25 2009 New Revision: 69527 Added: pypy/branch/compress-virtuals-resumedata2/ - copied from r69526, pypy/trunk/ Log: branch to try to rebase compress virtuals resumedata to test things before thinking of merging From pedronis at codespeak.net Mon Nov 23 11:05:17 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 23 Nov 2009 11:05:17 +0100 (CET) Subject: [pypy-svn] r69528 - in pypy/branch/compress-virtuals-resumedata2/pypy: . jit/backend/llgraph jit/backend/llvm/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test objspace/std/test translator/c/test Message-ID: <20091123100517.8C846168020@codespeak.net> Author: pedronis Date: Mon Nov 23 11:05:16 2009 New Revision: 69528 Modified: pypy/branch/compress-virtuals-resumedata2/pypy/ (props changed) pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/llimpl.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/runner.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/test/runner_test.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/test/test_virtual.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/compile.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/history.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/logger.py (props changed) pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/pyjitpl.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_virtual.py pypy/branch/compress-virtuals-resumedata2/pypy/objspace/std/test/test_setobject.py (props changed) pypy/branch/compress-virtuals-resumedata2/pypy/translator/c/test/test_refcount.py (props changed) Log: merge the work done on the original branch here Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/llimpl.py Mon Nov 23 11:05:16 2009 @@ -364,7 +364,10 @@ op = loop.operations[-1] if op.fail_args is None: op.fail_args = [] - op.fail_args.append(_variables[intvar]) + if intvar == -1: + op.fail_args.append(None) + else: + op.fail_args.append(_variables[intvar]) def compile_redirect_fail(old_loop, old_index, new_loop): old_loop = _from_opaque(old_loop) @@ -407,23 +410,27 @@ except GuardFailed: assert op.is_guard() _stats.exec_conditional_jumps += 1 - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args] - else: - args = [] if op.jump_target is not None: # a patched guard, pointing to further code + args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) operations = op.jump_target.operations opindex = 0 continue else: + fail_args = [] + if op.fail_args: + for fail_arg in op.fail_args: + if fail_arg is None: + fail_args.append(None) + else: + fail_args.append(self.getenv(fail_arg)) # a non-patched guard if self.verbose: log.trace('failed: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args + ', '.join(map(str, fail_args)),)) + self.fail_args = fail_args return op.fail_index #verbose = self.verbose assert (result is None) == (op.result is None) Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/llgraph/runner.py Mon Nov 23 11:05:16 2009 @@ -171,7 +171,10 @@ index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: - llimpl.compile_add_fail_arg(c, var2index[box]) + if box is not None: + llimpl.compile_add_fail_arg(c, var2index[box]) + else: + llimpl.compile_add_fail_arg(c, -1) x = op.result if x is not None: if isinstance(x, history.BoxInt): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/test/runner_test.py Mon Nov 23 11:05:16 2009 @@ -125,6 +125,27 @@ res = self.cpu.get_latest_value_int(0) assert res == 10 + def test_compile_with_holes_in_fail_args(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, None, i1, None] + + self.cpu.compile_loop(inputargs, operations, looptoken) + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(2) + assert res == 10 + def test_backends_dont_keep_loops_alive(self): import weakref, gc self.cpu.dont_keepalive_stuff = True @@ -183,6 +204,40 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_compile_bridge_with_holes(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, i1, None] + self.cpu.compile_loop(inputargs, operations, looptoken) + + i1b = BoxInt() + i3 = BoxInt() + bridge = [ + ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3), + ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2), + ResOperation(rop.JUMP, [i1b], None, descr=looptoken), + ] + bridge[1].fail_args = [i1b] + + self.cpu.compile_bridge(faildescr1, [i1b], bridge) + + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(0) + assert res == 20 + def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py Mon Nov 23 11:05:16 2009 @@ -738,7 +738,7 @@ """ assert that all args are actually Boxes """ for arg in args: - assert isinstance(arg, Box) + assert arg is None or isinstance(arg, Box) def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): @@ -754,6 +754,8 @@ pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] + if arg is None: + continue loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: @@ -767,6 +769,8 @@ mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] + if arg is None: + continue loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py Mon Nov 23 11:05:16 2009 @@ -203,7 +203,8 @@ def possibly_free_vars(self, vars): for var in vars: - self.possibly_free_var(var) + if var is not None: + self.possibly_free_var(var) def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, imm_fine=True, @@ -239,9 +240,12 @@ def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py used = {} - for i in range(len(inputargs)): + i = 0 + for loc in locs: + if loc is None: # xxx bit kludgy + continue arg = inputargs[i] - loc = locs[i] + i += 1 if arg.type == FLOAT: if isinstance(loc, REG): self.xrm.reg_bindings[arg] = loc @@ -360,6 +364,8 @@ longevity[arg] = (start_live[arg], i) if op.is_guard(): for arg in op.fail_args: + if arg is None: # hole + continue assert isinstance(arg, Box) if arg not in start_live: print "Bogus arg in guard %d at %d" % (op.opnum, i) @@ -373,6 +379,8 @@ return longevity def loc(self, v): + if v is None: + return None if v.type == FLOAT: return self.xrm.loc(v) return self.rm.loc(v) Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/test/test_virtual.py Mon Nov 23 11:05:16 2009 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.test.test_virtual import VirtualTests +from pypy.jit.metainterp.test.test_virtual import VirtualTests, VirtualMiscTests from pypy.jit.backend.x86.test.test_basic import Jit386Mixin class MyClass: @@ -13,3 +13,8 @@ @staticmethod def _new(): return MyClass() + +class TestsVirtualMisc(Jit386Mixin, VirtualMiscTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_virtual.py + pass Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/compile.py Mon Nov 23 11:05:16 2009 @@ -233,7 +233,12 @@ def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes self.guard_opnum = guard_op.opnum - self.fail_arg_types = [box.type for box in boxes] + fail_arg_types = [history.HOLE] * len(boxes) + for i in range(len(boxes)): + box = boxes[i] + if box: + fail_arg_types[i] = box.type + self.fail_arg_types = fail_arg_types def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/history.py Mon Nov 23 11:05:16 2009 @@ -15,6 +15,7 @@ INT = 'i' REF = 'r' FLOAT = 'f' +HOLE = '_' FAILARGS_LIMIT = 1000 @@ -756,8 +757,9 @@ ops = op.descr._debug_suboperations TreeLoop.check_consistency_of_branch(ops, seen.copy()) for box in op.fail_args or []: - assert isinstance(box, Box) - assert box in seen + if box is not None: + assert isinstance(box, Box) + assert box in seen else: assert op.fail_args is None box = op.result Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py Mon Nov 23 11:05:16 2009 @@ -20,6 +20,10 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +NVIRTUALS +NVHOLES +NVREUSED +NVRIGHTHOLES """ def _setup(): @@ -175,6 +179,10 @@ self._print_intline("forcings", cnt[OPT_FORCINGS]) self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) + self._print_intline("nvirtuals", cnt[NVIRTUALS]) + self._print_intline("nvholes", cnt[NVHOLES]) + self._print_intline("nvrightholes", cnt[NVRIGHTHOLES]) + self._print_intline("nvreused", cnt[NVREUSED]) def _print_line_time(self, string, i, tim): final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py Mon Nov 23 11:05:16 2009 @@ -2,7 +2,7 @@ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.metainterp.jitprof import OPT_OPS, OPT_GUARDS, OPT_FORCINGS +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode from pypy.jit.metainterp.specnode import AbstractVirtualStructSpecNode @@ -62,6 +62,9 @@ def get_args_for_fail(self, modifier): pass + def make_virtual_info(self, modifier, fieldnums): + raise NotImplementedError # should not be called on this level + def is_constant(self): return self.level == LEVEL_CONSTANT @@ -134,9 +137,10 @@ class AbstractVirtualValue(OptValue): - _attrs_ = ('optimizer', 'keybox', 'source_op') + _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo') box = None level = LEVEL_NONNULL + _cached_vinfo = None def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer @@ -155,6 +159,19 @@ self._really_force() return self.box + def make_virtual_info(self, modifier, fieldnums): + vinfo = self._cached_vinfo + if vinfo is not None and resume.tagged_list_eq( + vinfo.fieldnums, fieldnums): + return vinfo + vinfo = self._make_virtual(modifier) + vinfo.fieldnums = fieldnums + self._cached_vinfo = vinfo + return vinfo + + def _make_virtual(self, modifier): + raise NotImplementedError("abstract base") + class AbstractVirtualStructValue(AbstractVirtualValue): _attrs_ = ('_fields', '_cached_sorted_fields') @@ -207,14 +224,11 @@ # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - self._make_virtual(modifier, lst, fieldboxes) + modifier.register_virtual_fields(self.keybox, fieldboxes) for ofs in lst: fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - raise NotImplementedError - class VirtualValue(AbstractVirtualStructValue): level = LEVEL_KNOWNCLASS @@ -224,10 +238,9 @@ assert isinstance(known_class, Const) self.known_class = known_class - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_virtual(self.keybox, self.known_class, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_virtual(self.known_class, fielddescrs) class VStructValue(AbstractVirtualStructValue): @@ -235,10 +248,9 @@ AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op) self.structdescr = structdescr - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_vstruct(self.keybox, self.structdescr, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_vstruct(self.structdescr, fielddescrs) class VArrayValue(AbstractVirtualValue): @@ -282,11 +294,14 @@ const = self.optimizer.new_const_item(self.arraydescr) for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) - modifier.make_varray(self.keybox, self.arraydescr, itemboxes) + modifier.register_virtual_fields(self.keybox, itemboxes) for itemvalue in self._items: if itemvalue is not self.constvalue: itemvalue.get_args_for_fail(modifier) + def _make_virtual(self, modifier): + return modifier.make_varray(self.arraydescr) + class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): raise NotImplementedError @@ -359,7 +374,7 @@ self.bool_boxes = {} def forget_numberings(self, virtualbox): - self.metainterp_sd.profiler.count(OPT_FORCINGS) + self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS) self.resumedata_memo.forget_numberings(virtualbox) def getinterned(self, box): @@ -480,6 +495,8 @@ else: self.optimize_default(op) self.loop.operations = self.newoperations + # accumulate counters + self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) @@ -492,9 +509,9 @@ op = op.clone() must_clone = False op.args[i] = box - self.metainterp_sd.profiler.count(OPT_OPS) + self.metainterp_sd.profiler.count(jitprof.OPT_OPS) if op.is_guard(): - self.metainterp_sd.profiler.count(OPT_GUARDS) + self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS) self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True @@ -507,7 +524,7 @@ assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) - if len(newboxes) > self.metainterp_sd.options.failargs_limit: + if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) # Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/pyjitpl.py Mon Nov 23 11:05:16 2009 @@ -1683,21 +1683,20 @@ def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around - inputargs = self.load_values_from_failure(resumedescr) - warmrunnerstate = self.staticdata.state + inputargs_and_holes = self.load_values_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) - self.history.inputargs = inputargs + self.history.inputargs = [box for box in inputargs_and_holes if box] self.staticdata.profiler.start_tracing() else: self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true - self.rebuild_state_after_failure(resumedescr, inputargs) + self.rebuild_state_after_failure(resumedescr, inputargs_and_holes) def load_values_from_failure(self, resumedescr): cpu = self.cpu fail_arg_types = resumedescr.fail_arg_types - inputargs = [] + inputargs_and_holes = [] for i in range(len(fail_arg_types)): boxtype = fail_arg_types[i] if boxtype == history.INT: @@ -1706,10 +1705,12 @@ box = cpu.ts.BoxRef(cpu.get_latest_value_ref(i)) elif boxtype == history.FLOAT: box = history.BoxFloat(cpu.get_latest_value_float(i)) + elif boxtype == history.HOLE: + box = None else: assert False, "bad box type: num=%d" % ord(boxtype) - inputargs.append(box) - return inputargs + inputargs_and_holes.append(box) + return inputargs_and_holes def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py Mon Nov 23 11:05:16 2009 @@ -1,6 +1,7 @@ import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp import jitprof from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated @@ -84,12 +85,21 @@ # please rpython :( return rarithmetic.widen(x) == rarithmetic.widen(y) +def tagged_list_eq(tl1, tl2): + if len(tl1) != len(tl2): + return False + for i in range(len(tl1)): + if not tagged_eq(tl1[i], tl2[i]): + return False + return True + TAGCONST = 0 TAGINT = 1 TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-1, TAGBOX) +UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) @@ -101,6 +111,13 @@ self.large_ints = {} self.refs = cpu.ts.new_ref_dict_2() self.numberings = {} + self.cached_boxes = {} + self.cached_virtuals = {} + + self.nvirtuals = 0 + self.nvholes = 0 + self.nvrightholes = 0 + self.nvreused = 0 def getconst(self, const): if const.type == INT: @@ -135,6 +152,8 @@ self.consts.append(const) return result + # env numbering + def number(self, values, snapshot): if snapshot is None: return None, {}, 0 @@ -173,7 +192,42 @@ def forget_numberings(self, virtualbox): # XXX ideally clear only the affected numberings self.numberings.clear() + self.clear_box_virtual_numbers() + + # caching for virtuals and boxes inside them + + def num_cached_boxes(self): + return len(self.cached_boxes) + + def assign_number_to_box(self, box, boxes): + if box in self.cached_boxes: + num = self.cached_boxes[box] + boxes[-num-1] = box + else: + boxes.append(box) + num = -len(boxes) + self.cached_boxes[box] = num + return num + + def num_cached_virtuals(self): + return len(self.cached_virtuals) + + def assign_number_to_virtual(self, box): + if box in self.cached_virtuals: + num = self.cached_virtuals[box] + else: + num = self.cached_virtuals[box] = -len(self.cached_virtuals) - 1 + return num + def clear_box_virtual_numbers(self): + self.cached_boxes.clear() + self.cached_virtuals.clear() + + def update_counters(self, profiler): + profiler.count(jitprof.NVIRTUALS, self.nvirtuals) + profiler.count(jitprof.NVHOLES, self.nvholes) + profiler.count(jitprof.NVREUSED, self.nvreused) + profiler.count(jitprof.NVRIGHTHOLES, self.nvrightholes) _frame_info_placeholder = (None, 0, 0) @@ -185,30 +239,19 @@ #self.virtuals = [] #self.vfieldboxes = [] - def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): - vinfo = VirtualInfo(known_class, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_vstruct(self, virtualbox, typedescr, fielddescrs, fieldboxes): - vinfo = VStructInfo(typedescr, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_varray(self, virtualbox, arraydescr, itemboxes): - vinfo = VArrayInfo(arraydescr) - self._make_virtual(virtualbox, vinfo, itemboxes) - - def _make_virtual(self, virtualbox, vinfo, fieldboxes): - if virtualbox in self.liveboxes_from_env: - tagged = self.liveboxes_from_env[virtualbox] - i, _ = untag(tagged) - assert self.virtuals[i] is None - self.virtuals[i] = vinfo - self.vfieldboxes[i] = fieldboxes - else: - tagged = tag(len(self.virtuals), TAGVIRTUAL) - self.virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) + def make_virtual(self, known_class, fielddescrs): + return VirtualInfo(known_class, fielddescrs) + + def make_vstruct(self, typedescr, fielddescrs): + return VStructInfo(typedescr, fielddescrs) + + def make_varray(self, arraydescr): + return VArrayInfo(arraydescr) + + def register_virtual_fields(self, virtualbox, fieldboxes): + tagged = self.liveboxes_from_env.get(virtualbox, UNASSIGNEDVIRTUAL) self.liveboxes[virtualbox] = tagged + self.vfieldboxes[virtualbox] = fieldboxes self._register_boxes(fieldboxes) def register_box(self, box): @@ -234,7 +277,8 @@ def finish(self, values): # compute the numbering storage = self.storage - numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) + numb, liveboxes_from_env, v = self.memo.number(values, + storage.rd_snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} storage.rd_numb = numb @@ -243,8 +287,7 @@ # collect liveboxes and virtuals n = len(liveboxes_from_env) - v liveboxes = [None]*n - self.virtuals = [None]*v - self.vfieldboxes = [None]*v + self.vfieldboxes = {} for box, tagged in liveboxes_from_env.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -254,31 +297,57 @@ value = values[box] value.get_args_for_fail(self) - self._number_virtuals(liveboxes) + self._number_virtuals(liveboxes, values, v) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) return liveboxes[:] - def _number_virtuals(self, liveboxes): + def _number_virtuals(self, liveboxes, values, num_env_virtuals): + memo = self.memo + new_liveboxes = [None] * memo.num_cached_boxes() for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: + assert box not in self.liveboxes_from_env assert tagged_eq(tagged, UNASSIGNED) - self.liveboxes[box] = tag(len(liveboxes), TAGBOX) - liveboxes.append(box) + index = memo.assign_number_to_box(box, new_liveboxes) + self.liveboxes[box] = tag(index, TAGBOX) else: assert tagbits == TAGVIRTUAL + if tagged_eq(tagged, UNASSIGNEDVIRTUAL): + assert box not in self.liveboxes_from_env + index = memo.assign_number_to_virtual(box) + self.liveboxes[box] = tag(index, TAGVIRTUAL) + new_liveboxes.reverse() + liveboxes.extend(new_liveboxes) storage = self.storage storage.rd_virtuals = None - if len(self.virtuals) > 0: - storage.rd_virtuals = self.virtuals[:] - for i in range(len(storage.rd_virtuals)): - vinfo = storage.rd_virtuals[i] - fieldboxes = self.vfieldboxes[i] - vinfo.fieldnums = [self._gettagged(box) - for box in fieldboxes] + vfieldboxes = self.vfieldboxes + if vfieldboxes: + length = num_env_virtuals + memo.num_cached_virtuals() + virtuals = storage.rd_virtuals = [None] * length + memo.nvirtuals += length + memo.nvholes += length - len(vfieldboxes) + for virtualbox, fieldboxes in vfieldboxes.iteritems(): + num, _ = untag(self.liveboxes[virtualbox]) + value = values[virtualbox] + fieldnums = [self._gettagged(box) + for box in fieldboxes] + vinfo = value.make_virtual_info(self, fieldnums) + # if a new vinfo instance is made, we get the fieldnums list we + # pass in as an attribute. hackish. + if vinfo.fieldnums is not fieldnums: + memo.nvreused += 1 + virtuals[num] = vinfo + # count right holes + r = 0 + while r < length: + if virtuals[-1-r]: + break + r += 1 + memo.nvrightholes += r def _gettagged(self, box): if isinstance(box, Const): @@ -392,10 +461,15 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(metainterp) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): numb = self.cur_numb @@ -450,8 +524,14 @@ for const in storage.rd_consts: debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - debug_print('\tbox', box.repr_rpython()) + if box is None: + debug_print('\tbox', 'None') + else: + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - virtual.debug_prints() + if virtual is None: + debug_print('\t\t', 'None') + else: + virtual.debug_prints() debug_stop("jit-resume") Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py Mon Nov 23 11:05:16 2009 @@ -63,7 +63,8 @@ ] assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] - assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0] + assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, + 0, 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 23 11:05:16 2009 @@ -74,6 +74,22 @@ lst2 = virt1._get_field_descr_list() assert lst1 is lst2 +def test_reuse_vinfo(): + class FakeVInfo(object): + pass + class FakeVirtualValue(optimizeopt.AbstractVirtualValue): + def _make_virtual(self, *args): + return FakeVInfo() + v1 = FakeVirtualValue(None, None, None) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) + assert vinfo1 is vinfo2 + vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is not vinfo2 + vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is vinfo4 + + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_pyjitpl.py Mon Nov 23 11:05:16 2009 @@ -1,7 +1,8 @@ # some unit tests for the bytecode decoding -from pypy.jit.metainterp import pyjitpl, codewriter, resoperation +from pypy.jit.metainterp import pyjitpl, codewriter, resoperation, history +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt from pypy.jit.metainterp.history import History from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -154,3 +155,39 @@ assert metainterp_sd.get_name_from_address(123) == 'a' assert metainterp_sd.get_name_from_address(456) == 'b' assert metainterp_sd.get_name_from_address(789) == '' + +def test_initialize_state_from_guard_failure(): + from pypy.jit.metainterp.typesystem import llhelper + calls = [] + + class FakeCPU: + ts = llhelper + + def get_latest_value_int(self, index): + return index + + class FakeStaticData: + cpu = FakeCPU() + profiler = jitprof.EmptyProfiler() + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + + def rebuild_state_after_failure(descr, newboxes): + calls.append(newboxes) + metainterp.rebuild_state_after_failure = rebuild_state_after_failure + + class FakeResumeDescr: + pass + resumedescr = FakeResumeDescr() + resumedescr.fail_arg_types = [history.INT, history.HOLE, + history.INT, history.HOLE, + history.INT] + + metainterp.initialize_state_from_guard_failure(resumedescr, True) + + inp = metainterp.history.inputargs + assert len(inp) == 3 + assert [box.value for box in inp] == [0, 2, 4] + b0, b2, b4 = inp + assert len(calls) == 1 + assert calls[0] == [b0, None, b2, None, b4] Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py Mon Nov 23 11:05:16 2009 @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue +from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue +from pypy.jit.metainterp.optimizeopt import VStructValue from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr from pypy.jit.metainterp.history import ConstPtr, ConstFloat @@ -33,6 +34,12 @@ assert tagged_eq(UNASSIGNED, UNASSIGNED) assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) +def test_tagged_list_eq(): + assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)], + [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)]) + assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) + assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) + class MyMetaInterp: def __init__(self, cpu=None): if cpu is None: @@ -105,6 +112,22 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] + +def test_prepare_virtuals(): + class FakeVinfo(object): + def allocate(self, metainterp): + return "allocated" + def setfields(self, metainterp, virtual, func): + assert virtual == "allocated" + class FakeStorage(object): + rd_virtuals = [FakeVinfo(), None] + rd_numb = [] + rd_consts = [] + class FakeMetainterp(object): + cpu = None + reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) + assert reader.virtuals == ["allocated", None] + # ____________________________________________________________ @@ -356,20 +379,21 @@ modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(0, TAGCONST)] b6 = BoxPtr() v6 = virtual_value(b6, c2, None) v6.setfield(LLtypeMixin.nextdescr, v6) values = {b2: virtual_value(b2, b4, v6), b6: v6} + memo.clear_box_virtual_numbers() modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] # now on to resuming metainterp = MyMetaInterp() @@ -398,6 +422,33 @@ FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 +def test_rebuild_from_resumedata_two_guards_w_shared_virtuals(): + b1, b2, b3, b4, b5, b6 = [BoxPtr(), BoxPtr(), BoxInt(), BoxPtr(), BoxInt(), BoxInt()] + c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), + LLtypeMixin.nodebox.constbox()] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] + capture_resumedata(fs, None, storage) + + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + values = {b2: virtual_value(b2, b5, c4)} + modifier = ResumeDataVirtualAdder(storage, memo) + liveboxes = modifier.finish(values) + assert len(storage.rd_virtuals) == 1 + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), + tag(0, TAGCONST)] + + storage2 = Storage() + fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] + capture_resumedata(fs, None, storage2) + values[b4] = virtual_value(b4, b6, c4) + modifier = ResumeDataVirtualAdder(storage2, memo) + liveboxes = modifier.finish(values) + assert len(storage2.rd_virtuals) == 2 + assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums + assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] + + def test_resumedata_top_recursive_virtuals(): b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() @@ -413,9 +464,9 @@ liveboxes = modifier.finish(values) assert liveboxes == [b3] assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), tag(0, TAGVIRTUAL)] @@ -553,31 +604,76 @@ tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 +def test_ResumeDataLoopMemo_number_boxes(): + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + b1, b2 = [BoxInt(), BoxInt()] + assert memo.num_cached_boxes() == 0 + boxes = [] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + assert memo.num_cached_boxes() == 1 + boxes = [None] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [b1, b2] + + assert memo.num_cached_boxes() == 2 + boxes = [None, None] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [None, b2] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1, b2] + + memo.clear_box_virtual_numbers() + assert memo.num_cached_boxes() == 0 + +def test_ResumeDataLoopMemo_number_virtuals(): + memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + b1, b2 = [BoxInt(), BoxInt()] + assert memo.num_cached_virtuals() == 0 + num = memo.assign_number_to_virtual(b1) + assert num == -1 + assert memo.num_cached_virtuals() == 1 + num = memo.assign_number_to_virtual(b1) + assert num == -1 + num = memo.assign_number_to_virtual(b2) + assert num == -2 + + assert memo.num_cached_virtuals() == 2 + num = memo.assign_number_to_virtual(b2) + assert num == -2 + num = memo.assign_number_to_virtual(b1) + assert num == -1 + + memo.clear_box_virtual_numbers() + assert memo.num_cached_virtuals() == 0 -def test__make_virtual(): +def test_register_virtual_fields(): b1, b2 = BoxInt(), BoxInt() vbox = BoxPtr() modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_virtual(vbox, None, ['a', 'b'], [b1, b2]) - assert modifier.liveboxes == {vbox: tag(0, TAGVIRTUAL), b1: UNASSIGNED, + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2]) + assert modifier.liveboxes == {vbox: UNASSIGNEDVIRTUAL, b1: UNASSIGNED, b2: UNASSIGNED} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2]] + assert modifier.vfieldboxes == {vbox: [b1, b2]} modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} modifier.liveboxes = {} - modifier.virtuals = [None] - modifier.vfieldboxes = [None] - modifier.make_virtual(vbox, None, ['a', 'b', 'c'], [b1, b2, vbox]) + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2, vbox]) assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, vbox: tag(0, TAGVIRTUAL)} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2, vbox]] + assert modifier.vfieldboxes == {vbox: [b1, b2, vbox]} def _resume_remap(liveboxes, expected, *newvalues): newboxes = [] @@ -692,22 +788,26 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_virtual(b2s, - ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], - [b4s, c1s]) # new fields - modifier.make_virtual(b4s, - ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, - LLtypeMixin.otherdescr], - [b2s, b3s, b5s]) # new fields + modifier.vfieldboxes = {} + + v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), b2s) + v2._fields = {LLtypeMixin.nextdescr: b4s, + LLtypeMixin.valuedescr: c1s} + v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] + v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), b4s) + v4._fields = {LLtypeMixin.nextdescr: b2s, + LLtypeMixin.valuedescr: b3s, + LLtypeMixin.otherdescr: b5s} + v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, + LLtypeMixin.otherdescr] + modifier.register_virtual_fields(b2s, [b4s, c1s]) + modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) + values = {b2s: v2, b4s: v4} liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, values, 0) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -722,22 +822,25 @@ metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 2 - b2t = reader._decode_box(tag(0, TAGVIRTUAL)) - b4t = reader._decode_box(tag(1, TAGVIRTUAL)) + b2t = reader._decode_box(modifier._gettagged(b2s)) + b4t = reader._decode_box(modifier._gettagged(b4s)) trace = metainterp.trace - expected = [ - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, + b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu)], - b2t, None), - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, + b2t, None) + b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu)], - b4t, None), - (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr), - ] + b4t, None) + b2set = [(rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr)] + b4set = [(rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr)] + if untag(modifier._gettagged(b2s))[0] == -2: + expected = [b2new, b4new] + b2set + b4set + else: + expected = [b4new, b2new] + b4set + b2set + for x, y in zip(expected, trace): assert x == y ptr = b2t.value._obj.container._as_ptr() @@ -757,13 +860,17 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_varray(b2s, - LLtypeMixin.arraydescr, - [b4s, c1s]) # new fields + modifier.vfieldboxes = {} + + class FakeOptimizer(object): + def new_const_item(self, descr): + return None + v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) + v2._items = [b4s, c1s] + modifier.register_virtual_fields(b2s, [b4s, c1s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + values = {b2s: v2} + modifier._number_virtuals(liveboxes, values, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None @@ -803,14 +910,13 @@ modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_vstruct(b2s, - LLtypeMixin.ssize, - [LLtypeMixin.adescr, LLtypeMixin.bdescr], - [c1s, b4s]) # new fields + modifier.vfieldboxes = {} + v2 = VStructValue(None, LLtypeMixin.ssize, b2s) + v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} + v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] + modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, {b2s: v2}, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_virtual.py Mon Nov 23 11:05:16 2009 @@ -304,6 +304,8 @@ # ENTER - compile the leaving path self.check_enter_count(4) +class VirtualMiscTests: + def test_guards_around_forcing(self): class A(object): def __init__(self, x): @@ -329,6 +331,29 @@ return 0 self.meta_interp(f, [50]) + def test_guards_and_holes(self): + class A(object): + def __init__(self, x): + self.x = x + mydriver = JitDriver(reds = ['n', 'tot'], greens = []) + + def f(n): + tot = 0 + while n > 0: + mydriver.can_enter_jit(n=n, tot=tot) + mydriver.jit_merge_point(n=n, tot=tot) + a = A(n) + b = A(n+1) + if n % 9 == 0: + tot += (a.x + b.x) % 3 + c = A(n+1) + if n % 10 == 0: + tot -= (c.x + a.x) % 3 + n -= 1 + return tot + r = self.meta_interp(f, [70]) + expected = f(70) + assert r == expected # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class @@ -435,3 +460,11 @@ p = lltype.malloc(NODE2) p.parent.typeptr = vtable2 return p + +# misc + +class TestOOTypeMisc(VirtualMiscTests, OOJitMixin): + pass + +class TestLLTypeMisc(VirtualMiscTests, LLJitMixin): + pass From arigo at codespeak.net Mon Nov 23 11:35:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 11:35:27 +0100 (CET) Subject: [pypy-svn] r69529 - pypy/extradoc/planning Message-ID: <20091123103527.5168C168020@codespeak.net> Author: arigo Date: Mon Nov 23 11:35:26 2009 New Revision: 69529 Modified: pypy/extradoc/planning/jit.txt Log: Draft plan for killing the hacks in executioncontext.py. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 23 11:35:26 2009 @@ -125,3 +125,25 @@ - ResOperation - ResumeGuardDescr - ConstInt + + +Plan for killing the hacks in executioncontext.py +------------------------------------------------- + +1. Finish support for virtualizables. We need a way to ask, + typically from a recursive residual call, for the structure + in some parent JITted context. So the virtualizable and + all virtuals need a way to be forced. The idea is to reuse + as much as possible the backend logic of guard failure and + the corresponding frontend logic to rebuild virtuals, but + called in this case indirectly, in the middle of running a + residual call. + +2. Reverting to the old logic without "f_forward", we need to + make "f_back" and "topframe" pointers that can reference + virtuals. Probably implemented by having them point to a + stub object that contains the stack position and the index + of the virtual in the list of virtuals. If we need to + access its content, we force it as above. + +Open question: how can this be done on CLI? From arigo at codespeak.net Mon Nov 23 11:38:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 11:38:23 +0100 (CET) Subject: [pypy-svn] r69530 - pypy/extradoc/planning Message-ID: <20091123103823.8493E168008@codespeak.net> Author: arigo Date: Mon Nov 23 11:38:22 2009 New Revision: 69530 Modified: pypy/extradoc/planning/jit.txt Log: Add a sentence. Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 23 11:38:22 2009 @@ -144,6 +144,9 @@ virtuals. Probably implemented by having them point to a stub object that contains the stack position and the index of the virtual in the list of virtuals. If we need to - access its content, we force it as above. + access its content, we force it as above. For this to work, + we must make sure that no reference to such a stub can live + longer than the residual call -- either we are sure that it + dies, or we have forced it. Open question: how can this be done on CLI? From fijal at codespeak.net Mon Nov 23 11:38:43 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 11:38:43 +0100 (CET) Subject: [pypy-svn] r69531 - pypy/branch/unpackiterable-improvements Message-ID: <20091123103843.C10AA16800D@codespeak.net> Author: fijal Date: Mon Nov 23 11:38:43 2009 New Revision: 69531 Removed: pypy/branch/unpackiterable-improvements/ Log: Remove merged branch From fijal at codespeak.net Mon Nov 23 11:39:20 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 11:39:20 +0100 (CET) Subject: [pypy-svn] r69532 - pypy/branch/llop-copy-list Message-ID: <20091123103920.5A65F16801F@codespeak.net> Author: fijal Date: Mon Nov 23 11:39:19 2009 New Revision: 69532 Removed: pypy/branch/llop-copy-list/ Log: This branch is not going to be worked on now, kill From fijal at codespeak.net Mon Nov 23 11:39:55 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 11:39:55 +0100 (CET) Subject: [pypy-svn] r69533 - pypy/branch/no-trace-if-frame-escapes Message-ID: <20091123103955.71EBC16801F@codespeak.net> Author: fijal Date: Mon Nov 23 11:39:55 2009 New Revision: 69533 Removed: pypy/branch/no-trace-if-frame-escapes/ Log: This one also dies From arigo at codespeak.net Mon Nov 23 11:59:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 11:59:35 +0100 (CET) Subject: [pypy-svn] r69534 - pypy/branch/faster-raise-2 Message-ID: <20091123105935.252B4168016@codespeak.net> Author: arigo Date: Mon Nov 23 11:59:35 2009 New Revision: 69534 Added: pypy/branch/faster-raise-2/ - copied from r69533, pypy/trunk/ Log: A branch in which to merge part of faster-raise. From arigo at codespeak.net Mon Nov 23 12:00:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:00:42 +0100 (CET) Subject: [pypy-svn] r69535 - in pypy/trunk/py: . bin impl impl/cmdline impl/code impl/compat impl/io impl/log impl/path impl/path/gateway impl/process impl/test impl/test/dist impl/test/looponfail plugin Message-ID: <20091123110042.5AEB816801B@codespeak.net> Author: arigo Date: Mon Nov 23 12:00:42 2009 New Revision: 69535 Modified: pypy/trunk/py/ (props changed) pypy/trunk/py/__init__.py (props changed) pypy/trunk/py/apipkg.py (props changed) pypy/trunk/py/bin/ (props changed) pypy/trunk/py/bin/_findpy.py (props changed) pypy/trunk/py/bin/env.py (props changed) pypy/trunk/py/impl/ (props changed) pypy/trunk/py/impl/__init__.py (props changed) pypy/trunk/py/impl/_com.py (props changed) pypy/trunk/py/impl/_metainfo.py (props changed) pypy/trunk/py/impl/builtin.py (props changed) pypy/trunk/py/impl/cmdline/ (props changed) pypy/trunk/py/impl/cmdline/__init__.py (props changed) pypy/trunk/py/impl/cmdline/pycleanup.py (props changed) pypy/trunk/py/impl/cmdline/pyconvert_unittest.py (props changed) pypy/trunk/py/impl/cmdline/pycountloc.py (props changed) pypy/trunk/py/impl/cmdline/pylookup.py (props changed) pypy/trunk/py/impl/cmdline/pysvnwcrevert.py (props changed) pypy/trunk/py/impl/cmdline/pytest.py (props changed) pypy/trunk/py/impl/cmdline/pywhich.py (props changed) pypy/trunk/py/impl/code/ (props changed) pypy/trunk/py/impl/code/__init__.py (props changed) pypy/trunk/py/impl/code/_assertionnew.py (props changed) pypy/trunk/py/impl/code/_assertionold.py (props changed) pypy/trunk/py/impl/code/assertion.py (props changed) pypy/trunk/py/impl/code/code.py (props changed) pypy/trunk/py/impl/code/oldmagic.py (props changed) pypy/trunk/py/impl/code/oldmagic2.py (props changed) pypy/trunk/py/impl/code/source.py (props changed) pypy/trunk/py/impl/compat/ (props changed) pypy/trunk/py/impl/compat/__init__.py (props changed) pypy/trunk/py/impl/compat/dep_doctest.py (props changed) pypy/trunk/py/impl/compat/dep_optparse.py (props changed) pypy/trunk/py/impl/compat/dep_subprocess.py (props changed) pypy/trunk/py/impl/compat/dep_textwrap.py (props changed) pypy/trunk/py/impl/error.py (props changed) pypy/trunk/py/impl/io/ (props changed) pypy/trunk/py/impl/io/__init__.py (props changed) pypy/trunk/py/impl/io/capture.py (props changed) pypy/trunk/py/impl/io/terminalwriter.py (props changed) pypy/trunk/py/impl/log/ (props changed) pypy/trunk/py/impl/log/__init__.py (props changed) pypy/trunk/py/impl/log/log.py (props changed) pypy/trunk/py/impl/log/warning.py (props changed) pypy/trunk/py/impl/path/ (props changed) pypy/trunk/py/impl/path/__init__.py (props changed) pypy/trunk/py/impl/path/cacheutil.py (props changed) pypy/trunk/py/impl/path/common.py (props changed) pypy/trunk/py/impl/path/gateway/ (props changed) pypy/trunk/py/impl/path/gateway/__init__.py (props changed) pypy/trunk/py/impl/path/gateway/channeltest.py (props changed) pypy/trunk/py/impl/path/gateway/channeltest2.py (props changed) pypy/trunk/py/impl/path/gateway/remotepath.py (props changed) pypy/trunk/py/impl/path/local.py (props changed) pypy/trunk/py/impl/path/svnurl.py (props changed) pypy/trunk/py/impl/path/svnwc.py (props changed) pypy/trunk/py/impl/process/ (props changed) pypy/trunk/py/impl/process/__init__.py (props changed) pypy/trunk/py/impl/process/cmdexec.py (props changed) pypy/trunk/py/impl/process/forkedfunc.py (props changed) pypy/trunk/py/impl/process/killproc.py (props changed) pypy/trunk/py/impl/std.py (props changed) pypy/trunk/py/impl/test/ (props changed) pypy/trunk/py/impl/test/__init__.py (props changed) pypy/trunk/py/impl/test/cmdline.py (props changed) pypy/trunk/py/impl/test/collect.py (props changed) pypy/trunk/py/impl/test/compat.py (props changed) pypy/trunk/py/impl/test/config.py (props changed) pypy/trunk/py/impl/test/conftesthandle.py (props changed) pypy/trunk/py/impl/test/defaultconftest.py (props changed) pypy/trunk/py/impl/test/dist/ (props changed) pypy/trunk/py/impl/test/dist/__init__.py (props changed) pypy/trunk/py/impl/test/dist/dsession.py (props changed) pypy/trunk/py/impl/test/dist/gwmanage.py (props changed) pypy/trunk/py/impl/test/dist/mypickle.py (props changed) pypy/trunk/py/impl/test/dist/nodemanage.py (props changed) pypy/trunk/py/impl/test/dist/txnode.py (props changed) pypy/trunk/py/impl/test/funcargs.py (props changed) pypy/trunk/py/impl/test/looponfail/ (props changed) pypy/trunk/py/impl/test/looponfail/__init__.py (props changed) pypy/trunk/py/impl/test/looponfail/remote.py (props changed) pypy/trunk/py/impl/test/looponfail/util.py (props changed) pypy/trunk/py/impl/test/outcome.py (props changed) pypy/trunk/py/impl/test/parseopt.py (props changed) pypy/trunk/py/impl/test/pluginmanager.py (props changed) pypy/trunk/py/impl/test/pycollect.py (props changed) pypy/trunk/py/impl/test/session.py (props changed) pypy/trunk/py/impl/xmlgen.py (props changed) pypy/trunk/py/plugin/ (props changed) pypy/trunk/py/plugin/__init__.py (props changed) pypy/trunk/py/plugin/hookspec.py (props changed) pypy/trunk/py/plugin/pytest__pytest.py (props changed) pypy/trunk/py/plugin/pytest_assertion.py (props changed) pypy/trunk/py/plugin/pytest_capture.py (props changed) pypy/trunk/py/plugin/pytest_default.py (props changed) pypy/trunk/py/plugin/pytest_doctest.py (props changed) pypy/trunk/py/plugin/pytest_figleaf.py (props changed) pypy/trunk/py/plugin/pytest_helpconfig.py (props changed) pypy/trunk/py/plugin/pytest_hooklog.py (props changed) pypy/trunk/py/plugin/pytest_mark.py (props changed) pypy/trunk/py/plugin/pytest_monkeypatch.py (props changed) pypy/trunk/py/plugin/pytest_nose.py (props changed) pypy/trunk/py/plugin/pytest_pastebin.py (props changed) pypy/trunk/py/plugin/pytest_pdb.py (props changed) pypy/trunk/py/plugin/pytest_pylint.py (props changed) pypy/trunk/py/plugin/pytest_pytester.py (props changed) pypy/trunk/py/plugin/pytest_recwarn.py (props changed) pypy/trunk/py/plugin/pytest_restdoc.py (props changed) pypy/trunk/py/plugin/pytest_resultlog.py (props changed) pypy/trunk/py/plugin/pytest_runner.py (props changed) pypy/trunk/py/plugin/pytest_skipping.py (props changed) pypy/trunk/py/plugin/pytest_terminal.py (props changed) pypy/trunk/py/plugin/pytest_tmpdir.py (props changed) pypy/trunk/py/plugin/pytest_unittest.py (props changed) Log: fixeol From fijal at codespeak.net Mon Nov 23 12:05:00 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:05:00 +0100 (CET) Subject: [pypy-svn] r69536 - pypy/branch/stringbuilder/pypy/rpython/memory/gc Message-ID: <20091123110500.CEB9116801B@codespeak.net> Author: fijal Date: Mon Nov 23 12:04:59 2009 New Revision: 69536 Modified: pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py Log: An attempt to fix a segfault, also make tests happy and improve comments Modified: pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py Mon Nov 23 12:04:59 2009 @@ -240,7 +240,8 @@ result = llop.raw_realloc_grow(llmemory.Address, source_addr, old_tot_size, tot_size) else: - if old_tot_size == tot_size: + if oldlength == newlength: + # no need to call realloc result = source_addr else: result = llop.raw_realloc_shrink(llmemory.Address, source_addr, @@ -251,11 +252,15 @@ if grow: self.gen2_resizable_objects.append(result + size_gc_header) else: + # this means we can no longer access this as a resizable + # object. Instead it becomes a simple rawmalloc object self.gen2_rawmalloced_objects.append(result + size_gc_header) + (result + size_gc_header + lengthofs).signed[0] = newlength + # because thing below can call semispace collect, we need to + # set length correctly above self._check_rawsize_alloced(raw_malloc_usage(tot_size) - raw_malloc_usage(old_tot_size), can_collect = not grow) - (result + size_gc_header + lengthofs).signed[0] = newlength return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF) def can_move(self, addr): From arigo at codespeak.net Mon Nov 23 12:42:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:42:27 +0100 (CET) Subject: [pypy-svn] r69537 - in pypy/branch/faster-raise-2/pypy: annotation interpreter interpreter/test lib module/__builtin__ module/_file module/exceptions module/exceptions/test module/pypyjit module/pypyjit/test objspace/std objspace/std/test rlib tool Message-ID: <20091123114227.3D1EF16801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:42:24 2009 New Revision: 69537 Added: pypy/branch/faster-raise-2/pypy/module/exceptions/ (props changed) - copied from r69249, pypy/branch/faster-raise/pypy/module/exceptions/ Removed: pypy/branch/faster-raise-2/pypy/lib/_exceptions.py pypy/branch/faster-raise-2/pypy/tool/_enum_exceptions_broken.py Modified: pypy/branch/faster-raise-2/pypy/annotation/description.py pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise-2/pypy/interpreter/error.py pypy/branch/faster-raise-2/pypy/interpreter/pyframe.py pypy/branch/faster-raise-2/pypy/interpreter/pyopcode.py pypy/branch/faster-raise-2/pypy/interpreter/pytraceback.py pypy/branch/faster-raise-2/pypy/interpreter/test/test_module.py pypy/branch/faster-raise-2/pypy/interpreter/test/test_pyframe.py pypy/branch/faster-raise-2/pypy/interpreter/test/test_raise.py pypy/branch/faster-raise-2/pypy/interpreter/typedef.py pypy/branch/faster-raise-2/pypy/module/__builtin__/__init__.py pypy/branch/faster-raise-2/pypy/module/__builtin__/abstractinst.py pypy/branch/faster-raise-2/pypy/module/_file/interp_file.py pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise-2/pypy/module/exceptions/test/ (props changed) pypy/branch/faster-raise-2/pypy/module/pypyjit/interp_jit.py pypy/branch/faster-raise-2/pypy/module/pypyjit/policy.py pypy/branch/faster-raise-2/pypy/module/pypyjit/test/test_policy.py pypy/branch/faster-raise-2/pypy/objspace/std/objspace.py pypy/branch/faster-raise-2/pypy/objspace/std/stdtypedef.py pypy/branch/faster-raise-2/pypy/objspace/std/test/test_proxy_internals.py pypy/branch/faster-raise-2/pypy/objspace/std/transparent.py pypy/branch/faster-raise-2/pypy/objspace/std/typeobject.py pypy/branch/faster-raise-2/pypy/rlib/rwin32.py pypy/branch/faster-raise-2/pypy/tool/sourcetools.py Log: svn merge -r69169:69249 svn+ssh://codespeak.net/svn/pypy/branch/faster-raise . Modified: pypy/branch/faster-raise-2/pypy/annotation/description.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/annotation/description.py (original) +++ pypy/branch/faster-raise-2/pypy/annotation/description.py Mon Nov 23 12:42:24 2009 @@ -453,7 +453,7 @@ # is of type FunctionType. But bookkeeper.immutablevalue() # will do the right thing in s_get_value(). - if type(value) is MemberDescriptorType: + if type(value) in MemberDescriptorTypes: # skip __slots__, showing up in the class as 'member' objects return if name == '__init__' and self.is_builtin_exception_class(): @@ -896,5 +896,9 @@ class Sample(object): __slots__ = 'x' -MemberDescriptorType = type(Sample.x) +MemberDescriptorTypes = [type(Sample.x)] del Sample +try: + MemberDescriptorTypes.append(type(OSError.errno)) +except AttributeError: # on CPython <= 2.4 + pass Modified: pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py Mon Nov 23 12:42:24 2009 @@ -9,7 +9,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized from pypy.rlib.timer import DummyTimer, Timer -from pypy.rlib.jit import we_are_jitted, dont_look_inside +from pypy.rlib.jit import we_are_jitted, dont_look_inside, unroll_safe import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -389,12 +389,19 @@ def make_builtins(self): "NOT_RPYTHON: only for initializing the space." + from pypy.module.exceptions import Module + w_name_exceptions = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name_exceptions) + from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.setitem(w_modules, w_name_exceptions, + self.wrap(self.exceptions_module)) + from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) @@ -405,6 +412,8 @@ bootstrap_modules = ['sys', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] + self.export_builtin_exceptions() + # initialize with "bootstrap types" from objspace (e.g. w_None) for name, value in self.__dict__.items(): if name.startswith('w_') and not name.endswith('Type'): @@ -430,6 +439,18 @@ self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'), w_builtin_module_names) + def export_builtin_exceptions(self): + """NOT_RPYTHON""" + w_dic = self.exceptions_module.getdict() + names_w = self.unpackiterable(self.call_function(self.getattr(w_dic, self.wrap("keys")))) + + for w_name in names_w: + name = self.str_w(w_name) + if not name.startswith('__'): + excname = name + w_exc = self.getitem(w_dic, w_name) + setattr(self, "w_"+excname, w_exc) + def install_mixedmodule(self, mixedname, installed_builtin_modules): """NOT_RPYTHON""" modname = self.setbuiltinmodule(mixedname) @@ -679,12 +700,19 @@ """ return self.unpackiterable(w_iterable, expected_length) + @unroll_safe def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): return True # fast path (also here to handle string exceptions) try: - return self.abstract_issubclass_w(w_exc_type, w_check_class) + if self.is_true(self.isinstance(w_check_class, self.w_tuple)): + for w_t in self.fixedview(w_check_class): + if self.exception_match(w_exc_type, w_t): + return True + else: + return False + return self.exception_issubclass_w(w_exc_type, w_check_class) except OperationError, e: if e.match(self, self.w_TypeError): # string exceptions maybe return False @@ -806,34 +834,52 @@ w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + # The code below only works + # for the simple case (new-style instance). + # These methods are patched with the full logic by the __builtin__ + # module when it is loaded + def abstract_issubclass_w(self, w_cls1, w_cls2): - # Equivalent to 'issubclass(cls1, cls2)'. The code below only works - # for the simple case (new-style class, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'issubclass(cls1, cls2)'. return self.is_true(self.issubtype(w_cls1, w_cls2)) def abstract_isinstance_w(self, w_obj, w_cls): - # Equivalent to 'isinstance(obj, cls)'. The code below only works - # for the simple case (new-style instance, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, cls)'. return self.is_true(self.isinstance(w_obj, w_cls)) def abstract_isclass_w(self, w_obj): - # Equivalent to 'isinstance(obj, type)'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, type)'. return self.is_true(self.isinstance(w_obj, self.w_type)) def abstract_getclass(self, w_obj): - # Equivalent to 'obj.__class__'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'obj.__class__'. return self.type(w_obj) + # CPython rules allows old style classes or subclasses + # of BaseExceptions to be exceptions. + # This is slightly less general than the case above, so we prefix + # it with exception_ + + def exception_is_valid_obj_as_class_w(self, w_obj): + if not self.is_true(self.isinstance(w_obj, self.w_type)): + return False + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_obj, self.w_BaseException)) + + def exception_is_valid_class_w(self, w_cls): + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_cls, self.w_BaseException)) + + def exception_getclass(self, w_obj): + return self.type(w_obj) + + def exception_issubclass_w(self, w_cls1, w_cls2): + return self.is_true(self.issubtype(w_cls1, w_cls2)) + + # end of special support code + def eval(self, expression, w_globals, w_locals, hidden_applevel=False): "NOT_RPYTHON: For internal debugging." import types Modified: pypy/branch/faster-raise-2/pypy/interpreter/error.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/error.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/error.py Mon Nov 23 12:42:24 2009 @@ -73,13 +73,6 @@ else: return '%s: %s' % (exc_typename, exc_value) - def getframe(self): - "The frame this exception was raised in, or None." - if self.application_traceback: - return self.application_traceback.frame - else: - return None - def record_interpreter_traceback(self): """Records the current traceback inside the interpreter. This traceback is only useful to debug the interpreter, not the @@ -102,7 +95,7 @@ print >> file, "Traceback (application-level):" while tb is not None: co = tb.frame.pycode - lineno = tb.lineno + lineno = tb.get_lineno() fname = co.co_filename if fname.startswith('\n'): lines = fname.split('\n') @@ -177,16 +170,15 @@ while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if (space.abstract_isclass_w(w_type) and - is_valid_exception_class(space, w_type)): + if space.exception_is_valid_obj_as_class_w(w_type): # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): # raise Type: we assume we have to instantiate Type w_value = space.call_function(w_type) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) else: - w_valuetype = space.abstract_getclass(w_value) - if space.abstract_issubclass_w(w_valuetype, w_type): + w_valuetype = space.exception_getclass(w_value) + if space.exception_issubclass_w(w_valuetype, w_type): # raise Type, Instance: let etype be the exact type of value w_type = w_valuetype else: @@ -198,7 +190,7 @@ else: # raise Type, X: assume X is the constructor argument w_value = space.call_function(w_type, w_value) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) elif space.full_exceptions and space.is_w(space.type(w_type), space.w_str): @@ -208,8 +200,8 @@ else: # the only case left here is (inst, None), from a 'raise inst'. w_inst = w_type - w_instclass = space.abstract_getclass(w_inst) - if not is_valid_exception_class(space, w_instclass): + w_instclass = space.exception_getclass(w_inst) + if not space.exception_is_valid_class_w(w_instclass): instclassname = w_instclass.getname(space, '?') msg = ("exceptions must be classes, or instances," "or strings (deprecated) not %s" % (instclassname,)) @@ -240,23 +232,6 @@ except OperationError: pass # ignored - -def is_valid_exception_class(space, w_type): - """Assuming that 'w_type' is a new-style or old-style class, is it - correct to use it as the class of an exception? The answer is no - if it is a new-style class that doesn't inherit from BaseException. - """ - if not space.full_exceptions: - return True # always, for the flow space - try: - return space.is_true( - space.issubtype(w_type, space.w_BaseException)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - return True # assuming w_type is an old-style class - - # Utilities from pypy.tool.ansi_print import ansi_print Modified: pypy/branch/faster-raise-2/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/pyframe.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/pyframe.py Mon Nov 23 12:42:24 2009 @@ -56,7 +56,6 @@ self.valuestack_w = [None] * code.co_stacksize self.valuestackdepth = 0 self.lastblock = None - self.blockcount = 0 if space.config.objspace.honor__builtins__: self.builtin = space.builtin.pick_builtin(w_globals) # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. @@ -70,31 +69,29 @@ def append_block(self, block): block.previous = self.lastblock self.lastblock = block - self.blockcount += 1 def pop_block(self): block = self.lastblock self.lastblock = block.previous - self.blockcount -= 1 return block + def blockstack_non_empty(self): + return self.lastblock is not None + def get_blocklist(self): """Returns a list containing all the blocks in the frame""" - lst = [None] * self.blockcount + lst = [] block = self.lastblock - i = 0 while block is not None: - lst[i] = block - i += 1 + lst.append(block) block = block.previous return lst def set_blocklist(self, lst): self.lastblock = None - self.blockcount = 0 - i = len(lst) - while i > 0: - block = lst[i-1] + i = len(lst) - 1 + while i >= 0: + block = lst[i] i -= 1 self.append_block(block) @@ -162,9 +159,6 @@ raise if not we_are_jitted(): executioncontext.return_trace(self, w_exitvalue) - # on exit, we try to release self.last_exception -- breaks an - # obvious reference cycle, so it helps refcounting implementations - self.last_exception = None finally: executioncontext.leave(self) return w_exitvalue @@ -464,7 +458,7 @@ if self.w_f_trace is None: raise OperationError(space.w_ValueError, - space.wrap("f_lineo can only be set by a trace function.")) + space.wrap("f_lineno can only be set by a trace function.")) if new_lineno < self.pycode.co_firstlineno: raise OperationError(space.w_ValueError, @@ -559,7 +553,11 @@ else: addr += 1 - f_iblock = self.blockcount + f_iblock = 0 + block = self.lastblock + while block: + f_iblock += 1 + block = block.previous min_iblock = f_iblock + min_delta_iblock if new_lasti > self.last_instr: new_iblock = f_iblock + delta_iblock Modified: pypy/branch/faster-raise-2/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/pyopcode.py Mon Nov 23 12:42:24 2009 @@ -274,11 +274,8 @@ @jit.unroll_safe def unrollstack(self, unroller_kind): - n = self.blockcount - n = jit.hint(n, promote=True) - while n > 0: + while self.blockstack_non_empty(): block = self.pop_block() - n -= 1 if (block.handling_mask & unroller_kind) != 0: return block block.cleanupstack(self) Modified: pypy/branch/faster-raise-2/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/pytraceback.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/pytraceback.py Mon Nov 23 12:42:24 2009 @@ -5,20 +5,25 @@ class PyTraceback(baseobjspace.Wrappable): """Traceback object - Public fields: + Public app-level fields: * 'tb_frame' * 'tb_lasti' * 'tb_lineno' * 'tb_next' """ - def __init__(self, space, frame, lasti, lineno, next): + def __init__(self, space, frame, lasti, next): self.space = space self.frame = frame self.lasti = lasti - self.lineno = lineno self.next = next + def get_lineno(self): + return offset2lineno(self.frame.pycode, self.lasti) + + def descr_tb_lineno(space, self): + return space.wrap(self.get_lineno()) + def descr__reduce__(self, space): from pypy.interpreter.mixedmodule import MixedModule w_mod = space.getbuiltinmodule('_pickle_support') @@ -30,7 +35,6 @@ tup_state = [ w(self.frame), w(self.lasti), - w(self.lineno), w(self.next), ] nt = space.newtuple @@ -39,19 +43,17 @@ def descr__setstate__(self, space, w_args): from pypy.interpreter.pyframe import PyFrame args_w = space.unpackiterable(w_args) - w_frame, w_lasti, w_lineno, w_next = args_w + w_frame, w_lasti, w_next = args_w self.frame = space.interp_w(PyFrame, w_frame) self.lasti = space.int_w(w_lasti) - self.lineno = space.int_w(w_lineno) self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): frame.force_f_back() if frame.pycode.hidden_applevel: return - lineno = offset2lineno(frame.pycode, last_instruction) tb = operror.application_traceback - tb = PyTraceback(space, frame, last_instruction, lineno, tb) + tb = PyTraceback(space, frame, last_instruction, tb) operror.application_traceback = tb def offset2lineno(c, stopat): Modified: pypy/branch/faster-raise-2/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/test/test_module.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/test/test_module.py Mon Nov 23 12:42:24 2009 @@ -54,13 +54,13 @@ r = repr(sys) assert r == "" - import _exceptions # known to be in pypy/lib - r = repr(_exceptions) - assert r.startswith("') - nofile = type(_exceptions)('nofile', 'foo') + nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" - m = type(_exceptions).__new__(type(_exceptions)) + m = type(_pypy_interact).__new__(type(_pypy_interact)) assert repr(m).startswith(" Author: fijal Date: Mon Nov 23 12:42:26 2009 New Revision: 69538 Modified: pypy/trunk/pypy/objspace/std/stringobject.py Log: don't create a newlist here (it's also safer) Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Mon Nov 23 12:42:26 2009 @@ -361,7 +361,7 @@ if not space.is_true(space.isinstance(w_s, space.w_str)): if space.is_true(space.isinstance(w_s, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", space.newlist(list_w)) + return space.call_method(w_u, "join", w_list) raise OperationError( space.w_TypeError, space.wrap("sequence item %d: expected string, %s " From fijal at codespeak.net Mon Nov 23 12:44:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:44:08 +0100 (CET) Subject: [pypy-svn] r69539 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123114408.CAE44168021@codespeak.net> Author: fijal Date: Mon Nov 23 12:44:07 2009 New Revision: 69539 Modified: pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py Log: Fix also here Modified: pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py Mon Nov 23 12:44:07 2009 @@ -362,7 +362,7 @@ if not space.is_true(space.isinstance(w_s, space.w_str)): if space.is_true(space.isinstance(w_s, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", space.newlist(list_w)) + return space.call_method(w_u, "join", w_list) raise OperationError( space.w_TypeError, space.wrap("sequence item %d: expected string, %s " From arigo at codespeak.net Mon Nov 23 12:47:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:47:28 +0100 (CET) Subject: [pypy-svn] r69540 - in pypy/branch/faster-raise-2/pypy: interpreter interpreter/pyparser module/exceptions module/exceptions/test objspace/std Message-ID: <20091123114728.2537716801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:47:27 2009 New Revision: 69540 Modified: pypy/branch/faster-raise-2/pypy/interpreter/pyparser/error.py pypy/branch/faster-raise-2/pypy/interpreter/typedef.py pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py pypy/branch/faster-raise-2/pypy/objspace/std/stdtypedef.py Log: svn merge -r69274:69285 svn+ssh://codespeak.net/svn/pypy/branch/faster-raise . Modified: pypy/branch/faster-raise-2/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/pyparser/error.py Mon Nov 23 12:47:27 2009 @@ -8,7 +8,6 @@ self.offset = offset self.text = text self.filename = filename - self.print_file_and_line = False def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), Modified: pypy/branch/faster-raise-2/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/typedef.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/typedef.py Mon Nov 23 12:47:27 2009 @@ -562,6 +562,9 @@ def descr_set_dict(space, w_obj, w_dict): w_obj.setdict(space, w_dict) +def descr_del_dict(space, w_obj): # blame CPython for the existence of this one + w_obj.setdict(space, space.newdict()) + def descr_get_weakref(space, w_obj): lifeline = w_obj.getweakref() if lifeline is None: Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py Mon Nov 23 12:47:27 2009 @@ -72,8 +72,9 @@ from pypy.interpreter.baseobjspace import ObjSpace, Wrappable, W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w,\ - GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict -from pypy.interpreter.gateway import interp2app + GetSetProperty, interp_attrproperty, descr_get_dict, descr_set_dict,\ + descr_del_dict +from pypy.interpreter.gateway import interp2app, Arguments from pypy.interpreter.error import OperationError from pypy.rlib import rwin32 @@ -98,14 +99,19 @@ and will be deprecated at some point. """ w_dict = None + args_w = [] - def __init__(self, space, args_w): - self.args_w = args_w + def __init__(self, space): self.space = space + self.w_message = space.w_None + + def descr_init(self, space, args_w): + self.args_w = args_w if len(args_w) == 1: self.w_message = args_w[0] else: self.w_message = space.wrap("") + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): lgt = len(self.args_w) @@ -132,6 +138,10 @@ def descr_setargs(space, self, w_newargs): self.args_w = space.fixedview(w_newargs) + def descr_getitem(self, space, w_index): + return space.getitem(space.newtuple(self.args_w), w_index) + descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root] + def getdict(self): if self.w_dict is None: self.w_dict = self.space.newdict() @@ -142,15 +152,26 @@ raise OperationError( space.w_TypeError, space.wrap("setting exceptions's dictionary to a non-dict") ) self.w_dict = w_dict + def descr_reduce(self, space): + lst = [self.getclass(space), space.newtuple(self.args_w)] + if self.w_dict is not None and space.is_true(self.w_dict): + lst = lst + [self.w_dict] + return space.newtuple(lst) + descr_reduce.unwrap_spec = ['self', ObjSpace] + + def descr_setstate(self, space, w_dict): + w_olddict = self.getdict() + space.call_method(w_olddict, 'update', w_dict) + descr_setstate.unwrap_spec = ['self', ObjSpace, W_Root] def _new(cls, basecls=None): if basecls is None: basecls = cls - def descr_new_base_exception(space, w_subtype, args_w): + def descr_new_base_exception(space, w_subtype, __args__): exc = space.allocate_instance(cls, w_subtype) - basecls.__init__(exc, space, args_w) + basecls.__init__(exc, space) return space.wrap(exc) - descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, 'args_w'] + descr_new_base_exception.unwrap_spec = [ObjSpace, W_Root, Arguments] descr_new_base_exception.func_name = 'descr_new_' + cls.__name__ return interp2app(descr_new_base_exception) @@ -159,11 +180,15 @@ __doc__ = W_BaseException.__doc__, __module__ = 'exceptions', __new__ = _new(W_BaseException), + __init__ = interp2app(W_BaseException.descr_init), __str__ = interp2app(W_BaseException.descr_str), __repr__ = interp2app(W_BaseException.descr_repr), - __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, + __dict__ = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict, cls=W_BaseException), - message = interp_attrproperty_w('w_message', W_BaseException), + __getitem__ = interp2app(W_BaseException.descr_getitem), + __reduce__ = interp2app(W_BaseException.descr_reduce), + __setstate__ = interp2app(W_BaseException.descr_setstate), + message = readwrite_attrproperty_w('w_message', W_BaseException), args = GetSetProperty(W_BaseException.descr_getargs, W_BaseException.descr_setargs), ) @@ -219,13 +244,20 @@ class W_UnicodeTranslateError(W_UnicodeError): """Unicode translation error.""" - def __init__(self, space, w_object, w_start, w_end, w_reason): + object = u'' + start = 0 + end = 0 + reason = '' + + def descr_init(self, space, w_object, w_start, w_end, w_reason): self.object = space.unicode_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_object, w_start, - w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_object, w_start, + w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([space.wrap(self)], r"""(self): @@ -240,19 +272,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_translate_error(space, w_subtype, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeTranslateError, w_subtype) - W_UnicodeTranslateError.__init__(exc, space, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeTranslateError.typedef = TypeDef( 'UnicodeTranslateError', W_UnicodeError.typedef, __doc__ = W_UnicodeTranslateError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_translate_error), + __new__ = _new(W_UnicodeTranslateError), + __init__ = interp2app(W_UnicodeTranslateError.descr_init), __str__ = interp2app(W_UnicodeTranslateError.descr_str), object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), start = readwrite_attrproperty('start', W_UnicodeTranslateError, 'int_w'), @@ -289,17 +315,21 @@ class W_EnvironmentError(W_StandardError): """Base class for I/O related errors.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) + def __init__(self, space): self.w_errno = space.w_None self.w_strerror = space.w_None - self.w_filename = space.w_None + self.w_filename = space.w_None + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): + W_BaseException.descr_init(self, space, args_w) if 2 <= len(args_w) <= 3: self.w_errno = args_w[0] self.w_strerror = args_w[1] if len(args_w) == 3: self.w_filename = args_w[2] self.args_w = [args_w[0], args_w[1]] + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): if (not space.is_w(self.w_errno, space.w_None) and @@ -308,7 +338,7 @@ return space.wrap("[Errno %d] %s: %s" % ( space.int_w(self.w_errno), space.str_w(self.w_strerror), - space.str_w(self.w_filename))) + space.str_w(space.repr(self.w_filename)))) return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), space.str_w(self.w_strerror))) return W_BaseException.descr_str(self, space) @@ -320,6 +350,7 @@ __doc__ = W_EnvironmentError.__doc__, __module__ = 'exceptions', __new__ = _new(W_EnvironmentError), + __init__ = interp2app(W_EnvironmentError.descr_init), __str__ = interp2app(W_EnvironmentError.descr_str), errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), strerror = readwrite_attrproperty_w('w_strerror', W_EnvironmentError), @@ -332,10 +363,14 @@ class W_WindowsError(W_OSError): """MS-Windows OS system call failed.""" - def __init__(self, space, args_w): - W_OSError.__init__(self, space, args_w) + def __init__(self, space): + self.w_winerror = space.w_None + W_OSError.__init__(self, space) + + def descr_init(self, space, args_w): # Set errno to the POSIX errno, and winerror to the Win32 # error code. + W_OSError.descr_init(self, space, args_w) try: errno = space.int_w(self.w_errno) except OperationError: @@ -344,6 +379,7 @@ errno = self._winerror_to_errno.get(errno, self._default_errno) self.w_winerror = self.w_errno self.w_errno = space.wrap(errno) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): if (not space.is_w(self.w_winerror, space.w_None) and @@ -369,6 +405,7 @@ __doc__ = W_WindowsError.__doc__, __module__ = 'exceptions', __new__ = _new(W_WindowsError), + __init__ = interp2app(W_WindowsError.descr_init), __str__ = interp2app(W_WindowsError.descr_str), winerror = readwrite_attrproperty_w('w_winerror', W_WindowsError), ) @@ -395,24 +432,27 @@ class W_SyntaxError(W_StandardError): """Invalid syntax.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) + def __init__(self, space): + self.w_filename = space.w_None + self.w_lineno = space.w_None + self.w_offset = space.w_None + self.w_text = space.w_None + self.w_msg = space.wrap('') + self.w_print_file_and_line = space.w_None # what's that? + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): # that's not a self.w_message!!! if len(args_w) > 0: self.w_msg = args_w[0] - else: - self.w_msg = space.wrap('') if len(args_w) == 2: values_w = space.fixedview(args_w[1], 4) self.w_filename = values_w[0] self.w_lineno = values_w[1] self.w_offset = values_w[2] self.w_text = values_w[3] - else: - self.w_filename = space.w_None - self.w_lineno = space.w_None - self.w_offset = space.w_None - self.w_text = space.w_None + W_BaseException.descr_init(self, space, args_w) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] def descr_str(self, space): return space.appexec([self], """(self): @@ -440,6 +480,7 @@ 'SyntaxError', W_StandardError.typedef, __new__ = _new(W_SyntaxError), + __init__ = interp2app(W_SyntaxError.descr_init), __str__ = interp2app(W_SyntaxError.descr_str), __doc__ = W_SyntaxError.__doc__, __module__ = 'exceptions', @@ -448,6 +489,8 @@ lineno = readwrite_attrproperty_w('w_lineno', W_SyntaxError), offset = readwrite_attrproperty_w('w_offset', W_SyntaxError), text = readwrite_attrproperty_w('w_text', W_SyntaxError), + print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', + W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, @@ -456,19 +499,23 @@ class W_SystemExit(W_BaseException): """Request to exit from the interpreter.""" - def __init__(self, space, args_w): - W_BaseException.__init__(self, space, args_w) - if len(args_w) == 0: - self.w_code = space.w_None - elif len(args_w) == 1: + def __init__(self, space): + self.w_code = space.w_None + W_BaseException.__init__(self, space) + + def descr_init(self, space, args_w): + if len(args_w) == 1: self.w_code = args_w[0] - else: + elif len(args_w) > 1: self.w_code = space.newtuple(args_w) + W_BaseException.descr_init(self, space, args_w) + descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] W_SystemExit.typedef = TypeDef( 'SystemExit', W_BaseException.typedef, __new__ = _new(W_SystemExit), + __init__ = interp2app(W_SystemExit.descr_init), __doc__ = W_SystemExit.__doc__, __module__ = 'exceptions', code = readwrite_attrproperty_w('w_code', W_SystemExit) @@ -497,15 +544,22 @@ class W_UnicodeDecodeError(W_UnicodeError): """Unicode decoding error.""" + encoding = '' + object = '' + start = 0 + end = 0 + reason = '' - def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): self.encoding = space.str_w(w_encoding) self.object = space.str_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_encoding, w_object, - w_start, w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([self], """(self): @@ -518,19 +572,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_decode_error(space, w_subtype, w_encoding, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeDecodeError, w_subtype) - W_UnicodeDecodeError.__init__(exc, space, w_encoding, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeDecodeError.typedef = TypeDef( 'UnicodeDecodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeDecodeError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_decode_error), + __new__ = _new(W_UnicodeDecodeError), + __init__ = interp2app(W_UnicodeDecodeError.descr_init), __str__ = interp2app(W_UnicodeDecodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), object = readwrite_attrproperty('object', W_UnicodeDecodeError, 'str_w'), @@ -582,14 +630,22 @@ class W_UnicodeEncodeError(W_UnicodeError): """Unicode encoding error.""" - def __init__(self, space, w_encoding, w_object, w_start, w_end, w_reason): + encoding = '' + object = u'' + start = 0 + end = 0 + reason = '' + + def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): self.encoding = space.str_w(w_encoding) self.object = space.unicode_w(w_object) self.start = space.int_w(w_start) self.end = space.int_w(w_end) self.reason = space.str_w(w_reason) - W_BaseException.__init__(self, space, [w_encoding, w_object, - w_start, w_end, w_reason]) + W_BaseException.descr_init(self, space, [w_encoding, w_object, + w_start, w_end, w_reason]) + descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, + W_Root] def descr_str(self, space): return space.appexec([self], r"""(self): @@ -608,19 +664,13 @@ """) descr_str.unwrap_spec = ['self', ObjSpace] -def descr_new_unicode_encode_error(space, w_subtype, w_encoding, w_object, - w_start, w_end, w_reason): - exc = space.allocate_instance(W_UnicodeEncodeError, w_subtype) - W_UnicodeEncodeError.__init__(exc, space, w_encoding, w_object, w_start, - w_end, w_reason) - return space.wrap(exc) - W_UnicodeEncodeError.typedef = TypeDef( 'UnicodeEncodeError', W_UnicodeError.typedef, __doc__ = W_UnicodeEncodeError.__doc__, __module__ = 'exceptions', - __new__ = interp2app(descr_new_unicode_encode_error), + __new__ = _new(W_UnicodeEncodeError), + __init__ = interp2app(W_UnicodeEncodeError.descr_init), __str__ = interp2app(W_UnicodeEncodeError.descr_str), encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), object = readwrite_attrproperty('object', W_UnicodeEncodeError, 'unicode_w'), Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py Mon Nov 23 12:47:27 2009 @@ -25,6 +25,29 @@ assert x.xyz == 3 x.args = [42] assert x.args == (42,) + assert x[0] == 42 + x.args = (1, 2, 3) + assert x[1:2] == (2,) + x.message = "xyz" + assert x.message == "xyz" + + def test_kwargs(self): + from exceptions import Exception + class X(Exception): + def __init__(self, x=3): + self.x = x + + x = X(x=8) + assert x.x == 8 + + def test_catch_with_unpack(self): + from exceptions import LookupError + + try: + raise LookupError(1, 2) + except LookupError, (one, two): + assert one == 1 + assert two == 2 def test_exc(self): from exceptions import Exception, BaseException @@ -73,7 +96,7 @@ def test_environment_error(self): from exceptions import EnvironmentError ee = EnvironmentError(3, "x", "y") - assert str(ee) == "[Errno 3] x: y" + assert str(ee) == "[Errno 3] x: 'y'" assert str(EnvironmentError(3, "x")) == "[Errno 3] x" assert ee.errno == 3 assert ee.strerror == "x" @@ -172,3 +195,21 @@ if isinstance(e, type) and issubclass(e, exceptions.BaseException): assert e.__doc__, e assert e.__module__ == 'exceptions', e + + def test_reduce(self): + from exceptions import LookupError + + le = LookupError(1, 2, "a") + assert le.__reduce__() == (LookupError, (1, 2, "a")) + le.xyz = (1, 2) + assert le.__reduce__() == (LookupError, (1, 2, "a"), {"xyz": (1, 2)}) + + def test_setstate(self): + from exceptions import FutureWarning + + fw = FutureWarning() + fw.__setstate__({"xyz": (1, 2)}) + assert fw.xyz == (1, 2) + fw.__setstate__({'z': 1}) + assert fw.z == 1 + assert fw.xyz == (1, 2) Modified: pypy/branch/faster-raise-2/pypy/objspace/std/stdtypedef.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/objspace/std/stdtypedef.py (original) +++ pypy/branch/faster-raise-2/pypy/objspace/std/stdtypedef.py Mon Nov 23 12:47:27 2009 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, GetSetProperty, Member from pypy.interpreter.typedef import descr_get_dict, descr_set_dict -from pypy.interpreter.typedef import no_hash_descr +from pypy.interpreter.typedef import no_hash_descr, descr_del_dict from pypy.interpreter.baseobjspace import SpaceCache from pypy.objspace.std.model import StdObjSpaceMultiMethod from pypy.objspace.std.multimethod import FailedToImplement @@ -39,9 +39,6 @@ a = a.base return True -def descr_del_dict(space, w_obj): # blame CPython for the existence of this one - w_obj.setdict(space, space.newdict()) - std_dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict) std_dict_descr.name = '__dict__' From fijal at codespeak.net Mon Nov 23 12:48:01 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:48:01 +0100 (CET) Subject: [pypy-svn] r69541 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123114801.6AC4E168021@codespeak.net> Author: fijal Date: Mon Nov 23 12:48:01 2009 New Revision: 69541 Modified: pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py Log: Re-read the length of the list since we can invoke arbitrary python code in between Modified: pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py Mon Nov 23 12:48:01 2009 @@ -198,7 +198,8 @@ # now we know it's a list of unicode or string lgt += len(delim) * (len(l_w) - 1) builder = UnicodeBuilder(lgt) - for i in range(len(l_w)): + i = 0 + while i < space.int_w(space.len(w_list)): w_item = l_w[i] if isinstance(w_item, W_UnicodeObject): builder.append(w_item._value) @@ -206,6 +207,7 @@ builder.append(space.unicode_w(w_item)) if i != len(l_w) - 1: builder.append(delim) + i += 1 return W_UnicodeObject(builder.build()) def hash__Unicode(space, w_uni): From arigo at codespeak.net Mon Nov 23 12:49:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:49:21 +0100 (CET) Subject: [pypy-svn] r69542 - in pypy/branch/faster-raise-2/pypy: interpreter module/exceptions module/exceptions/test Message-ID: <20091123114921.8299C16801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:49:20 2009 New Revision: 69542 Modified: pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py Log: svn merge -r69296:69353 svn+ssh://codespeak.net/svn/pypy/branch/faster-raise . Modified: pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/faster-raise-2/pypy/interpreter/baseobjspace.py Mon Nov 23 12:49:20 2009 @@ -1037,6 +1037,21 @@ buffer = self.buffer_w(w_obj) return buffer.as_str() + def realstr_w(self, w_obj): + # Like str_w, but only works if w_obj is really of type 'str'. + if not self.is_true(self.isinstance(w_obj, self.w_str)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a string')) + return self.str_w(w_obj) + + def realunicode_w(self, w_obj): + # Like unicode_w, but only works if w_obj is really of type + # 'unicode'. + if not self.is_true(self.isinstance(w_obj, self.w_unicode)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a unicode')) + return self.unicode_w(w_obj) + def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py Mon Nov 23 12:49:20 2009 @@ -78,13 +78,6 @@ from pypy.interpreter.error import OperationError from pypy.rlib import rwin32 -def readwrite_attrproperty(name, cls, unwrapname): - def fget(space, obj): - return space.wrap(getattr(obj, name)) - def fset(space, obj, w_val): - setattr(obj, name, getattr(space, unwrapname)(w_val)) - return GetSetProperty(fget, fset, cls=cls) - def readwrite_attrproperty_w(name, cls): def fget(space, obj): return getattr(obj, name) @@ -244,16 +237,22 @@ class W_UnicodeTranslateError(W_UnicodeError): """Unicode translation error.""" - object = u'' - start = 0 - end = 0 - reason = '' + object = None + start = None + end = None + reason = None def descr_init(self, space, w_object, w_start, w_end, w_reason): - self.object = space.unicode_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.realunicode_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.realstr_w(w_reason) + # assign attributes + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, @@ -280,10 +279,10 @@ __new__ = _new(W_UnicodeTranslateError), __init__ = interp2app(W_UnicodeTranslateError.descr_init), __str__ = interp2app(W_UnicodeTranslateError.descr_str), - object = readwrite_attrproperty('object', W_UnicodeTranslateError, 'unicode_w'), - start = readwrite_attrproperty('start', W_UnicodeTranslateError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeTranslateError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeTranslateError, 'str_w'), + object = readwrite_attrproperty_w('w_object', W_UnicodeTranslateError), + start = readwrite_attrproperty_w('w_start', W_UnicodeTranslateError), + end = readwrite_attrproperty_w('w_end', W_UnicodeTranslateError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeTranslateError), ) W_LookupError = _new_exception('LookupError', W_StandardError, @@ -331,16 +330,29 @@ self.args_w = [args_w[0], args_w[1]] descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] + # since we rebind args_w, we need special reduce, grump + def descr_reduce(self, space): + if not space.is_w(self.w_filename, space.w_None): + lst = [self.getclass(space), space.newtuple( + self.args_w + [self.w_filename])] + else: + lst = [self.getclass(space), space.newtuple(self.args_w)] + if self.w_dict is not None and space.is_true(self.w_dict): + lst = lst + [self.w_dict] + return space.newtuple(lst) + descr_reduce.unwrap_spec = ['self', ObjSpace] + def descr_str(self, space): if (not space.is_w(self.w_errno, space.w_None) and not space.is_w(self.w_strerror, space.w_None)): if not space.is_w(self.w_filename, space.w_None): - return space.wrap("[Errno %d] %s: %s" % ( - space.int_w(self.w_errno), + return space.wrap("[Errno %s] %s: %s" % ( + space.str_w(space.str(self.w_errno)), space.str_w(self.w_strerror), space.str_w(space.repr(self.w_filename)))) - return space.wrap("[Errno %d] %s" % (space.int_w(self.w_errno), - space.str_w(self.w_strerror))) + return space.wrap("[Errno %s] %s" % + (space.str_w(space.str(self.w_errno)), + space.str_w(self.w_strerror))) return W_BaseException.descr_str(self, space) descr_str.unwrap_spec = ['self', ObjSpace] @@ -350,6 +362,7 @@ __doc__ = W_EnvironmentError.__doc__, __module__ = 'exceptions', __new__ = _new(W_EnvironmentError), + __reduce__ = interp2app(W_EnvironmentError.descr_reduce), __init__ = interp2app(W_EnvironmentError.descr_init), __str__ = interp2app(W_EnvironmentError.descr_str), errno = readwrite_attrproperty_w('w_errno', W_EnvironmentError), @@ -544,18 +557,25 @@ class W_UnicodeDecodeError(W_UnicodeError): """Unicode decoding error.""" - encoding = '' - object = '' - start = 0 - end = 0 - reason = '' + w_encoding = None + w_object = None + w_start = None + w_end = None + w_reason = None def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): - self.encoding = space.str_w(w_encoding) - self.object = space.str_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.realstr_w(w_encoding) + space.realstr_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.realstr_w(w_reason) + # assign attributes + self.w_encoding = w_encoding + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_encoding, w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, @@ -580,11 +600,11 @@ __new__ = _new(W_UnicodeDecodeError), __init__ = interp2app(W_UnicodeDecodeError.descr_init), __str__ = interp2app(W_UnicodeDecodeError.descr_str), - encoding = readwrite_attrproperty('encoding', W_UnicodeDecodeError, 'str_w'), - object = readwrite_attrproperty('object', W_UnicodeDecodeError, 'str_w'), - start = readwrite_attrproperty('start', W_UnicodeDecodeError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeDecodeError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeDecodeError, 'str_w'), + encoding = readwrite_attrproperty_w('w_encoding', W_UnicodeDecodeError), + object = readwrite_attrproperty_w('w_object', W_UnicodeDecodeError), + start = readwrite_attrproperty_w('w_start', W_UnicodeDecodeError), + end = readwrite_attrproperty_w('w_end', W_UnicodeDecodeError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeDecodeError), ) W_TypeError = _new_exception('TypeError', W_StandardError, @@ -629,19 +649,25 @@ class W_UnicodeEncodeError(W_UnicodeError): """Unicode encoding error.""" - - encoding = '' - object = u'' - start = 0 - end = 0 - reason = '' + w_encoding = None + w_object = None + w_start = None + w_end = None + w_reason = None def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason): - self.encoding = space.str_w(w_encoding) - self.object = space.unicode_w(w_object) - self.start = space.int_w(w_start) - self.end = space.int_w(w_end) - self.reason = space.str_w(w_reason) + # typechecking + space.realstr_w(w_encoding) + space.realunicode_w(w_object) + space.int_w(w_start) + space.int_w(w_end) + space.realstr_w(w_reason) + # assign attributes + self.w_encoding = w_encoding + self.w_object = w_object + self.w_start = w_start + self.w_end = w_end + self.w_reason = w_reason W_BaseException.descr_init(self, space, [w_encoding, w_object, w_start, w_end, w_reason]) descr_init.unwrap_spec = ['self', ObjSpace, W_Root, W_Root, W_Root, W_Root, @@ -672,9 +698,9 @@ __new__ = _new(W_UnicodeEncodeError), __init__ = interp2app(W_UnicodeEncodeError.descr_init), __str__ = interp2app(W_UnicodeEncodeError.descr_str), - encoding = readwrite_attrproperty('encoding', W_UnicodeEncodeError, 'str_w'), - object = readwrite_attrproperty('object', W_UnicodeEncodeError, 'unicode_w'), - start = readwrite_attrproperty('start', W_UnicodeEncodeError, 'int_w'), - end = readwrite_attrproperty('end', W_UnicodeEncodeError, 'int_w'), - reason = readwrite_attrproperty('reason', W_UnicodeEncodeError, 'str_w'), + encoding = readwrite_attrproperty_w('w_encoding', W_UnicodeEncodeError), + object = readwrite_attrproperty_w('w_object', W_UnicodeEncodeError), + start = readwrite_attrproperty_w('w_start', W_UnicodeEncodeError), + end = readwrite_attrproperty_w('w_end', W_UnicodeEncodeError), + reason = readwrite_attrproperty_w('w_reason', W_UnicodeEncodeError), ) Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py Mon Nov 23 12:49:20 2009 @@ -56,6 +56,7 @@ assert isinstance(Exception(), Exception) assert isinstance(Exception(), BaseException) assert repr(Exception(3, "x")) == "Exception(3, 'x')" + assert str(IOError("foo", "bar")) == "[Errno foo] bar" def test_custom_class(self): from exceptions import Exception, BaseException, LookupError @@ -87,6 +88,8 @@ ut.start = 4 ut.object = u'012345' assert str(ut) == "can't translate character u'\\x34' in position 4: bah" + ut.object = [] + assert ut.object == [] def test_key_error(self): from exceptions import KeyError @@ -163,6 +166,10 @@ assert str(ue) == "'x' codec can't encode characters in position 1-4: bah" ue.end = 2 assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah" + ue.object = [] + assert ue.object == [] + raises(TypeError, UnicodeEncodeError, "x", "y", 1, 5, "bah") + raises(TypeError, UnicodeEncodeError, u"x", u"y", 1, 5, "bah") def test_multiple_inheritance(self): from exceptions import LookupError, ValueError, Exception @@ -197,12 +204,14 @@ assert e.__module__ == 'exceptions', e def test_reduce(self): - from exceptions import LookupError + from exceptions import LookupError, EnvironmentError le = LookupError(1, 2, "a") assert le.__reduce__() == (LookupError, (1, 2, "a")) le.xyz = (1, 2) assert le.__reduce__() == (LookupError, (1, 2, "a"), {"xyz": (1, 2)}) + ee = EnvironmentError(1, 2, "a") + assert ee.__reduce__() == (EnvironmentError, (1, 2, "a")) def test_setstate(self): from exceptions import FutureWarning @@ -213,3 +222,4 @@ fw.__setstate__({'z': 1}) assert fw.z == 1 assert fw.xyz == (1, 2) + From arigo at codespeak.net Mon Nov 23 12:50:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:50:37 +0100 (CET) Subject: [pypy-svn] r69543 - pypy/branch/faster-raise-2/pypy/doc/config Message-ID: <20091123115037.05FB316801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:50:37 2009 New Revision: 69543 Added: pypy/branch/faster-raise-2/pypy/doc/config/objspace.usemodules.exceptions.txt - copied unchanged from r69462, pypy/branch/faster-raise/pypy/doc/config/objspace.usemodules.exceptions.txt Log: svn merge -r69461:69462 svn+ssh://codespeak.net/svn/pypy/branch/faster-raise . From arigo at codespeak.net Mon Nov 23 12:51:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:51:56 +0100 (CET) Subject: [pypy-svn] r69544 - in pypy/branch/faster-raise-2/pypy: doc lib/app_test module/exceptions module/exceptions/test Message-ID: <20091123115156.CC9AC16801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:51:56 2009 New Revision: 69544 Modified: pypy/branch/faster-raise-2/pypy/doc/geninterp.txt pypy/branch/faster-raise-2/pypy/lib/app_test/test_exception_extra.py pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py Log: svn merge -r69489:69493 svn+ssh://codespeak.net/svn/pypy/branch/faster-raise . Modified: pypy/branch/faster-raise-2/pypy/doc/geninterp.txt ============================================================================== --- pypy/branch/faster-raise-2/pypy/doc/geninterp.txt (original) +++ pypy/branch/faster-raise-2/pypy/doc/geninterp.txt Mon Nov 23 12:51:56 2009 @@ -18,20 +18,15 @@ any longer to execute this code. .. _`application-level`: coding-guide.html#app-preferable -.. _exceptions: ../../pypy/lib/_exceptions.py -An example is exceptions_. They are -needed in a very early phase of bootstrapping StdObjspace, but -for simplicity, they are written as RPythonic application -level code. This implies that the interpreter must be quite -completely initialized to execute this code, which is -impossible in the early phase, where we have neither -exceptions implemented nor classes available. +Bootstrap issue ++++++++++++++++ -Solution -++++++++ +One issue we had so far was of bootstrapping: some pieces of the +interpreter (e.g. exceptions) were written in geninterped code. +It is unclear how much of it is left, thought. -This bootstrap issue is solved by invoking a new bytecode interpreter +That bootstrap issue is (was?) solved by invoking a new bytecode interpreter which runs on FlowObjspace. FlowObjspace is complete without complicated initialization. It is able to do abstract interpretation of any Rpythonic code, without actually implementing anything. It just @@ -176,10 +171,8 @@ Interplevel Snippets in the Sources +++++++++++++++++++++++++++++++++++ -.. _`_exceptions.py`: ../../pypy/lib/_exceptions.py - Code written in application space can consist of complete files -to be translated (e.g. `_exceptions.py`_), or they +to be translated, or they can be tiny snippets scattered all over a source file, similar to our example from above. Modified: pypy/branch/faster-raise-2/pypy/lib/app_test/test_exception_extra.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/lib/app_test/test_exception_extra.py (original) +++ pypy/branch/faster-raise-2/pypy/lib/app_test/test_exception_extra.py Mon Nov 23 12:51:56 2009 @@ -6,7 +6,7 @@ e = ex.EnvironmentError(1, "hello") assert str(e) == "[Errno 1] hello" e = ex.EnvironmentError(1, "hello", "world") - assert str(e) == "[Errno 1] hello: world" + assert str(e) == "[Errno 1] hello: 'world'" def app_test_import(): import exceptions Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/interp_exceptions.py Mon Nov 23 12:51:56 2009 @@ -111,7 +111,7 @@ if lgt == 0: return space.wrap('') elif lgt == 1: - return space.str(self.w_message) + return space.str(self.args_w[0]) else: return space.str(space.newtuple(self.args_w)) descr_str.unwrap_spec = ['self', ObjSpace] Modified: pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py (original) +++ pypy/branch/faster-raise-2/pypy/module/exceptions/test/test_exc.py Mon Nov 23 12:51:56 2009 @@ -25,6 +25,7 @@ assert x.xyz == 3 x.args = [42] assert x.args == (42,) + assert str(x) == '42' assert x[0] == 42 x.args = (1, 2, 3) assert x[1:2] == (2,) From arigo at codespeak.net Mon Nov 23 12:54:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 12:54:27 +0100 (CET) Subject: [pypy-svn] r69545 - pypy/branch/faster-raise-2/pypy/module/pypyjit/test Message-ID: <20091123115427.ECA6716801F@codespeak.net> Author: arigo Date: Mon Nov 23 12:54:27 2009 New Revision: 69545 Modified: pypy/branch/faster-raise-2/pypy/module/pypyjit/test/test_pypy_c.py Log: Add these tests from r69498, but skip them for now. Modified: pypy/branch/faster-raise-2/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/faster-raise-2/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/faster-raise-2/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 23 12:54:27 2009 @@ -373,6 +373,51 @@ assert len(bytecode.get_opnames("call")) == 1 # the call to append assert len(bytecode.get_opnames("guard")) == 1 # guard_no_exception after the call + def test_exception_inside_loop_1(self): + py.test.skip("exceptions: in-progress") + self.run_source(''' + def main(n): + while n: + try: + raise ValueError + except ValueError: + pass + n -= 1 + return n + ''', + ([30], 0)) + + bytecode, = self.get_by_bytecode("SETUP_EXCEPT") + #assert not bytecode.get_opnames("new") -- currently, we have + # new_with_vtable(pypy.interpreter.pyopcode.ExceptBlock) + bytecode, = self.get_by_bytecode("RAISE_VARARGS") + assert not bytecode.get_opnames("new") + bytecode, = self.get_by_bytecode("COMPARE_OP") + assert not bytecode.get_opnames() + + def test_exception_inside_loop_2(self): + py.test.skip("exceptions: in-progress") + self.run_source(''' + def g(n): + raise ValueError(n) + def f(n): + g(n) + def main(n): + while n: + try: + f(n) + except ValueError: + pass + n -= 1 + return n + ''', + ([30], 0)) + + bytecode, = self.get_by_bytecode("RAISE_VARARGS") + assert not bytecode.get_opnames("new") + bytecode, = self.get_by_bytecode("COMPARE_OP") + assert len(bytecode.get_opnames()) <= 2 # oois, guard_true + class AppTestJIT(PyPyCJITTests): def setup_class(cls): From fijal at codespeak.net Mon Nov 23 12:54:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:54:32 +0100 (CET) Subject: [pypy-svn] r69546 - pypy/branch/stringbuilder/pypy/module/select Message-ID: <20091123115432.4E103168025@codespeak.net> Author: fijal Date: Mon Nov 23 12:54:30 2009 New Revision: 69546 Modified: pypy/branch/stringbuilder/pypy/module/select/interp_select.py Log: listview is not safe here Modified: pypy/branch/stringbuilder/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/stringbuilder/pypy/module/select/interp_select.py (original) +++ pypy/branch/stringbuilder/pypy/module/select/interp_select.py Mon Nov 23 12:54:30 2009 @@ -113,9 +113,9 @@ On Windows, only sockets are supported; on Unix, all file descriptors. """ - iwtd_w = space.listview(w_iwtd) - owtd_w = space.listview(w_owtd) - ewtd_w = space.listview(w_ewtd) + iwtd_w = space.unpackiterable(w_iwtd) + owtd_w = space.unpackiterable(w_owtd) + ewtd_w = space.unpackiterable(w_ewtd) iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] owtd = [as_fd_w(space, w_f) for w_f in owtd_w] ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] From fijal at codespeak.net Mon Nov 23 12:57:30 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:57:30 +0100 (CET) Subject: [pypy-svn] r69547 - pypy/branch/stringbuilder/pypy/interpreter Message-ID: <20091123115730.B7A12168025@codespeak.net> Author: fijal Date: Mon Nov 23 12:57:29 2009 New Revision: 69547 Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Log: kill expected_length argument to listview and fixedview Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Mon Nov 23 12:57:29 2009 @@ -668,16 +668,15 @@ (i, plural)) return items - def fixedview(self, w_iterable, expected_length=-1): + def fixedview(self, w_iterable): """ A fixed list view of w_iterable. Don't modify the result """ - return make_sure_not_resized(self.unpackiterable(w_iterable, - expected_length)[:]) + return make_sure_not_resized(self.unpackiterable(w_iterable)[:]) - def listview(self, w_iterable, expected_length=-1): + def listview(self, w_iterable): """ A non-fixed view of w_iterable. Don't modify the result """ - return self.unpackiterable(w_iterable, expected_length) + return self.unpackiterable(w_iterable) def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" From fijal at codespeak.net Mon Nov 23 12:57:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 12:57:56 +0100 (CET) Subject: [pypy-svn] r69548 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123115756.58DF9168025@codespeak.net> Author: fijal Date: Mon Nov 23 12:57:55 2009 New Revision: 69548 Modified: pypy/branch/stringbuilder/pypy/objspace/std/typetype.py Log: unsafe (and pointless?) use of listview Modified: pypy/branch/stringbuilder/pypy/objspace/std/typetype.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/typetype.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/typetype.py Mon Nov 23 12:57:55 2009 @@ -38,7 +38,7 @@ name = space.str_w(w_name) assert isinstance(name, str) dict_w = {} - dictkeys_w = space.listview(w_dict) + dictkeys_w = space.unpackiterable(w_dict) for w_key in dictkeys_w: key = space.str_w(w_key) dict_w[key] = space.getitem(w_dict, w_key) From fijal at codespeak.net Mon Nov 23 13:07:45 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 13:07:45 +0100 (CET) Subject: [pypy-svn] r69549 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123120745.4D593168021@codespeak.net> Author: fijal Date: Mon Nov 23 13:07:44 2009 New Revision: 69549 Modified: pypy/branch/stringbuilder/pypy/objspace/std/objspace.py Log: Kill expected_length argument here as well Modified: pypy/branch/stringbuilder/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/objspace.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/objspace.py Mon Nov 23 13:07:44 2009 @@ -643,7 +643,7 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def fixedview(self, w_obj, expected_length=-1): + def fixedview(self, w_obj): """ Fast paths """ if isinstance(w_obj, W_TupleObject): @@ -651,20 +651,16 @@ elif isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.fixedview(self, w_obj, expected_length) - if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + return ObjSpace.fixedview(self, w_obj) return t - def listview(self, w_obj, expected_length=-1): + def listview(self, w_obj): if isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems elif isinstance(w_obj, W_TupleObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.listview(self, w_obj, expected_length) - if expected_length != -1 and len(t) != expected_length: - raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) + return ObjSpace.listview(self, w_obj) return t def sliceindices(self, w_slice, w_length): From fijal at codespeak.net Mon Nov 23 13:09:29 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 13:09:29 +0100 (CET) Subject: [pypy-svn] r69550 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123120929.CD0CF168021@codespeak.net> Author: fijal Date: Mon Nov 23 13:09:29 2009 New Revision: 69550 Modified: pypy/branch/stringbuilder/pypy/objspace/std/ropeunicodeobject.py Log: I don't care too much about ropes, be on the safe side Modified: pypy/branch/stringbuilder/pypy/objspace/std/ropeunicodeobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/ropeunicodeobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/ropeunicodeobject.py Mon Nov 23 13:09:29 2009 @@ -234,7 +234,7 @@ return space.contains(unicode_from_string(space, w_container), w_item ) def unicode_join__RopeUnicode_ANY(space, w_self, w_list): - l_w = space.listview(w_list) + l_w = space.unpackiterable(w_list) delim = w_self._node totlen = 0 if len(l_w) == 0: From fijal at codespeak.net Mon Nov 23 14:08:38 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 14:08:38 +0100 (CET) Subject: [pypy-svn] r69551 - in pypy/branch/stringbuilder/pypy: interpreter objspace/std Message-ID: <20091123130838.8CCB6168020@codespeak.net> Author: fijal Date: Mon Nov 23 14:08:37 2009 New Revision: 69551 Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py pypy/branch/stringbuilder/pypy/objspace/std/objspace.py Log: grumble. indeed, someone uses expected_length Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Mon Nov 23 14:08:37 2009 @@ -668,7 +668,7 @@ (i, plural)) return items - def fixedview(self, w_iterable): + def fixedview(self, w_iterable, expected_lenght=-1): """ A fixed list view of w_iterable. Don't modify the result """ return make_sure_not_resized(self.unpackiterable(w_iterable)[:]) Modified: pypy/branch/stringbuilder/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/objspace.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/objspace.py Mon Nov 23 14:08:37 2009 @@ -643,7 +643,7 @@ raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t - def fixedview(self, w_obj): + def fixedview(self, w_obj, expected_length=-1): """ Fast paths """ if isinstance(w_obj, W_TupleObject): @@ -652,6 +652,8 @@ t = w_obj.wrappeditems[:] else: return ObjSpace.fixedview(self, w_obj) + if expected_length != -1 and len(t) != expected_length: + raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t def listview(self, w_obj): From fijal at codespeak.net Mon Nov 23 14:14:28 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 14:14:28 +0100 (CET) Subject: [pypy-svn] r69552 - pypy/branch/stringbuilder/pypy/objspace/std Message-ID: <20091123131428.17721168024@codespeak.net> Author: fijal Date: Mon Nov 23 14:14:27 2009 New Revision: 69552 Modified: pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py Log: Revert reusin list, breaks tests. We're safe since unicodejoin is aware of people changing lists. Semantics of this is messy, but at least consistent with CPython. Modified: pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/stringobject.py Mon Nov 23 14:14:27 2009 @@ -362,7 +362,7 @@ if not space.is_true(space.isinstance(w_s, space.w_str)): if space.is_true(space.isinstance(w_s, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", w_list) + return space.call_method(w_u, "join", space.newlist(list_w)) raise OperationError( space.w_TypeError, space.wrap("sequence item %d: expected string, %s " From arigo at codespeak.net Mon Nov 23 16:42:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 16:42:02 +0100 (CET) Subject: [pypy-svn] r69554 - in pypy/trunk/pypy: annotation doc doc/config interpreter interpreter/pyparser interpreter/test lib lib/app_test module/__builtin__ module/_file module/exceptions module/exceptions/test module/pypyjit module/pypyjit/test objspace/std objspace/std/test rlib tool Message-ID: <20091123154202.A4A7C16801F@codespeak.net> Author: arigo Date: Mon Nov 23 16:42:00 2009 New Revision: 69554 Added: pypy/trunk/pypy/doc/config/objspace.usemodules.exceptions.txt - copied unchanged from r69545, pypy/branch/faster-raise-2/pypy/doc/config/objspace.usemodules.exceptions.txt pypy/trunk/pypy/module/exceptions/ (props changed) - copied from r69545, pypy/branch/faster-raise-2/pypy/module/exceptions/ Removed: pypy/trunk/pypy/lib/_exceptions.py pypy/trunk/pypy/tool/_enum_exceptions_broken.py Modified: pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/doc/geninterp.txt pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/error.py pypy/trunk/pypy/interpreter/pyframe.py pypy/trunk/pypy/interpreter/pyopcode.py pypy/trunk/pypy/interpreter/pyparser/error.py pypy/trunk/pypy/interpreter/pytraceback.py pypy/trunk/pypy/interpreter/test/test_module.py pypy/trunk/pypy/interpreter/test/test_pyframe.py pypy/trunk/pypy/interpreter/test/test_raise.py pypy/trunk/pypy/interpreter/typedef.py pypy/trunk/pypy/lib/app_test/test_exception_extra.py pypy/trunk/pypy/module/__builtin__/__init__.py pypy/trunk/pypy/module/__builtin__/abstractinst.py pypy/trunk/pypy/module/_file/interp_file.py pypy/trunk/pypy/module/exceptions/test/ (props changed) pypy/trunk/pypy/module/pypyjit/interp_jit.py pypy/trunk/pypy/module/pypyjit/policy.py pypy/trunk/pypy/module/pypyjit/test/test_policy.py pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py pypy/trunk/pypy/objspace/std/objspace.py pypy/trunk/pypy/objspace/std/stdtypedef.py pypy/trunk/pypy/objspace/std/test/test_proxy_internals.py pypy/trunk/pypy/objspace/std/transparent.py pypy/trunk/pypy/objspace/std/typeobject.py pypy/trunk/pypy/rlib/rwin32.py pypy/trunk/pypy/tool/sourcetools.py Log: Merge the branch/faster-raise-2: * move exceptions to interp-level. * add helpers space.exception_getclass() & co. * other similar small fixes all over the place. Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Mon Nov 23 16:42:00 2009 @@ -453,7 +453,7 @@ # is of type FunctionType. But bookkeeper.immutablevalue() # will do the right thing in s_get_value(). - if type(value) is MemberDescriptorType: + if type(value) in MemberDescriptorTypes: # skip __slots__, showing up in the class as 'member' objects return if name == '__init__' and self.is_builtin_exception_class(): @@ -896,5 +896,9 @@ class Sample(object): __slots__ = 'x' -MemberDescriptorType = type(Sample.x) +MemberDescriptorTypes = [type(Sample.x)] del Sample +try: + MemberDescriptorTypes.append(type(OSError.errno)) +except AttributeError: # on CPython <= 2.4 + pass Modified: pypy/trunk/pypy/doc/geninterp.txt ============================================================================== --- pypy/trunk/pypy/doc/geninterp.txt (original) +++ pypy/trunk/pypy/doc/geninterp.txt Mon Nov 23 16:42:00 2009 @@ -18,20 +18,15 @@ any longer to execute this code. .. _`application-level`: coding-guide.html#app-preferable -.. _exceptions: ../../pypy/lib/_exceptions.py -An example is exceptions_. They are -needed in a very early phase of bootstrapping StdObjspace, but -for simplicity, they are written as RPythonic application -level code. This implies that the interpreter must be quite -completely initialized to execute this code, which is -impossible in the early phase, where we have neither -exceptions implemented nor classes available. +Bootstrap issue ++++++++++++++++ -Solution -++++++++ +One issue we had so far was of bootstrapping: some pieces of the +interpreter (e.g. exceptions) were written in geninterped code. +It is unclear how much of it is left, thought. -This bootstrap issue is solved by invoking a new bytecode interpreter +That bootstrap issue is (was?) solved by invoking a new bytecode interpreter which runs on FlowObjspace. FlowObjspace is complete without complicated initialization. It is able to do abstract interpretation of any Rpythonic code, without actually implementing anything. It just @@ -176,10 +171,8 @@ Interplevel Snippets in the Sources +++++++++++++++++++++++++++++++++++ -.. _`_exceptions.py`: ../../pypy/lib/_exceptions.py - Code written in application space can consist of complete files -to be translated (e.g. `_exceptions.py`_), or they +to be translated, or they can be tiny snippets scattered all over a source file, similar to our example from above. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Mon Nov 23 16:42:00 2009 @@ -9,7 +9,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import make_sure_not_resized from pypy.rlib.timer import DummyTimer, Timer -from pypy.rlib.jit import we_are_jitted, dont_look_inside +from pypy.rlib.jit import we_are_jitted, dont_look_inside, unroll_safe import os, sys __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -389,12 +389,19 @@ def make_builtins(self): "NOT_RPYTHON: only for initializing the space." + from pypy.module.exceptions import Module + w_name_exceptions = self.wrap('exceptions') + self.exceptions_module = Module(self, w_name_exceptions) + from pypy.module.sys import Module w_name = self.wrap('sys') self.sys = Module(self, w_name) w_modules = self.sys.get('modules') self.setitem(w_modules, w_name, self.wrap(self.sys)) + self.setitem(w_modules, w_name_exceptions, + self.wrap(self.exceptions_module)) + from pypy.module.__builtin__ import Module w_name = self.wrap('__builtin__') self.builtin = Module(self, w_name) @@ -405,6 +412,8 @@ bootstrap_modules = ['sys', '__builtin__', 'exceptions'] installed_builtin_modules = bootstrap_modules[:] + self.export_builtin_exceptions() + # initialize with "bootstrap types" from objspace (e.g. w_None) for name, value in self.__dict__.items(): if name.startswith('w_') and not name.endswith('Type'): @@ -430,6 +439,18 @@ self.setitem(self.sys.w_dict, self.wrap('builtin_module_names'), w_builtin_module_names) + def export_builtin_exceptions(self): + """NOT_RPYTHON""" + w_dic = self.exceptions_module.getdict() + names_w = self.unpackiterable(self.call_function(self.getattr(w_dic, self.wrap("keys")))) + + for w_name in names_w: + name = self.str_w(w_name) + if not name.startswith('__'): + excname = name + w_exc = self.getitem(w_dic, w_name) + setattr(self, "w_"+excname, w_exc) + def install_mixedmodule(self, mixedname, installed_builtin_modules): """NOT_RPYTHON""" modname = self.setbuiltinmodule(mixedname) @@ -679,12 +700,19 @@ """ return self.unpackiterable(w_iterable, expected_length) + @unroll_safe def exception_match(self, w_exc_type, w_check_class): """Checks if the given exception type matches 'w_check_class'.""" if self.is_w(w_exc_type, w_check_class): return True # fast path (also here to handle string exceptions) try: - return self.abstract_issubclass_w(w_exc_type, w_check_class) + if self.is_true(self.isinstance(w_check_class, self.w_tuple)): + for w_t in self.fixedview(w_check_class): + if self.exception_match(w_exc_type, w_t): + return True + else: + return False + return self.exception_issubclass_w(w_exc_type, w_check_class) except OperationError, e: if e.match(self, self.w_TypeError): # string exceptions maybe return False @@ -806,34 +834,52 @@ w_objtype = self.type(w_obj) return self.issubtype(w_objtype, w_type) + # The code below only works + # for the simple case (new-style instance). + # These methods are patched with the full logic by the __builtin__ + # module when it is loaded + def abstract_issubclass_w(self, w_cls1, w_cls2): - # Equivalent to 'issubclass(cls1, cls2)'. The code below only works - # for the simple case (new-style class, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'issubclass(cls1, cls2)'. return self.is_true(self.issubtype(w_cls1, w_cls2)) def abstract_isinstance_w(self, w_obj, w_cls): - # Equivalent to 'isinstance(obj, cls)'. The code below only works - # for the simple case (new-style instance, new-style class). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, cls)'. return self.is_true(self.isinstance(w_obj, w_cls)) def abstract_isclass_w(self, w_obj): - # Equivalent to 'isinstance(obj, type)'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'isinstance(obj, type)'. return self.is_true(self.isinstance(w_obj, self.w_type)) def abstract_getclass(self, w_obj): - # Equivalent to 'obj.__class__'. The code below only works - # for the simple case (new-style instance without special stuff). - # This method is patched with the full logic by the __builtin__ - # module when it is loaded. + # Equivalent to 'obj.__class__'. return self.type(w_obj) + # CPython rules allows old style classes or subclasses + # of BaseExceptions to be exceptions. + # This is slightly less general than the case above, so we prefix + # it with exception_ + + def exception_is_valid_obj_as_class_w(self, w_obj): + if not self.is_true(self.isinstance(w_obj, self.w_type)): + return False + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_obj, self.w_BaseException)) + + def exception_is_valid_class_w(self, w_cls): + if not self.full_exceptions: + return True + return self.is_true(self.issubtype(w_cls, self.w_BaseException)) + + def exception_getclass(self, w_obj): + return self.type(w_obj) + + def exception_issubclass_w(self, w_cls1, w_cls2): + return self.is_true(self.issubtype(w_cls1, w_cls2)) + + # end of special support code + def eval(self, expression, w_globals, w_locals, hidden_applevel=False): "NOT_RPYTHON: For internal debugging." import types @@ -991,6 +1037,21 @@ buffer = self.buffer_w(w_obj) return buffer.as_str() + def realstr_w(self, w_obj): + # Like str_w, but only works if w_obj is really of type 'str'. + if not self.is_true(self.isinstance(w_obj, self.w_str)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a string')) + return self.str_w(w_obj) + + def realunicode_w(self, w_obj): + # Like unicode_w, but only works if w_obj is really of type + # 'unicode'. + if not self.is_true(self.isinstance(w_obj, self.w_unicode)): + raise OperationError(self.w_TypeError, + self.wrap('argument must be a unicode')) + return self.unicode_w(w_obj) + def bool_w(self, w_obj): # Unwraps a bool, also accepting an int for compatibility. # This is here mostly just for gateway.int_unwrapping_space_method(). Modified: pypy/trunk/pypy/interpreter/error.py ============================================================================== --- pypy/trunk/pypy/interpreter/error.py (original) +++ pypy/trunk/pypy/interpreter/error.py Mon Nov 23 16:42:00 2009 @@ -73,13 +73,6 @@ else: return '%s: %s' % (exc_typename, exc_value) - def getframe(self): - "The frame this exception was raised in, or None." - if self.application_traceback: - return self.application_traceback.frame - else: - return None - def record_interpreter_traceback(self): """Records the current traceback inside the interpreter. This traceback is only useful to debug the interpreter, not the @@ -102,7 +95,7 @@ print >> file, "Traceback (application-level):" while tb is not None: co = tb.frame.pycode - lineno = tb.lineno + lineno = tb.get_lineno() fname = co.co_filename if fname.startswith('\n'): lines = fname.split('\n') @@ -177,16 +170,15 @@ while space.is_true(space.isinstance(w_type, space.w_tuple)): w_type = space.getitem(w_type, space.wrap(0)) - if (space.abstract_isclass_w(w_type) and - is_valid_exception_class(space, w_type)): + if space.exception_is_valid_obj_as_class_w(w_type): # this is for all cases of the form (Class, something) if space.is_w(w_value, space.w_None): # raise Type: we assume we have to instantiate Type w_value = space.call_function(w_type) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) else: - w_valuetype = space.abstract_getclass(w_value) - if space.abstract_issubclass_w(w_valuetype, w_type): + w_valuetype = space.exception_getclass(w_value) + if space.exception_issubclass_w(w_valuetype, w_type): # raise Type, Instance: let etype be the exact type of value w_type = w_valuetype else: @@ -198,7 +190,7 @@ else: # raise Type, X: assume X is the constructor argument w_value = space.call_function(w_type, w_value) - w_type = space.abstract_getclass(w_value) + w_type = space.exception_getclass(w_value) elif space.full_exceptions and space.is_w(space.type(w_type), space.w_str): @@ -208,8 +200,8 @@ else: # the only case left here is (inst, None), from a 'raise inst'. w_inst = w_type - w_instclass = space.abstract_getclass(w_inst) - if not is_valid_exception_class(space, w_instclass): + w_instclass = space.exception_getclass(w_inst) + if not space.exception_is_valid_class_w(w_instclass): instclassname = w_instclass.getname(space, '?') msg = ("exceptions must be classes, or instances," "or strings (deprecated) not %s" % (instclassname,)) @@ -240,23 +232,6 @@ except OperationError: pass # ignored - -def is_valid_exception_class(space, w_type): - """Assuming that 'w_type' is a new-style or old-style class, is it - correct to use it as the class of an exception? The answer is no - if it is a new-style class that doesn't inherit from BaseException. - """ - if not space.full_exceptions: - return True # always, for the flow space - try: - return space.is_true( - space.issubtype(w_type, space.w_BaseException)) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - return True # assuming w_type is an old-style class - - # Utilities from pypy.tool.ansi_print import ansi_print Modified: pypy/trunk/pypy/interpreter/pyframe.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyframe.py (original) +++ pypy/trunk/pypy/interpreter/pyframe.py Mon Nov 23 16:42:00 2009 @@ -56,7 +56,6 @@ self.valuestack_w = [None] * code.co_stacksize self.valuestackdepth = 0 self.lastblock = None - self.blockcount = 0 if space.config.objspace.honor__builtins__: self.builtin = space.builtin.pick_builtin(w_globals) # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. @@ -70,31 +69,29 @@ def append_block(self, block): block.previous = self.lastblock self.lastblock = block - self.blockcount += 1 def pop_block(self): block = self.lastblock self.lastblock = block.previous - self.blockcount -= 1 return block + def blockstack_non_empty(self): + return self.lastblock is not None + def get_blocklist(self): """Returns a list containing all the blocks in the frame""" - lst = [None] * self.blockcount + lst = [] block = self.lastblock - i = 0 while block is not None: - lst[i] = block - i += 1 + lst.append(block) block = block.previous return lst def set_blocklist(self, lst): self.lastblock = None - self.blockcount = 0 - i = len(lst) - while i > 0: - block = lst[i-1] + i = len(lst) - 1 + while i >= 0: + block = lst[i] i -= 1 self.append_block(block) @@ -162,9 +159,6 @@ raise if not we_are_jitted(): executioncontext.return_trace(self, w_exitvalue) - # on exit, we try to release self.last_exception -- breaks an - # obvious reference cycle, so it helps refcounting implementations - self.last_exception = None finally: executioncontext.leave(self) return w_exitvalue @@ -464,7 +458,7 @@ if self.w_f_trace is None: raise OperationError(space.w_ValueError, - space.wrap("f_lineo can only be set by a trace function.")) + space.wrap("f_lineno can only be set by a trace function.")) if new_lineno < self.pycode.co_firstlineno: raise OperationError(space.w_ValueError, @@ -559,7 +553,11 @@ else: addr += 1 - f_iblock = self.blockcount + f_iblock = 0 + block = self.lastblock + while block: + f_iblock += 1 + block = block.previous min_iblock = f_iblock + min_delta_iblock if new_lasti > self.last_instr: new_iblock = f_iblock + delta_iblock Modified: pypy/trunk/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyopcode.py (original) +++ pypy/trunk/pypy/interpreter/pyopcode.py Mon Nov 23 16:42:00 2009 @@ -274,11 +274,8 @@ @jit.unroll_safe def unrollstack(self, unroller_kind): - n = self.blockcount - n = jit.hint(n, promote=True) - while n > 0: + while self.blockstack_non_empty(): block = self.pop_block() - n -= 1 if (block.handling_mask & unroller_kind) != 0: return block block.cleanupstack(self) Modified: pypy/trunk/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/error.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/error.py Mon Nov 23 16:42:00 2009 @@ -8,7 +8,6 @@ self.offset = offset self.text = text self.filename = filename - self.print_file_and_line = False def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), Modified: pypy/trunk/pypy/interpreter/pytraceback.py ============================================================================== --- pypy/trunk/pypy/interpreter/pytraceback.py (original) +++ pypy/trunk/pypy/interpreter/pytraceback.py Mon Nov 23 16:42:00 2009 @@ -5,20 +5,25 @@ class PyTraceback(baseobjspace.Wrappable): """Traceback object - Public fields: + Public app-level fields: * 'tb_frame' * 'tb_lasti' * 'tb_lineno' * 'tb_next' """ - def __init__(self, space, frame, lasti, lineno, next): + def __init__(self, space, frame, lasti, next): self.space = space self.frame = frame self.lasti = lasti - self.lineno = lineno self.next = next + def get_lineno(self): + return offset2lineno(self.frame.pycode, self.lasti) + + def descr_tb_lineno(space, self): + return space.wrap(self.get_lineno()) + def descr__reduce__(self, space): from pypy.interpreter.mixedmodule import MixedModule w_mod = space.getbuiltinmodule('_pickle_support') @@ -30,7 +35,6 @@ tup_state = [ w(self.frame), w(self.lasti), - w(self.lineno), w(self.next), ] nt = space.newtuple @@ -39,19 +43,17 @@ def descr__setstate__(self, space, w_args): from pypy.interpreter.pyframe import PyFrame args_w = space.unpackiterable(w_args) - w_frame, w_lasti, w_lineno, w_next = args_w + w_frame, w_lasti, w_next = args_w self.frame = space.interp_w(PyFrame, w_frame) self.lasti = space.int_w(w_lasti) - self.lineno = space.int_w(w_lineno) self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) def record_application_traceback(space, operror, frame, last_instruction): frame.force_f_back() if frame.pycode.hidden_applevel: return - lineno = offset2lineno(frame.pycode, last_instruction) tb = operror.application_traceback - tb = PyTraceback(space, frame, last_instruction, lineno, tb) + tb = PyTraceback(space, frame, last_instruction, tb) operror.application_traceback = tb def offset2lineno(c, stopat): Modified: pypy/trunk/pypy/interpreter/test/test_module.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_module.py (original) +++ pypy/trunk/pypy/interpreter/test/test_module.py Mon Nov 23 16:42:00 2009 @@ -54,13 +54,13 @@ r = repr(sys) assert r == "" - import _exceptions # known to be in pypy/lib - r = repr(_exceptions) - assert r.startswith("') - nofile = type(_exceptions)('nofile', 'foo') + nofile = type(_pypy_interact)('nofile', 'foo') assert repr(nofile) == "" - m = type(_exceptions).__new__(type(_exceptions)) + m = type(_pypy_interact).__new__(type(_pypy_interact)) assert repr(m).startswith(" Author: arigo Date: Mon Nov 23 16:42:23 2009 New Revision: 69555 Removed: pypy/branch/faster-raise-2/ Log: Kill merged branch. From pedronis at codespeak.net Mon Nov 23 17:02:21 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 23 Nov 2009 17:02:21 +0100 (CET) Subject: [pypy-svn] r69556 - in pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp: . test Message-ID: <20091123160221.8D83C16801F@codespeak.net> Author: pedronis Date: Mon Nov 23 17:02:21 2009 New Revision: 69556 Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/simple_optimize.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py Log: experimental heuristic to clear the numbering cache Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/optimizeopt.py Mon Nov 23 17:02:21 2009 @@ -369,7 +369,7 @@ self.loop = loop self.values = {} self.interned_refs = self.cpu.ts.new_ref_dict() - self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) + self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd) self.heap_op_optimizer = HeapOpOptimizer(self) self.bool_boxes = {} Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py Mon Nov 23 17:02:21 2009 @@ -105,11 +105,12 @@ class ResumeDataLoopMemo(object): - def __init__(self, cpu): - self.cpu = cpu + def __init__(self, metainterp_sd): + self.metainterp_sd = metainterp_sd + self.cpu = metainterp_sd.cpu self.consts = [] self.large_ints = {} - self.refs = cpu.ts.new_ref_dict_2() + self.refs = self.cpu.ts.new_ref_dict_2() self.numberings = {} self.cached_boxes = {} self.cached_virtuals = {} @@ -306,6 +307,7 @@ def _number_virtuals(self, liveboxes, values, num_env_virtuals): memo = self.memo new_liveboxes = [None] * memo.num_cached_boxes() + count = 0 for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -313,6 +315,7 @@ assert tagged_eq(tagged, UNASSIGNED) index = memo.assign_number_to_box(box, new_liveboxes) self.liveboxes[box] = tag(index, TAGBOX) + count += 1 else: assert tagbits == TAGVIRTUAL if tagged_eq(tagged, UNASSIGNEDVIRTUAL): @@ -321,6 +324,7 @@ self.liveboxes[box] = tag(index, TAGVIRTUAL) new_liveboxes.reverse() liveboxes.extend(new_liveboxes) + nholes = len(new_liveboxes) - count storage = self.storage storage.rd_virtuals = None @@ -349,6 +353,19 @@ r += 1 memo.nvrightholes += r + + if self._invalidation_needed(len(liveboxes), nholes): + memo.clear_box_virtual_numbers() + + def _invalidation_needed(self, nliveboxes, nholes): + memo = self.memo + # xxx heuristic a bit out of thin air + failargs_limit = memo.metainterp_sd.options.failargs_limit + if nliveboxes > (failargs_limit // 2): + if nholes > nliveboxes//3: + return True + return False + def _gettagged(self, box): if isinstance(box, Const): return self.memo.getconst(box) Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/simple_optimize.py Mon Nov 23 17:02:21 2009 @@ -16,7 +16,7 @@ # we need it since the backend can modify those lists, which make # get_guard_op in compile.py invalid # in fact, x86 modifies this list for moving GCs - memo = resume.ResumeDataLoopMemo(metainterp_sd.cpu) + memo = resume.ResumeDataLoopMemo(metainterp_sd) newoperations = [] for op in loop.operations: if op.is_guard(): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_resume.py Mon Nov 23 17:02:21 2009 @@ -261,6 +261,13 @@ assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env + +class FakeMetaInterpStaticData: + cpu = LLtypeMixin.cpu + + class options: + failargs_limit = 100 + def test_rebuild_from_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] @@ -269,7 +276,7 @@ FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] capture_resumedata(fs, None, storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) metainterp = MyMetaInterp() @@ -293,7 +300,7 @@ FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] capture_resumedata(fs, [b4], storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) metainterp = MyMetaInterp() @@ -321,7 +328,7 @@ fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] capture_resumedata(fs, None, storage2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -353,7 +360,7 @@ def virtual_value(keybox, value, next): vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), keybox) + LLtypeMixin.cpu), keybox) if not isinstance(next, OptValue): next = OptValue(next) vv.setfield(LLtypeMixin.nextdescr, next) @@ -374,7 +381,7 @@ fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] capture_resumedata(fs, None, storage2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) @@ -430,7 +437,7 @@ fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] capture_resumedata(fs, None, storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) @@ -455,7 +462,7 @@ fs = [FakeFrame("code0", 0, -1, b1, b2)] capture_resumedata(fs, None, storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) v1 = virtual_value(b1, b3, None) v2 = virtual_value(b2, b3, v1) v1.setfield(LLtypeMixin.nextdescr, v2) @@ -474,7 +481,7 @@ def test_ResumeDataLoopMemo_ints(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) tagged = memo.getconst(ConstInt(44)) assert untag(tagged) == (44, TAGINT) tagged = memo.getconst(ConstInt(-3)) @@ -496,7 +503,7 @@ def test_ResumeDataLoopMemo_refs(): cpu = LLtypeMixin.cpu - memo = ResumeDataLoopMemo(cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) const = cpu.ts.ConstRef(demo55o) tagged = memo.getconst(const) index, tagbits = untag(tagged) @@ -514,7 +521,7 @@ assert tagged == NULLREF def test_ResumeDataLoopMemo_other(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) const = ConstFloat(-1.0) tagged = memo.getconst(const) index, tagbits = untag(tagged) @@ -532,7 +539,7 @@ env2 = [c3, b3, b1, c3] snap2 = Snapshot(snap, env2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) numb, liveboxes, v = memo.number({}, snap1) assert v == 0 @@ -605,7 +612,7 @@ assert numb5.prev is numb4 def test_ResumeDataLoopMemo_number_boxes(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) b1, b2 = [BoxInt(), BoxInt()] assert memo.num_cached_boxes() == 0 boxes = [] @@ -634,7 +641,7 @@ assert memo.num_cached_boxes() == 0 def test_ResumeDataLoopMemo_number_virtuals(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) b1, b2 = [BoxInt(), BoxInt()] assert memo.num_cached_virtuals() == 0 num = memo.assign_number_to_virtual(b1) @@ -695,7 +702,7 @@ def test_virtual_adder_int_constants(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) assert storage.rd_snapshot is None @@ -714,7 +721,7 @@ def test_virtual_adder_memo_const_sharing(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.finish({}) assert len(memo.consts) == 2 @@ -731,7 +738,7 @@ def test_virtual_adder_no_op_renaming(): b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) b1_2 = BoxInt() class FakeValue(object): @@ -763,7 +770,7 @@ b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] @@ -784,7 +791,7 @@ b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} @@ -856,7 +863,7 @@ b2s, b4s = [BoxPtr(), BoxInt(4)] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} @@ -906,7 +913,7 @@ b2s, b4s = [BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} @@ -943,3 +950,23 @@ assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S) assert ptr.a == 111 assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) + +def test_invalidation_needed(): + class options: + failargs_limit = 10 + + metainterp_sd = FakeMetaInterpStaticData() + metainterp_sd.options = options + memo = ResumeDataLoopMemo(metainterp_sd) + modifier = ResumeDataVirtualAdder(None, memo) + + for i in range(5): + assert not modifier._invalidation_needed(5, i) + + assert not modifier._invalidation_needed(7, 2) + assert modifier._invalidation_needed(7, 3) + + assert not modifier._invalidation_needed(10, 2) + assert not modifier._invalidation_needed(10, 3) + assert modifier._invalidation_needed(10, 4) + From pedronis at codespeak.net Mon Nov 23 17:02:58 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 23 Nov 2009 17:02:58 +0100 (CET) Subject: [pypy-svn] r69557 - in pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp: . test Message-ID: <20091123160258.0C1DC16801F@codespeak.net> Author: pedronis Date: Mon Nov 23 17:02:57 2009 New Revision: 69557 Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/logger.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/oparser.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_logger.py Log: adjust logger/oparser to deal with holes in fail_args Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/logger.py Mon Nov 23 17:02:57 2009 @@ -63,6 +63,8 @@ if not name: name = 'cls' + str(mv) return 'ConstClass(' + name + ')' + elif arg is None: + return 'None' else: return '?' Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/oparser.py Mon Nov 23 17:02:57 2009 @@ -205,10 +205,15 @@ if i < j: for arg in line[i:j].split(','): arg = arg.strip() - try: - fail_args.append(self.vars[arg]) - except KeyError: - raise ParseError("Unknown var in fail_args: %s" % arg) + if arg == 'None': + fail_arg = None + else: + try: + fail_arg = self.vars[arg] + except KeyError: + raise ParseError( + "Unknown var in fail_args: %s" % arg) + fail_args.append(fail_arg) if descr is None and self.invent_fail_descr: descr = self.invent_fail_descr(fail_args) if hasattr(descr, '_oparser_uses_descr_of_guard'): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_logger.py Mon Nov 23 17:02:57 2009 @@ -83,6 +83,15 @@ ''' self.reparse(inp) + def test_guard_w_hole(self): + inp = ''' + [i0] + i1 = int_add(i0, 1) + guard_true(i0) [i0, None, i1] + finish(i1) + ''' + self.reparse(inp) + def test_debug_merge_point(self): inp = ''' [] @@ -112,7 +121,7 @@ assert output.splitlines()[-1] == "jump(i0, descr=)" pure_parse(output) - def test_guard(self): + def test_guard_descr(self): namespace = {'fdescr': BasicFailDescr(4)} inp = ''' [i0] From fijal at codespeak.net Mon Nov 23 17:13:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 17:13:08 +0100 (CET) Subject: [pypy-svn] r69558 - in pypy/branch/stringbuilder/pypy/objspace/std: . test Message-ID: <20091123161308.761D616801F@codespeak.net> Author: fijal Date: Mon Nov 23 17:13:07 2009 New Revision: 69558 Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_unicodeobject.py pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py Log: Give up and use unpackiterable + a test Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_unicodeobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/test/test_unicodeobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/test/test_unicodeobject.py Mon Nov 23 17:13:07 2009 @@ -799,3 +799,7 @@ b = unicode(a) assert type(b) is unicode assert b == u'hello \u1234' + + def test_join_genexp(self): + s = [u'a', u'b', u'c'] + assert u".".join(i for i in s) == u"a.b.c" Modified: pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/unicodeobject.py Mon Nov 23 17:13:07 2009 @@ -175,7 +175,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l_w = space.listview(w_list) + l_w = space.unpackiterable(w_list) delim = w_self._value if len(l_w) == 0: return W_UnicodeObject.EMPTY @@ -199,7 +199,7 @@ lgt += len(delim) * (len(l_w) - 1) builder = UnicodeBuilder(lgt) i = 0 - while i < space.int_w(space.len(w_list)): + for i in range(len(l_w)): w_item = l_w[i] if isinstance(w_item, W_UnicodeObject): builder.append(w_item._value) From pedronis at codespeak.net Mon Nov 23 17:30:40 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 23 Nov 2009 17:30:40 +0100 (CET) Subject: [pypy-svn] r69559 - pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86 Message-ID: <20091123163040.DD44016801F@codespeak.net> Author: pedronis Date: Mon Nov 23 17:30:40 2009 New Revision: 69559 Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py Log: comment places that need some rethinking when merged with the shorter-guard-path, not clear is worth improving them now Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/assembler.py Mon Nov 23 17:30:40 2009 @@ -738,7 +738,7 @@ """ assert that all args are actually Boxes """ for arg in args: - assert arg is None or isinstance(arg, Box) + assert arg is None or isinstance(arg, Box) # hole def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): @@ -754,7 +754,7 @@ pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] - if arg is None: + if arg is None: # hole continue loc = locs[i] if isinstance(loc, REG): @@ -769,7 +769,7 @@ mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] - if arg is None: + if arg is None: # hole continue loc = locs[i] if not isinstance(loc, REG): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/backend/x86/regalloc.py Mon Nov 23 17:30:40 2009 @@ -203,7 +203,7 @@ def possibly_free_vars(self, vars): for var in vars: - if var is not None: + if var is not None: # xxx kludgy self.possibly_free_var(var) def make_sure_var_in_reg(self, var, forbidden_vars=[], @@ -379,7 +379,7 @@ return longevity def loc(self, v): - if v is None: + if v is None: # xxx kludgy return None if v.type == FLOAT: return self.xrm.loc(v) From arigo at codespeak.net Mon Nov 23 17:48:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 17:48:40 +0100 (CET) Subject: [pypy-svn] r69560 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091123164840.6F143168020@codespeak.net> Author: arigo Date: Mon Nov 23 17:48:39 2009 New Revision: 69560 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: We have os.popen() on pypy-c nowadays. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 23 17:48:39 2009 @@ -104,12 +104,10 @@ print >> f, "print 'OK :-)'" f.close() - # we don't have os.popen() yet on pypy-c... if sys.platform.startswith('win'): py.test.skip("XXX this is not Windows-friendly") - child_stdin, child_stdout = os.popen2('PYPYLOG=":%s" "%s" "%s"' % ( - logfilepath, self.pypy_c, filepath)) - child_stdin.close() + child_stdout = os.popen('PYPYLOG=":%s" "%s" "%s"' % ( + logfilepath, self.pypy_c, filepath), 'r') result = child_stdout.read() child_stdout.close() assert result From arigo at codespeak.net Mon Nov 23 18:04:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 23 Nov 2009 18:04:22 +0100 (CET) Subject: [pypy-svn] r69561 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20091123170422.9F15416801F@codespeak.net> Author: arigo Date: Mon Nov 23 18:04:22 2009 New Revision: 69561 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Print the list of operations recorded when the number exceed the limit. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Mon Nov 23 18:04:22 2009 @@ -113,7 +113,10 @@ assert result assert result.splitlines()[-1].strip() == 'OK :-)' self.parse_loops(logfilepath) - assert self.total_ops <= expected_max_ops + if self.total_ops > expected_max_ops: + self.print_loops() + assert 0, "too many operations: got %d, expected maximum %d" % ( + self.total_ops, expected_max_ops) def parse_loops(self, opslogfile): from pypy.jit.metainterp.test.oparser import parse @@ -146,6 +149,16 @@ def get_by_bytecode(self, name): return [ops for ops in self.sliced_loops if ops.bytecode == name] + def print_loops(self): + for loop in self.loops: + print + print '@' * 79 + print + for op in loop.operations: + print op + print + print '@' * 79 + def test_f1(self): self.run_source(''' def main(n): From santagada at codespeak.net Mon Nov 23 18:48:44 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 23 Nov 2009 18:48:44 +0100 (CET) Subject: [pypy-svn] r69562 - pypy/branch/force-arch-darwin Message-ID: <20091123174844.DDFD5168007@codespeak.net> Author: santagada Date: Mon Nov 23 18:48:43 2009 New Revision: 69562 Added: pypy/branch/force-arch-darwin/ - copied from r69561, pypy/trunk/ Log: a branch to force compilation to a specific architecture on darwin/osx From santagada at codespeak.net Mon Nov 23 19:10:08 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 23 Nov 2009 19:10:08 +0100 (CET) Subject: [pypy-svn] r69563 - in pypy/branch/force-arch-darwin/pypy/translator/platform: . test Message-ID: <20091123181008.ADBF8168008@codespeak.net> Author: santagada Date: Mon Nov 23 19:10:07 2009 New Revision: 69563 Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/__init__.py pypy/branch/force-arch-darwin/pypy/translator/platform/darwin.py pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Log: force compilation to an specific architecture, so on snow leopard it would not try to compile with the kernel architecture but use the python one. This fixes problems with python int, c long size mismatch (and other problems related to this). Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/translator/platform/__init__.py (original) +++ pypy/branch/force-arch-darwin/pypy/translator/platform/__init__.py Mon Nov 23 19:10:07 2009 @@ -171,8 +171,15 @@ else: host_factory = Linux64 elif sys.platform == 'darwin': - from pypy.translator.platform.darwin import Darwin - host_factory = Darwin + from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 + import platform + if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 + else: + host_factory = Darwin elif sys.platform == 'freebsd7': from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 import platform Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/translator/platform/darwin.py (original) +++ pypy/branch/force-arch-darwin/pypy/translator/platform/darwin.py Mon Nov 23 19:10:07 2009 @@ -4,7 +4,7 @@ class Darwin(posix.BasePosix): name = "darwin" - + link_flags = ['-mmacosx-version-min=10.4'] cflags = ['-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] standalone_only = ['-mdynamic-no-pic'] @@ -44,3 +44,13 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) +class Darwin_i386(Darwin): + name = "darwin_i386" + link_flags = ['-arch', 'i386', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'i386', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + +class Darwin_x86_64(Darwin): + name = "darwin_x86_64" + link_flags = ['-arch', 'x86_64', '-mmacosx-version-min=10.4'] + cflags = ['-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4'] + Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Mon Nov 23 19:10:07 2009 @@ -2,17 +2,25 @@ """ File containing darwin platform tests """ -import py, sys +import py, sys, platform if sys.platform != 'darwin': py.test.skip("Darwin only") from pypy.tool.udir import udir -from pypy.translator.platform.darwin import Darwin +from pypy.translator.platform.darwin import Darwin_i386, Darwin_x86_64 from pypy.translator.platform.test.test_platform import TestPlatform as BasicTest from pypy.translator.tool.cbuild import ExternalCompilationInfo +if platform.machine() == 'i386': + if sys.maxint <= 2147483647: + host_factory = Darwin_i386 + else: + host_factory = Darwin_x86_64 +else: + host_factory = Darwin + class TestDarwin(BasicTest): - platform = Darwin() + platform = host_factory() def test_frameworks(self): objcfile = udir.join('test_simple.m') @@ -39,3 +47,80 @@ res = self.platform.execute(executable) self.check_res(res) + def test_64_32_results(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = plat32.compile([cfile], eci) + res = plat32.execute(executable) + self.check_res(res, '0\n') + executable = plat64.compile([cfile], eci) + res = plat64.execute(executable) + self.check_res(res, '1\n') + + def test_longsize(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + cfile = udir.join('test_int_size.c') + cfile.write(r''' + #include + #include + + int main() { + printf("%d\n", LONG_MAX); + return 0; + } + ''') + eci = ExternalCompilationInfo() + executable = self.platform.compile([cfile], eci) + res = self.platform.execute(executable) + self.check_res(res, str(sys.maxint) + '\n') + + def test_32bit_makefile(self): + if platform.machine() != 'i386': + py.test.skip("i386 only") + plat32 = Darwin_i386() + plat64 = Darwin_x86_64() + eci = ExternalCompilationInfo() + cfile_content =r''' + #include + #include + + int main() { + printf("%d\n", INT_MAX < LONG_MAX); + return 0; + } + ''' + + tmpdir = udir.join('32_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat32.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat32.execute_makefile(mk) + res = plat32.execute(tmpdir.join('test_int_size')) + self.check_res(res, '0\n') + + tmpdir = udir.join('64_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat64.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat64.execute_makefile(mk) + res = plat64.execute(tmpdir.join('test_int_size')) + self.check_res(res, '1\n') + From santagada at codespeak.net Mon Nov 23 19:14:31 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 23 Nov 2009 19:14:31 +0100 (CET) Subject: [pypy-svn] r69564 - pypy/branch/force-arch-darwin/pypy/translator/platform/test Message-ID: <20091123181431.D558B168008@codespeak.net> Author: santagada Date: Mon Nov 23 19:14:30 2009 New Revision: 69564 Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Log: actually print a long, test passes on 64/32bit now. Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Mon Nov 23 19:14:30 2009 @@ -79,7 +79,7 @@ #include int main() { - printf("%d\n", LONG_MAX); + printf("%ld\n", LONG_MAX); return 0; } ''') From antocuni at codespeak.net Mon Nov 23 20:26:54 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 23 Nov 2009 20:26:54 +0100 (CET) Subject: [pypy-svn] r69565 - pypy/trunk/pypy/translator/oosupport Message-ID: <20091123192654.1C73316800D@codespeak.net> Author: antocuni Date: Mon Nov 23 20:26:54 2009 New Revision: 69565 Modified: pypy/trunk/pypy/translator/oosupport/constant.py Log: reduce the maximum number of constants per step, else pypy-cli-jit might crash at startup Modified: pypy/trunk/pypy/translator/oosupport/constant.py ============================================================================== --- pypy/trunk/pypy/translator/oosupport/constant.py (original) +++ pypy/trunk/pypy/translator/oosupport/constant.py Mon Nov 23 20:26:54 2009 @@ -29,7 +29,7 @@ from pypy.rpython.lltypesystem import rffi import operator -MAX_CONST_PER_STEP = 50 +MAX_CONST_PER_STEP = 40 PRIMITIVE_TYPES = set([ootype.Void, ootype.Bool, ootype.Char, ootype.UniChar, ootype.Float, ootype.Signed, ootype.Unsigned, From fijal at codespeak.net Mon Nov 23 21:46:04 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 21:46:04 +0100 (CET) Subject: [pypy-svn] r69566 - pypy/branch/stringbuilder/pypy/translator/c/src Message-ID: <20091123204604.E3C22168008@codespeak.net> Author: fijal Date: Mon Nov 23 21:46:02 2009 New Revision: 69566 Modified: pypy/branch/stringbuilder/pypy/translator/c/src/mem.h Log: Make sure realloc_grow cleans up memory. A bit hard to test :-/ Modified: pypy/branch/stringbuilder/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/stringbuilder/pypy/translator/c/src/mem.h (original) +++ pypy/branch/stringbuilder/pypy/translator/c/src/mem.h Mon Nov 23 21:46:02 2009 @@ -103,8 +103,23 @@ #define OP_RAW_REALLOC_SHRINK(p, old_size, size, r) r = PyObject_Realloc((void*)p, size) +#if RAW_MALLOC_ZERO_FILLED + +#define OP_RAW_REALLOC_GROW(p, old_size, size, r) { \ + r = PyObject_Realloc((void*)p, size); \ + if (r != NULL) { \ + memset((void*)r + old_size, 0, size - old_size); \ + } \ +} + +#else + #define OP_RAW_REALLOC_GROW(p, old_size, size, r) r = PyObject_Realloc((void*)p, size) +#endif + + + #ifdef MS_WINDOWS #define alloca _alloca #endif From fijal at codespeak.net Mon Nov 23 22:53:13 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 23 Nov 2009 22:53:13 +0100 (CET) Subject: [pypy-svn] r69567 - pypy/branch/stringbuilder/pypy/rpython/memory/gc Message-ID: <20091123215313.5ABFA168008@codespeak.net> Author: fijal Date: Mon Nov 23 22:53:12 2009 New Revision: 69567 Modified: pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py Log: Also, cleanup memory in case raw_realloc does not clean it (which is the default) Modified: pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py ============================================================================== --- pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py (original) +++ pypy/branch/stringbuilder/pypy/rpython/memory/gc/hybrid.py Mon Nov 23 22:53:12 2009 @@ -239,6 +239,9 @@ if grow: result = llop.raw_realloc_grow(llmemory.Address, source_addr, old_tot_size, tot_size) + llmemory.raw_memclear(result + size_gc_header + + fixedsize + oldlength * itemsize, + (newlength - oldlength) * itemsize) else: if oldlength == newlength: # no need to call realloc From afa at codespeak.net Tue Nov 24 09:57:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 09:57:22 +0100 (CET) Subject: [pypy-svn] r69568 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091124085722.BBBDF16800D@codespeak.net> Author: afa Date: Tue Nov 24 09:57:21 2009 New Revision: 69568 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_numbervar.py pypy/trunk/pypy/module/oracle/transform.py Log: Fix many (not all!) translations failures Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Tue Nov 24 09:57:21 2009 @@ -21,8 +21,8 @@ self.environment = None self.autocommit = False - self.inputTypeHandler = None - self.outputTypeHandler = None + self.w_inputTypeHandler = None + self.w_outputTypeHandler = None self.w_version = None @@ -204,7 +204,7 @@ self.environment.checkForError( status, "Connection_Connect(): begin session") except: - self.sessionHandle = None + self.sessionHandle = lltype.nullptr(roci.OCISession.TO) raise def _checkConnected(self, space): @@ -233,7 +233,7 @@ status, "Connection_Close(): end session") roci.OCIHandleFree(self.handle, roci.OCI_HTYPE_SVCCTX) - self.handle = None + self.handle = lltype.nullptr(roci.OCISvcCtx.TO) close.unwrap_spec = ['self', ObjSpace] def commit(self, space): @@ -255,7 +255,7 @@ status = roci.OCITransRollback( self.handle, self.environment.errorHandle, - self.OCI_DEFAULT) + roci.OCI_DEFAULT) self.environment.checkForError( status, "Connection_Rollback()") rollback.unwrap_spec = ['self', ObjSpace] Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 24 09:57:21 2009 @@ -12,6 +12,9 @@ from pypy.module.oracle import interp_variable from pypy.module.oracle.interp_error import get +# XXX are those "assert isinstance(xxx, interp_variable.W_Variable)" necessary? +# the bindList should annotate to SomeList(SomeInstance(W_Variable)) + class W_Cursor(Wrappable): def __init__(self, space, connection): self.connection = connection @@ -19,7 +22,7 @@ self.w_statement = None self.statementType = -1 - self.handle = None + self.handle = lltype.nullptr(roci.OCIStmt.TO) self.isOpen = True self.isOwned = False @@ -33,8 +36,8 @@ self.outputSize = -1 self.outputSizeColumn = -1 - self.inputTypeHandler = None - self.outputTypeHandler = None + self.w_inputTypeHandler = None + self.w_outputTypeHandler = None self.w_rowFactory = None def execute(self, space, w_stmt, __args__): @@ -129,7 +132,7 @@ # queries are not supported as the result is undefined if self.statementType == roci.OCI_STMT_SELECT: raise OperationError( - get(space).w_NotSupportedErrorException, + get(space).w_NotSupportedError, space.wrap("queries not supported: results undefined")) # perform binds @@ -160,7 +163,7 @@ self.freeHandle(space, raiseError=True) self.isOpen = False - self.handle = None + self.handle = lltype.nullptr(roci.OCIStmt.TO) close.unwrap_spec = ['self', ObjSpace] def callfunc(self, space, name, w_returnType, w_parameters=None): @@ -181,11 +184,12 @@ self._call(space, name, None, w_parameters) # create the return value + ret_w = [] if self.bindList: - ret_w = [v.getValue(space, 0) for v in self.bindList] - return space.newlist(ret_w) - else: - return space.newlist([]) + for v in self.bindList: + assert isinstance(v, interp_variable.W_Variable) + ret_w.append(v.getValue(space, 0)) + return space.newlist(ret_w) callproc.unwrap_spec = ['self', ObjSpace, str, W_Root] @@ -557,8 +561,8 @@ "handle positional binds" # make sure positional and named binds are not being intermixed if self.bindDict is not None: - raise OperationalError( - get(space).w_ProgrammingErrorException, + raise OperationError( + get(space).w_ProgrammingError, space.wrap("positional and named binds cannot be intermixed")) if self.bindList is None: @@ -587,8 +591,8 @@ "handle named binds" # make sure positional and named binds are not being intermixed if self.bindList is not None: - raise OperationalError( - get(space).w_ProgrammingErrorException, + raise OperationError( + get(space).w_ProgrammingError, space.wrap("positional and named binds cannot be intermixed")) if self.bindDict is None: @@ -607,13 +611,16 @@ numElements, arrayPos, defer): valueIsVariable = space.is_true(space.isinstance(w_value, get(space).w_Variable)) + newVar = None # handle case where variable is already bound if origVar: + assert isinstance(origVar, interp_variable.W_Variable) # if the value is a variable object, rebind it if necessary if valueIsVariable: newVar = space.interp_w(interp_variable.W_Variable, w_value) + assert isinstance(newVar, interp_variable.W_Variable) if newVar == origVar: newVar = None @@ -621,12 +628,13 @@ # this is only necessary for executemany() since execute() always # passes a value of 1 for the number of elements elif numElements > origVar.allocatedElements: - newVar = type(origVar)(self, numElements, origVar.size) + newVar = origVar.clone( + self, numElements, origVar.size) + assert isinstance(newVar, interp_variable.W_Variable) newVar.setValue(space, arrayPos, w_value) # otherwise, attempt to set the value else: - newVar = None try: origVar.setValue(space, arrayPos, w_value) except OperationError, e: @@ -644,6 +652,7 @@ # if the value is a variable object, bind it directly if valueIsVariable: newVar = space.interp_w(interp_variable.W_Variable, w_value) + assert isinstance(newVar, interp_variable.W_Variable) newVar.boundPos = 0 newVar.boundName = None @@ -653,8 +662,10 @@ newVar = interp_variable.newVariableByValue(space, self, w_value, numElements) + assert isinstance(newVar, interp_variable.W_Variable) newVar.setValue(space, arrayPos, w_value) + assert newVar is None or isinstance(newVar, interp_variable.W_Variable) return newVar def _performBind(self, space): @@ -662,12 +673,14 @@ if self.bindList: for i in range(len(self.bindList)): var = self.bindList[i] + assert isinstance(var, interp_variable.W_Variable) var.bind(space, self, None, i + 1) if self.bindDict: items_w = space.fixedview( space.call_method(self.bindDict, "iteritems")) for w_item in items_w: w_key, var = space.fixedview(w_item, 2) + assert isinstance(var, interp_variable.W_Variable) var.bind(space, self, w_key, 0) # ensure that input sizes are reset @@ -723,6 +736,7 @@ self.fetchArraySize = self.arraySize for i in range(numParams): var = interp_variable.define(self, i+1, self.fetchArraySize) + assert isinstance(var, interp_variable.W_Variable) self.fetchVariables.append(var) def _verifyFetch(self, space): @@ -772,7 +786,7 @@ # verify fetch can be performed self._verifyFetch(space) - return self._multiFetch(space, limit=None) + return self._multiFetch(space, limit=0) fetchall.unwrap_spec = ['self', ObjSpace] def descr_iter(self, space): @@ -820,6 +834,7 @@ "Cursor_InternalFetch(): fetch") for var in self.fetchVariables: + assert isinstance(var, interp_variable.W_Variable) var.internalFetchNum += 1 attrptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, @@ -840,12 +855,12 @@ finally: lltype.free(attrptr, flavor='raw') - def _multiFetch(self, space, limit=None): + def _multiFetch(self, space, limit=0): results_w = [] rowNum = 0 # fetch as many rows as possible - while limit is None or rowNum < limit: + while limit == 0 or rowNum < limit: rowNum += 1 if not self._moreRows(space): break @@ -859,6 +874,7 @@ # acquire the value for each item for var in self.fetchVariables: + assert isinstance(var, interp_variable.W_Variable) w_item = var.getValue(space, self.rowNum) items_w.append(w_item) @@ -979,7 +995,7 @@ numElements = space.int_w(w_value) else: raise OperationError( - get(space).w_NotSupportedErrorException, + get(space).w_NotSupportedError, space.wrap("expecting integer or list of values")) # create the variable @@ -1025,7 +1041,7 @@ for i in range(len(args_w)): w_value = args_w[i] if space.is_w(w_value, space.w_None): - var = space.w_None + var = None else: var = interp_variable.newVariableByType( space, self, w_value, self.bindArraySize) Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Tue Nov 24 09:57:21 2009 @@ -21,12 +21,15 @@ status = roci.OCIEnvNlsCreate( handleptr, mode, None, - lltype.nullptr(lltype.FuncType( # malocfp - [roci.dvoidp, roci.size_t], roci.dvoidp)), - lltype.nullptr(lltype.FuncType( # ralocfp - [roci.dvoidp, roci.dvoidp, roci.size_t], roci.dvoidp)), - lltype.nullptr(lltype.FuncType( # mfreefp - [roci.dvoidp, roci.dvoidp], lltype.Void)), + rffi.cast(rffi.CCallback( # malocfp + (roci.dvoidp, roci.size_t), roci.dvoidp), + 0), + rffi.cast(rffi.CCallback( # ralocfp + (roci.dvoidp, roci.dvoidp, roci.size_t), roci.dvoidp), + 0), + rffi.cast(rffi.CCallback( # mfreefp + (roci.dvoidp, roci.dvoidp), lltype.Void), + 0), 0, lltype.nullptr(rffi.CArray(roci.dvoidp)), config.CHARSETID, config.CHARSETID) Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 24 09:57:21 2009 @@ -1,7 +1,9 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.rpython.lltypesystem import rffi, lltype -from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError from pypy.module.oracle import roci, config class State: @@ -16,6 +18,7 @@ self.w_ProgrammingError = get('ProgrammingError') self.w_NotSupportedError = get('NotSupportedError') self.w_IntegrityError = get('IntegrityError') + self.w_InternalError = get('InternalError') self.w_Variable = get('Variable') w_import = space.builtin.get('__import__') @@ -52,14 +55,11 @@ textbuf, BUFSIZE, handleType) if status != roci.OCI_SUCCESS: raise OperationError( - w_InternalErrorException, + get(space).w_InternalError, space.wrap("No Oracle error?")) self.code = codeptr[0] - self.message = config.w_string( - space, - rffi.str_from_buffer(textbuf, text, - BUFSIZE, BUFSIZE)) + self.w_message = config.w_string(space, textbuf) finally: lltype.free(codeptr, flavor='raw') rffi.keep_buffer_alive_until_here(textbuf, text) @@ -75,6 +75,6 @@ 'Error', __str__ = interp2app(W_Error.desc_str), code = interp_attrproperty('code', W_Error), - message = interp_attrproperty('message', W_Error)) + message = interp_attrproperty_w('w_message', W_Error)) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 09:57:21 2009 @@ -9,7 +9,7 @@ import sys from pypy.module.oracle import roci, config, transform -from pypy.module.oracle.interp_error import get +from pypy.module.oracle.interp_error import W_Error, get from pypy.module.oracle.config import string_w, StringBuffer def define(cursor, position, numElements): @@ -33,7 +33,7 @@ roci.OCIDescriptorFree(param, roci.OCI_DTYPE_PARAM) return var - + def _defineHelper(cursor, param, position, numElements): # determine data type varType = typeByOracleDescriptor(param, cursor.environment) @@ -71,21 +71,23 @@ size = cursor.outputSize # call the procedure to set values prior to define - varType = varType.preDefine(param, cursor.environment) + varType2 = varType.preDefine(varType, param, cursor.environment) # create a variable of the correct type - if cursor.outputTypeHandler: - var = _newByOutputTypeHandler( + if 0 and cursor.w_outputTypeHandler: # XXX + var = newByOutputTypeHandler( cursor, param, - cursor.outputTypeHandler, - varType, size, numElements) - elif cursor.connection.outputTypeHandler: - var = _newByOutputTypeHandler( + cursor.w_outputTypeHandler, + varType2, size, numElements) + elif 0 and cursor.connection.w_outputTypeHandler: # XXX + var = newByOutputTypeHandler( cursor, param, - cursor.connection.outputTypeHandler, - varType, size, numElements) + cursor.connection.w_outputTypeHandler, + varType2, size, numElements) else: - var = varType(cursor, numElements, size) + var = varType2(cursor, numElements, size) + + assert isinstance(var, W_Variable) # perform the define handleptr = lltype.malloc(roci.Ptr(roci.OCIDefine).TO, 1, flavor='raw') @@ -118,7 +120,7 @@ charsetForm = roci.SQLCS_IMPLICIT isVariableLength = False canBeInArray = True - + def __init__(self, cursor, numElements, size=0): self.environment = cursor.environment self.boundCursorHandle = lltype.nullptr(roci.OCIStmt.TO) @@ -211,7 +213,7 @@ # force rebinding if self.boundName or self.boundPos: - self._internalBind() + self._internalBind(space) def makeArray(self, space): if not self.canBeInArray: @@ -227,7 +229,7 @@ def finalize(self): pass - @classmethod + @staticmethod def preDefine(cls, param, environment): return cls @@ -313,7 +315,7 @@ # ensure we do not exceed the number of allocated elements if pos >= self.allocatedElements: raise OperationError( - space.w_PyExc_IndexError, + space.w_IndexError, space.wrap("Variable_GetSingleValue: array size exceeded")) # check for a NULL value @@ -465,6 +467,15 @@ raise OperationError( space.w_TypeError, space.wrap("expecting string or buffer data")) + else: + if space.is_true(space.isinstance(w_value, space.w_unicode)): + buf = config.StringBuffer() + buf.fill(space, w_value) + size = buf.size + else: + raise OperationError( + space.w_TypeError, + space.wrap("expecting unicode data")) try: if buf.size > self.environment.maxStringBytes: @@ -545,7 +556,7 @@ oracleType = roci.SQLT_VNU size = rffi.sizeof(roci.OCINumber) - @classmethod + @staticmethod def preDefine(cls, param, environment): # if the return type has not already been specified, check to # see if the number can fit inside an integer by looking at @@ -608,12 +619,12 @@ self.environment.checkForError( status, "NumberVar_GetValue(): as integer") if isinstance(self, VT_Boolean): - return space.newbool(integerValuePtr[0]) + return space.newbool(bool(integerValuePtr[0])) else: return space.wrap(integerValuePtr[0]) finally: lltype.free(integerValuePtr, flavor='raw') - elif isinstance(self, (VT_NumberAsString, VT_LongInteger)): + elif isinstance(self, VT_NumberAsString) or isinstance(self, VT_LongInteger): format_buf = config.StringBuffer() format_buf.fill(space, space.wrap("TM9")) sizeptr = lltype.malloc(rffi.CArray(roci.ub4), 1, flavor='raw') @@ -671,7 +682,7 @@ finally: lltype.free(integerValuePtr, flavor='raw') return - if space.is_true(space.isinstance(w_value, space.w_long)): + elif space.is_true(space.isinstance(w_value, space.w_long)): text_buf = config.StringBuffer() text_buf.fill(space, space.str(w_value)) format_buf = config.StringBuffer() @@ -685,8 +696,7 @@ self.environment.checkForError( status, "NumberVar_SetValue(): from long") return - elif space.is_true(space.isinstance(w_value, space.w_bool)): - XXX + # XXX The bool case was already processed above elif space.is_true(space.isinstance(w_value, space.w_float)): doubleValuePtr = lltype.malloc(roci.Ptr(lltype.Float).TO, 1, flavor='raw') @@ -852,10 +862,15 @@ for name, cls in globals().items(): if not name.startswith('VT_') or not isinstance(cls, type): continue - cls.typedef = TypeDef( - cls.__name__, W_Variable.typedef, - ) - variableTypeByTypedef[cls.typedef] = cls + def register_variable_class(cls): + def clone(self, cursor, numElements, size): + return cls(cursor, numElements, size) + cls.clone = clone + cls.typedef = TypeDef( + cls.__name__, W_Variable.typedef, + ) + variableTypeByTypedef[cls.typedef] = cls + register_variable_class(cls) def typeByOracleDescriptor(param, environment): # retrieve datatype of the parameter @@ -929,9 +944,9 @@ def _typeByOracleDataType(dataType, charsetForm): if charsetForm == roci.SQLCS_NCHAR: - varType = variableTypeNChar.get(dataType) + varType = variableTypeNChar.get(dataType, None) else: - varType = variableType.get(dataType) + varType = variableType.get(dataType, None) if varType is None: raise ValueError("Variable_TypeByOracleDataType: " @@ -941,11 +956,14 @@ def typeByPythonType(space, cursor, w_type): """Return a variable type given a Python type object""" + from pypy.objspace.std.typeobject import W_TypeObject + moduledict = get(space) if not space.is_true(space.isinstance(w_type, space.w_type)): raise OperationError( space.w_TypeError, space.wrap("Variable_TypeByPythonType(): type expected")) + assert isinstance(w_type, W_TypeObject) if w_type.instancetypedef in variableTypeByTypedef: return variableTypeByTypedef[w_type.instancetypedef] if space.is_w(w_type, space.w_int): @@ -1017,21 +1035,38 @@ space.wrap("Variable_TypeByValue(): unhandled data type %s" % (space.type(w_value).getname(space, '?'),))) +def newByInputTypeHandler(space, cursor, w_inputTypeHandler, w_value, numElements): + w_var = space.call(w_inputTypeHandler, + space.wrap(cursor), + w_value, + space.wrap(numElements)) + if not space.is_true(space.isinstance(w_var, + get(space).w_Variable)): + raise OperationError( + space.w_TypeError, + space.wrap("expecting variable from input type handler")) + return space.interp_w(W_Variable, w_var) + def newVariableByValue(space, cursor, w_value, numElements): - if cursor.inputTypeHandler: - return newByInputTypeHandler( - cursor, cursor.inputTypeHandler, + var = space.w_None + + if cursor.w_inputTypeHandler: + var = newByInputTypeHandler( + space, cursor, cursor.w_inputTypeHandler, w_value, numElements) - elif cursor.connection.inputTypeHandler: - return newByInputTypeHandler( - cursor, cursor.connection.inputTypeHandler, + elif cursor.connection.w_inputTypeHandler: + var = newByInputTypeHandler( + space, cursor, cursor.connection.w_inputTypeHandler, w_value, numElements) - else: + + if space.is_w(var, space.w_None): varType, size, numElements = typeByValue(space, w_value, numElements) var = varType(cursor, numElements, size) if space.is_true(space.isinstance(w_value, space.w_list)): var.makeArray(space) - return var + + assert isinstance(var, W_Variable) + return var def newArrayVariableByType(space, cursor, w_value): "Allocate a new PL/SQL array by looking at the Python data type." Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Tue Nov 24 09:57:21 2009 @@ -58,7 +58,7 @@ OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET - OCI_NTV_SYNTAX + OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER Modified: pypy/trunk/pypy/module/oracle/test/test_numbervar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_numbervar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_numbervar.py Tue Nov 24 09:57:21 2009 @@ -15,6 +15,13 @@ b=0.5) assert var.getvalue() == 1.25 + def test_bool(self): + cur = self.cnx.cursor() + var = cur.var(oracle.NUMBER) + cur.execute("begin :a := :b*2.5; end;", a=var, + b=True) + assert var.getvalue() == 2.5 + def test_decimal(self): import decimal cur = self.cnx.cursor() Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Tue Nov 24 09:57:21 2009 @@ -16,7 +16,6 @@ lltype.free(doubleptr, flavor='raw') def OracleDateToPythonDate(environment, valueptr): - print valueptr.OCIDateYYYY, valueptr.OCIDateMM, valueptr.OCIDateDD yearptr = lltype.malloc(roci.Ptr(roci.sb2).TO, 1, flavor='raw') monthptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') dayptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') @@ -77,7 +76,7 @@ text += '-' for i in xrange(0, num_digits + scale): format += '9' - if i < numdigits: + if i < num_digits: digit = space.int_w(digits_w[i]) text += "0123456789"[digit] else: From pedronis at codespeak.net Tue Nov 24 10:59:43 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 10:59:43 +0100 (CET) Subject: [pypy-svn] r69569 - in pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp: . test Message-ID: <20091124095943.0D375168012@codespeak.net> Author: pedronis Date: Tue Nov 24 10:59:42 2009 New Revision: 69569 Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py Log: don't think about right holes, not clear their space can be optimized away Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/jitprof.py Tue Nov 24 10:59:42 2009 @@ -23,7 +23,6 @@ NVIRTUALS NVHOLES NVREUSED -NVRIGHTHOLES """ def _setup(): @@ -181,7 +180,6 @@ self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) - self._print_intline("nvrightholes", cnt[NVRIGHTHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) def _print_line_time(self, string, i, tim): Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/resume.py Tue Nov 24 10:59:42 2009 @@ -117,7 +117,6 @@ self.nvirtuals = 0 self.nvholes = 0 - self.nvrightholes = 0 self.nvreused = 0 def getconst(self, const): @@ -228,7 +227,6 @@ profiler.count(jitprof.NVIRTUALS, self.nvirtuals) profiler.count(jitprof.NVHOLES, self.nvholes) profiler.count(jitprof.NVREUSED, self.nvreused) - profiler.count(jitprof.NVRIGHTHOLES, self.nvrightholes) _frame_info_placeholder = (None, 0, 0) @@ -345,14 +343,6 @@ if vinfo.fieldnums is not fieldnums: memo.nvreused += 1 virtuals[num] = vinfo - # count right holes - r = 0 - while r < length: - if virtuals[-1-r]: - break - r += 1 - memo.nvrightholes += r - if self._invalidation_needed(len(liveboxes), nholes): memo.clear_box_virtual_numbers() Modified: pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/branch/compress-virtuals-resumedata2/pypy/jit/metainterp/test/test_jitprof.py Tue Nov 24 10:59:42 2009 @@ -64,7 +64,7 @@ assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, - 0, 0, 0, 0] + 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside From pedronis at codespeak.net Tue Nov 24 11:08:44 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 11:08:44 +0100 (CET) Subject: [pypy-svn] r69570 - in pypy/trunk/pypy: . interpreter jit/backend/llgraph jit/backend/llvm/test jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test module/exceptions objspace/std/test translator/c/test Message-ID: <20091124100844.B0C5B168012@codespeak.net> Author: pedronis Date: Tue Nov 24 11:08:43 2009 New Revision: 69570 Modified: pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/interpreter/executioncontext.py pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llvm/test/conftest.py (props changed) pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (props changed) pypy/trunk/pypy/jit/backend/x86/test/test_virtual.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/jitprof.py pypy/trunk/pypy/jit/metainterp/logger.py (contents, props changed) pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/resume.py pypy/trunk/pypy/jit/metainterp/simple_optimize.py pypy/trunk/pypy/jit/metainterp/test/oparser.py pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py pypy/trunk/pypy/jit/metainterp/test/test_logger.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py pypy/trunk/pypy/jit/metainterp/test/test_virtual.py pypy/trunk/pypy/module/exceptions/ (props changed) pypy/trunk/pypy/objspace/std/test/test_setobject.py (props changed) pypy/trunk/pypy/translator/c/test/test_refcount.py (props changed) Log: merge compress-virtuals-resumedata(2) (cfbolz, pedronis) reuse vinfos by making virtuals and boxes out of them (and not the environment) have more stable numberings by introducing holes (None) in fail_args Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Tue Nov 24 11:08:43 2009 @@ -142,6 +142,8 @@ #assert frame is self.gettopframe() --- slowish if self.some_frame is frame: self.some_frame = frame.f_back_some + if self.some_frame and self.some_frame.f_forward: + raise Exception() else: f_back = frame.f_back() if f_back is not None: Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Tue Nov 24 11:08:43 2009 @@ -364,7 +364,10 @@ op = loop.operations[-1] if op.fail_args is None: op.fail_args = [] - op.fail_args.append(_variables[intvar]) + if intvar == -1: + op.fail_args.append(None) + else: + op.fail_args.append(_variables[intvar]) def compile_redirect_fail(old_loop, old_index, new_loop): old_loop = _from_opaque(old_loop) @@ -407,23 +410,27 @@ except GuardFailed: assert op.is_guard() _stats.exec_conditional_jumps += 1 - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args] - else: - args = [] if op.jump_target is not None: # a patched guard, pointing to further code + args = [self.getenv(v) for v in op.fail_args if v] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) operations = op.jump_target.operations opindex = 0 continue else: + fail_args = [] + if op.fail_args: + for fail_arg in op.fail_args: + if fail_arg is None: + fail_args.append(None) + else: + fail_args.append(self.getenv(fail_arg)) # a non-patched guard if self.verbose: log.trace('failed: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args + ', '.join(map(str, fail_args)),)) + self.fail_args = fail_args return op.fail_index #verbose = self.verbose assert (result is None) == (op.result is None) Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Tue Nov 24 11:08:43 2009 @@ -171,7 +171,10 @@ index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: - llimpl.compile_add_fail_arg(c, var2index[box]) + if box is not None: + llimpl.compile_add_fail_arg(c, var2index[box]) + else: + llimpl.compile_add_fail_arg(c, -1) x = op.result if x is not None: if isinstance(x, history.BoxInt): Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Nov 24 11:08:43 2009 @@ -125,6 +125,27 @@ res = self.cpu.get_latest_value_int(0) assert res == 10 + def test_compile_with_holes_in_fail_args(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, None, i1, None] + + self.cpu.compile_loop(inputargs, operations, looptoken) + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(2) + assert res == 10 + def test_backends_dont_keep_loops_alive(self): import weakref, gc self.cpu.dont_keepalive_stuff = True @@ -183,6 +204,40 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + def test_compile_bridge_with_holes(self): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].fail_args = [None, i1, None] + self.cpu.compile_loop(inputargs, operations, looptoken) + + i1b = BoxInt() + i3 = BoxInt() + bridge = [ + ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3), + ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2), + ResOperation(rop.JUMP, [i1b], None, descr=looptoken), + ] + bridge[1].fail_args = [i1b] + + self.cpu.compile_bridge(faildescr1, [i1b], bridge) + + self.cpu.set_future_value_int(0, 2) + fail = self.cpu.execute_token(looptoken) + assert fail == 2 + res = self.cpu.get_latest_value_int(0) + assert res == 20 + def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Nov 24 11:08:43 2009 @@ -738,7 +738,7 @@ """ assert that all args are actually Boxes """ for arg in args: - assert isinstance(arg, Box) + assert arg is None or isinstance(arg, Box) # hole def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): @@ -754,6 +754,8 @@ pos = mc.tell() for i in range(len(failargs)): arg = failargs[i] + if arg is None: # hole + continue loc = locs[i] if isinstance(loc, REG): if arg.type == FLOAT: @@ -767,6 +769,8 @@ mc.MOV(heap(adr), loc) for i in range(len(failargs)): arg = failargs[i] + if arg is None: # hole + continue loc = locs[i] if not isinstance(loc, REG): if arg.type == FLOAT: Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Tue Nov 24 11:08:43 2009 @@ -203,7 +203,8 @@ def possibly_free_vars(self, vars): for var in vars: - self.possibly_free_var(var) + if var is not None: # xxx kludgy + self.possibly_free_var(var) def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, imm_fine=True, @@ -239,9 +240,12 @@ def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py used = {} - for i in range(len(inputargs)): + i = 0 + for loc in locs: + if loc is None: # xxx bit kludgy + continue arg = inputargs[i] - loc = locs[i] + i += 1 if arg.type == FLOAT: if isinstance(loc, REG): self.xrm.reg_bindings[arg] = loc @@ -360,6 +364,8 @@ longevity[arg] = (start_live[arg], i) if op.is_guard(): for arg in op.fail_args: + if arg is None: # hole + continue assert isinstance(arg, Box) if arg not in start_live: print "Bogus arg in guard %d at %d" % (op.opnum, i) @@ -373,6 +379,8 @@ return longevity def loc(self, v): + if v is None: # xxx kludgy + return None if v.type == FLOAT: return self.xrm.loc(v) return self.rm.loc(v) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_virtual.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_virtual.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_virtual.py Tue Nov 24 11:08:43 2009 @@ -1,4 +1,4 @@ -from pypy.jit.metainterp.test.test_virtual import VirtualTests +from pypy.jit.metainterp.test.test_virtual import VirtualTests, VirtualMiscTests from pypy.jit.backend.x86.test.test_basic import Jit386Mixin class MyClass: @@ -13,3 +13,8 @@ @staticmethod def _new(): return MyClass() + +class TestsVirtualMisc(Jit386Mixin, VirtualMiscTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_virtual.py + pass Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Tue Nov 24 11:08:43 2009 @@ -233,7 +233,12 @@ def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes self.guard_opnum = guard_op.opnum - self.fail_arg_types = [box.type for box in boxes] + fail_arg_types = [history.HOLE] * len(boxes) + for i in range(len(boxes)): + box = boxes[i] + if box: + fail_arg_types[i] = box.type + self.fail_arg_types = fail_arg_types def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Tue Nov 24 11:08:43 2009 @@ -15,6 +15,7 @@ INT = 'i' REF = 'r' FLOAT = 'f' +HOLE = '_' FAILARGS_LIMIT = 1000 @@ -756,8 +757,9 @@ ops = op.descr._debug_suboperations TreeLoop.check_consistency_of_branch(ops, seen.copy()) for box in op.fail_args or []: - assert isinstance(box, Box) - assert box in seen + if box is not None: + assert isinstance(box, Box) + assert box in seen else: assert op.fail_args is None box = op.result Modified: pypy/trunk/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/jitprof.py Tue Nov 24 11:08:43 2009 @@ -20,6 +20,9 @@ OPT_FORCINGS ABORT_TOO_LONG ABORT_BRIDGE +NVIRTUALS +NVHOLES +NVREUSED """ def _setup(): @@ -175,6 +178,9 @@ self._print_intline("forcings", cnt[OPT_FORCINGS]) self._print_intline("trace too long", cnt[ABORT_TOO_LONG]) self._print_intline("bridge abort", cnt[ABORT_BRIDGE]) + self._print_intline("nvirtuals", cnt[NVIRTUALS]) + self._print_intline("nvholes", cnt[NVHOLES]) + self._print_intline("nvreused", cnt[NVREUSED]) def _print_line_time(self, string, i, tim): final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Tue Nov 24 11:08:43 2009 @@ -63,6 +63,8 @@ if not name: name = 'cls' + str(mv) return 'ConstClass(' + name + ')' + elif arg is None: + return 'None' else: return '?' Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Tue Nov 24 11:08:43 2009 @@ -2,7 +2,7 @@ ConstFloat from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj, REF from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.metainterp.jitprof import OPT_OPS, OPT_GUARDS, OPT_FORCINGS +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.executor import execute_nonspec from pypy.jit.metainterp.specnode import SpecNode, NotSpecNode, ConstantSpecNode from pypy.jit.metainterp.specnode import AbstractVirtualStructSpecNode @@ -62,6 +62,9 @@ def get_args_for_fail(self, modifier): pass + def make_virtual_info(self, modifier, fieldnums): + raise NotImplementedError # should not be called on this level + def is_constant(self): return self.level == LEVEL_CONSTANT @@ -134,9 +137,10 @@ class AbstractVirtualValue(OptValue): - _attrs_ = ('optimizer', 'keybox', 'source_op') + _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo') box = None level = LEVEL_NONNULL + _cached_vinfo = None def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer @@ -155,6 +159,19 @@ self._really_force() return self.box + def make_virtual_info(self, modifier, fieldnums): + vinfo = self._cached_vinfo + if vinfo is not None and resume.tagged_list_eq( + vinfo.fieldnums, fieldnums): + return vinfo + vinfo = self._make_virtual(modifier) + vinfo.fieldnums = fieldnums + self._cached_vinfo = vinfo + return vinfo + + def _make_virtual(self, modifier): + raise NotImplementedError("abstract base") + class AbstractVirtualStructValue(AbstractVirtualValue): _attrs_ = ('_fields', '_cached_sorted_fields') @@ -207,14 +224,11 @@ # we have already seen the very same keybox lst = self._get_field_descr_list() fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - self._make_virtual(modifier, lst, fieldboxes) + modifier.register_virtual_fields(self.keybox, fieldboxes) for ofs in lst: fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - raise NotImplementedError - class VirtualValue(AbstractVirtualStructValue): level = LEVEL_KNOWNCLASS @@ -224,10 +238,9 @@ assert isinstance(known_class, Const) self.known_class = known_class - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_virtual(self.keybox, self.known_class, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_virtual(self.known_class, fielddescrs) class VStructValue(AbstractVirtualStructValue): @@ -235,10 +248,9 @@ AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op) self.structdescr = structdescr - def _make_virtual(self, modifier, fielddescrs, fieldboxes): - modifier.make_vstruct(self.keybox, self.structdescr, - fielddescrs, fieldboxes) - + def _make_virtual(self, modifier): + fielddescrs = self._get_field_descr_list() + return modifier.make_vstruct(self.structdescr, fielddescrs) class VArrayValue(AbstractVirtualValue): @@ -282,11 +294,14 @@ const = self.optimizer.new_const_item(self.arraydescr) for itemvalue in self._items: itemboxes.append(itemvalue.get_key_box()) - modifier.make_varray(self.keybox, self.arraydescr, itemboxes) + modifier.register_virtual_fields(self.keybox, itemboxes) for itemvalue in self._items: if itemvalue is not self.constvalue: itemvalue.get_args_for_fail(modifier) + def _make_virtual(self, modifier): + return modifier.make_varray(self.arraydescr) + class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): raise NotImplementedError @@ -354,12 +369,12 @@ self.loop = loop self.values = {} self.interned_refs = self.cpu.ts.new_ref_dict() - self.resumedata_memo = resume.ResumeDataLoopMemo(self.cpu) + self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd) self.heap_op_optimizer = HeapOpOptimizer(self) self.bool_boxes = {} def forget_numberings(self, virtualbox): - self.metainterp_sd.profiler.count(OPT_FORCINGS) + self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS) self.resumedata_memo.forget_numberings(virtualbox) def getinterned(self, box): @@ -480,6 +495,8 @@ else: self.optimize_default(op) self.loop.operations = self.newoperations + # accumulate counters + self.resumedata_memo.update_counters(self.metainterp_sd.profiler) def emit_operation(self, op, must_clone=True): self.heap_op_optimizer.emitting_operation(op) @@ -492,9 +509,9 @@ op = op.clone() must_clone = False op.args[i] = box - self.metainterp_sd.profiler.count(OPT_OPS) + self.metainterp_sd.profiler.count(jitprof.OPT_OPS) if op.is_guard(): - self.metainterp_sd.profiler.count(OPT_GUARDS) + self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS) self.store_final_boxes_in_guard(op) elif op.can_raise(): self.exception_might_have_happened = True @@ -507,7 +524,7 @@ assert isinstance(descr, compile.ResumeGuardDescr) modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo) newboxes = modifier.finish(self.values) - if len(newboxes) > self.metainterp_sd.options.failargs_limit: + if len(newboxes) > self.metainterp_sd.options.failargs_limit: # XXX be careful here raise compile.GiveUp descr.store_final_boxes(op, newboxes) # Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Nov 24 11:08:43 2009 @@ -1683,21 +1683,20 @@ def initialize_state_from_guard_failure(self, resumedescr, must_compile): # guard failure: rebuild a complete MIFrame stack self.in_recursion = -1 # always one portal around - inputargs = self.load_values_from_failure(resumedescr) - warmrunnerstate = self.staticdata.state + inputargs_and_holes = self.load_values_from_failure(resumedescr) if must_compile: self.history = history.History(self.cpu) - self.history.inputargs = inputargs + self.history.inputargs = [box for box in inputargs_and_holes if box] self.staticdata.profiler.start_tracing() else: self.staticdata.profiler.start_blackhole() self.history = None # this means that is_blackholing() is true - self.rebuild_state_after_failure(resumedescr, inputargs) + self.rebuild_state_after_failure(resumedescr, inputargs_and_holes) def load_values_from_failure(self, resumedescr): cpu = self.cpu fail_arg_types = resumedescr.fail_arg_types - inputargs = [] + inputargs_and_holes = [] for i in range(len(fail_arg_types)): boxtype = fail_arg_types[i] if boxtype == history.INT: @@ -1706,10 +1705,12 @@ box = cpu.ts.BoxRef(cpu.get_latest_value_ref(i)) elif boxtype == history.FLOAT: box = history.BoxFloat(cpu.get_latest_value_float(i)) + elif boxtype == history.HOLE: + box = None else: assert False, "bad box type: num=%d" % ord(boxtype) - inputargs.append(box) - return inputargs + inputargs_and_holes.append(box) + return inputargs_and_holes def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info Modified: pypy/trunk/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/resume.py Tue Nov 24 11:08:43 2009 @@ -1,6 +1,7 @@ import sys, os from pypy.jit.metainterp.history import Box, Const, ConstInt, INT, REF from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp import jitprof from pypy.rpython.lltypesystem import rffi from pypy.rlib import rarithmetic from pypy.rlib.objectmodel import we_are_translated @@ -84,23 +85,39 @@ # please rpython :( return rarithmetic.widen(x) == rarithmetic.widen(y) +def tagged_list_eq(tl1, tl2): + if len(tl1) != len(tl2): + return False + for i in range(len(tl1)): + if not tagged_eq(tl1[i], tl2[i]): + return False + return True + TAGCONST = 0 TAGINT = 1 TAGBOX = 2 TAGVIRTUAL = 3 -UNASSIGNED = tag(-1, TAGBOX) +UNASSIGNED = tag(-2 ** 12 - 1, TAGBOX) +UNASSIGNEDVIRTUAL = tag(-2 ** 12 - 1, TAGVIRTUAL) NULLREF = tag(-1, TAGCONST) class ResumeDataLoopMemo(object): - def __init__(self, cpu): - self.cpu = cpu + def __init__(self, metainterp_sd): + self.metainterp_sd = metainterp_sd + self.cpu = metainterp_sd.cpu self.consts = [] self.large_ints = {} - self.refs = cpu.ts.new_ref_dict_2() + self.refs = self.cpu.ts.new_ref_dict_2() self.numberings = {} + self.cached_boxes = {} + self.cached_virtuals = {} + + self.nvirtuals = 0 + self.nvholes = 0 + self.nvreused = 0 def getconst(self, const): if const.type == INT: @@ -135,6 +152,8 @@ self.consts.append(const) return result + # env numbering + def number(self, values, snapshot): if snapshot is None: return None, {}, 0 @@ -173,7 +192,41 @@ def forget_numberings(self, virtualbox): # XXX ideally clear only the affected numberings self.numberings.clear() + self.clear_box_virtual_numbers() + + # caching for virtuals and boxes inside them + + def num_cached_boxes(self): + return len(self.cached_boxes) + + def assign_number_to_box(self, box, boxes): + if box in self.cached_boxes: + num = self.cached_boxes[box] + boxes[-num-1] = box + else: + boxes.append(box) + num = -len(boxes) + self.cached_boxes[box] = num + return num + + def num_cached_virtuals(self): + return len(self.cached_virtuals) + + def assign_number_to_virtual(self, box): + if box in self.cached_virtuals: + num = self.cached_virtuals[box] + else: + num = self.cached_virtuals[box] = -len(self.cached_virtuals) - 1 + return num + def clear_box_virtual_numbers(self): + self.cached_boxes.clear() + self.cached_virtuals.clear() + + def update_counters(self, profiler): + profiler.count(jitprof.NVIRTUALS, self.nvirtuals) + profiler.count(jitprof.NVHOLES, self.nvholes) + profiler.count(jitprof.NVREUSED, self.nvreused) _frame_info_placeholder = (None, 0, 0) @@ -185,30 +238,19 @@ #self.virtuals = [] #self.vfieldboxes = [] - def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): - vinfo = VirtualInfo(known_class, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_vstruct(self, virtualbox, typedescr, fielddescrs, fieldboxes): - vinfo = VStructInfo(typedescr, fielddescrs) - self._make_virtual(virtualbox, vinfo, fieldboxes) - - def make_varray(self, virtualbox, arraydescr, itemboxes): - vinfo = VArrayInfo(arraydescr) - self._make_virtual(virtualbox, vinfo, itemboxes) - - def _make_virtual(self, virtualbox, vinfo, fieldboxes): - if virtualbox in self.liveboxes_from_env: - tagged = self.liveboxes_from_env[virtualbox] - i, _ = untag(tagged) - assert self.virtuals[i] is None - self.virtuals[i] = vinfo - self.vfieldboxes[i] = fieldboxes - else: - tagged = tag(len(self.virtuals), TAGVIRTUAL) - self.virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) + def make_virtual(self, known_class, fielddescrs): + return VirtualInfo(known_class, fielddescrs) + + def make_vstruct(self, typedescr, fielddescrs): + return VStructInfo(typedescr, fielddescrs) + + def make_varray(self, arraydescr): + return VArrayInfo(arraydescr) + + def register_virtual_fields(self, virtualbox, fieldboxes): + tagged = self.liveboxes_from_env.get(virtualbox, UNASSIGNEDVIRTUAL) self.liveboxes[virtualbox] = tagged + self.vfieldboxes[virtualbox] = fieldboxes self._register_boxes(fieldboxes) def register_box(self, box): @@ -234,7 +276,8 @@ def finish(self, values): # compute the numbering storage = self.storage - numb, liveboxes_from_env, v = self.memo.number(values, storage.rd_snapshot) + numb, liveboxes_from_env, v = self.memo.number(values, + storage.rd_snapshot) self.liveboxes_from_env = liveboxes_from_env self.liveboxes = {} storage.rd_numb = numb @@ -243,8 +286,7 @@ # collect liveboxes and virtuals n = len(liveboxes_from_env) - v liveboxes = [None]*n - self.virtuals = [None]*v - self.vfieldboxes = [None]*v + self.vfieldboxes = {} for box, tagged in liveboxes_from_env.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: @@ -254,31 +296,65 @@ value = values[box] value.get_args_for_fail(self) - self._number_virtuals(liveboxes) + self._number_virtuals(liveboxes, values, v) storage.rd_consts = self.memo.consts dump_storage(storage, liveboxes) return liveboxes[:] - def _number_virtuals(self, liveboxes): + def _number_virtuals(self, liveboxes, values, num_env_virtuals): + memo = self.memo + new_liveboxes = [None] * memo.num_cached_boxes() + count = 0 for box, tagged in self.liveboxes.iteritems(): i, tagbits = untag(tagged) if tagbits == TAGBOX: + assert box not in self.liveboxes_from_env assert tagged_eq(tagged, UNASSIGNED) - self.liveboxes[box] = tag(len(liveboxes), TAGBOX) - liveboxes.append(box) + index = memo.assign_number_to_box(box, new_liveboxes) + self.liveboxes[box] = tag(index, TAGBOX) + count += 1 else: assert tagbits == TAGVIRTUAL + if tagged_eq(tagged, UNASSIGNEDVIRTUAL): + assert box not in self.liveboxes_from_env + index = memo.assign_number_to_virtual(box) + self.liveboxes[box] = tag(index, TAGVIRTUAL) + new_liveboxes.reverse() + liveboxes.extend(new_liveboxes) + nholes = len(new_liveboxes) - count storage = self.storage storage.rd_virtuals = None - if len(self.virtuals) > 0: - storage.rd_virtuals = self.virtuals[:] - for i in range(len(storage.rd_virtuals)): - vinfo = storage.rd_virtuals[i] - fieldboxes = self.vfieldboxes[i] - vinfo.fieldnums = [self._gettagged(box) - for box in fieldboxes] + vfieldboxes = self.vfieldboxes + if vfieldboxes: + length = num_env_virtuals + memo.num_cached_virtuals() + virtuals = storage.rd_virtuals = [None] * length + memo.nvirtuals += length + memo.nvholes += length - len(vfieldboxes) + for virtualbox, fieldboxes in vfieldboxes.iteritems(): + num, _ = untag(self.liveboxes[virtualbox]) + value = values[virtualbox] + fieldnums = [self._gettagged(box) + for box in fieldboxes] + vinfo = value.make_virtual_info(self, fieldnums) + # if a new vinfo instance is made, we get the fieldnums list we + # pass in as an attribute. hackish. + if vinfo.fieldnums is not fieldnums: + memo.nvreused += 1 + virtuals[num] = vinfo + + if self._invalidation_needed(len(liveboxes), nholes): + memo.clear_box_virtual_numbers() + + def _invalidation_needed(self, nliveboxes, nholes): + memo = self.memo + # xxx heuristic a bit out of thin air + failargs_limit = memo.metainterp_sd.options.failargs_limit + if nliveboxes > (failargs_limit // 2): + if nholes > nliveboxes//3: + return True + return False def _gettagged(self, box): if isinstance(box, Const): @@ -392,10 +468,15 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(metainterp) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): numb = self.cur_numb @@ -450,8 +531,14 @@ for const in storage.rd_consts: debug_print('\tconst', const.repr_rpython()) for box in liveboxes: - debug_print('\tbox', box.repr_rpython()) + if box is None: + debug_print('\tbox', 'None') + else: + debug_print('\tbox', box.repr_rpython()) if storage.rd_virtuals is not None: for virtual in storage.rd_virtuals: - virtual.debug_prints() + if virtual is None: + debug_print('\t\t', 'None') + else: + virtual.debug_prints() debug_stop("jit-resume") Modified: pypy/trunk/pypy/jit/metainterp/simple_optimize.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/simple_optimize.py (original) +++ pypy/trunk/pypy/jit/metainterp/simple_optimize.py Tue Nov 24 11:08:43 2009 @@ -16,7 +16,7 @@ # we need it since the backend can modify those lists, which make # get_guard_op in compile.py invalid # in fact, x86 modifies this list for moving GCs - memo = resume.ResumeDataLoopMemo(metainterp_sd.cpu) + memo = resume.ResumeDataLoopMemo(metainterp_sd) newoperations = [] for op in loop.operations: if op.is_guard(): Modified: pypy/trunk/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/oparser.py Tue Nov 24 11:08:43 2009 @@ -205,10 +205,15 @@ if i < j: for arg in line[i:j].split(','): arg = arg.strip() - try: - fail_args.append(self.vars[arg]) - except KeyError: - raise ParseError("Unknown var in fail_args: %s" % arg) + if arg == 'None': + fail_arg = None + else: + try: + fail_arg = self.vars[arg] + except KeyError: + raise ParseError( + "Unknown var in fail_args: %s" % arg) + fail_args.append(fail_arg) if descr is None and self.invent_fail_descr: descr = self.invent_fail_descr(fail_args) if hasattr(descr, '_oparser_uses_descr_of_guard'): Modified: pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_jitprof.py Tue Nov 24 11:08:43 2009 @@ -63,7 +63,8 @@ ] assert profiler.events == expected assert profiler.times == [2, 1, 1, 1] - assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0] + assert profiler.counters == [1, 1, 1, 1, 4, 3, 1, 1, 7, 1, 0, 0, 0, + 0, 0, 0] def test_simple_loop_with_call(self): @dont_look_inside Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Tue Nov 24 11:08:43 2009 @@ -83,6 +83,15 @@ ''' self.reparse(inp) + def test_guard_w_hole(self): + inp = ''' + [i0] + i1 = int_add(i0, 1) + guard_true(i0) [i0, None, i1] + finish(i1) + ''' + self.reparse(inp) + def test_debug_merge_point(self): inp = ''' [] @@ -112,7 +121,7 @@ assert output.splitlines()[-1] == "jump(i0, descr=)" pure_parse(output) - def test_guard(self): + def test_guard_descr(self): namespace = {'fdescr': BasicFailDescr(4)} inp = ''' [i0] Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Nov 24 11:08:43 2009 @@ -74,6 +74,22 @@ lst2 = virt1._get_field_descr_list() assert lst1 is lst2 +def test_reuse_vinfo(): + class FakeVInfo(object): + pass + class FakeVirtualValue(optimizeopt.AbstractVirtualValue): + def _make_virtual(self, *args): + return FakeVInfo() + v1 = FakeVirtualValue(None, None, None) + vinfo1 = v1.make_virtual_info(None, [1, 2, 4]) + vinfo2 = v1.make_virtual_info(None, [1, 2, 4]) + assert vinfo1 is vinfo2 + vinfo3 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is not vinfo2 + vinfo4 = v1.make_virtual_info(None, [1, 2, 6]) + assert vinfo3 is vinfo4 + + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): Modified: pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py Tue Nov 24 11:08:43 2009 @@ -1,7 +1,8 @@ # some unit tests for the bytecode decoding -from pypy.jit.metainterp import pyjitpl, codewriter, resoperation +from pypy.jit.metainterp import pyjitpl, codewriter, resoperation, history +from pypy.jit.metainterp import jitprof from pypy.jit.metainterp.history import AbstractFailDescr, BoxInt, ConstInt from pypy.jit.metainterp.history import History from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -154,3 +155,39 @@ assert metainterp_sd.get_name_from_address(123) == 'a' assert metainterp_sd.get_name_from_address(456) == 'b' assert metainterp_sd.get_name_from_address(789) == '' + +def test_initialize_state_from_guard_failure(): + from pypy.jit.metainterp.typesystem import llhelper + calls = [] + + class FakeCPU: + ts = llhelper + + def get_latest_value_int(self, index): + return index + + class FakeStaticData: + cpu = FakeCPU() + profiler = jitprof.EmptyProfiler() + + metainterp = pyjitpl.MetaInterp(FakeStaticData()) + + def rebuild_state_after_failure(descr, newboxes): + calls.append(newboxes) + metainterp.rebuild_state_after_failure = rebuild_state_after_failure + + class FakeResumeDescr: + pass + resumedescr = FakeResumeDescr() + resumedescr.fail_arg_types = [history.INT, history.HOLE, + history.INT, history.HOLE, + history.INT] + + metainterp.initialize_state_from_guard_failure(resumedescr, True) + + inp = metainterp.history.inputargs + assert len(inp) == 3 + assert [box.value for box in inp] == [0, 2, 4] + b0, b2, b4 = inp + assert len(calls) == 1 + assert calls[0] == [b0, None, b2, None, b4] Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Tue Nov 24 11:08:43 2009 @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue +from pypy.jit.metainterp.optimizeopt import VirtualValue, OptValue, VArrayValue +from pypy.jit.metainterp.optimizeopt import VStructValue from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr from pypy.jit.metainterp.history import ConstPtr, ConstFloat @@ -33,6 +34,12 @@ assert tagged_eq(UNASSIGNED, UNASSIGNED) assert not tagged_eq(tag(1, TAGBOX), UNASSIGNED) +def test_tagged_list_eq(): + assert tagged_list_eq([UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)], + [UNASSIGNED, tag(1, TAGBOX), tag(-2, TAGVIRTUAL)]) + assert not tagged_list_eq([tag(1, TAGBOX)], [tag(-2, TAGBOX)]) + assert not tagged_list_eq([tag(1, TAGBOX), tag(-2, TAGBOX)], [tag(1, TAGBOX)]) + class MyMetaInterp: def __init__(self, cpu=None): if cpu is None: @@ -105,6 +112,22 @@ lst = reader.consume_boxes() assert lst == [b1s, b2s, b3s] + +def test_prepare_virtuals(): + class FakeVinfo(object): + def allocate(self, metainterp): + return "allocated" + def setfields(self, metainterp, virtual, func): + assert virtual == "allocated" + class FakeStorage(object): + rd_virtuals = [FakeVinfo(), None] + rd_numb = [] + rd_consts = [] + class FakeMetainterp(object): + cpu = None + reader = ResumeDataReader(FakeStorage(), [], FakeMetainterp()) + assert reader.virtuals == ["allocated", None] + # ____________________________________________________________ @@ -238,6 +261,13 @@ assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2].env + +class FakeMetaInterpStaticData: + cpu = LLtypeMixin.cpu + + class options: + failargs_limit = 100 + def test_rebuild_from_resumedata(): b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] @@ -246,7 +276,7 @@ FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] capture_resumedata(fs, None, storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) metainterp = MyMetaInterp() @@ -270,7 +300,7 @@ FakeFrame("code1", 3, 7, b3, c2, b1), FakeFrame("code2", 9, -1, c3, b2)] capture_resumedata(fs, [b4], storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) metainterp = MyMetaInterp() @@ -298,7 +328,7 @@ fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] capture_resumedata(fs, None, storage2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) @@ -330,7 +360,7 @@ def virtual_value(keybox, value, next): vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), keybox) + LLtypeMixin.cpu), keybox) if not isinstance(next, OptValue): next = OptValue(next) vv.setfield(LLtypeMixin.nextdescr, next) @@ -351,25 +381,26 @@ fs = fs[:-1] + [FakeFrame("code2", 10, -1, c3, b2, b4)] capture_resumedata(fs, None, storage2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) values = {b2: virtual_value(b2, b5, c4)} modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish(values) assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(len(liveboxes)-1, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(0, TAGCONST)] b6 = BoxPtr() v6 = virtual_value(b6, c2, None) v6.setfield(LLtypeMixin.nextdescr, v6) values = {b2: virtual_value(b2, b4, v6), b6: v6} + memo.clear_box_virtual_numbers() modifier = ResumeDataVirtualAdder(storage2, memo) liveboxes2 = modifier.finish(values) assert len(storage2.rd_virtuals) == 2 assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), - tag(1, TAGVIRTUAL)] + tag(-1, TAGVIRTUAL)] # now on to resuming metainterp = MyMetaInterp() @@ -398,13 +429,40 @@ FakeFrame("code2", 10, -1, c3, b2t, b4t)] assert metainterp.framestack == fs2 +def test_rebuild_from_resumedata_two_guards_w_shared_virtuals(): + b1, b2, b3, b4, b5, b6 = [BoxPtr(), BoxPtr(), BoxInt(), BoxPtr(), BoxInt(), BoxInt()] + c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), + LLtypeMixin.nodebox.constbox()] + storage = Storage() + fs = [FakeFrame("code0", 0, -1, c1, b2, b3)] + capture_resumedata(fs, None, storage) + + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + values = {b2: virtual_value(b2, b5, c4)} + modifier = ResumeDataVirtualAdder(storage, memo) + liveboxes = modifier.finish(values) + assert len(storage.rd_virtuals) == 1 + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), + tag(0, TAGCONST)] + + storage2 = Storage() + fs = [FakeFrame("code0", 0, -1, b1, b4, b2)] + capture_resumedata(fs, None, storage2) + values[b4] = virtual_value(b4, b6, c4) + modifier = ResumeDataVirtualAdder(storage2, memo) + liveboxes = modifier.finish(values) + assert len(storage2.rd_virtuals) == 2 + assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums + assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] + + def test_resumedata_top_recursive_virtuals(): b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()] storage = Storage() fs = [FakeFrame("code0", 0, -1, b1, b2)] capture_resumedata(fs, None, storage) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) v1 = virtual_value(b1, b3, None) v2 = virtual_value(b2, b3, v1) v1.setfield(LLtypeMixin.nextdescr, v2) @@ -413,9 +471,9 @@ liveboxes = modifier.finish(values) assert liveboxes == [b3] assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums == [tag(0, TAGBOX), + assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), tag(0, TAGVIRTUAL)] @@ -423,7 +481,7 @@ def test_ResumeDataLoopMemo_ints(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) tagged = memo.getconst(ConstInt(44)) assert untag(tagged) == (44, TAGINT) tagged = memo.getconst(ConstInt(-3)) @@ -445,7 +503,7 @@ def test_ResumeDataLoopMemo_refs(): cpu = LLtypeMixin.cpu - memo = ResumeDataLoopMemo(cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) const = cpu.ts.ConstRef(demo55o) tagged = memo.getconst(const) index, tagbits = untag(tagged) @@ -463,7 +521,7 @@ assert tagged == NULLREF def test_ResumeDataLoopMemo_other(): - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) const = ConstFloat(-1.0) tagged = memo.getconst(const) index, tagbits = untag(tagged) @@ -481,7 +539,7 @@ env2 = [c3, b3, b1, c3] snap2 = Snapshot(snap, env2) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) numb, liveboxes, v = memo.number({}, snap1) assert v == 0 @@ -553,31 +611,76 @@ tag(1, TAGVIRTUAL)] assert numb5.prev is numb4 +def test_ResumeDataLoopMemo_number_boxes(): + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + b1, b2 = [BoxInt(), BoxInt()] + assert memo.num_cached_boxes() == 0 + boxes = [] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + assert memo.num_cached_boxes() == 1 + boxes = [None] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [b1, b2] + + assert memo.num_cached_boxes() == 2 + boxes = [None, None] + num = memo.assign_number_to_box(b2, boxes) + assert num == -2 + assert boxes == [None, b2] + num = memo.assign_number_to_box(b1, boxes) + assert num == -1 + assert boxes == [b1, b2] + + memo.clear_box_virtual_numbers() + assert memo.num_cached_boxes() == 0 + +def test_ResumeDataLoopMemo_number_virtuals(): + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + b1, b2 = [BoxInt(), BoxInt()] + assert memo.num_cached_virtuals() == 0 + num = memo.assign_number_to_virtual(b1) + assert num == -1 + assert memo.num_cached_virtuals() == 1 + num = memo.assign_number_to_virtual(b1) + assert num == -1 + num = memo.assign_number_to_virtual(b2) + assert num == -2 + + assert memo.num_cached_virtuals() == 2 + num = memo.assign_number_to_virtual(b2) + assert num == -2 + num = memo.assign_number_to_virtual(b1) + assert num == -1 -def test__make_virtual(): + memo.clear_box_virtual_numbers() + assert memo.num_cached_virtuals() == 0 + +def test_register_virtual_fields(): b1, b2 = BoxInt(), BoxInt() vbox = BoxPtr() modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_virtual(vbox, None, ['a', 'b'], [b1, b2]) - assert modifier.liveboxes == {vbox: tag(0, TAGVIRTUAL), b1: UNASSIGNED, + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2]) + assert modifier.liveboxes == {vbox: UNASSIGNEDVIRTUAL, b1: UNASSIGNED, b2: UNASSIGNED} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2]] + assert modifier.vfieldboxes == {vbox: [b1, b2]} modifier = ResumeDataVirtualAdder(None, None) modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)} modifier.liveboxes = {} - modifier.virtuals = [None] - modifier.vfieldboxes = [None] - modifier.make_virtual(vbox, None, ['a', 'b', 'c'], [b1, b2, vbox]) + modifier.vfieldboxes = {} + modifier.register_virtual_fields(vbox, [b1, b2, vbox]) assert modifier.liveboxes == {b1: UNASSIGNED, b2: UNASSIGNED, vbox: tag(0, TAGVIRTUAL)} - assert len(modifier.virtuals) == 1 - assert modifier.vfieldboxes == [[b1, b2, vbox]] + assert modifier.vfieldboxes == {vbox: [b1, b2, vbox]} def _resume_remap(liveboxes, expected, *newvalues): newboxes = [] @@ -599,7 +702,7 @@ def test_virtual_adder_int_constants(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) assert storage.rd_snapshot is None @@ -618,7 +721,7 @@ def test_virtual_adder_memo_const_sharing(): b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.finish({}) assert len(memo.consts) == 2 @@ -635,7 +738,7 @@ def test_virtual_adder_no_op_renaming(): b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)] storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) b1_2 = BoxInt() class FakeValue(object): @@ -667,7 +770,7 @@ b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] b1s = ConstInt(111) storage = make_storage(b1s, b2s, b3s) - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) liveboxes = modifier.finish({}) b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] @@ -688,26 +791,30 @@ b2s, b3s, b4s, b5s = [BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_virtual(b2s, - ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], - [b4s, c1s]) # new fields - modifier.make_virtual(b4s, - ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu), - [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, - LLtypeMixin.otherdescr], - [b2s, b3s, b5s]) # new fields + modifier.vfieldboxes = {} + + v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), b2s) + v2._fields = {LLtypeMixin.nextdescr: b4s, + LLtypeMixin.valuedescr: c1s} + v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] + v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), b4s) + v4._fields = {LLtypeMixin.nextdescr: b2s, + LLtypeMixin.valuedescr: b3s, + LLtypeMixin.otherdescr: b5s} + v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, + LLtypeMixin.otherdescr] + modifier.register_virtual_fields(b2s, [b4s, c1s]) + modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) + values = {b2s: v2, b4s: v4} liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, values, 0) storage.rd_consts = memo.consts[:] storage.rd_numb = None # resume @@ -722,22 +829,25 @@ metainterp = MyMetaInterp() reader = ResumeDataReader(storage, newboxes, metainterp) assert len(reader.virtuals) == 2 - b2t = reader._decode_box(tag(0, TAGVIRTUAL)) - b4t = reader._decode_box(tag(1, TAGVIRTUAL)) + b2t = reader._decode_box(modifier._gettagged(b2s)) + b4t = reader._decode_box(modifier._gettagged(b4s)) trace = metainterp.trace - expected = [ - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, + b2new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu)], - b2t, None), - (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, + b2t, None) + b4new = (rop.NEW_WITH_VTABLE, [ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu)], - b4t, None), - (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), - (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), - (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr), - ] + b4t, None) + b2set = [(rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b2t, c1s], None, LLtypeMixin.valuedescr)] + b4set = [(rop.SETFIELD_GC, [b4t, b2t], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b4t, b3t], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4t, b5t], None, LLtypeMixin.otherdescr)] + if untag(modifier._gettagged(b2s))[0] == -2: + expected = [b2new, b4new] + b2set + b4set + else: + expected = [b4new, b2new] + b4set + b2set + for x, y in zip(expected, trace): assert x == y ptr = b2t.value._obj.container._as_ptr() @@ -753,17 +863,21 @@ b2s, b4s = [BoxPtr(), BoxInt(4)] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_varray(b2s, - LLtypeMixin.arraydescr, - [b4s, c1s]) # new fields + modifier.vfieldboxes = {} + + class FakeOptimizer(object): + def new_const_item(self, descr): + return None + v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) + v2._items = [b4s, c1s] + modifier.register_virtual_fields(b2s, [b4s, c1s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + values = {b2s: v2} + modifier._number_virtuals(liveboxes, values, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None @@ -799,18 +913,17 @@ b2s, b4s = [BoxPtr(), BoxPtr()] c1s = ConstInt(111) storage = Storage() - memo = ResumeDataLoopMemo(LLtypeMixin.cpu) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) modifier = ResumeDataVirtualAdder(storage, memo) modifier.liveboxes_from_env = {} modifier.liveboxes = {} - modifier.virtuals = [] - modifier.vfieldboxes = [] - modifier.make_vstruct(b2s, - LLtypeMixin.ssize, - [LLtypeMixin.adescr, LLtypeMixin.bdescr], - [c1s, b4s]) # new fields + modifier.vfieldboxes = {} + v2 = VStructValue(None, LLtypeMixin.ssize, b2s) + v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} + v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] + modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] - modifier._number_virtuals(liveboxes) + modifier._number_virtuals(liveboxes, {b2s: v2}, 0) dump_storage(storage, liveboxes) storage.rd_consts = memo.consts[:] storage.rd_numb = None @@ -837,3 +950,23 @@ assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S) assert ptr.a == 111 assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) + +def test_invalidation_needed(): + class options: + failargs_limit = 10 + + metainterp_sd = FakeMetaInterpStaticData() + metainterp_sd.options = options + memo = ResumeDataLoopMemo(metainterp_sd) + modifier = ResumeDataVirtualAdder(None, memo) + + for i in range(5): + assert not modifier._invalidation_needed(5, i) + + assert not modifier._invalidation_needed(7, 2) + assert modifier._invalidation_needed(7, 3) + + assert not modifier._invalidation_needed(10, 2) + assert not modifier._invalidation_needed(10, 3) + assert modifier._invalidation_needed(10, 4) + Modified: pypy/trunk/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_virtual.py Tue Nov 24 11:08:43 2009 @@ -304,6 +304,8 @@ # ENTER - compile the leaving path self.check_enter_count(4) +class VirtualMiscTests: + def test_guards_around_forcing(self): class A(object): def __init__(self, x): @@ -329,6 +331,29 @@ return 0 self.meta_interp(f, [50]) + def test_guards_and_holes(self): + class A(object): + def __init__(self, x): + self.x = x + mydriver = JitDriver(reds = ['n', 'tot'], greens = []) + + def f(n): + tot = 0 + while n > 0: + mydriver.can_enter_jit(n=n, tot=tot) + mydriver.jit_merge_point(n=n, tot=tot) + a = A(n) + b = A(n+1) + if n % 9 == 0: + tot += (a.x + b.x) % 3 + c = A(n+1) + if n % 10 == 0: + tot -= (c.x + a.x) % 3 + n -= 1 + return tot + r = self.meta_interp(f, [70]) + expected = f(70) + assert r == expected # ____________________________________________________________ # Run 1: all the tests instantiate a real RPython class @@ -435,3 +460,11 @@ p = lltype.malloc(NODE2) p.parent.typeptr = vtable2 return p + +# misc + +class TestOOTypeMisc(VirtualMiscTests, OOJitMixin): + pass + +class TestLLTypeMisc(VirtualMiscTests, LLJitMixin): + pass From pedronis at codespeak.net Tue Nov 24 11:09:25 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 11:09:25 +0100 (CET) Subject: [pypy-svn] r69571 - pypy/branch/compress-virtuals-resumedata2 Message-ID: <20091124100925.6239A168012@codespeak.net> Author: pedronis Date: Tue Nov 24 11:09:24 2009 New Revision: 69571 Removed: pypy/branch/compress-virtuals-resumedata2/ Log: kill merged branch From pedronis at codespeak.net Tue Nov 24 11:09:44 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 11:09:44 +0100 (CET) Subject: [pypy-svn] r69572 - pypy/branch/compress-virtuals-resumedata Message-ID: <20091124100944.0C5FD168012@codespeak.net> Author: pedronis Date: Tue Nov 24 11:09:43 2009 New Revision: 69572 Removed: pypy/branch/compress-virtuals-resumedata/ Log: kill merged feature branch From pedronis at codespeak.net Tue Nov 24 11:10:48 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 11:10:48 +0100 (CET) Subject: [pypy-svn] r69573 - pypy/trunk/pypy/jit/backend/x86 Message-ID: <20091124101048.025DB168012@codespeak.net> Author: pedronis Date: Tue Nov 24 11:10:48 2009 New Revision: 69573 Removed: pypy/trunk/pypy/jit/backend/x86/TODO Log: outdated From afa at codespeak.net Tue Nov 24 11:51:01 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 11:51:01 +0100 (CET) Subject: [pypy-svn] r69574 - pypy/trunk/pypy/module/oracle Message-ID: <20091124105101.D1364168012@codespeak.net> Author: afa Date: Tue Nov 24 11:51:01 2009 New Revision: 69574 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: more translation fixes Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 24 11:51:01 2009 @@ -446,7 +446,7 @@ self.environment.checkForError( status, "Cursor_ItemDescription(): internal size") - internalSize = attrptr[0] + internalSize = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -471,7 +471,7 @@ lltype.free(lenptr, flavor='raw') # lookup precision and scale - if varType == interp_variable.VT_Float: + if varType is interp_variable.VT_Float: attrptr = lltype.malloc(rffi.CArrayPtr(roci.sb1).TO, 1, flavor='raw') try: Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 11:51:01 2009 @@ -62,7 +62,7 @@ # use the length from Oracle directly if available if sizeFromOracle: - size = sizeFromOracle + size = rffi.cast(lltype.Signed, sizeFromOracle) # otherwise, use the value set with the setoutputsize() parameter elif cursor.outputSize >= 0: @@ -299,7 +299,7 @@ # Verifies that truncation or other problems did not take place on # retrieve. if self.isVariableLength: - if self.returnCode[pos] != 0: + if rffi.cast(lltype.Signed, self.returnCode[pos]) != 0: error = W_Error(space, self.environment, "Variable_VerifyFetch()", 0) error.code = self.returnCode[pos] From afa at codespeak.net Tue Nov 24 13:13:42 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 13:13:42 +0100 (CET) Subject: [pypy-svn] r69575 - pypy/trunk/pypy/module/oracle Message-ID: <20091124121342.AA2DF168006@codespeak.net> Author: afa Date: Tue Nov 24 13:13:41 2009 New Revision: 69575 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: More translation fixes. Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 24 13:13:41 2009 @@ -484,7 +484,7 @@ self.environment.checkForError( status, "Cursor_ItemDescription(): scale") - scale = attrptr[0] + scale = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -500,7 +500,7 @@ self.environment.checkForError( status, "Cursor_ItemDescription(): precision") - precision = attrptr[0] + precision = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') else: Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 24 13:13:41 2009 @@ -6,7 +6,8 @@ from pypy.interpreter.error import OperationError from pypy.module.oracle import roci, config -class State: +class State: + # XXX move to another file def __init__(self, space): w_module = space.getbuiltinmodule('cx_Oracle') def get(name): @@ -30,6 +31,11 @@ self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) self.w_DateType = space.getattr(w_datetime, space.wrap("date")) + from pypy.module.oracle.interp_variable import all_variable_types + self.variableTypeByPythonType = {} + for varType in all_variable_types: + w_type = space.gettypeobject(varType.typedef) + self.variableTypeByPythonType[w_type] = varType def get(space): return space.fromcache(State) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 13:13:41 2009 @@ -858,7 +858,7 @@ class VT_Object(W_Variable): canBeInArray = False -variableTypeByTypedef = {} +all_variable_types = [] for name, cls in globals().items(): if not name.startswith('VT_') or not isinstance(cls, type): continue @@ -869,7 +869,7 @@ cls.typedef = TypeDef( cls.__name__, W_Variable.typedef, ) - variableTypeByTypedef[cls.typedef] = cls + all_variable_types.append(cls) register_variable_class(cls) def typeByOracleDescriptor(param, environment): @@ -964,8 +964,8 @@ space.w_TypeError, space.wrap("Variable_TypeByPythonType(): type expected")) assert isinstance(w_type, W_TypeObject) - if w_type.instancetypedef in variableTypeByTypedef: - return variableTypeByTypedef[w_type.instancetypedef] + if w_type in get(space).variableTypeByPythonType: + return get(space).variableTypeByPythonType[w_type] if space.is_w(w_type, space.w_int): return VT_Integer raise OperationError( From afa at codespeak.net Tue Nov 24 13:20:25 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 13:20:25 +0100 (CET) Subject: [pypy-svn] r69576 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091124122025.63880168006@codespeak.net> Author: afa Date: Tue Nov 24 13:20:24 2009 New Revision: 69576 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/test/test_datetimevar.py pypy/trunk/pypy/module/oracle/transform.py Log: Test and fix for DATE variables Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Tue Nov 24 13:20:24 2009 @@ -9,6 +9,7 @@ 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', 'DATETIME': 'interp_variable.VT_DateTime', + 'DATE': 'interp_variable.VT_Date', 'BINARY': 'interp_variable.VT_Binary', 'LONG_STRING': 'interp_variable.VT_LongString', 'LONG_BINARY': 'interp_variable.VT_LongBinary', Modified: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_datetimevar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Tue Nov 24 13:20:24 2009 @@ -18,7 +18,7 @@ data = cur.fetchall() assert data == [('20021213-000000',)] - def test_variable(self): + def test_variable_datetime(self): import datetime cur = self.cnx.cursor() var = cur.var(oracle.DATETIME) @@ -30,6 +30,17 @@ var.setvalue(0, value) assert var.getvalue() == datetime.datetime(2002, 12, 13) + def test_variable_date(self): + import datetime + cur = self.cnx.cursor() + var = cur.var(oracle.DATE) + value = datetime.date(2002, 12, 13) + var.setvalue(0, value) + assert var.getvalue() == value + + var.setvalue(0, datetime.datetime(2002, 12, 13, 9, 36, 15)) + assert var.getvalue() == value + def test_arrayvar(self): import datetime cur = self.cnx.cursor() Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Tue Nov 24 13:20:24 2009 @@ -16,25 +16,21 @@ lltype.free(doubleptr, flavor='raw') def OracleDateToPythonDate(environment, valueptr): - yearptr = lltype.malloc(roci.Ptr(roci.sb2).TO, 1, flavor='raw') - monthptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') - dayptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') - - try: - roci.OCIDateGetDate( - valueptr, - yearptr, - monthptr, - dayptr) - - space = environment.space - w = space.wrap - - return space.call_w(w_date, [w(yearptr[0]), w(monthptr[0]), w(dayptr[0])]) - finally: - lltype.free(yearptr, flavor='raw') - lltype.free(monthptr, flavor='raw') - lltype.free(dayptr, flavor='raw') + space = environment.space + w = space.wrap + + # XXX check that this does not copy the whole structure + date = valueptr[0] + + w_date = space.getattr( + space.getbuiltinmodule('datetime'), + w('date')) + + return space.call_function( + w_date, + w(date.c_OCIDateYYYY), + w(date.c_OCIDateMM), + w(date.c_OCIDateDD)) def OracleDateToPythonDateTime(environment, valueptr): space = environment.space From arigo at codespeak.net Tue Nov 24 14:16:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 14:16:45 +0100 (CET) Subject: [pypy-svn] r69577 - pypy/branch/shorter-guard-path-2 Message-ID: <20091124131645.4AF92168008@codespeak.net> Author: arigo Date: Tue Nov 24 14:16:44 2009 New Revision: 69577 Added: pypy/branch/shorter-guard-path-2/ - copied from r69576, pypy/trunk/ Log: A branch in which to merge shorter-guard-path. From arigo at codespeak.net Tue Nov 24 14:29:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 14:29:48 +0100 (CET) Subject: [pypy-svn] r69578 - in pypy/branch/shorter-guard-path-2/pypy: jit/backend jit/backend/cli jit/backend/llgraph jit/backend/llsupport jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test objspace/std Message-ID: <20091124132948.6DCD3168008@codespeak.net> Author: arigo Date: Tue Nov 24 14:29:46 2009 New Revision: 69578 Added: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_model.py - copied unchanged from r69483, pypy/branch/shorter-guard-path/pypy/jit/backend/test/test_model.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py - copied unchanged from r69483, pypy/branch/shorter-guard-path/pypy/jit/backend/x86/test/test_assembler.py Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/method.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/runner.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/llgraph/runner.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/model.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_ll_random.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_random.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/regalloc.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/ri386setup.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/runner.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/support.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_recompilation.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_runner.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/compile.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/history.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/logger.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/pyjitpl.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_compile.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_logger.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/warmstate.py pypy/branch/shorter-guard-path-2/pypy/objspace/std/multimethod.py Log: Merge the branch/shorter-guard-path. Needs a bit of fixing. Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/method.py Tue Nov 24 14:29:46 2009 @@ -268,9 +268,6 @@ try: return self.cpu.failing_ops.index(op) except ValueError: - descr = op.descr - assert isinstance(descr, history.AbstractFailDescr) - descr.get_index() self.cpu.failing_ops.append(op) return len(self.cpu.failing_ops)-1 Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/cli/runner.py Tue Nov 24 14:29:46 2009 @@ -135,9 +135,7 @@ func = cliloop.funcbox.holder.GetFunc() func(self.get_inputargs()) op = self.failing_ops[self.inputargs.get_failed_op()] - descr = op.descr - assert isinstance(descr, AbstractFailDescr) - return descr.get_index() + return op.descr def set_future_value_int(self, index, intvalue): self.get_inputargs().set_int(index, intvalue) Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/llgraph/runner.py Tue Nov 24 14:29:46 2009 @@ -88,6 +88,7 @@ def __init__(self, rtyper, stats=None, translate_support_code=False, annmixlevel=None, gcdescr=None): + model.AbstractCPU.__init__(self) self.rtyper = rtyper self.translate_support_code = translate_support_code self.stats = stats or MiniStats() @@ -167,7 +168,7 @@ if op.is_guard(): faildescr = op.descr assert isinstance(faildescr, history.AbstractFailDescr) - fail_index = faildescr.get_index() + fail_index = self.get_fail_descr_number(faildescr) index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: @@ -195,8 +196,7 @@ llimpl.compile_add_jump_target(c, compiled_version) elif op.opnum == rop.FINISH: faildescr = op.descr - assert isinstance(faildescr, history.AbstractFailDescr) - index = faildescr.get_index() + index = self.get_fail_descr_number(faildescr) llimpl.compile_add_fail(c, index) else: assert False, "unknown operation" @@ -213,7 +213,7 @@ fail_index = llimpl.frame_execute(frame) # we hit a FAIL operation. self.latest_frame = frame - return fail_index + return self.get_fail_descr_from_number(fail_index) def set_future_value_int(self, index, intvalue): llimpl.set_future_value_int(index, intvalue) Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/llsupport/llmodel.py Tue Nov 24 14:29:46 2009 @@ -20,6 +20,7 @@ def __init__(self, rtyper, stats, translate_support_code=False, gcdescr=None): from pypy.jit.backend.llsupport.gc import get_ll_description + AbstractCPU.__init__(self) self.rtyper = rtyper self.stats = stats self.translate_support_code = translate_support_code @@ -149,8 +150,8 @@ on_leave_jitted_hook() def on_leave_jitted_save_exc(): - on_leave_jitted_hook() save_exception() + on_leave_jitted_hook() self.on_leave_jitted_noexc = on_leave_jitted_noexc self.on_leave_jitted_save_exc = on_leave_jitted_save_exc Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/model.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/model.py Tue Nov 24 14:29:46 2009 @@ -1,6 +1,25 @@ +from pypy.jit.metainterp import history + + class AbstractCPU(object): supports_floats = False + def __init__(self): + self.fail_descr_list = [] + + def get_fail_descr_number(self, descr): + assert isinstance(descr, history.AbstractFailDescr) + n = descr.index + if n < 0: + lst = self.fail_descr_list + n = len(lst) + lst.append(descr) + descr.index = n + return n + + def get_fail_descr_from_number(self, n): + return self.fail_descr_list[n] + def set_class_sizes(self, class_sizes): self.class_sizes = class_sizes Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py Tue Nov 24 14:29:46 2009 @@ -39,7 +39,7 @@ else: raise NotImplementedError(box) res = self.cpu.execute_token(looptoken) - if res == operations[-1].descr.index: + if res is operations[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -102,7 +102,7 @@ fail = self.cpu.execute_token(looptoken) res = self.cpu.get_latest_value_int(0) assert res == 3 - assert fail == 1 + assert fail.identifier == 1 def test_compile_loop(self): i0 = BoxInt() @@ -121,7 +121,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 10 @@ -200,7 +200,7 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 20 @@ -242,9 +242,9 @@ i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): def __setattr__(self, name, value): + if name == 'index': + return AbstractFailDescr.__setattr__(self, name, value) py.test.fail("finish descrs should not be touched") - def get_index(self): - return 7 faildescr = UntouchableFailDescr() # to check that is not touched looptoken = LoopToken() operations = [ @@ -253,7 +253,7 @@ self.cpu.compile_loop([i0], operations, looptoken) self.cpu.set_future_value_int(0, 99) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 99 @@ -263,7 +263,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 42 @@ -273,7 +273,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr def test_execute_operations_in_env(self): cpu = self.cpu @@ -357,9 +357,9 @@ self.cpu.set_future_value_int(1, y) fail = self.cpu.execute_token(looptoken) if (z == boom) ^ reversed: - assert fail == 1 + assert fail.identifier == 1 else: - assert fail == 2 + assert fail.identifier == 2 if z != boom: assert self.cpu.get_latest_value_int(0) == z assert not self.cpu.get_exception() @@ -832,7 +832,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 15 + assert fail.identifier == 15 # dstvalues = values[:] for _ in range(11): @@ -884,7 +884,7 @@ for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_float(0) assert res == 8.5 for i in range(1, len(fboxes)): @@ -935,7 +935,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 1 + assert fail.identifier == 1 class LLtypeBackendTest(BaseBackendTest): Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_ll_random.py Tue Nov 24 14:29:46 2009 @@ -4,6 +4,7 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import ConstInt, ConstPtr from pypy.jit.metainterp.history import ConstAddr, BoxPtr, BoxInt +from pypy.jit.metainterp.history import BasicFailDescr from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rarithmetic import intmask from pypy.rpython.llinterp import LLException @@ -464,7 +465,7 @@ descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -486,7 +487,7 @@ _, vtableptr = builder.get_random_structure_type_and_vtable(r) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = builder.subset_of_intvars(r) op._exc_box = None builder.should_fail_by = op @@ -509,7 +510,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -527,7 +528,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op @@ -553,7 +554,7 @@ break other_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [other_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_random.py Tue Nov 24 14:29:46 2009 @@ -18,8 +18,6 @@ self.operations = subops class OperationBuilder(object): - failnumbering = 0 - def __init__(self, cpu, loop, vars): self.cpu = cpu self.loop = loop @@ -36,12 +34,6 @@ self.counter = 0 assert len(self.intvars) == len(dict.fromkeys(self.intvars)) - @classmethod - def make_fail_descr(cls): - descr = BasicFailDescr(cls.failnumbering) - cls.failnumbering += 1 - return descr - def fork(self, cpu, loop, vars): fork = self.__class__(cpu, loop, vars) fork.prebuilt_ptr_consts = self.prebuilt_ptr_consts @@ -282,7 +274,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = fail_subset builder.loop.operations.append(op) @@ -343,7 +335,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = builder.subset_of_intvars(r) if not passing: builder.should_fail_by = op @@ -559,7 +551,7 @@ endvars.append(v) r.shuffle(endvars) loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr(-2))) + descr=BasicFailDescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -605,7 +597,7 @@ else: raise NotImplementedError(box) fail = cpu.execute_token(self.loop.token) - assert fail == self.should_fail_by.descr.index + assert fail is self.should_fail_by.descr for i, v in enumerate(self.get_fail_args()): if isinstance(v, (BoxFloat, ConstFloat)): value = cpu.get_latest_value_float(i) @@ -634,7 +626,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.descr = self.builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = [] return op Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py Tue Nov 24 14:29:46 2009 @@ -1,11 +1,12 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF, FLOAT +from pypy.jit.metainterp.history import Const, Box, BoxPtr, INT, REF, FLOAT from pypy.jit.metainterp.history import AbstractFailDescr from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs @@ -16,6 +17,7 @@ from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ CHUNK_SIZE +from pypy.rlib.debug import debug_print # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -33,11 +35,24 @@ return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) class MachineCodeBlockWrapper(object): - MC_SIZE = 1024*1024 + MC_DEFAULT_SIZE = 1024*1024 - def __init__(self): + def __init__(self, bigsize): self.old_mcs = [] # keepalive - self._mc = codebuf.MachineCodeBlock(self.MC_SIZE) + self.bigsize = bigsize + self._mc = codebuf.MachineCodeBlock(bigsize) + + def bytes_free(self): + return self._mc._size - self._mc._pos + + def make_new_mc(self): + new_mc = codebuf.MachineCodeBlock(self.bigsize) + debug_print('[new machine code block at', new_mc.tell(), ']') + self._mc.JMP(rel32(new_mc.tell())) + self._mc.done() + self.old_mcs.append(self._mc) + self._mc = new_mc + make_new_mc._dont_inline_ = True def tell(self): return self._mc.tell() @@ -49,12 +64,8 @@ def method(self, *args): # XXX er.... pretty random number, just to be sure # not to write half-instruction - if self._mc._pos + 64 >= self._mc._size: - new_mc = codebuf.MachineCodeBlock(self.MC_SIZE) - self._mc.JMP(rel32(new_mc.tell())) - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc + if self.bytes_free() < 64: + self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method @@ -66,7 +77,7 @@ class Assembler386(object): mc = None mc2 = None - debug_markers = True + mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE def __init__(self, cpu, translate_support_code=False): self.cpu = cpu @@ -79,11 +90,9 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.setup_failure_recovery() def leave_jitted_hook(self): - # XXX BIG FAT WARNING XXX - # At this point, we should not call anyone here, because - # RPython-level exception might be set. Here be dragons i = 0 while i < self.fail_boxes_ptr.lgt: chunk = self.fail_boxes_ptr.chunks[i] @@ -113,8 +122,13 @@ # done # we generate the loop body in 'mc' # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper() - self.mc2 = MachineCodeBlockWrapper() + self.mc = MachineCodeBlockWrapper(self.mc_size) + self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -137,8 +151,12 @@ def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() + arglocs = self.rebuild_faillocs_from_descr( + faildescr._x86_failure_recovery_bytecode) + if not we_are_translated(): + assert ([loc.assembler() for loc in arglocs] == + [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - arglocs = faildescr._x86_faillocs fail_stack_depth = faildescr._x86_current_stack_depth regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, operations) @@ -150,9 +168,12 @@ # for the benefit of tests faildescr._x86_bridge_stack_depth = stack_depth # patch the jump from original guard + self.patch_jump(faildescr, adr_bridge) + + def patch_jump(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_bridge - adr_jump_offset - 4)) + mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) mc.valgrind_invalidated() mc.done() @@ -743,50 +764,270 @@ def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): self._no_const_locs(failargs) - addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = fail_locs - self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) + return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + + def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + """Generate the initial code for handling a failure. We try to + keep it as compact as possible. The idea is that this code is + executed at most once (and very often, zero times); when + executed, it generates a more complete piece of code which can + really handle recovery from this particular failure. + """ + fail_index = self.cpu.get_fail_descr_number(faildescr) + bytes_needed = 20 + 5 * len(failargs) # conservative estimate + if self.mc2.bytes_free() < bytes_needed: + self.mc2.make_new_mc() + mc = self.mc2._mc + addr = mc.tell() + withfloats = False + for box in failargs: + if box.type == FLOAT: + withfloats = True + break + mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + # write tight data that describes the failure recovery + faildescr._x86_failure_recovery_bytecode = mc.tell() + self.write_failure_recovery_description(mc, failargs, fail_locs) + # write the fail_index too + mc.write(packimm32(fail_index)) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + mc.writechr(0xCC) + faildescr._x86_debug_faillocs = fail_locs return addr - def generate_failure(self, mc, faildescr, failargs, locs, exc): - pos = mc.tell() + DESCR_REF = 0x00 + DESCR_INT = 0x01 + DESCR_FLOAT = 0x02 + DESCR_FROMSTACK = 8 + DESCR_STOP = DESCR_INT + 4*esp.op + + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] - if arg is None: # hole - continue + if arg.type == REF: + kind = self.DESCR_REF + elif arg.type == INT: + kind = self.DESCR_INT + elif arg.type == FLOAT: + kind = self.DESCR_FLOAT + else: + raise AssertionError("bogus kind") + loc = locs[i] + if isinstance(loc, MODRM): + n = self.DESCR_FROMSTACK + loc.position + else: + assert isinstance(loc, REG) + n = loc.op + n = kind + 4*n + while n > 0x7F: + mc.writechr((n & 0x7F) | 0x80) + n >>= 7 + mc.writechr(n) + mc.writechr(self.DESCR_STOP) + # preallocate the fail_boxes + i = len(failargs) - 1 + if i >= 0: + self.fail_boxes_int.get_addr_for_num(i) + self.fail_boxes_ptr.get_addr_for_num(i) + if self.cpu.supports_floats: + self.fail_boxes_float.get_addr_for_num(i) + + def rebuild_faillocs_from_descr(self, bytecode): + from pypy.jit.backend.x86.regalloc import X86StackManager + bytecode = rffi.cast(rffi.UCHARP, bytecode) + arglocs = [] + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + # 'code' identifies a stack location + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + if kind == self.DESCR_FLOAT: + size = 2 + else: + size = 1 + loc = X86StackManager.stack_pos(code, size) + elif code == self.DESCR_STOP: + break + else: + # 'code' identifies a register + kind = code & 3 + code >>= 2 + if kind == self.DESCR_FLOAT: + loc = xmm_registers[code] + else: + loc = registers[code] + arglocs.append(loc) + return arglocs[:] + + def setup_failure_recovery(self): + + def failure_recovery_func(registers): + # no malloc allowed here!! + # 'registers' is a pointer to a structure containing the + # original value of the registers, optionally the original + # value of XMM registers, and finally a reference to the + # recovery bytecode. See _build_failure_recovery() for details. + stack_at_ebp = registers[ebp.op] + bytecode = rffi.cast(rffi.UCHARP, registers[8]) + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + stackloc = stack_at_ebp + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - 4)[0] + elif code == self.DESCR_STOP: + break + else: + # 'code' identifies a register: load its value + kind = code & 3 + code >>= 2 + if kind == self.DESCR_FLOAT: + xmmregisters = rffi.ptradd(registers, -16) + value = xmmregisters[2*code] + value_hi = xmmregisters[2*code + 1] + else: + value = registers[code] + + # store the loaded value into fail_boxes_ + if kind == self.DESCR_INT: + tgt = self.fail_boxes_int.get_addr_for_num(num) + elif kind == self.DESCR_REF: + tgt = self.fail_boxes_ptr.get_addr_for_num(num) + elif kind == self.DESCR_FLOAT: + tgt = self.fail_boxes_float.get_addr_for_num(num) + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + else: + assert 0, "bogus kind" + rffi.cast(rffi.LONGP, tgt)[0] = value + num += 1 + # + if not we_are_translated(): + assert bytecode[4] == 0xCC + fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + return fail_index + + self.failure_recovery_func = failure_recovery_func + self.failure_recovery_code = [0, 0, 0, 0] + + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], + lltype.Signed)) + + def _build_failure_recovery(self, exc, withfloats=False): + failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, + self.failure_recovery_func) + failure_recovery_func = rffi.cast(lltype.Signed, + failure_recovery_func) + mc = self.mc2._mc + # Assume that we are called at the beginning, when there is no risk + # that 'mc' runs out of space. Checked by asserts in mc.write(). + recovery_addr = mc.tell() + mc.PUSH(edi) + mc.PUSH(esi) + mc.PUSH(ebp) + mc.PUSH(esp) # <-- not really used, but needed to take up the space + mc.PUSH(ebx) + mc.PUSH(edx) + mc.PUSH(ecx) + mc.PUSH(eax) + mc.MOV(esi, esp) + if withfloats: + mc.SUB(esp, imm(8*8)) + for i in range(8): + mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + + # we call a provided function that will + # - call our on_leave_jitted_hook which will mark + # the fail_boxes_ptr array as pointing to young objects to + # avoid unwarranted freeing + # - optionally save exception depending on the flag + addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) + mc.CALL(rel32(addr)) + + # the following call saves all values from the stack and from + # registers to the right 'fail_boxes_' location. + # Note that the registers are saved so far in esi[0] to esi[7], + # as pushed above, plus optionally in esi[-16] to esi[-1] for + # the XMM registers. Moreover, esi[8] is a pointer to the recovery + # bytecode, pushed just before by the CALL instruction written by + # generate_quick_failure(). + mc.PUSH(esi) + mc.CALL(rel32(failure_recovery_func)) + # returns in eax the fail_index + + # now we return from the complete frame, which starts from + # _assemble_bootstrap_code(). The LEA below throws away most + # of the frame, including all the PUSHes that we did just above. + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) + mc.POP(edi) + mc.POP(esi) + mc.POP(ebx) + mc.POP(ebp) + mc.RET() + self.mc2.done() + self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + + def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): + for i in range(len(locs)): loc = locs[i] if isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), loc) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(heap(adr), loc) - for i in range(len(failargs)): - arg = failargs[i] - if arg is None: # hole - continue + for i in range(len(locs)): loc = locs[i] if not isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: mc.MOVSD(xmm0, loc) adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), xmm0) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) mc.MOV(heap(adr), eax) - if self.debug_markers: - mc.MOV(eax, imm(pos)) - mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -798,9 +1039,7 @@ # don't break the following code sequence! mc = mc._mc - mc.LEA(esp, addr_add(imm(0), ebp, (-RET_BP + 2) * WORD)) - assert isinstance(faildescr, AbstractFailDescr) - fail_index = faildescr.get_index() + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) mc.MOV(eax, imm(fail_index)) mc.POP(edi) mc.POP(esi) Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/regalloc.py Tue Nov 24 14:29:46 2009 @@ -397,8 +397,10 @@ def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] - self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, - locs, self.exc) + locs_are_ref = [v.type == REF for v in op.args] + fail_index = self.assembler.cpu.get_fail_descr_number(op.descr) + self.assembler.generate_failure(self.assembler.mc, fail_index, locs, + self.exc, locs_are_ref) self.possibly_free_vars(op.args) def consider_guard_no_exception(self, op, ignored): Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/ri386setup.py Tue Nov 24 14:29:46 2009 @@ -459,6 +459,13 @@ SAHF = Instruction() SAHF.mode0(['\x9E']) +LODSB = Instruction() +LODSB.mode0(['\xAC']) + +LODSD = Instruction() +LODSD.mode0(['\xAD']) +LODSD.as_alias = "LODSL" + # ------------------------- floating point instructions ------------------ FLDL = Instruction() Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/runner.py Tue Nov 24 14:29:46 2009 @@ -63,7 +63,7 @@ addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) fail_index = self._execute_call(func) - return fail_index + return self.get_fail_descr_from_number(fail_index) def _execute_call(self, func): # help flow objspace Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/support.py Tue Nov 24 14:29:46 2009 @@ -3,7 +3,8 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated -CHUNK_SIZE = 1000 +CHUNK_SIZE_BITS = 8 +CHUNK_SIZE = 1 << CHUNK_SIZE_BITS def new_nonmovable_growable_array(TP): ATP = lltype.GcArray(TP) @@ -34,7 +35,7 @@ def _no_of(self, i): while i >= len(self.chunks) * CHUNK_SIZE: self._grow() - return i / CHUNK_SIZE, i % CHUNK_SIZE + return i >> CHUNK_SIZE_BITS, i & (CHUNK_SIZE-1) _no_of._always_inline_ = True def setitem(self, i, v): Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_recompilation.py Tue Nov 24 14:29:46 2009 @@ -21,7 +21,7 @@ bridge = self.attach_bridge(ops, loop, -2) self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 def test_compile_bridge_deeper(self): @@ -52,7 +52,7 @@ assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 assert self.getint(1) == 22 assert self.getint(2) == 23 @@ -78,7 +78,7 @@ bridge = self.attach_bridge(ops, other_loop, 0, looptoken=loop.token) self.cpu.set_future_value_int(0, 1) fail = self.run(other_loop) - assert fail == 1 + assert fail.identifier == 1 def test_bridge_jumps_to_self_deeper(self): loop = self.interpret(''' Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Tue Nov 24 14:29:46 2009 @@ -92,7 +92,7 @@ return [pick1(i386.memSIB64) for i in range(COUNT2)] def xmm_tests(): - return i386.xmm_registers + return i386.xmm_registers[:] def modrm8_tests(): return i386.registers8 + [pick1(i386.memSIB8) for i in range(COUNT2)] Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_runner.py Tue Nov 24 14:29:46 2009 @@ -260,7 +260,7 @@ self.cpu.set_future_value_int(0, b.value) else: self.cpu.set_future_value_ref(0, b.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) if guard == rop.GUARD_FALSE: assert result == execute(self.cpu, op, None, b).value @@ -306,7 +306,7 @@ self.cpu.compile_loop(inputargs, ops, looptoken) for i, box in enumerate(inputargs): self.cpu.set_future_value_int(i, box.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) expected = execute(self.cpu, op, None, a, b).value if guard == rop.GUARD_FALSE: @@ -314,31 +314,27 @@ else: assert result != expected - def test_overflow_mc(self): - from pypy.jit.backend.x86.assembler import MachineCodeBlockWrapper - orig_size = MachineCodeBlockWrapper.MC_SIZE - MachineCodeBlockWrapper.MC_SIZE = 1024 - old_mc = self.cpu.assembler.mc - old_mc2 = self.cpu.assembler.mc2 - self.cpu.assembler.mc = None - try: - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc != old_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - finally: - MachineCodeBlockWrapper.MC_SIZE = orig_size - self.cpu.assembler.mc = old_mc - self.cpu.assembler.mc2 = old_mc2 +class TestX86OverflowMC(TestX86): + + def setup_class(cls): + cls.cpu = CPU(rtyper=None, stats=FakeStats()) + cls.cpu.assembler.mc_size = 1024 + + def test_overflow_mc(self): + ops = [] + base_v = BoxInt() + v = base_v + for i in range(1024): + next_v = BoxInt() + ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) + v = next_v + ops.append(ResOperation(rop.FINISH, [v], None, + descr=BasicFailDescr())) + looptoken = LoopToken() + old_mc_mc = self.cpu.assembler.mc._mc + self.cpu.compile_loop([base_v], ops, looptoken) + assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed + self.cpu.set_future_value_int(0, base_v.value) + self.cpu.execute_token(looptoken) + assert self.cpu.get_latest_value_int(0) == 1024 Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/compile.py Tue Nov 24 14:29:46 2009 @@ -113,7 +113,7 @@ metainterp_sd.log("compiled new " + type) def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): - n = faildescr.get_index() + n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): show_loop(metainterp_sd) @@ -133,14 +133,7 @@ # ____________________________________________________________ class _DoneWithThisFrameDescr(AbstractFailDescr): - - def __init__(self, lst): - "NOT_RPYTHON" - self.index = len(lst) - lst.append(self) - - def get_index(self): - return self.index + pass class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): def handle_fail(self, metainterp_sd): @@ -173,18 +166,6 @@ raise metainterp_sd.ExitFrameWithExceptionRef(cpu, value) -done_with_this_frame_descrs = [] -done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid( - done_with_this_frame_descrs) -done_with_this_frame_descr_int = DoneWithThisFrameDescrInt( - done_with_this_frame_descrs) -done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef( - done_with_this_frame_descrs) -done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat( - done_with_this_frame_descrs) -exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef( - done_with_this_frame_descrs) - prebuiltNotSpecNode = NotSpecNode() class TerminatingLoopToken(LoopToken): @@ -194,22 +175,30 @@ self.specnodes = [prebuiltNotSpecNode]*nargs self.finishdescr = finishdescr -# pseudo loop tokens to make the life of optimize.py easier -loop_tokens_done_with_this_frame_int = [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ] -loop_tokens_done_with_this_frame_ref = [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ] -loop_tokens_done_with_this_frame_float = [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ] -loop_tokens_done_with_this_frame_void = [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ] -loop_tokens_exit_frame_with_exception_ref = [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ] +def make_done_loop_tokens(): + done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() + done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() + done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() + done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() + exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() + + # pseudo loop tokens to make the life of optimize.py easier + return {'loop_tokens_done_with_this_frame_int': [ + TerminatingLoopToken(1, done_with_this_frame_descr_int) + ], + 'loop_tokens_done_with_this_frame_ref': [ + TerminatingLoopToken(1, done_with_this_frame_descr_ref) + ], + 'loop_tokens_done_with_this_frame_float': [ + TerminatingLoopToken(1, done_with_this_frame_descr_float) + ], + 'loop_tokens_done_with_this_frame_void': [ + TerminatingLoopToken(0, done_with_this_frame_descr_void) + ], + 'loop_tokens_exit_frame_with_exception_ref': [ + TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) + ], + } class ResumeDescr(AbstractFailDescr): def __init__(self, original_greenkey): @@ -222,13 +211,6 @@ def __init__(self, metainterp_sd, original_greenkey): ResumeDescr.__init__(self, original_greenkey) self.metainterp_sd = metainterp_sd - self.index = -1 - - def get_index(self): - if self.index == -1: - globaldata = self.metainterp_sd.globaldata - self.index = globaldata.get_fail_descr_number(self) - return self.index def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/history.py Tue Nov 24 14:29:46 2009 @@ -123,21 +123,16 @@ return '%r' % (self,) class AbstractFailDescr(AbstractDescr): + index = -1 - def get_index(self): - raise NotImplementedError def handle_fail(self, metainterp_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError class BasicFailDescr(AbstractFailDescr): - - def __init__(self, index=-1): - self.index = index - - def get_index(self): - return self.index + def __init__(self, identifier=None): + self.identifier = identifier # for testing class AbstractMethDescr(AbstractDescr): # the base class of the result of cpu.methdescrof() Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/logger.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/logger.py Tue Nov 24 14:29:46 2009 @@ -90,8 +90,8 @@ if op.descr is not None: descr = op.descr if is_guard and self.guard_number: - assert isinstance(descr, AbstractFailDescr) - r = "" % descr.get_index() + index = self.metainterp_sd.cpu.get_fail_descr_number(descr) + r = "" % index else: r = self.repr_of_descr(descr) args += ', descr=' + r Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/pyjitpl.py Tue Nov 24 14:29:46 2009 @@ -1022,6 +1022,8 @@ self._addr2name_keys = [] self._addr2name_values = [] + self.__dict__.update(compile.make_done_loop_tokens()) + def _freeze_(self): return True @@ -1044,8 +1046,7 @@ else: self.num_green_args = 0 self.state = None - self.globaldata = MetaInterpGlobalData( - self, compile.done_with_this_frame_descrs) + self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" @@ -1136,11 +1137,10 @@ # ____________________________________________________________ class MetaInterpGlobalData(object): - def __init__(self, staticdata, prebuilt_fail_descr_list): + def __init__(self, staticdata): self.initialized = False self.indirectcall_dict = None self.addr2name = None - self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # state = staticdata.state @@ -1164,16 +1164,6 @@ cell.compiled_merge_points = [] return cell.compiled_merge_points - def get_fail_descr_number(self, descr): - assert isinstance(descr, history.AbstractFailDescr) - lst = self.fail_descr_list - n = len(lst) - lst.append(descr) - return n - - def get_fail_descr_from_number(self, n): - return self.fail_descr_list[n] - # ____________________________________________________________ class MetaInterp(object): @@ -1622,16 +1612,16 @@ if sd.result_type == 'void': assert exitbox is None exits = [] - loop_tokens = compile.loop_tokens_done_with_this_frame_void + loop_tokens = sd.loop_tokens_done_with_this_frame_void elif sd.result_type == 'int': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_int + loop_tokens = sd.loop_tokens_done_with_this_frame_int elif sd.result_type == 'ref': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_ref + loop_tokens = sd.loop_tokens_done_with_this_frame_ref elif sd.result_type == 'float': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_float + loop_tokens = sd.loop_tokens_done_with_this_frame_float else: assert False self.history.record(rop.JUMP, exits, None) @@ -1643,7 +1633,8 @@ self.gen_store_back_in_virtualizable() # temporarily put a JUMP to a pseudo-loop self.history.record(rop.JUMP, [valuebox], None) - loop_tokens = compile.loop_tokens_exit_frame_with_exception_ref + sd = self.staticdata + loop_tokens = sd.loop_tokens_exit_frame_with_exception_ref target_loop_token = compile.compile_new_bridge(self, loop_tokens, self.resumekey) assert target_loop_token is loop_tokens[0] Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_compile.py Tue Nov 24 14:29:46 2009 @@ -103,22 +103,3 @@ assert loop_tokens == [loop_token] assert len(cpu.seen) == 0 assert staticdata.globaldata.loopnumbering == 2 - -def test_resumeguarddescr_get_index(): - from pypy.jit.metainterp.pyjitpl import MetaInterpGlobalData - - class FakeStaticData: - state = None - virtualizable_info = None - gd = MetaInterpGlobalData(FakeStaticData, []) - FakeStaticData.globaldata = gd - - rgd = ResumeGuardDescr(FakeStaticData, ()) - - fail_index = rgd.get_index() - fail_index1 = rgd.get_index() - - assert fail_index == fail_index1 - - assert gd.get_fail_descr_from_number(fail_index) is rgd - Modified: pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/metainterp/test/test_logger.py Tue Nov 24 14:29:46 2009 @@ -6,6 +6,8 @@ from StringIO import StringIO from pypy.jit.metainterp.test.test_optimizeopt import equaloplists from pypy.jit.metainterp.history import AbstractDescr, LoopToken, BasicFailDescr +from pypy.jit.backend.model import AbstractCPU + class Descr(AbstractDescr): pass @@ -36,8 +38,8 @@ def make_metainterp_sd(self): class FakeMetaInterpSd: - class cpu: - ts = self.ts + cpu = AbstractCPU() + cpu.ts = self.ts def get_name_from_address(self, addr): return 'Name' return FakeMetaInterpSd() @@ -122,7 +124,7 @@ pure_parse(output) def test_guard_descr(self): - namespace = {'fdescr': BasicFailDescr(4)} + namespace = {'fdescr': BasicFailDescr()} inp = ''' [i0] guard_true(i0, descr=fdescr) [i0] @@ -130,15 +132,14 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" + assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) - def boom(): - raise Exception - namespace['fdescr'].get_index = boom logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) - assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + lastline = output.splitlines()[-1] + assert lastline.startswith("guard_true(i0, descr=<") + assert not lastline.startswith("guard_true(i0, descr=' % (self.w_type, self.w_value) + def raiseFailedToImplement(): raise FailedToImplement From arigo at codespeak.net Tue Nov 24 14:55:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 14:55:26 +0100 (CET) Subject: [pypy-svn] r69579 - in pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86: . test Message-ID: <20091124135526.B0294168008@codespeak.net> Author: arigo Date: Tue Nov 24 14:55:26 2009 New Revision: 69579 Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py Log: Put holes in test_assembler, and fix the code. Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py Tue Nov 24 14:55:26 2009 @@ -755,15 +755,8 @@ # return self.implement_guard(addr, self.mc.JNE) - def _no_const_locs(self, args): - """ assert that all args are actually Boxes - """ - for arg in args: - assert arg is None or isinstance(arg, Box) # hole - def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): - self._no_const_locs(failargs) exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) @@ -783,7 +776,7 @@ addr = mc.tell() withfloats = False for box in failargs: - if box.type == FLOAT: + if box is not None and box.type == FLOAT: withfloats = True break mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) @@ -801,30 +794,35 @@ DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 + DESCR_SPECIAL = 0x03 DESCR_FROMSTACK = 8 - DESCR_STOP = DESCR_INT + 4*esp.op + DESCR_STOP = 0 | DESCR_SPECIAL + DESCR_HOLE = 4 | DESCR_SPECIAL def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] - if arg.type == REF: - kind = self.DESCR_REF - elif arg.type == INT: - kind = self.DESCR_INT - elif arg.type == FLOAT: - kind = self.DESCR_FLOAT - else: - raise AssertionError("bogus kind") - loc = locs[i] - if isinstance(loc, MODRM): - n = self.DESCR_FROMSTACK + loc.position + if arg is not None: + if arg.type == REF: + kind = self.DESCR_REF + elif arg.type == INT: + kind = self.DESCR_INT + elif arg.type == FLOAT: + kind = self.DESCR_FLOAT + else: + raise AssertionError("bogus kind") + loc = locs[i] + if isinstance(loc, MODRM): + n = self.DESCR_FROMSTACK + loc.position + else: + assert isinstance(loc, REG) + n = loc.op + n = kind + 4*n + while n > 0x7F: + mc.writechr((n & 0x7F) | 0x80) + n >>= 7 else: - assert isinstance(loc, REG) - n = loc.op - n = kind + 4*n - while n > 0x7F: - mc.writechr((n & 0x7F) | 0x80) - n >>= 7 + n = self.DESCR_HOLE mc.writechr(n) mc.writechr(self.DESCR_STOP) # preallocate the fail_boxes @@ -864,6 +862,8 @@ loc = X86StackManager.stack_pos(code, size) elif code == self.DESCR_STOP: break + elif code == self.DESCR_HOLE: + continue else: # 'code' identifies a register kind = code & 3 @@ -910,11 +910,15 @@ if kind == self.DESCR_FLOAT: value_hi = value value = rffi.cast(rffi.LONGP, stackloc - 4)[0] - elif code == self.DESCR_STOP: - break else: # 'code' identifies a register: load its value kind = code & 3 + if kind == self.DESCR_SPECIAL: + if code == self.DESCR_HOLE: + num += 1 + continue + assert code == self.DESCR_STOP + break code >>= 2 if kind == self.DESCR_FLOAT: xmmregisters = rffi.ptradd(registers, -16) Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py Tue Nov 24 14:55:26 2009 @@ -21,15 +21,20 @@ assembler = Assembler386(FakeCPU()) mc = FakeMC() failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 + failargs.insert(6, None) + failargs.insert(7, None) locs = [X86StackManager.stack_pos(0, 1), X86StackManager.stack_pos(1, 1), X86StackManager.stack_pos(10, 2), X86StackManager.stack_pos(100, 1), X86StackManager.stack_pos(101, 1), X86StackManager.stack_pos(110, 2), + None, + None, ebx, esi, xmm2] + assert len(failargs) == len(locs) assembler.write_failure_recovery_description(mc, failargs, locs) nums = [Assembler386.DESCR_INT + 4*(8+0), Assembler386.DESCR_REF + 4*(8+1), @@ -37,6 +42,8 @@ Assembler386.DESCR_INT + 4*(8+100), Assembler386.DESCR_REF + 4*(8+101), Assembler386.DESCR_FLOAT + 4*(8+110), + Assembler386.DESCR_HOLE, + Assembler386.DESCR_HOLE, Assembler386.DESCR_INT + 4*ebx.op, Assembler386.DESCR_REF + 4*esi.op, Assembler386.DESCR_FLOAT + 4*xmm2.op] @@ -47,7 +54,8 @@ assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + [assembler.DESCR_STOP]) - # also test rebuild_faillocs_from_descr() + # also test rebuild_faillocs_from_descr(), which should not + # reproduce the holes at all bytecode = lltype.malloc(rffi.UCHARP.TO, len(mc.content), flavor='raw') for i in range(len(mc.content)): assert 0 <= mc.content[i] <= 255 @@ -55,7 +63,7 @@ bytecode_addr = rffi.cast(lltype.Signed, bytecode) newlocs = assembler.rebuild_faillocs_from_descr(bytecode_addr) assert ([loc.assembler() for loc in newlocs] == - [loc.assembler() for loc in locs]) + [loc.assembler() for loc in locs if loc is not None]) # ____________________________________________________________ @@ -83,26 +91,29 @@ rffi.cast(rffi.DOUBLEP, tmp)[0] = value return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] - # memory locations: 30 integers, 30 pointers, 26 floats + # memory locations: 26 integers, 26 pointers, 26 floats # main registers: half of them as signed and the other half as ptrs # xmm registers: all floats, from xmm0 to xmm7 + # holes: 8 locations = [] baseloc = 4 - for i in range(30+30+26): + for i in range(26+26+26): if baseloc < 128: baseloc += random.randrange(2, 20) else: baseloc += random.randrange(2, 1000) locations.append(baseloc) random.shuffle(locations) - content = ([('int', locations.pop()) for _ in range(30)] + - [('ptr', locations.pop()) for _ in range(30)] + + content = ([('int', locations.pop()) for _ in range(26)] + + [('ptr', locations.pop()) for _ in range(26)] + [(['int', 'ptr'][random.randrange(0, 2)], reg) for reg in [eax, ecx, edx, ebx, esi, edi]]) if withfloats: content += ([('float', locations.pop()) for _ in range(26)] + [('float', reg) for reg in [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7]]) + for i in range(8): + content.append(('hole', None)) random.shuffle(content) # prepare the expected target arrays, the descr_bytecode, @@ -124,40 +135,43 @@ descr_bytecode = [] for i, (kind, loc) in enumerate(content): - if kind == 'float': - value, lo, hi = get_random_float() - expected_floats[i] = value - kind = Assembler386.DESCR_FLOAT - if isinstance(loc, REG): - xmmregisters[2*loc.op] = lo - xmmregisters[2*loc.op+1] = hi - else: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) + if kind == 'hole': + num = Assembler386.DESCR_HOLE else: - if kind == 'int': - value = get_random_int() - expected_ints[i] = value - kind = Assembler386.DESCR_INT - elif kind == 'ptr': - value = get_random_ptr() - expected_ptrs[i] = value - kind = Assembler386.DESCR_REF - value = rffi.cast(rffi.LONG, value) + if kind == 'float': + value, lo, hi = get_random_float() + expected_floats[i] = value + kind = Assembler386.DESCR_FLOAT + if isinstance(loc, REG): + xmmregisters[2*loc.op] = lo + xmmregisters[2*loc.op+1] = hi + else: + write_in_stack(loc, hi) + write_in_stack(loc+1, lo) else: - assert 0, kind + if kind == 'int': + value = get_random_int() + expected_ints[i] = value + kind = Assembler386.DESCR_INT + elif kind == 'ptr': + value = get_random_ptr() + expected_ptrs[i] = value + kind = Assembler386.DESCR_REF + value = rffi.cast(rffi.LONG, value) + else: + assert 0, kind + if isinstance(loc, REG): + registers[loc.op] = value + else: + write_in_stack(loc, value) + if isinstance(loc, REG): - registers[loc.op] = value + num = kind + 4*loc.op else: - write_in_stack(loc, value) - - if isinstance(loc, REG): - num = kind + 4*loc.op - else: - num = kind + 4*(8+loc) - while num >= 0x80: - descr_bytecode.append((num & 0x7F) | 0x80) - num >>= 7 + num = kind + 4*(8+loc) + while num >= 0x80: + descr_bytecode.append((num & 0x7F) | 0x80) + num >>= 7 descr_bytecode.append(num) descr_bytecode.append(Assembler386.DESCR_STOP) From arigo at codespeak.net Tue Nov 24 15:02:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 15:02:23 +0100 (CET) Subject: [pypy-svn] r69580 - in pypy/branch/shorter-guard-path-2/pypy/jit/backend: test x86 Message-ID: <20091124140223.AA2A1168008@codespeak.net> Author: arigo Date: Tue Nov 24 15:02:23 2009 New Revision: 69580 Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py Log: Small fixes. Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/runner_test.py Tue Nov 24 15:02:23 2009 @@ -142,7 +142,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(2) assert res == 10 @@ -234,7 +234,7 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 20 Modified: pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/assembler.py Tue Nov 24 15:02:23 2009 @@ -788,7 +788,8 @@ # for testing the decoding, write a final byte 0xCC if not we_are_translated(): mc.writechr(0xCC) - faildescr._x86_debug_faillocs = fail_locs + faildescr._x86_debug_faillocs = [loc for loc in fail_locs + if loc is not None] return addr DESCR_REF = 0x00 From arigo at codespeak.net Tue Nov 24 15:35:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 15:35:33 +0100 (CET) Subject: [pypy-svn] r69581 - in pypy/trunk/pypy: jit/backend jit/backend/cli jit/backend/llgraph jit/backend/llsupport jit/backend/test jit/backend/x86 jit/backend/x86/test jit/metainterp jit/metainterp/test objspace/std Message-ID: <20091124143533.E795B168008@codespeak.net> Author: arigo Date: Tue Nov 24 15:35:31 2009 New Revision: 69581 Added: pypy/trunk/pypy/jit/backend/test/test_model.py - copied unchanged from r69580, pypy/branch/shorter-guard-path-2/pypy/jit/backend/test/test_model.py pypy/trunk/pypy/jit/backend/x86/test/test_assembler.py - copied unchanged from r69580, pypy/branch/shorter-guard-path-2/pypy/jit/backend/x86/test/test_assembler.py Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/cli/runner.py pypy/trunk/pypy/jit/backend/llgraph/runner.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/model.py pypy/trunk/pypy/jit/backend/test/runner_test.py pypy/trunk/pypy/jit/backend/test/test_ll_random.py pypy/trunk/pypy/jit/backend/test/test_random.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/ri386setup.py pypy/trunk/pypy/jit/backend/x86/runner.py pypy/trunk/pypy/jit/backend/x86/support.py pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py pypy/trunk/pypy/jit/backend/x86/test/test_runner.py pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/history.py pypy/trunk/pypy/jit/metainterp/logger.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_compile.py pypy/trunk/pypy/jit/metainterp/test/test_logger.py pypy/trunk/pypy/jit/metainterp/test/test_pyjitpl.py pypy/trunk/pypy/jit/metainterp/warmstate.py pypy/trunk/pypy/objspace/std/multimethod.py Log: Merge the branch shorter-guard-path: * Cancel a previous refactoring that moved the numbering of the guard descrs to the front-end. Instead, move that counting to AbstractCPU. * Generate much shorter failure paths from the x86 backend: generate_quick_failure() writes just a jump to control code which can itself do the guard recovery by reading some byte-compressed data. * Guards in x86 don't need to store the attribute _x86_faillocs any more. * Improve the testing of "machine code block full". Although it is still a bit random, now it runs all runner tests a second time with a 1KB limit. Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Tue Nov 24 15:35:31 2009 @@ -268,9 +268,6 @@ try: return self.cpu.failing_ops.index(op) except ValueError: - descr = op.descr - assert isinstance(descr, history.AbstractFailDescr) - descr.get_index() self.cpu.failing_ops.append(op) return len(self.cpu.failing_ops)-1 Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Tue Nov 24 15:35:31 2009 @@ -135,9 +135,7 @@ func = cliloop.funcbox.holder.GetFunc() func(self.get_inputargs()) op = self.failing_ops[self.inputargs.get_failed_op()] - descr = op.descr - assert isinstance(descr, AbstractFailDescr) - return descr.get_index() + return op.descr def set_future_value_int(self, index, intvalue): self.get_inputargs().set_int(index, intvalue) Modified: pypy/trunk/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/runner.py Tue Nov 24 15:35:31 2009 @@ -88,6 +88,7 @@ def __init__(self, rtyper, stats=None, translate_support_code=False, annmixlevel=None, gcdescr=None): + model.AbstractCPU.__init__(self) self.rtyper = rtyper self.translate_support_code = translate_support_code self.stats = stats or MiniStats() @@ -167,7 +168,7 @@ if op.is_guard(): faildescr = op.descr assert isinstance(faildescr, history.AbstractFailDescr) - fail_index = faildescr.get_index() + fail_index = self.get_fail_descr_number(faildescr) index = llimpl.compile_add_fail(c, fail_index) faildescr._compiled_fail = c, index for box in op.fail_args: @@ -195,8 +196,7 @@ llimpl.compile_add_jump_target(c, compiled_version) elif op.opnum == rop.FINISH: faildescr = op.descr - assert isinstance(faildescr, history.AbstractFailDescr) - index = faildescr.get_index() + index = self.get_fail_descr_number(faildescr) llimpl.compile_add_fail(c, index) else: assert False, "unknown operation" @@ -213,7 +213,7 @@ fail_index = llimpl.frame_execute(frame) # we hit a FAIL operation. self.latest_frame = frame - return fail_index + return self.get_fail_descr_from_number(fail_index) def set_future_value_int(self, index, intvalue): llimpl.set_future_value_int(index, intvalue) Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Tue Nov 24 15:35:31 2009 @@ -20,6 +20,7 @@ def __init__(self, rtyper, stats, translate_support_code=False, gcdescr=None): from pypy.jit.backend.llsupport.gc import get_ll_description + AbstractCPU.__init__(self) self.rtyper = rtyper self.stats = stats self.translate_support_code = translate_support_code @@ -149,8 +150,8 @@ on_leave_jitted_hook() def on_leave_jitted_save_exc(): - on_leave_jitted_hook() save_exception() + on_leave_jitted_hook() self.on_leave_jitted_noexc = on_leave_jitted_noexc self.on_leave_jitted_save_exc = on_leave_jitted_save_exc Modified: pypy/trunk/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/pypy/jit/backend/model.py (original) +++ pypy/trunk/pypy/jit/backend/model.py Tue Nov 24 15:35:31 2009 @@ -1,6 +1,25 @@ +from pypy.jit.metainterp import history + + class AbstractCPU(object): supports_floats = False + def __init__(self): + self.fail_descr_list = [] + + def get_fail_descr_number(self, descr): + assert isinstance(descr, history.AbstractFailDescr) + n = descr.index + if n < 0: + lst = self.fail_descr_list + n = len(lst) + lst.append(descr) + descr.index = n + return n + + def get_fail_descr_from_number(self, n): + return self.fail_descr_list[n] + def set_class_sizes(self, class_sizes): self.class_sizes = class_sizes Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Tue Nov 24 15:35:31 2009 @@ -39,7 +39,7 @@ else: raise NotImplementedError(box) res = self.cpu.execute_token(looptoken) - if res == operations[-1].descr.index: + if res is operations[-1].descr: self.guard_failed = False else: self.guard_failed = True @@ -102,7 +102,7 @@ fail = self.cpu.execute_token(looptoken) res = self.cpu.get_latest_value_int(0) assert res == 3 - assert fail == 1 + assert fail.identifier == 1 def test_compile_loop(self): i0 = BoxInt() @@ -121,7 +121,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 10 @@ -142,7 +142,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(2) assert res == 10 @@ -200,7 +200,7 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 20 @@ -234,7 +234,7 @@ self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_int(0) assert res == 20 @@ -242,9 +242,9 @@ i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): def __setattr__(self, name, value): + if name == 'index': + return AbstractFailDescr.__setattr__(self, name, value) py.test.fail("finish descrs should not be touched") - def get_index(self): - return 7 faildescr = UntouchableFailDescr() # to check that is not touched looptoken = LoopToken() operations = [ @@ -253,7 +253,7 @@ self.cpu.compile_loop([i0], operations, looptoken) self.cpu.set_future_value_int(0, 99) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 99 @@ -263,7 +263,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr res = self.cpu.get_latest_value_int(0) assert res == 42 @@ -273,7 +273,7 @@ ] self.cpu.compile_loop([], operations, looptoken) fail = self.cpu.execute_token(looptoken) - assert fail == 7 + assert fail is faildescr def test_execute_operations_in_env(self): cpu = self.cpu @@ -357,9 +357,9 @@ self.cpu.set_future_value_int(1, y) fail = self.cpu.execute_token(looptoken) if (z == boom) ^ reversed: - assert fail == 1 + assert fail.identifier == 1 else: - assert fail == 2 + assert fail.identifier == 2 if z != boom: assert self.cpu.get_latest_value_int(0) == z assert not self.cpu.get_exception() @@ -832,7 +832,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 15 + assert fail.identifier == 15 # dstvalues = values[:] for _ in range(11): @@ -884,7 +884,7 @@ for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) fail = self.cpu.execute_token(looptoken) - assert fail == 2 + assert fail.identifier == 2 res = self.cpu.get_latest_value_float(0) assert res == 8.5 for i in range(1, len(fboxes)): @@ -935,7 +935,7 @@ assert 0 # fail = self.cpu.execute_token(looptoken) - assert fail == 1 + assert fail.identifier == 1 class LLtypeBackendTest(BaseBackendTest): Modified: pypy/trunk/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_ll_random.py Tue Nov 24 15:35:31 2009 @@ -4,6 +4,7 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import ConstInt, ConstPtr from pypy.jit.metainterp.history import ConstAddr, BoxPtr, BoxInt +from pypy.jit.metainterp.history import BasicFailDescr from pypy.rpython.annlowlevel import llhelper from pypy.rlib.rarithmetic import intmask from pypy.rpython.llinterp import LLException @@ -464,7 +465,7 @@ descr = builder.cpu.calldescrof(TP, TP.ARGS, TP.RESULT) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -486,7 +487,7 @@ _, vtableptr = builder.get_random_structure_type_and_vtable(r) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = builder.subset_of_intvars(r) op._exc_box = None builder.should_fail_by = op @@ -509,7 +510,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op.fail_args = fail_subset builder.loop.operations.append(op) @@ -527,7 +528,7 @@ assert builder.cpu.get_exception() builder.cpu.clear_exception() op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op @@ -553,7 +554,7 @@ break other_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [other_box], BoxPtr(), - descr=builder.make_fail_descr()) + descr=BasicFailDescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.fail_args = builder.subset_of_intvars(r) builder.should_fail_by = op Modified: pypy/trunk/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/test_random.py (original) +++ pypy/trunk/pypy/jit/backend/test/test_random.py Tue Nov 24 15:35:31 2009 @@ -18,8 +18,6 @@ self.operations = subops class OperationBuilder(object): - failnumbering = 0 - def __init__(self, cpu, loop, vars): self.cpu = cpu self.loop = loop @@ -36,12 +34,6 @@ self.counter = 0 assert len(self.intvars) == len(dict.fromkeys(self.intvars)) - @classmethod - def make_fail_descr(cls): - descr = BasicFailDescr(cls.failnumbering) - cls.failnumbering += 1 - return descr - def fork(self, cpu, loop, vars): fork = self.__class__(cpu, loop, vars) fork.prebuilt_ptr_consts = self.prebuilt_ptr_consts @@ -282,7 +274,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = fail_subset builder.loop.operations.append(op) @@ -343,7 +335,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.descr = builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = builder.subset_of_intvars(r) if not passing: builder.should_fail_by = op @@ -559,7 +551,7 @@ endvars.append(v) r.shuffle(endvars) loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr(-2))) + descr=BasicFailDescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -605,7 +597,7 @@ else: raise NotImplementedError(box) fail = cpu.execute_token(self.loop.token) - assert fail == self.should_fail_by.descr.index + assert fail is self.should_fail_by.descr for i, v in enumerate(self.get_fail_args()): if isinstance(v, (BoxFloat, ConstFloat)): value = cpu.get_latest_value_float(i) @@ -634,7 +626,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.descr = self.builder.make_fail_descr() + op.descr = BasicFailDescr() op.fail_args = [] return op Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Tue Nov 24 15:35:31 2009 @@ -1,11 +1,12 @@ import sys, os import ctypes from pypy.jit.backend.llsupport import symbolic -from pypy.jit.metainterp.history import Const, Box, BoxPtr, REF, FLOAT +from pypy.jit.metainterp.history import Const, Box, BoxPtr, INT, REF, FLOAT from pypy.jit.metainterp.history import AbstractFailDescr from pypy.rpython.lltypesystem import lltype, rffi, ll2ctypes, rstr, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs @@ -16,6 +17,7 @@ from pypy.jit.backend.x86.support import NonmovableGrowableArrayFloat,\ NonmovableGrowableArraySigned, NonmovableGrowableArrayGCREF,\ CHUNK_SIZE +from pypy.rlib.debug import debug_print # our calling convention - we pass first 6 args in registers # and the rest stays on the stack @@ -33,11 +35,24 @@ return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) class MachineCodeBlockWrapper(object): - MC_SIZE = 1024*1024 + MC_DEFAULT_SIZE = 1024*1024 - def __init__(self): + def __init__(self, bigsize): self.old_mcs = [] # keepalive - self._mc = codebuf.MachineCodeBlock(self.MC_SIZE) + self.bigsize = bigsize + self._mc = codebuf.MachineCodeBlock(bigsize) + + def bytes_free(self): + return self._mc._size - self._mc._pos + + def make_new_mc(self): + new_mc = codebuf.MachineCodeBlock(self.bigsize) + debug_print('[new machine code block at', new_mc.tell(), ']') + self._mc.JMP(rel32(new_mc.tell())) + self._mc.done() + self.old_mcs.append(self._mc) + self._mc = new_mc + make_new_mc._dont_inline_ = True def tell(self): return self._mc.tell() @@ -49,12 +64,8 @@ def method(self, *args): # XXX er.... pretty random number, just to be sure # not to write half-instruction - if self._mc._pos + 64 >= self._mc._size: - new_mc = codebuf.MachineCodeBlock(self.MC_SIZE) - self._mc.JMP(rel32(new_mc.tell())) - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc + if self.bytes_free() < 64: + self.make_new_mc() getattr(self._mc, name)(*args) method.func_name = name return method @@ -66,7 +77,7 @@ class Assembler386(object): mc = None mc2 = None - debug_markers = True + mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE def __init__(self, cpu, translate_support_code=False): self.cpu = cpu @@ -79,11 +90,9 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.setup_failure_recovery() def leave_jitted_hook(self): - # XXX BIG FAT WARNING XXX - # At this point, we should not call anyone here, because - # RPython-level exception might be set. Here be dragons i = 0 while i < self.fail_boxes_ptr.lgt: chunk = self.fail_boxes_ptr.chunks[i] @@ -113,8 +122,13 @@ # done # we generate the loop body in 'mc' # 'mc2' is for guard recovery code - self.mc = MachineCodeBlockWrapper() - self.mc2 = MachineCodeBlockWrapper() + self.mc = MachineCodeBlockWrapper(self.mc_size) + self.mc2 = MachineCodeBlockWrapper(self.mc_size) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) def assemble_loop(self, inputargs, operations, looptoken): """adds the following attributes to looptoken: @@ -137,8 +151,12 @@ def assemble_bridge(self, faildescr, inputargs, operations): self.make_sure_mc_exists() + arglocs = self.rebuild_faillocs_from_descr( + faildescr._x86_failure_recovery_bytecode) + if not we_are_translated(): + assert ([loc.assembler() for loc in arglocs] == + [loc.assembler() for loc in faildescr._x86_debug_faillocs]) regalloc = RegAlloc(self, self.cpu.translate_support_code) - arglocs = faildescr._x86_faillocs fail_stack_depth = faildescr._x86_current_stack_depth regalloc.prepare_bridge(fail_stack_depth, inputargs, arglocs, operations) @@ -150,9 +168,12 @@ # for the benefit of tests faildescr._x86_bridge_stack_depth = stack_depth # patch the jump from original guard + self.patch_jump(faildescr, adr_bridge) + + def patch_jump(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) - mc.write(packimm32(adr_bridge - adr_jump_offset - 4)) + mc.write(packimm32(adr_new_target - adr_jump_offset - 4)) mc.valgrind_invalidated() mc.done() @@ -734,59 +755,284 @@ # return self.implement_guard(addr, self.mc.JNE) - def _no_const_locs(self, args): - """ assert that all args are actually Boxes - """ - for arg in args: - assert arg is None or isinstance(arg, Box) # hole - def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): - self._no_const_locs(failargs) - addr = self.mc2.tell() exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION) - faildescr._x86_faillocs = fail_locs - self.generate_failure(self.mc2, faildescr, failargs, fail_locs, exc) + return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) + + def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): + """Generate the initial code for handling a failure. We try to + keep it as compact as possible. The idea is that this code is + executed at most once (and very often, zero times); when + executed, it generates a more complete piece of code which can + really handle recovery from this particular failure. + """ + fail_index = self.cpu.get_fail_descr_number(faildescr) + bytes_needed = 20 + 5 * len(failargs) # conservative estimate + if self.mc2.bytes_free() < bytes_needed: + self.mc2.make_new_mc() + mc = self.mc2._mc + addr = mc.tell() + withfloats = False + for box in failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + mc.CALL(rel32(self.failure_recovery_code[exc + 2 * withfloats])) + # write tight data that describes the failure recovery + faildescr._x86_failure_recovery_bytecode = mc.tell() + self.write_failure_recovery_description(mc, failargs, fail_locs) + # write the fail_index too + mc.write(packimm32(fail_index)) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + mc.writechr(0xCC) + faildescr._x86_debug_faillocs = [loc for loc in fail_locs + if loc is not None] return addr - def generate_failure(self, mc, faildescr, failargs, locs, exc): - pos = mc.tell() + DESCR_REF = 0x00 + DESCR_INT = 0x01 + DESCR_FLOAT = 0x02 + DESCR_SPECIAL = 0x03 + DESCR_FROMSTACK = 8 + DESCR_STOP = 0 | DESCR_SPECIAL + DESCR_HOLE = 4 | DESCR_SPECIAL + + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] - if arg is None: # hole + if arg is not None: + if arg.type == REF: + kind = self.DESCR_REF + elif arg.type == INT: + kind = self.DESCR_INT + elif arg.type == FLOAT: + kind = self.DESCR_FLOAT + else: + raise AssertionError("bogus kind") + loc = locs[i] + if isinstance(loc, MODRM): + n = self.DESCR_FROMSTACK + loc.position + else: + assert isinstance(loc, REG) + n = loc.op + n = kind + 4*n + while n > 0x7F: + mc.writechr((n & 0x7F) | 0x80) + n >>= 7 + else: + n = self.DESCR_HOLE + mc.writechr(n) + mc.writechr(self.DESCR_STOP) + # preallocate the fail_boxes + i = len(failargs) - 1 + if i >= 0: + self.fail_boxes_int.get_addr_for_num(i) + self.fail_boxes_ptr.get_addr_for_num(i) + if self.cpu.supports_floats: + self.fail_boxes_float.get_addr_for_num(i) + + def rebuild_faillocs_from_descr(self, bytecode): + from pypy.jit.backend.x86.regalloc import X86StackManager + bytecode = rffi.cast(rffi.UCHARP, bytecode) + arglocs = [] + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + # 'code' identifies a stack location + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + if kind == self.DESCR_FLOAT: + size = 2 + else: + size = 1 + loc = X86StackManager.stack_pos(code, size) + elif code == self.DESCR_STOP: + break + elif code == self.DESCR_HOLE: continue + else: + # 'code' identifies a register + kind = code & 3 + code >>= 2 + if kind == self.DESCR_FLOAT: + loc = xmm_registers[code] + else: + loc = registers[code] + arglocs.append(loc) + return arglocs[:] + + def setup_failure_recovery(self): + + def failure_recovery_func(registers): + # no malloc allowed here!! + # 'registers' is a pointer to a structure containing the + # original value of the registers, optionally the original + # value of XMM registers, and finally a reference to the + # recovery bytecode. See _build_failure_recovery() for details. + stack_at_ebp = registers[ebp.op] + bytecode = rffi.cast(rffi.UCHARP, registers[8]) + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + stackloc = stack_at_ebp + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - 4)[0] + else: + # 'code' identifies a register: load its value + kind = code & 3 + if kind == self.DESCR_SPECIAL: + if code == self.DESCR_HOLE: + num += 1 + continue + assert code == self.DESCR_STOP + break + code >>= 2 + if kind == self.DESCR_FLOAT: + xmmregisters = rffi.ptradd(registers, -16) + value = xmmregisters[2*code] + value_hi = xmmregisters[2*code + 1] + else: + value = registers[code] + + # store the loaded value into fail_boxes_ + if kind == self.DESCR_INT: + tgt = self.fail_boxes_int.get_addr_for_num(num) + elif kind == self.DESCR_REF: + tgt = self.fail_boxes_ptr.get_addr_for_num(num) + elif kind == self.DESCR_FLOAT: + tgt = self.fail_boxes_float.get_addr_for_num(num) + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + else: + assert 0, "bogus kind" + rffi.cast(rffi.LONGP, tgt)[0] = value + num += 1 + # + if not we_are_translated(): + assert bytecode[4] == 0xCC + fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + return fail_index + + self.failure_recovery_func = failure_recovery_func + self.failure_recovery_code = [0, 0, 0, 0] + + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], + lltype.Signed)) + + def _build_failure_recovery(self, exc, withfloats=False): + failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, + self.failure_recovery_func) + failure_recovery_func = rffi.cast(lltype.Signed, + failure_recovery_func) + mc = self.mc2._mc + # Assume that we are called at the beginning, when there is no risk + # that 'mc' runs out of space. Checked by asserts in mc.write(). + recovery_addr = mc.tell() + mc.PUSH(edi) + mc.PUSH(esi) + mc.PUSH(ebp) + mc.PUSH(esp) # <-- not really used, but needed to take up the space + mc.PUSH(ebx) + mc.PUSH(edx) + mc.PUSH(ecx) + mc.PUSH(eax) + mc.MOV(esi, esp) + if withfloats: + mc.SUB(esp, imm(8*8)) + for i in range(8): + mc.MOVSD(mem64(esp, 8*i), xmm_registers[i]) + + # we call a provided function that will + # - call our on_leave_jitted_hook which will mark + # the fail_boxes_ptr array as pointing to young objects to + # avoid unwarranted freeing + # - optionally save exception depending on the flag + addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) + mc.CALL(rel32(addr)) + + # the following call saves all values from the stack and from + # registers to the right 'fail_boxes_' location. + # Note that the registers are saved so far in esi[0] to esi[7], + # as pushed above, plus optionally in esi[-16] to esi[-1] for + # the XMM registers. Moreover, esi[8] is a pointer to the recovery + # bytecode, pushed just before by the CALL instruction written by + # generate_quick_failure(). + mc.PUSH(esi) + mc.CALL(rel32(failure_recovery_func)) + # returns in eax the fail_index + + # now we return from the complete frame, which starts from + # _assemble_bootstrap_code(). The LEA below throws away most + # of the frame, including all the PUSHes that we did just above. + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) + mc.POP(edi) + mc.POP(esi) + mc.POP(ebx) + mc.POP(ebp) + mc.RET() + self.mc2.done() + self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + + def generate_failure(self, mc, fail_index, locs, exc, locs_are_ref): + for i in range(len(locs)): loc = locs[i] if isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), loc) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(heap(adr), loc) - for i in range(len(failargs)): - arg = failargs[i] - if arg is None: # hole - continue + for i in range(len(locs)): loc = locs[i] if not isinstance(loc, REG): - if arg.type == FLOAT: + if loc.width == 8: mc.MOVSD(xmm0, loc) adr = self.fail_boxes_float.get_addr_for_num(i) mc.MOVSD(heap64(adr), xmm0) else: - if arg.type == REF: + if locs_are_ref[i]: adr = self.fail_boxes_ptr.get_addr_for_num(i) else: adr = self.fail_boxes_int.get_addr_for_num(i) mc.MOV(eax, loc) mc.MOV(heap(adr), eax) - if self.debug_markers: - mc.MOV(eax, imm(pos)) - mc.MOV(heap(self.fail_boxes_int.get_addr_for_num(len(locs))), eax) # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -798,9 +1044,7 @@ # don't break the following code sequence! mc = mc._mc - mc.LEA(esp, addr_add(imm(0), ebp, (-RET_BP + 2) * WORD)) - assert isinstance(faildescr, AbstractFailDescr) - fail_index = faildescr.get_index() + mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) mc.MOV(eax, imm(fail_index)) mc.POP(edi) mc.POP(esi) Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Tue Nov 24 15:35:31 2009 @@ -397,8 +397,10 @@ def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] - self.assembler.generate_failure(self.assembler.mc, op.descr, op.args, - locs, self.exc) + locs_are_ref = [v.type == REF for v in op.args] + fail_index = self.assembler.cpu.get_fail_descr_number(op.descr) + self.assembler.generate_failure(self.assembler.mc, fail_index, locs, + self.exc, locs_are_ref) self.possibly_free_vars(op.args) def consider_guard_no_exception(self, op, ignored): Modified: pypy/trunk/pypy/jit/backend/x86/ri386setup.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/ri386setup.py (original) +++ pypy/trunk/pypy/jit/backend/x86/ri386setup.py Tue Nov 24 15:35:31 2009 @@ -459,6 +459,13 @@ SAHF = Instruction() SAHF.mode0(['\x9E']) +LODSB = Instruction() +LODSB.mode0(['\xAC']) + +LODSD = Instruction() +LODSD.mode0(['\xAD']) +LODSD.as_alias = "LODSL" + # ------------------------- floating point instructions ------------------ FLDL = Instruction() Modified: pypy/trunk/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/runner.py Tue Nov 24 15:35:31 2009 @@ -63,7 +63,7 @@ addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) fail_index = self._execute_call(func) - return fail_index + return self.get_fail_descr_from_number(fail_index) def _execute_call(self, func): # help flow objspace Modified: pypy/trunk/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/support.py (original) +++ pypy/trunk/pypy/jit/backend/x86/support.py Tue Nov 24 15:35:31 2009 @@ -3,7 +3,8 @@ from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated -CHUNK_SIZE = 1000 +CHUNK_SIZE_BITS = 8 +CHUNK_SIZE = 1 << CHUNK_SIZE_BITS def new_nonmovable_growable_array(TP): ATP = lltype.GcArray(TP) @@ -34,7 +35,7 @@ def _no_of(self, i): while i >= len(self.chunks) * CHUNK_SIZE: self._grow() - return i / CHUNK_SIZE, i % CHUNK_SIZE + return i >> CHUNK_SIZE_BITS, i & (CHUNK_SIZE-1) _no_of._always_inline_ = True def setitem(self, i, v): Modified: pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_recompilation.py Tue Nov 24 15:35:31 2009 @@ -21,7 +21,7 @@ bridge = self.attach_bridge(ops, loop, -2) self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 def test_compile_bridge_deeper(self): @@ -52,7 +52,7 @@ assert new > previous self.cpu.set_future_value_int(0, 0) fail = self.run(loop) - assert fail == 2 + assert fail.identifier == 2 assert self.getint(0) == 21 assert self.getint(1) == 22 assert self.getint(2) == 23 @@ -78,7 +78,7 @@ bridge = self.attach_bridge(ops, other_loop, 0, looptoken=loop.token) self.cpu.set_future_value_int(0, 1) fail = self.run(other_loop) - assert fail == 1 + assert fail.identifier == 1 def test_bridge_jumps_to_self_deeper(self): loop = self.interpret(''' Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ri386_auto_encoding.py Tue Nov 24 15:35:31 2009 @@ -92,7 +92,7 @@ return [pick1(i386.memSIB64) for i in range(COUNT2)] def xmm_tests(): - return i386.xmm_registers + return i386.xmm_registers[:] def modrm8_tests(): return i386.registers8 + [pick1(i386.memSIB8) for i in range(COUNT2)] Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Tue Nov 24 15:35:31 2009 @@ -260,7 +260,7 @@ self.cpu.set_future_value_int(0, b.value) else: self.cpu.set_future_value_ref(0, b.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) if guard == rop.GUARD_FALSE: assert result == execute(self.cpu, op, None, b).value @@ -306,7 +306,7 @@ self.cpu.compile_loop(inputargs, ops, looptoken) for i, box in enumerate(inputargs): self.cpu.set_future_value_int(i, box.value) - r = self.cpu.execute_token(looptoken) + self.cpu.execute_token(looptoken) result = self.cpu.get_latest_value_int(0) expected = execute(self.cpu, op, None, a, b).value if guard == rop.GUARD_FALSE: @@ -314,31 +314,27 @@ else: assert result != expected - def test_overflow_mc(self): - from pypy.jit.backend.x86.assembler import MachineCodeBlockWrapper - orig_size = MachineCodeBlockWrapper.MC_SIZE - MachineCodeBlockWrapper.MC_SIZE = 1024 - old_mc = self.cpu.assembler.mc - old_mc2 = self.cpu.assembler.mc2 - self.cpu.assembler.mc = None - try: - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc != old_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - finally: - MachineCodeBlockWrapper.MC_SIZE = orig_size - self.cpu.assembler.mc = old_mc - self.cpu.assembler.mc2 = old_mc2 +class TestX86OverflowMC(TestX86): + + def setup_class(cls): + cls.cpu = CPU(rtyper=None, stats=FakeStats()) + cls.cpu.assembler.mc_size = 1024 + + def test_overflow_mc(self): + ops = [] + base_v = BoxInt() + v = base_v + for i in range(1024): + next_v = BoxInt() + ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) + v = next_v + ops.append(ResOperation(rop.FINISH, [v], None, + descr=BasicFailDescr())) + looptoken = LoopToken() + old_mc_mc = self.cpu.assembler.mc._mc + self.cpu.compile_loop([base_v], ops, looptoken) + assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed + self.cpu.set_future_value_int(0, base_v.value) + self.cpu.execute_token(looptoken) + assert self.cpu.get_latest_value_int(0) == 1024 Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Tue Nov 24 15:35:31 2009 @@ -113,7 +113,7 @@ metainterp_sd.log("compiled new " + type) def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): - n = faildescr.get_index() + n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): show_loop(metainterp_sd) @@ -133,14 +133,7 @@ # ____________________________________________________________ class _DoneWithThisFrameDescr(AbstractFailDescr): - - def __init__(self, lst): - "NOT_RPYTHON" - self.index = len(lst) - lst.append(self) - - def get_index(self): - return self.index + pass class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): def handle_fail(self, metainterp_sd): @@ -173,18 +166,6 @@ raise metainterp_sd.ExitFrameWithExceptionRef(cpu, value) -done_with_this_frame_descrs = [] -done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid( - done_with_this_frame_descrs) -done_with_this_frame_descr_int = DoneWithThisFrameDescrInt( - done_with_this_frame_descrs) -done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef( - done_with_this_frame_descrs) -done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat( - done_with_this_frame_descrs) -exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef( - done_with_this_frame_descrs) - prebuiltNotSpecNode = NotSpecNode() class TerminatingLoopToken(LoopToken): @@ -194,22 +175,30 @@ self.specnodes = [prebuiltNotSpecNode]*nargs self.finishdescr = finishdescr -# pseudo loop tokens to make the life of optimize.py easier -loop_tokens_done_with_this_frame_int = [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ] -loop_tokens_done_with_this_frame_ref = [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ] -loop_tokens_done_with_this_frame_float = [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ] -loop_tokens_done_with_this_frame_void = [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ] -loop_tokens_exit_frame_with_exception_ref = [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ] +def make_done_loop_tokens(): + done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() + done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() + done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() + done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() + exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() + + # pseudo loop tokens to make the life of optimize.py easier + return {'loop_tokens_done_with_this_frame_int': [ + TerminatingLoopToken(1, done_with_this_frame_descr_int) + ], + 'loop_tokens_done_with_this_frame_ref': [ + TerminatingLoopToken(1, done_with_this_frame_descr_ref) + ], + 'loop_tokens_done_with_this_frame_float': [ + TerminatingLoopToken(1, done_with_this_frame_descr_float) + ], + 'loop_tokens_done_with_this_frame_void': [ + TerminatingLoopToken(0, done_with_this_frame_descr_void) + ], + 'loop_tokens_exit_frame_with_exception_ref': [ + TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) + ], + } class ResumeDescr(AbstractFailDescr): def __init__(self, original_greenkey): @@ -222,13 +211,6 @@ def __init__(self, metainterp_sd, original_greenkey): ResumeDescr.__init__(self, original_greenkey) self.metainterp_sd = metainterp_sd - self.index = -1 - - def get_index(self): - if self.index == -1: - globaldata = self.metainterp_sd.globaldata - self.index = globaldata.get_fail_descr_number(self) - return self.index def store_final_boxes(self, guard_op, boxes): guard_op.fail_args = boxes Modified: pypy/trunk/pypy/jit/metainterp/history.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/history.py (original) +++ pypy/trunk/pypy/jit/metainterp/history.py Tue Nov 24 15:35:31 2009 @@ -123,21 +123,16 @@ return '%r' % (self,) class AbstractFailDescr(AbstractDescr): + index = -1 - def get_index(self): - raise NotImplementedError def handle_fail(self, metainterp_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError class BasicFailDescr(AbstractFailDescr): - - def __init__(self, index=-1): - self.index = index - - def get_index(self): - return self.index + def __init__(self, identifier=None): + self.identifier = identifier # for testing class AbstractMethDescr(AbstractDescr): # the base class of the result of cpu.methdescrof() Modified: pypy/trunk/pypy/jit/metainterp/logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/logger.py Tue Nov 24 15:35:31 2009 @@ -90,8 +90,8 @@ if op.descr is not None: descr = op.descr if is_guard and self.guard_number: - assert isinstance(descr, AbstractFailDescr) - r = "" % descr.get_index() + index = self.metainterp_sd.cpu.get_fail_descr_number(descr) + r = "" % index else: r = self.repr_of_descr(descr) args += ', descr=' + r Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Nov 24 15:35:31 2009 @@ -1022,6 +1022,8 @@ self._addr2name_keys = [] self._addr2name_values = [] + self.__dict__.update(compile.make_done_loop_tokens()) + def _freeze_(self): return True @@ -1044,8 +1046,7 @@ else: self.num_green_args = 0 self.state = None - self.globaldata = MetaInterpGlobalData( - self, compile.done_with_this_frame_descrs) + self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" @@ -1136,11 +1137,10 @@ # ____________________________________________________________ class MetaInterpGlobalData(object): - def __init__(self, staticdata, prebuilt_fail_descr_list): + def __init__(self, staticdata): self.initialized = False self.indirectcall_dict = None self.addr2name = None - self.fail_descr_list = prebuilt_fail_descr_list[:] self.loopnumbering = 0 # state = staticdata.state @@ -1164,16 +1164,6 @@ cell.compiled_merge_points = [] return cell.compiled_merge_points - def get_fail_descr_number(self, descr): - assert isinstance(descr, history.AbstractFailDescr) - lst = self.fail_descr_list - n = len(lst) - lst.append(descr) - return n - - def get_fail_descr_from_number(self, n): - return self.fail_descr_list[n] - # ____________________________________________________________ class MetaInterp(object): @@ -1622,16 +1612,16 @@ if sd.result_type == 'void': assert exitbox is None exits = [] - loop_tokens = compile.loop_tokens_done_with_this_frame_void + loop_tokens = sd.loop_tokens_done_with_this_frame_void elif sd.result_type == 'int': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_int + loop_tokens = sd.loop_tokens_done_with_this_frame_int elif sd.result_type == 'ref': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_ref + loop_tokens = sd.loop_tokens_done_with_this_frame_ref elif sd.result_type == 'float': exits = [exitbox] - loop_tokens = compile.loop_tokens_done_with_this_frame_float + loop_tokens = sd.loop_tokens_done_with_this_frame_float else: assert False self.history.record(rop.JUMP, exits, None) @@ -1643,7 +1633,8 @@ self.gen_store_back_in_virtualizable() # temporarily put a JUMP to a pseudo-loop self.history.record(rop.JUMP, [valuebox], None) - loop_tokens = compile.loop_tokens_exit_frame_with_exception_ref + sd = self.staticdata + loop_tokens = sd.loop_tokens_exit_frame_with_exception_ref target_loop_token = compile.compile_new_bridge(self, loop_tokens, self.resumekey) assert target_loop_token is loop_tokens[0] Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Tue Nov 24 15:35:31 2009 @@ -103,22 +103,3 @@ assert loop_tokens == [loop_token] assert len(cpu.seen) == 0 assert staticdata.globaldata.loopnumbering == 2 - -def test_resumeguarddescr_get_index(): - from pypy.jit.metainterp.pyjitpl import MetaInterpGlobalData - - class FakeStaticData: - state = None - virtualizable_info = None - gd = MetaInterpGlobalData(FakeStaticData, []) - FakeStaticData.globaldata = gd - - rgd = ResumeGuardDescr(FakeStaticData, ()) - - fail_index = rgd.get_index() - fail_index1 = rgd.get_index() - - assert fail_index == fail_index1 - - assert gd.get_fail_descr_from_number(fail_index) is rgd - Modified: pypy/trunk/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_logger.py Tue Nov 24 15:35:31 2009 @@ -6,6 +6,8 @@ from StringIO import StringIO from pypy.jit.metainterp.test.test_optimizeopt import equaloplists from pypy.jit.metainterp.history import AbstractDescr, LoopToken, BasicFailDescr +from pypy.jit.backend.model import AbstractCPU + class Descr(AbstractDescr): pass @@ -36,8 +38,8 @@ def make_metainterp_sd(self): class FakeMetaInterpSd: - class cpu: - ts = self.ts + cpu = AbstractCPU() + cpu.ts = self.ts def get_name_from_address(self, addr): return 'Name' return FakeMetaInterpSd() @@ -122,7 +124,7 @@ pure_parse(output) def test_guard_descr(self): - namespace = {'fdescr': BasicFailDescr(4)} + namespace = {'fdescr': BasicFailDescr()} inp = ''' [i0] guard_true(i0, descr=fdescr) [i0] @@ -130,15 +132,14 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" + assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" pure_parse(output) - def boom(): - raise Exception - namespace['fdescr'].get_index = boom logger = Logger(self.make_metainterp_sd(), guard_number=False) output = logger.log_loop(loop) - assert output.splitlines()[-1].startswith("guard_true(i0, descr=<") + lastline = output.splitlines()[-1] + assert lastline.startswith("guard_true(i0, descr=<") + assert not lastline.startswith("guard_true(i0, descr=' % (self.w_type, self.w_value) + def raiseFailedToImplement(): raise FailedToImplement From arigo at codespeak.net Tue Nov 24 15:37:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 15:37:40 +0100 (CET) Subject: [pypy-svn] r69582 - in pypy/trunk/pypy/objspace/flow: . test Message-ID: <20091124143740.4400B16800D@codespeak.net> Author: arigo Date: Tue Nov 24 15:37:39 2009 New Revision: 69582 Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py pypy/trunk/pypy/objspace/flow/test/test_objspace.py Log: A test and fix in the flow object space. This is a situation in which we want to get an error, but the error was swallowed. Modified: pypy/trunk/pypy/objspace/flow/flowcontext.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/flowcontext.py (original) +++ pypy/trunk/pypy/objspace/flow/flowcontext.py Tue Nov 24 15:37:39 2009 @@ -399,3 +399,10 @@ return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) def argument_factory(self, *args): return ArgumentsForTranslation(self.space, *args) + + def handle_operation_error(self, ec, operr, *args, **kwds): + # see test_propagate_attribute_error for why this is here + if isinstance(operr, OperationThatShouldNotBePropagatedError): + raise operr + return pyframe.PyFrame.handle_operation_error(self, ec, operr, + *args, **kwds) Modified: pypy/trunk/pypy/objspace/flow/test/test_objspace.py ============================================================================== --- pypy/trunk/pypy/objspace/flow/test/test_objspace.py (original) +++ pypy/trunk/pypy/objspace/flow/test/test_objspace.py Tue Nov 24 15:37:39 2009 @@ -883,6 +883,15 @@ return s[-3:] check(f3, 'llo') + def test_propagate_attribute_error(self): + def f(x): + try: + "".invalid + finally: + if x and 0: + raise TypeError() + py.test.raises(Exception, self.codetest, f) + class TestFlowObjSpaceDelay(Base): def setup_class(cls): From pedronis at codespeak.net Tue Nov 24 15:47:21 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 15:47:21 +0100 (CET) Subject: [pypy-svn] r69583 - pypy/extradoc/planning Message-ID: <20091124144721.4F6BD168012@codespeak.net> Author: pedronis Date: Tue Nov 24 15:47:20 2009 New Revision: 69583 Modified: pypy/extradoc/planning/jit.txt Log: mark some stuff as done Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 24 15:47:20 2009 @@ -1,7 +1,7 @@ TASKS ----- -- compress the virtuals part of resume data more +- compress the virtuals part of resume data more DONE - sort out a benchmark infrastructure. graphs! @@ -9,7 +9,7 @@ - directly call assembler for residual portal calls -- make the assembler produced by generate_failure smaller +- make the assembler produced by generate_failure smaller DONE - jit/asmgcc + threads? [DONE probably. Needs a bit more testing.] From pedronis at codespeak.net Tue Nov 24 15:48:52 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 24 Nov 2009 15:48:52 +0100 (CET) Subject: [pypy-svn] r69584 - pypy/extradoc/planning Message-ID: <20091124144852.37648168012@codespeak.net> Author: pedronis Date: Tue Nov 24 15:48:51 2009 New Revision: 69584 Modified: pypy/extradoc/planning/jit.txt Log: update Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 24 15:48:51 2009 @@ -1,22 +1,22 @@ TASKS ----- -- compress the virtuals part of resume data more DONE - - sort out a benchmark infrastructure. graphs! - improve on predictability: don't trace into signals ... but produce just a conditional call - directly call assembler for residual portal calls -- make the assembler produced by generate_failure smaller DONE - - jit/asmgcc + threads? [DONE probably. Needs a bit more testing.] - think about code memory management - forcing virtualizables should only force fields affected, not everything +- share lists of field descrs + +- compress resume data (?) + - think out looking into functions or not, based on arguments, for example contains__Tuple should be unrolled if tuple is of constant length @@ -25,6 +25,7 @@ engine would do it) and not spend all the time in _trace_and_drag_out_of_nursery + Python interpreter: - goal: on average <=5 guards per original bytecode From fijal at codespeak.net Tue Nov 24 15:51:26 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 15:51:26 +0100 (CET) Subject: [pypy-svn] r69585 - pypy/extradoc/planning Message-ID: <20091124145126.8ACA1168012@codespeak.net> Author: fijal Date: Tue Nov 24 15:51:26 2009 New Revision: 69585 Modified: pypy/extradoc/planning/jit.txt Log: Two issues with interpreter Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Tue Nov 24 15:51:26 2009 @@ -41,6 +41,12 @@ - we would like probably enabling sys.settrace() to leave the jit instead of just being ignored +- list copy as ll operations, to avoid calling write barriers again and + again while growing lists (lists of pointers are extra slow on pypy) + +- improve ''.join and u''.join by using stringbuilder, enable realloc + for hybrid GC (on stringbuilder branch so far). + META ----- From fijal at codespeak.net Tue Nov 24 16:01:59 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 16:01:59 +0100 (CET) Subject: [pypy-svn] r69586 - pypy/trunk/pypy/objspace/std Message-ID: <20091124150159.7E918168008@codespeak.net> Author: fijal Date: Tue Nov 24 16:01:58 2009 New Revision: 69586 Modified: pypy/trunk/pypy/objspace/std/stringobject.py Log: Fix the fix, fixes tests. We can't freely pass w_list around, since multiple calls to __iter__ might do different things. Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Tue Nov 24 16:01:58 2009 @@ -361,7 +361,7 @@ if not space.is_true(space.isinstance(w_s, space.w_str)): if space.is_true(space.isinstance(w_s, space.w_unicode)): w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", w_list) + return space.call_method(w_u, "join", space.newlist(list_w)) raise OperationError( space.w_TypeError, space.wrap("sequence item %d: expected string, %s " From fijal at codespeak.net Tue Nov 24 16:05:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 16:05:14 +0100 (CET) Subject: [pypy-svn] r69587 - pypy/trunk/pypy/objspace/std Message-ID: <20091124150514.00A19168008@codespeak.net> Author: fijal Date: Tue Nov 24 16:05:14 2009 New Revision: 69587 Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py Log: Port the fix from stringbuilder branch. unicode join should use unpackiterable Modified: pypy/trunk/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/unicodeobject.py (original) +++ pypy/trunk/pypy/objspace/std/unicodeobject.py Tue Nov 24 16:05:14 2009 @@ -174,7 +174,7 @@ return space.newbool(container.find(item) != -1) def unicode_join__Unicode_ANY(space, w_self, w_list): - l = space.listview(w_list) + l = space.unpackiterable(w_list) delim = w_self._value totlen = 0 if len(l) == 0: From arigo at codespeak.net Tue Nov 24 16:08:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 24 Nov 2009 16:08:06 +0100 (CET) Subject: [pypy-svn] r69588 - pypy/trunk/pypy/objspace/std Message-ID: <20091124150806.0CE91168008@codespeak.net> Author: arigo Date: Tue Nov 24 16:08:06 2009 New Revision: 69588 Modified: pypy/trunk/pypy/objspace/std/stringobject.py Log: Fix the fix the fix. Please go on the branch. Seriously. Modified: pypy/trunk/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/stringobject.py Tue Nov 24 16:08:06 2009 @@ -360,8 +360,11 @@ w_s = list_w[i] if not space.is_true(space.isinstance(w_s, space.w_str)): if space.is_true(space.isinstance(w_s, space.w_unicode)): + # we need to rebuild w_list here, because the original + # w_list might be an iterable which we already consumed + w_list = space.newlist(list_w) w_u = space.call_function(space.w_unicode, w_self) - return space.call_method(w_u, "join", space.newlist(list_w)) + return space.call_method(w_u, "join", w_list) raise OperationError( space.w_TypeError, space.wrap("sequence item %d: expected string, %s " From afa at codespeak.net Tue Nov 24 16:15:58 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 16:15:58 +0100 (CET) Subject: [pypy-svn] r69589 - pypy/trunk/pypy/module/oracle Message-ID: <20091124151558.AC455168012@codespeak.net> Author: afa Date: Tue Nov 24 16:15:57 2009 New Revision: 69589 Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: The module translates now. See how many casts to lltype.Signed were needed... Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Tue Nov 24 16:15:57 2009 @@ -369,7 +369,7 @@ self.environment.checkForError( status, "Cursor_GetStatementType()") - self.statementType = attrptr[0] + self.statementType = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -520,29 +520,29 @@ self.environment.checkForError( status, "Cursor_ItemDescription(): nullable") - nullable = attrptr[0] != 0 + nullable = rffi.cast(lltype.Signed, attrptr[0]) != 0 finally: lltype.free(attrptr, flavor='raw') # set display size based on data type - if varType == interp_variable.VT_String: + if varType is interp_variable.VT_String: displaySize = internalSize - elif varType == interp_variable.VT_NationalCharString: + elif varType is interp_variable.VT_NationalCharString: displaySize = internalSize / 2 - elif varType == interp_variable.VT_Binary: + elif varType is interp_variable.VT_Binary: displaySize = internalSize - elif varType == interp_variable.VT_FixedChar: + elif varType is interp_variable.VT_FixedChar: displaySize = internalSize - elif varType == interp_variable.VT_FixedNationalChar: + elif varType is interp_variable.VT_FixedNationalChar: displaySize = internalSize / 2 - elif varType == interp_variable.VT_Float: + elif varType is interp_variable.VT_Float: if precision: displaySize = precision + 1 if scale > 0: displaySize += scale + 1 else: displaySize = 127 - elif varType == interp_variable.VT_DateTime: + elif varType is interp_variable.VT_DateTime: displaySize = 23 else: displaySize = -1 @@ -706,7 +706,7 @@ self.environment.checkForError( status, "Cursor_SetRowCount()") - self.rowCount = attrptr[0] + self.rowCount = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') else: @@ -850,7 +850,8 @@ self.environment.checkForError( status, "Cursor_InternalFetch(): row count") - self.actualRows = attrptr[0] - self.rowCount + self.actualRows = (rffi.cast(lltype.Signed, attrptr[0]) + - self.rowCount) self.rowNum = 0 finally: lltype.free(attrptr, flavor='raw') @@ -932,10 +933,12 @@ names_w = [] # process the bind information returned for i in range(foundElementsPtr[0]): - if duplicate[i]: + if rffi.cast(lltype.Signed, duplicate[i]): continue names_w.append( - w_string(space, bindNames[i], bindNameLengths[i])) + w_string(space, + bindNames[i], + rffi.cast(lltype.Signed, bindNameLengths[i]))) return 0, names_w finally: Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 16:15:57 2009 @@ -56,7 +56,7 @@ cursor.environment.checkForError( status, "Variable_Define(): data size") - sizeFromOracle = attrptr[0] + sizeFromOracle = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -293,7 +293,9 @@ lltype.free(bindHandlePtr, flavor='raw') def isNull(self, pos): - return self.indicator[pos] == roci.OCI_IND_NULL + return (rffi.cast(lltype.Signed, self.indicator[pos]) + == + rffi.cast(lltype.Signed, roci.OCI_IND_NULL)) def verifyFetch(self, space, pos): # Verifies that truncation or other problems did not take place on @@ -305,7 +307,8 @@ error.code = self.returnCode[pos] error.message = space.wrap( "column at array pos %d fetched with error: %d" % - (pos, self.returnCode[pos])) + (pos, + rffi.cast(lltype.Signed, self.returnCode[pos]))) w_error = get(space).w_DatabaseError raise OperationError(get(space).w_DatabaseError, @@ -423,7 +426,7 @@ def getValueProc(self, space, pos): offset = pos * self.bufferSize - length = self.actualLength[pos] + length = rffi.cast(lltype.Signed, self.actualLength[pos]) l = [] i = 0 @@ -574,7 +577,7 @@ environment.checkForError( status, "NumberVar_PreDefine(): scale") - scale = attrptr[0] + scale = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -590,7 +593,7 @@ environment.checkForError( status, "NumberVar_PreDefine(): precision") - precision = attrptr[0] + precision = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -885,7 +888,7 @@ environment.checkForError( status, "Variable_TypeByOracleDescriptor(): data type") - dataType = attrptr[0] + dataType = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') @@ -904,7 +907,7 @@ environment.checkForError( status, "Variable_TypeByOracleDescriptor(): charset form") - charsetForm = attrptr[0] + charsetForm = rffi.cast(lltype.Signed, attrptr[0]) finally: lltype.free(attrptr, flavor='raw') From afa at codespeak.net Tue Nov 24 16:56:24 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 16:56:24 +0100 (CET) Subject: [pypy-svn] r69590 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091124155624.795F3168008@codespeak.net> Author: afa Date: Tue Nov 24 16:56:24 2009 New Revision: 69590 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Log: Implement ref cursors retrieved from a PL/SQL package. Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Tue Nov 24 16:56:24 2009 @@ -14,6 +14,7 @@ 'LONG_STRING': 'interp_variable.VT_LongString', 'LONG_BINARY': 'interp_variable.VT_LongBinary', 'FIXED_CHAR': 'interp_variable.VT_FixedChar', + 'CURSOR': 'interp_variable.VT_Cursor', 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', 'Date': 'interp_error.get(space).w_DateType', Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 24 16:56:24 2009 @@ -75,7 +75,7 @@ pass def desc_str(self): - return self.message + return self.w_message W_Error.typedef = TypeDef( 'Error', Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 16:56:24 2009 @@ -835,6 +835,12 @@ i) dataptr[0] = tempCursor.handle + def getValueProc(self, space, pos): + from pypy.module.oracle import interp_cursor + w_cursor = self.cursors_w[pos] + space.interp_w(interp_cursor.W_Cursor, w_cursor).statementType = -1 + return w_cursor + def setValueProc(self, space, pos, w_value): from pypy.module.oracle import interp_cursor w_CursorType = space.gettypeobject(interp_cursor.W_Cursor.typedef) Modified: pypy/trunk/pypy/module/oracle/test/test_cursorvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_cursorvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_cursorvar.py Tue Nov 24 16:56:24 2009 @@ -17,3 +17,29 @@ [('NUMBERCOL', oracle.NUMBER, 127, 2, 0, -127, 1)]) data = cursor.fetchall() assert data == [(1,)] + + def test_bind_frompackage(self): + cur = self.cnx.cursor() + # create package + try: + cur.execute("drop package pypy_temp_cursorpkg") + except oracle.DatabaseError: + pass + cur.execute(""" + create package pypy_temp_cursorpkg as + type refcur is ref cursor; + procedure test_cursor(cur out refcur); + end;""") + cur.execute(""" + create package body pypy_temp_cursorpkg as + procedure test_cursor(cur out refcur) is + begin + open cur for + select level-1 intcol + from dual connect by level-1<42; + end; + end;""") + cursor = self.cnx.cursor() + cur.callproc("pypy_temp_cursorpkg.test_cursor", (cursor,)) + data = cursor.fetchall() + assert data == [(x,) for x in range(42)] From fijal at codespeak.net Tue Nov 24 17:22:10 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 17:22:10 +0100 (CET) Subject: [pypy-svn] r69591 - pypy/branch/stringbuilder/pypy/objspace/std/test Message-ID: <20091124162210.79F35168008@codespeak.net> Author: fijal Date: Tue Nov 24 17:22:09 2009 New Revision: 69591 Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py Log: A test crashing inlinedict Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py Tue Nov 24 17:22:09 2009 @@ -125,3 +125,21 @@ assert self.space.int_w(w_a.content['x']) == 12 assert self.space.int_w(w_a.content['y']) == 13 +class AppTestSharingDict(object): + def setup_class(cls): + cls.space = gettestobjspace(withsharingdict=True, withinlineddict=True) + + def test_bug(self): + skip("Fails") + class X(object): + __slots__ = 'a', 'b', 'c' + + class Y(X): + def __setattr__(self, name, value): + d = object.__getattribute__(self, '__dict__') + object.__setattr__(self, '__dict__', d) + return object.__setattr__(self, name, value) + + x = Y() + x.number = 42 + assert x.number == 42 From fijal at codespeak.net Tue Nov 24 17:35:23 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 17:35:23 +0100 (CET) Subject: [pypy-svn] r69592 - pypy/branch/stringbuilder/pypy/objspace/std/test Message-ID: <20091124163523.35597168008@codespeak.net> Author: fijal Date: Tue Nov 24 17:35:22 2009 New Revision: 69592 Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py Log: Revert, this test belongs to trunk Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/test/test_inlinedict.py Tue Nov 24 17:35:22 2009 @@ -124,22 +124,3 @@ assert w_a.w__dict__ is None assert self.space.int_w(w_a.content['x']) == 12 assert self.space.int_w(w_a.content['y']) == 13 - -class AppTestSharingDict(object): - def setup_class(cls): - cls.space = gettestobjspace(withsharingdict=True, withinlineddict=True) - - def test_bug(self): - skip("Fails") - class X(object): - __slots__ = 'a', 'b', 'c' - - class Y(X): - def __setattr__(self, name, value): - d = object.__getattribute__(self, '__dict__') - object.__setattr__(self, '__dict__', d) - return object.__setattr__(self, name, value) - - x = Y() - x.number = 42 - assert x.number == 42 From fijal at codespeak.net Tue Nov 24 17:38:34 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 17:38:34 +0100 (CET) Subject: [pypy-svn] r69593 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091124163834.B30A7168008@codespeak.net> Author: fijal Date: Tue Nov 24 17:38:33 2009 New Revision: 69593 Modified: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Log: XFail this test on trunk Modified: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Tue Nov 24 17:38:33 2009 @@ -1,3 +1,4 @@ +import py from pypy.conftest import gettestobjspace from pypy.objspace.std.inlinedict import make_inlinedict_mixin from pypy.objspace.std.dictmultiobject import StrDictImplementation @@ -125,3 +126,21 @@ assert self.space.int_w(w_a.content['x']) == 12 assert self.space.int_w(w_a.content['y']) == 13 +class AppTestSharingDict(object): + def setup_class(cls): + cls.space = gettestobjspace(withsharingdict=True, withinlineddict=True) + + @py.test.mark.xfail() + def test_bug(self): + class X(object): + __slots__ = 'a', 'b', 'c' + + class Y(X): + def __setattr__(self, name, value): + d = object.__getattribute__(self, '__dict__') + object.__setattr__(self, '__dict__', d) + return object.__setattr__(self, name, value) + + x = Y() + x.number = 42 + assert x.number == 42 From fijal at codespeak.net Tue Nov 24 17:55:09 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 17:55:09 +0100 (CET) Subject: [pypy-svn] r69594 - pypy/branch/stringbuilder/pypy/interpreter/test Message-ID: <20091124165509.E8905168012@codespeak.net> Author: fijal Date: Tue Nov 24 17:55:09 2009 New Revision: 69594 Modified: pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py Log: Fix and improve tests Modified: pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py Tue Nov 24 17:55:09 2009 @@ -62,19 +62,16 @@ w = self.space.wrap l = [w(1), w(2), w(3), w(4)] w_l = self.space.newtuple(l) - assert self.space.fixedview(w_l) == l - assert self.space.fixedview(w_l, 4) == l + assert self.space.fixedview(w_l) is l + assert self.space.fixedview(w_l, 4) is l raises(ValueError, self.space.fixedview, w_l, 3) raises(ValueError, self.space.fixedview, w_l, 5) def test_listview(self): w = self.space.wrap l = [w(1), w(2), w(3), w(4)] - w_l = self.space.newtuple(l) - assert self.space.listview(w_l) == l - assert self.space.listview(w_l, 4) == l - raises(ValueError, self.space.listview, w_l, 3) - raises(ValueError, self.space.listview, w_l, 5) + w_l = self.space.newlist(l) + assert self.space.listview(w_l) is l def test_exception_match(self): assert self.space.exception_match(self.space.w_ValueError, From antocuni at codespeak.net Tue Nov 24 18:08:40 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 24 Nov 2009 18:08:40 +0100 (CET) Subject: [pypy-svn] r69595 - pypy/trunk/pypy/jit/backend/cli Message-ID: <20091124170840.29856168012@codespeak.net> Author: antocuni Date: Tue Nov 24 18:08:39 2009 New Revision: 69595 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/cli/runner.py Log: consider 'holes' in fail_args. This fixes the tests broken by the merging of the compress-virtuals-resumedata branch Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Tue Nov 24 18:08:39 2009 @@ -459,7 +459,8 @@ # store the latest values i = 0 for box in args: - self.store_inputarg(i, box.type, box.getCliType(self), box) + if box is not None: + self.store_inputarg(i, box.type, box.getCliType(self), box) i+=1 def emit_guard_bool(self, op, opcode): Modified: pypy/trunk/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/runner.py (original) +++ pypy/trunk/pypy/jit/backend/cli/runner.py Tue Nov 24 18:08:39 2009 @@ -45,6 +45,7 @@ def __init__(self, rtyper, stats, translate_support_code=False, mixlevelann=None, gcdescr=None): + model.AbstractCPU.__init__(self) self.rtyper = rtyper if rtyper: assert rtyper.type_system.name == "ootypesystem" From afa at codespeak.net Tue Nov 24 18:30:44 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 18:30:44 +0100 (CET) Subject: [pypy-svn] r69596 - pypy/trunk/pypy/module/oracle Message-ID: <20091124173044.1E79E168006@codespeak.net> Author: afa Date: Tue Nov 24 18:30:43 2009 New Revision: 69596 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/transform.py Log: Add the TIMESTAMP datatype Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Tue Nov 24 18:30:43 2009 @@ -10,6 +10,7 @@ 'STRING': 'interp_variable.VT_String', 'DATETIME': 'interp_variable.VT_DateTime', 'DATE': 'interp_variable.VT_Date', + 'TIMESTAMP': 'interp_variable.VT_Timestamp', 'BINARY': 'interp_variable.VT_Binary', 'LONG_STRING': 'interp_variable.VT_LongString', 'LONG_BINARY': 'interp_variable.VT_LongBinary', Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Tue Nov 24 18:30:43 2009 @@ -82,6 +82,6 @@ error = W_Error(self.space, self, context, 0) error.code = 0 - error.message = self.space.wrap("Invalid handle!") + error.w_message = self.space.wrap("Invalid handle!") raise OperationError(get(self.space).w_DatabaseError, self.space.wrap(error)) Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 24 18:30:43 2009 @@ -20,6 +20,7 @@ self.w_NotSupportedError = get('NotSupportedError') self.w_IntegrityError = get('IntegrityError') self.w_InternalError = get('InternalError') + self.w_DataError = get('DataError') self.w_Variable = get('Variable') w_import = space.builtin.get('__import__') Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 18:30:43 2009 @@ -799,7 +799,85 @@ return transform.OracleDateToPythonDate(self.environment, dataptr) class VT_Timestamp(W_Variable): - pass + oracleType = roci.SQLT_TIMESTAMP + size = rffi.sizeof(roci.OCIDateTime_p) + + def initialize(self, space, cursor): + # initialize the Timestamp descriptors + for i in range(self.allocatedElements): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + i) + status = roci.OCIDescriptorAlloc( + self.environment.handle, + dataptr, + roci.OCI_DTYPE_TIMESTAMP, + 0, None) + self.environment.checkForError( + status, "TimestampVar_Initialize()") + + def finalize(self): + dataptr = rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data) + for i in range(self.allocatedElements): + if dataptr[i]: + roci.OCIDescriptorFree( + dataptr[i], roci.OCI_DTYPE_TIMESTAMP) + + def getValueProc(self, space, pos): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + pos) + + return transform.OracleTimestampToPythonDate( + self.environment, dataptr) + + def setValueProc(self, space, pos, w_value): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + pos) + + # make sure a timestamp is being bound + if not space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): + raise OperationError( + space.w_TypeError, + space.wrap("expecting timestamp data")) + + year = space.int_w(space.getattr(w_value, space.wrap('year'))) + month = space.int_w(space.getattr(w_value, space.wrap('month'))) + day = space.int_w(space.getattr(w_value, space.wrap('day'))) + hour = space.int_w(space.getattr(w_value, space.wrap('hour'))) + minute = space.int_w(space.getattr(w_value, space.wrap('minute'))) + second = space.int_w(space.getattr(w_value, space.wrap('second'))) + microsecond = space.int_w(space.getattr(w_value, space.wrap('microsecond'))) + + status = roci.OCIDateTimeConstruct( + self.environment.handle, + self.environment.errorHandle, + dataptr[0], + year, month, day, hour, minute, second, microsecond * 1000, + None, 0) + + self.environment.checkForError( + status, "TimestampVar_SetValue(): create structure") + + validptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') + try: + status = roci.OCIDateTimeCheck( + self.environment.handle, + self.environment.errorHandle, + dataptr[0], + validptr); + self.environment.checkForError( + status, + "TimestampVar_SetValue()") + valid = rffi.cast(lltype.Signed, validptr[0]) + finally: + lltype.free(validptr, flavor='raw') + + if valid != 0: + raise OperationError( + get(space).w_DataError, + space.wrap("invalid date")) class VT_Interval(W_Variable): pass Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Tue Nov 24 18:30:43 2009 @@ -45,12 +45,14 @@ ('OCIDateTime', OCITime), ]) + OCIDateTime_p = platform.SimpleType('OCIDateTime*', rffi.VOIDP) + constants = ''' OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV - OCI_DTYPE_PARAM + OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT @@ -165,6 +167,15 @@ OCIError], # errhp sword) +OCIDescriptorAlloc = external( + 'OCIDescriptorAlloc', + [dvoidp, # parenth + dvoidpp, # descpp + ub4, # type + size_t, # xtramem_sz + dvoidp], # usrmempp + sword) + OCIDescriptorFree = external( 'OCIDescriptorFree', [dvoidp, # descp @@ -336,6 +347,53 @@ ub4], # type sword) +# OCI Date, Datetime, and Interval Functions + +OCIDateTimeCheck = external( + 'OCIDateTimeCheck', + [dvoidp, # hndl + OCIError, # err + OCIDateTime_p, # date + Ptr(ub4)], # valid + sword) + +OCIDateTimeConstruct = external( + 'OCIDateTimeConstruct', + [dvoidp, # hndl + OCIError, # errhp + OCIDateTime_p, # datetime + sb2, # year + ub1, # month + ub1, # day + ub1, # hour + ub1, # min + ub1, # src + ub4, # fsec + oratext, # timezone + size_t], # timezone_length + sword) + +OCIDateTimeGetDate = external( + 'OCIDateTimeGetDate', + [dvoidp, # hndl + OCIError, # errhp + OCIDateTime_p, # datetime + Ptr(sb2), # year + Ptr(ub1), # month + Ptr(ub1)], # day + sword) + +OCIDateTimeGetTime = external( + 'OCIDateTimeGetTime', + [dvoidp, # hndl + OCIError, # errhp + OCIDateTime_p, # datetime + Ptr(ub1), # hour + Ptr(ub1), # minute + Ptr(ub1), # second + Ptr(ub4)], # fsec + sword) + # OCI Number Functions OCINumberFromInt = external( Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Tue Nov 24 18:30:43 2009 @@ -53,6 +53,52 @@ w(time.c_OCITimeMI), w(time.c_OCITimeSS)) +def OracleTimestampToPythonDate(environment, valueptr): + yearptr = lltype.malloc(roci.Ptr(roci.sb2).TO, 1, flavor='raw') + monthptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + dayptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + hourptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + minuteptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + secondptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, flavor='raw') + fsecondptr = lltype.malloc(roci.Ptr(roci.ub4).TO, 1, flavor='raw') + + try: + status = roci.OCIDateTimeGetDate( + environment.handle, environment.errorHandle, + valueptr[0], + yearptr, monthptr, dayptr) + + environment.checkForError( + status, "OracleTimestampToPythonDate(): date portion") + + status = roci.OCIDateTimeGetTime( + environment.handle, environment.errorHandle, + valueptr[0], + hourptr, minuteptr, secondptr, fsecondptr) + + environment.checkForError( + status, "OracleTimestampToPythonDate(): time portion") + + space = environment.space + w = space.wrap + w_datetime = space.getattr( + space.getbuiltinmodule('datetime'), + w('datetime')) + + return space.call_function( + w_datetime, + w(yearptr[0]), w(monthptr[0]), w(dayptr[0]), + w(hourptr[0]), w(minuteptr[0]), w(secondptr[0]), + w(fsecondptr[0] / 1000)) + finally: + lltype.free(yearptr, flavor='raw') + lltype.free(monthptr, flavor='raw') + lltype.free(dayptr, flavor='raw') + lltype.free(hourptr, flavor='raw') + lltype.free(minuteptr, flavor='raw') + lltype.free(secondptr, flavor='raw') + lltype.free(fsecondptr, flavor='raw') + def DecimalToFormatAndText(environment, w_value): space = environment.space w_tuple_value = space.call_method(w_value, "as_tuple") From fijal at codespeak.net Tue Nov 24 19:04:26 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 19:04:26 +0100 (CET) Subject: [pypy-svn] r69608 - in pypy/branch/stringbuilder/pypy: interpreter objspace/std objspace/std/test Message-ID: <20091124180426.E2C28168008@codespeak.net> Author: fijal Date: Tue Nov 24 19:04:26 2009 New Revision: 69608 Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py pypy/branch/stringbuilder/pypy/objspace/std/objspace.py pypy/branch/stringbuilder/pypy/objspace/std/test/test_iterobject.py Log: A test and a fix Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Tue Nov 24 19:04:26 2009 @@ -671,7 +671,8 @@ def fixedview(self, w_iterable, expected_lenght=-1): """ A fixed list view of w_iterable. Don't modify the result """ - return make_sure_not_resized(self.unpackiterable(w_iterable)[:]) + return make_sure_not_resized(self.unpackiterable(w_iterable, + expected_lenght)[:]) def listview(self, w_iterable): """ A non-fixed view of w_iterable. Don't modify the result Modified: pypy/branch/stringbuilder/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/objspace.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/objspace.py Tue Nov 24 19:04:26 2009 @@ -651,7 +651,7 @@ elif isinstance(w_obj, W_ListObject): t = w_obj.wrappeditems[:] else: - return ObjSpace.fixedview(self, w_obj) + return ObjSpace.fixedview(self, w_obj, expected_length) if expected_length != -1 and len(t) != expected_length: raise UnpackValueError("Expected length %d, got %d" % (expected_length, len(t))) return t Modified: pypy/branch/stringbuilder/pypy/objspace/std/test/test_iterobject.py ============================================================================== --- pypy/branch/stringbuilder/pypy/objspace/std/test/test_iterobject.py (original) +++ pypy/branch/stringbuilder/pypy/objspace/std/test/test_iterobject.py Tue Nov 24 19:04:26 2009 @@ -57,6 +57,31 @@ raises(TypeError, iter, C()) + + def test_unpacking_iter(self): + class BasicIterClass: + def __init__(self, n): + self.n = n + self.i = 0 + def next(self): + res = self.i + if res >= self.n: + raise StopIteration + self.i = res + 1 + return res + + class IteratingSequenceClass: + def __init__(self, n): + self.n = n + def __iter__(self): + return BasicIterClass(self.n) + + try: + a, b = IteratingSequenceClass(3) + except ValueError: + pass + else: + raise Exception("did not raise") class AppTest_IterObject(object): def test_no_len_on_list_iter(self): From afa at codespeak.net Tue Nov 24 19:12:17 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 19:12:17 +0100 (CET) Subject: [pypy-svn] r69610 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091124181217.CE89E168008@codespeak.net> Author: afa Date: Tue Nov 24 19:12:17 2009 New Revision: 69610 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_datetimevar.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py pypy/trunk/pypy/module/oracle/transform.py Log: submit more tests, and implement INTERVAL datatype (mapped to datetime.timedelta) Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Tue Nov 24 19:12:17 2009 @@ -11,6 +11,7 @@ 'DATETIME': 'interp_variable.VT_DateTime', 'DATE': 'interp_variable.VT_Date', 'TIMESTAMP': 'interp_variable.VT_Timestamp', + 'INTERVAL': 'interp_variable.VT_Interval', 'BINARY': 'interp_variable.VT_Binary', 'LONG_STRING': 'interp_variable.VT_LongString', 'LONG_BINARY': 'interp_variable.VT_LongBinary', Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Tue Nov 24 19:12:17 2009 @@ -31,6 +31,7 @@ [space.wrap('datetime')])) self.w_DateTimeType = space.getattr(w_datetime, space.wrap("datetime")) self.w_DateType = space.getattr(w_datetime, space.wrap("date")) + self.w_TimedeltaType = space.getattr(w_datetime, space.wrap("timedelta")) from pypy.module.oracle.interp_variable import all_variable_types self.variableTypeByPythonType = {} Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 19:12:17 2009 @@ -880,7 +880,67 @@ space.wrap("invalid date")) class VT_Interval(W_Variable): - pass + oracleType = roci.SQLT_INTERVAL_DS + size = rffi.sizeof(roci.OCIInterval_p) + + def initialize(self, space, cursor): + # initialize the Timestamp descriptors + for i in range(self.allocatedElements): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + i) + status = roci.OCIDescriptorAlloc( + self.environment.handle, + dataptr, + roci.OCI_DTYPE_INTERVAL_DS, + 0, None) + self.environment.checkForError( + status, "TimestampVar_Initialize()") + + def finalize(self): + dataptr = rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data) + for i in range(self.allocatedElements): + if dataptr[i]: + roci.OCIDescriptorFree( + dataptr[i], roci.OCI_DTYPE_INTERVAL_DS) + + def getValueProc(self, space, pos): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + pos) + + return transform.OracleIntervalToPythonDelta( + self.environment, dataptr) + + def setValueProc(self, space, pos, w_value): + dataptr = rffi.ptradd( + rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + pos) + + if not space.is_true(space.isinstance(w_value, + get(space).w_TimedeltaType)): + raise OperationError( + space.w_TypeError, + space.wrap("expecting timedelta data")) + + days = space.int_w(space.getattr(w_value, space.wrap('days'))) + seconds = space.int_w(space.getattr(w_value, space.wrap('seconds'))) + hours = seconds / 3600 + seconds -= hours * 3600 + minutes = seconds / 60 + seconds -= minutes * 60 + microseconds = space.int_w( + space.getattr(w_value, space.wrap('microseconds'))) + + status = roci.OCIIntervalSetDaySecond( + self.environment.handle, + self.environment.errorHandle, + days, hours, minutes, seconds, microseconds, + dataptr[0]) + + self.environment.checkForError( + status, "IntervalVar_SetValue()") + class VT_CLOB(W_Variable): pass Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Tue Nov 24 19:12:17 2009 @@ -46,13 +46,14 @@ ]) OCIDateTime_p = platform.SimpleType('OCIDateTime*', rffi.VOIDP) + OCIInterval_p = platform.SimpleType('OCIInterval*', rffi.VOIDP) constants = ''' OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV - OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP + OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT @@ -394,6 +395,30 @@ Ptr(ub4)], # fsec sword) +OCIIntervalGetDaySecond = external( + 'OCIIntervalGetDaySecond', + [dvoidp, # hndl + OCIError, # errhp + Ptr(sb4), # dy + Ptr(sb4), # hr + Ptr(sb4), # mm + Ptr(sb4), # ss + Ptr(sb4), # fsec + OCIInterval_p], # result + sword) + +OCIIntervalSetDaySecond = external( + 'OCIIntervalSetDaySecond', + [dvoidp, # hndl + OCIError, # errhp + sb4, # dy + sb4, # hr + sb4, # mm + sb4, # ss + sb4, # fsec + OCIInterval_p], # result + sword) + # OCI Number Functions OCINumberFromInt = external( Modified: pypy/trunk/pypy/module/oracle/test/test_datetimevar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_datetimevar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_datetimevar.py Tue Nov 24 19:12:17 2009 @@ -49,3 +49,37 @@ datetime.datetime(2003, 7, 22, 4, 24, 32)] var.setvalue(0, values) assert var.getvalue() == values + +class AppTestTimestamp(OracleTestBase): + + def test_bind_timestamp(self): + import datetime + cur = self.cnx.cursor() + + value = datetime.datetime(2002, 12, 13, 9, 36, 15, 123000) + var = cur.var(oracle.TIMESTAMP) + var.setvalue(0, value) + assert var.getvalue() == value + + cur.setinputsizes(value=oracle.TIMESTAMP) + cur.execute("select :value from dual", + value=value) + data = cur.fetchall() + assert data == [(value,)] + +class AppTestInterval(OracleTestBase): + + def test_bind_interval(self): + import datetime + cur = self.cnx.cursor() + + value = datetime.timedelta(days=5, hours=6, minutes=10, seconds=18) + var = cur.var(oracle.INTERVAL) + var.setvalue(0, value) + assert var.getvalue() == value + + cur.setinputsizes(value=oracle.INTERVAL) + cur.execute("select :value from dual", + value=value) + data = cur.fetchall() + assert data == [(value,)] Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Tue Nov 24 19:12:17 2009 @@ -130,3 +130,20 @@ bigString="X" * 10000, output=output) assert output.getvalue() == "X" * 10000 +class AppTestLongVar(OracleTestBase): + + def test_long(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_table") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_table (longcol long)") + + cur.setinputsizes(p=oracle.LONG_STRING) + cur.execute("insert into pypy_temp_table values (:p)", + p="long string") + cur.execute("select * from pypy_temp_table") + data = cur.fetchall() + assert data == [("long string",)] + Modified: pypy/trunk/pypy/module/oracle/transform.py ============================================================================== --- pypy/trunk/pypy/module/oracle/transform.py (original) +++ pypy/trunk/pypy/module/oracle/transform.py Tue Nov 24 19:12:17 2009 @@ -99,6 +99,41 @@ lltype.free(secondptr, flavor='raw') lltype.free(fsecondptr, flavor='raw') +def OracleIntervalToPythonDelta(environment, valueptr): + daysptr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, flavor='raw') + hoursptr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, flavor='raw') + minutesptr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, flavor='raw') + secondsptr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, flavor='raw') + fsecondsptr = lltype.malloc(roci.Ptr(roci.sb4).TO, 1, flavor='raw') + + try: + status = roci.OCIIntervalGetDaySecond( + environment.handle, environment.errorHandle, + daysptr, hoursptr, minutesptr, secondsptr, fsecondsptr, + valueptr[0]) + environment.checkForError( + status, "OracleIntervalToPythonDelta()") + + space = environment.space + w = space.wrap + w_timedelta = space.getattr( + space.getbuiltinmodule('datetime'), + w('timedelta')) + + days = daysptr[0] + seconds = hoursptr[0] * 3600 + minutesptr[0] * 60 + secondsptr[0] + microseconds = fsecondsptr[0] / 1000 + + return space.call_function( + w_timedelta, + w(days), w(seconds), w(microseconds)) + finally: + lltype.free(daysptr, flavor='raw') + lltype.free(hoursptr, flavor='raw') + lltype.free(minutesptr, flavor='raw') + lltype.free(secondsptr, flavor='raw') + lltype.free(fsecondsptr, flavor='raw') + def DecimalToFormatAndText(environment, w_value): space = environment.space w_tuple_value = space.call_method(w_value, "as_tuple") From fijal at codespeak.net Tue Nov 24 19:14:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 19:14:54 +0100 (CET) Subject: [pypy-svn] r69611 - pypy/branch/stringbuilder/pypy/interpreter/test Message-ID: <20091124181454.720CC168008@codespeak.net> Author: fijal Date: Tue Nov 24 19:14:53 2009 New Revision: 69611 Modified: pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py Log: increase a bit test coverage here Modified: pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/test/test_objspace.py Tue Nov 24 19:14:53 2009 @@ -12,7 +12,7 @@ INT32_MAX = 2147483648 -class TestObjSpace: +class TestObjSpace: def test_newlist(self): w = self.space.wrap l = range(10) @@ -66,6 +66,8 @@ assert self.space.fixedview(w_l, 4) is l raises(ValueError, self.space.fixedview, w_l, 3) raises(ValueError, self.space.fixedview, w_l, 5) + w_d = self.space.newdict({}) + assert self.space.fixedview(w_d) == [] def test_listview(self): w = self.space.wrap From fijal at codespeak.net Tue Nov 24 19:15:06 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 19:15:06 +0100 (CET) Subject: [pypy-svn] r69612 - pypy/branch/stringbuilder/pypy/interpreter Message-ID: <20091124181506.6B1B7168008@codespeak.net> Author: fijal Date: Tue Nov 24 19:15:05 2009 New Revision: 69612 Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Log: typo, thanks amaury :) Modified: pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/stringbuilder/pypy/interpreter/baseobjspace.py Tue Nov 24 19:15:05 2009 @@ -668,11 +668,11 @@ (i, plural)) return items - def fixedview(self, w_iterable, expected_lenght=-1): + def fixedview(self, w_iterable, expected_length=-1): """ A fixed list view of w_iterable. Don't modify the result """ return make_sure_not_resized(self.unpackiterable(w_iterable, - expected_lenght)[:]) + expected_length)[:]) def listview(self, w_iterable): """ A non-fixed view of w_iterable. Don't modify the result From afa at codespeak.net Tue Nov 24 19:34:23 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 24 Nov 2009 19:34:23 +0100 (CET) Subject: [pypy-svn] r69613 - pypy/trunk/pypy/module/oracle Message-ID: <20091124183423.98C79168012@codespeak.net> Author: afa Date: Tue Nov 24 19:34:23 2009 New Revision: 69613 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: A bit of refactoring Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Tue Nov 24 19:34:23 2009 @@ -798,44 +798,45 @@ pos) return transform.OracleDateToPythonDate(self.environment, dataptr) -class VT_Timestamp(W_Variable): - oracleType = roci.SQLT_TIMESTAMP - size = rffi.sizeof(roci.OCIDateTime_p) +class W_VariableWithDescriptor(W_Variable): + size = rffi.sizeof(roci.dvoidp) def initialize(self, space, cursor): - # initialize the Timestamp descriptors + # initialize the descriptors for i in range(self.allocatedElements): dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + rffi.cast(roci.Ptr(roci.dvoidp), self.data), i) status = roci.OCIDescriptorAlloc( self.environment.handle, dataptr, - roci.OCI_DTYPE_TIMESTAMP, + self.descriptorType, 0, None) self.environment.checkForError( - status, "TimestampVar_Initialize()") + status, self.descriptionText + "_Initialize()") def finalize(self): - dataptr = rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data) + dataptr = rffi.cast(roci.Ptr(roci.dvoidp), self.data) for i in range(self.allocatedElements): if dataptr[i]: roci.OCIDescriptorFree( - dataptr[i], roci.OCI_DTYPE_TIMESTAMP) + dataptr[i], self.descriptorType) - def getValueProc(self, space, pos): - dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), + def getDataptr(self, pos): + return rffi.ptradd( + rffi.cast(roci.Ptr(roci.dvoidp), self.data), pos) +class VT_Timestamp(W_VariableWithDescriptor): + oracleType = roci.SQLT_TIMESTAMP + descriptorType = roci.OCI_DTYPE_TIMESTAMP + descriptionText = "TimestampVar" + + def getValueProc(self, space, pos): return transform.OracleTimestampToPythonDate( - self.environment, dataptr) + self.environment, self.getDataptr(pos)) def setValueProc(self, space, pos, w_value): - dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), - pos) - # make sure a timestamp is being bound if not space.is_true(space.isinstance(w_value, get(space).w_DateTimeType)): raise OperationError( @@ -853,7 +854,7 @@ status = roci.OCIDateTimeConstruct( self.environment.handle, self.environment.errorHandle, - dataptr[0], + self.getDataptr(pos)[0], year, month, day, hour, minute, second, microsecond * 1000, None, 0) @@ -865,7 +866,7 @@ status = roci.OCIDateTimeCheck( self.environment.handle, self.environment.errorHandle, - dataptr[0], + self.getDataptr(pos)[0], validptr); self.environment.checkForError( status, @@ -879,44 +880,16 @@ get(space).w_DataError, space.wrap("invalid date")) -class VT_Interval(W_Variable): +class VT_Interval(W_VariableWithDescriptor): oracleType = roci.SQLT_INTERVAL_DS - size = rffi.sizeof(roci.OCIInterval_p) - - def initialize(self, space, cursor): - # initialize the Timestamp descriptors - for i in range(self.allocatedElements): - dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), - i) - status = roci.OCIDescriptorAlloc( - self.environment.handle, - dataptr, - roci.OCI_DTYPE_INTERVAL_DS, - 0, None) - self.environment.checkForError( - status, "TimestampVar_Initialize()") - - def finalize(self): - dataptr = rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data) - for i in range(self.allocatedElements): - if dataptr[i]: - roci.OCIDescriptorFree( - dataptr[i], roci.OCI_DTYPE_INTERVAL_DS) + descriptorType = roci.OCI_DTYPE_INTERVAL_DS + descriptionText = "TimestampVar" def getValueProc(self, space, pos): - dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), - pos) - return transform.OracleIntervalToPythonDelta( - self.environment, dataptr) + self.environment, self.getDataptr(pos)) def setValueProc(self, space, pos, w_value): - dataptr = rffi.ptradd( - rffi.cast(roci.Ptr(roci.OCIDateTime_p), self.data), - pos) - if not space.is_true(space.isinstance(w_value, get(space).w_TimedeltaType)): raise OperationError( @@ -936,7 +909,7 @@ self.environment.handle, self.environment.errorHandle, days, hours, minutes, seconds, microseconds, - dataptr[0]) + self.getDataptr(pos)[0]) self.environment.checkForError( status, "IntervalVar_SetValue()") Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Tue Nov 24 19:34:23 2009 @@ -45,9 +45,6 @@ ('OCIDateTime', OCITime), ]) - OCIDateTime_p = platform.SimpleType('OCIDateTime*', rffi.VOIDP) - OCIInterval_p = platform.SimpleType('OCIInterval*', rffi.VOIDP) - constants = ''' OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA @@ -94,6 +91,8 @@ OCIBind = rffi.VOIDP OCIDefine = rffi.VOIDP OCISnapshot = rffi.VOIDP +OCIDateTime = rffi.VOIDP +OCIInterval = rffi.VOIDP void = lltype.Void dvoidp = rffi.VOIDP @@ -354,7 +353,7 @@ 'OCIDateTimeCheck', [dvoidp, # hndl OCIError, # err - OCIDateTime_p, # date + OCIDateTime, # date Ptr(ub4)], # valid sword) @@ -362,7 +361,7 @@ 'OCIDateTimeConstruct', [dvoidp, # hndl OCIError, # errhp - OCIDateTime_p, # datetime + OCIDateTime, # datetime sb2, # year ub1, # month ub1, # day @@ -378,7 +377,7 @@ 'OCIDateTimeGetDate', [dvoidp, # hndl OCIError, # errhp - OCIDateTime_p, # datetime + OCIDateTime, # datetime Ptr(sb2), # year Ptr(ub1), # month Ptr(ub1)], # day @@ -388,7 +387,7 @@ 'OCIDateTimeGetTime', [dvoidp, # hndl OCIError, # errhp - OCIDateTime_p, # datetime + OCIDateTime, # datetime Ptr(ub1), # hour Ptr(ub1), # minute Ptr(ub1), # second @@ -404,7 +403,7 @@ Ptr(sb4), # mm Ptr(sb4), # ss Ptr(sb4), # fsec - OCIInterval_p], # result + OCIInterval], # result sword) OCIIntervalSetDaySecond = external( @@ -416,7 +415,7 @@ sb4, # mm sb4, # ss sb4, # fsec - OCIInterval_p], # result + OCIInterval], # result sword) # OCI Number Functions From fijal at codespeak.net Tue Nov 24 19:52:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 19:52:36 +0100 (CET) Subject: [pypy-svn] r69614 - pypy/trunk/pypy/interpreter Message-ID: <20091124185236.5746116800B@codespeak.net> Author: fijal Date: Tue Nov 24 19:52:35 2009 New Revision: 69614 Modified: pypy/trunk/pypy/interpreter/executioncontext.py Log: Comment this out for now. It breaks a bit everything. I know this is broken somehow, but actually f_back_some can have f_forward set to not-None, in case we have any virtual frames left. In this case f_back_some should point to virtualizable above those virtuals. Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Tue Nov 24 19:52:35 2009 @@ -142,8 +142,8 @@ #assert frame is self.gettopframe() --- slowish if self.some_frame is frame: self.some_frame = frame.f_back_some - if self.some_frame and self.some_frame.f_forward: - raise Exception() + #if self.some_frame and self.some_frame.f_forward: + # raise Exception() else: f_back = frame.f_back() if f_back is not None: From fijal at codespeak.net Tue Nov 24 21:16:38 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 24 Nov 2009 21:16:38 +0100 (CET) Subject: [pypy-svn] r69615 - pypy/trunk/pypy/interpreter Message-ID: <20091124201638.168F0168015@codespeak.net> Author: fijal Date: Tue Nov 24 21:16:36 2009 New Revision: 69615 Modified: pypy/trunk/pypy/interpreter/executioncontext.py Log: Improve situation overall: * Change Exception into FrameChainMismatch to avoid confusion * Move from catching superset of cases into catching subset of cases. The check should really be "next frame on the stack should have f_forward set to None", not sure if it's relevant since we're moving away from it anyway. Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Tue Nov 24 21:16:36 2009 @@ -11,6 +11,9 @@ space.wrap(frame), space.wrap(event), w_arg) +class FrameChainMismatch(Exception): + pass + class ExecutionContext(object): """An ExecutionContext holds the state of an execution thread in the Python interpreter.""" @@ -142,8 +145,11 @@ #assert frame is self.gettopframe() --- slowish if self.some_frame is frame: self.some_frame = frame.f_back_some - #if self.some_frame and self.some_frame.f_forward: - # raise Exception() + if self.some_frame and self.some_frame.f_forward: + if self.some_frame.f_forward is frame: + # and anything that actually goes forward from there, + # but this is both inefficient and breaks tests + raise FrameChainMismatch() else: f_back = frame.f_back() if f_back is not None: From xoraxax at codespeak.net Wed Nov 25 00:02:12 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Wed, 25 Nov 2009 00:02:12 +0100 (CET) Subject: [pypy-svn] r69618 - pypy/trunk/pypy/lib Message-ID: <20091124230212.DE5CA16802D@codespeak.net> Author: xoraxax Date: Wed Nov 25 00:02:11 2009 New Revision: 69618 Modified: pypy/trunk/pypy/lib/msvcrt.py Log: Correctly raise an import error on platforms without msvcrt support. Modified: pypy/trunk/pypy/lib/msvcrt.py ============================================================================== --- pypy/trunk/pypy/lib/msvcrt.py (original) +++ pypy/trunk/pypy/lib/msvcrt.py Wed Nov 25 00:02:11 2009 @@ -9,7 +9,11 @@ import ctypes from ctypes_support import standard_c_lib as _c -open_osfhandle = _c._open_osfhandle +try: + open_osfhandle = _c._open_osfhandle +except AttributeError: # we are not on windows + raise ImportError + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int From santagada at codespeak.net Wed Nov 25 01:45:52 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 25 Nov 2009 01:45:52 +0100 (CET) Subject: [pypy-svn] r69619 - pypy/branch/force-arch-darwin/pypy/translator/platform/test Message-ID: <20091125004552.DF9BE16802A@codespeak.net> Author: santagada Date: Wed Nov 25 01:45:50 2009 New Revision: 69619 Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Log: don't try to compile a 64 bit on a 32 bit only machine Modified: pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py (original) +++ pypy/branch/force-arch-darwin/pypy/translator/platform/test/test_darwin.py Wed Nov 25 01:45:50 2009 @@ -66,9 +66,10 @@ executable = plat32.compile([cfile], eci) res = plat32.execute(executable) self.check_res(res, '0\n') - executable = plat64.compile([cfile], eci) - res = plat64.execute(executable) - self.check_res(res, '1\n') + if host_factory == Darwin_x86_64: + executable = plat64.compile([cfile], eci) + res = plat64.execute(executable) + self.check_res(res, '1\n') def test_longsize(self): if platform.machine() != 'i386': @@ -113,14 +114,14 @@ plat32.execute_makefile(mk) res = plat32.execute(tmpdir.join('test_int_size')) self.check_res(res, '0\n') - - tmpdir = udir.join('64_makefile' + self.__class__.__name__).ensure(dir=1) - cfile = tmpdir.join('test_int_size.c') - cfile.write(cfile_content) - mk = plat64.gen_makefile([cfile], ExternalCompilationInfo(), - path=tmpdir) - mk.write() - plat64.execute_makefile(mk) - res = plat64.execute(tmpdir.join('test_int_size')) - self.check_res(res, '1\n') + if host_factory == Darwin_x86_64: + tmpdir = udir.join('64_makefile' + self.__class__.__name__).ensure(dir=1) + cfile = tmpdir.join('test_int_size.c') + cfile.write(cfile_content) + mk = plat64.gen_makefile([cfile], ExternalCompilationInfo(), + path=tmpdir) + mk.write() + plat64.execute_makefile(mk) + res = plat64.execute(tmpdir.join('test_int_size')) + self.check_res(res, '1\n') From santagada at codespeak.net Wed Nov 25 01:50:26 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 25 Nov 2009 01:50:26 +0100 (CET) Subject: [pypy-svn] r69620 - pypy/branch/force-arch-darwin/pypy/jit/backend/x86 Message-ID: <20091125005026.1D0E916802A@codespeak.net> Author: santagada Date: Wed Nov 25 01:50:25 2009 New Revision: 69620 Modified: pypy/branch/force-arch-darwin/pypy/jit/backend/x86/viewcode.py Log: support mac osx in viewcode (would be cool to support using otool also) Modified: pypy/branch/force-arch-darwin/pypy/jit/backend/x86/viewcode.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/jit/backend/x86/viewcode.py (original) +++ pypy/branch/force-arch-darwin/pypy/jit/backend/x86/viewcode.py Wed Nov 25 01:50:25 2009 @@ -34,7 +34,11 @@ def machine_code_dump(data, originaddr): # the disassembler to use. 'objdump' writes GNU-style instructions. # 'ndisasm' would use Intel syntax, but you need to fix the output parsing. - objdump = 'objdump -b binary -m i386 --adjust-vma=%(origin)d -D %(file)s' + if sys.platform == 'darwin': + objdump_exe = 'gobjdump' + else: + objdump_exe = 'objdump' + objdump = objdump_exe + ' -b binary -m i386 --adjust-vma=%(origin)d -D %(file)s' # f = open(tmpfile, 'wb') f.write(data) From santagada at codespeak.net Wed Nov 25 01:52:22 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 25 Nov 2009 01:52:22 +0100 (CET) Subject: [pypy-svn] r69621 - pypy/branch/force-arch-darwin/pypy/module/_socket/test Message-ID: <20091125005222.09CE816802A@codespeak.net> Author: santagada Date: Wed Nov 25 01:52:22 2009 New Revision: 69621 Modified: pypy/branch/force-arch-darwin/pypy/module/_socket/test/test_sock_app.py Log: fix the test, mac os x has a really tight limit on AF_UNIX socket name Modified: pypy/branch/force-arch-darwin/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_socket/test/test_sock_app.py Wed Nov 25 01:52:22 2009 @@ -416,7 +416,8 @@ import _socket, os if not hasattr(_socket, 'AF_UNIX'): skip('AF_UNIX not supported.') - sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect') + # Mac OS X has a very small limit on socket names + sockpath = os.path.join(self.udir, 'app_unix') serversock = _socket.socket(_socket.AF_UNIX) serversock.bind(sockpath) From santagada at codespeak.net Wed Nov 25 01:54:59 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 25 Nov 2009 01:54:59 +0100 (CET) Subject: [pypy-svn] r69622 - pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/ctypes Message-ID: <20091125005459.157D816802A@codespeak.net> Author: santagada Date: Wed Nov 25 01:54:58 2009 New Revision: 69622 Modified: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/ctypes/__init__.py Log: comment out osx 10.3 support code (we build with 10.4 sdk anyway). Modified: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/ctypes/__init__.py ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/ctypes/__init__.py (original) +++ pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/ctypes/__init__.py Wed Nov 25 01:54:58 2009 @@ -23,21 +23,23 @@ from _ctypes import FormatError DEFAULT_MODE = RTLD_LOCAL -if _os.name == "posix" and _sys.platform == "darwin": - import gestalt - - # gestalt.gestalt("sysv") returns the version number of the - # currently active system file as BCD. - # On OS X 10.4.6 -> 0x1046 - # On OS X 10.2.8 -> 0x1028 - # See also http://www.rgaros.nl/gestalt/ - # - # On OS X 10.3, we use RTLD_GLOBAL as default mode - # because RTLD_LOCAL does not work at least on some - # libraries. - - if gestalt.gestalt("sysv") < 0x1040: - DEFAULT_MODE = RTLD_GLOBAL +# XXX lets just not support osx 10.3 for now +# if _os.name == "posix" and _sys.platform == "darwin": +# +# import gestalt +# +# # gestalt.gestalt("sysv") returns the version number of the +# # currently active system file as BCD. +# # On OS X 10.4.6 -> 0x1046 +# # On OS X 10.2.8 -> 0x1028 +# # See also http://www.rgaros.nl/gestalt/ +# # +# # On OS X 10.3, we use RTLD_GLOBAL as default mode +# # because RTLD_LOCAL does not work at least on some +# # libraries. +# +# if gestalt.gestalt("sysv") < 0x1040: +# DEFAULT_MODE = RTLD_GLOBAL from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \ FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI From santagada at codespeak.net Wed Nov 25 02:00:20 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 25 Nov 2009 02:00:20 +0100 (CET) Subject: [pypy-svn] r69623 - in pypy/branch/force-arch-darwin: lib-python lib-python/2.5.2 lib-python/2.5.2/test lib-python/modified-2.5.2/test pypy/module/_locale pypy/module/_locale/test Message-ID: <20091125010020.4017C168027@codespeak.net> Author: santagada Date: Wed Nov 25 02:00:19 2009 New Revision: 69623 Added: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py - copied, changed from r69562, pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py Removed: pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py Modified: pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py pypy/branch/force-arch-darwin/lib-python/conftest.py pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py Log: Adding support for locale (probably another 10.4+ only patch). Modified: pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py (original) +++ pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py Wed Nov 25 02:00:19 2009 @@ -30,7 +30,7 @@ from _locale import * except ImportError: - + raise # Locale emulation CHAR_MAX = 127 Modified: pypy/branch/force-arch-darwin/lib-python/conftest.py ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/conftest.py (original) +++ pypy/branch/force-arch-darwin/lib-python/conftest.py Wed Nov 25 02:00:19 2009 @@ -282,7 +282,7 @@ RegrTest('test_largefile.py'), RegrTest('test_linuxaudiodev.py', skip="unsupported extension module"), RegrTest('test_list.py', core=True), - RegrTest('test_locale.py'), + RegrTest('test_locale.py', usemodules='_rawffi'), RegrTest('test_logging.py', usemodules='thread'), RegrTest('test_long.py', core=True), RegrTest('test_long_future.py', core=True), Copied: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py (from r69562, pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py) ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py (original) +++ pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py Wed Nov 25 02:00:19 2009 @@ -2,8 +2,6 @@ import locale import sys -if sys.platform == 'darwin': - raise TestSkipped("Locale support on MacOSX is minimal and cannot be tested") oldlocale = locale.setlocale(locale.LC_NUMERIC) if sys.platform.startswith("win"): Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py Wed Nov 25 02:00:19 2009 @@ -12,7 +12,7 @@ 'strxfrm': 'interp_locale.strxfrm', } - if sys.platform in ('win32', 'darwin'): + if sys.platform == 'win32': interpleveldefs.update({ '_getdefaultlocale': 'interp_locale.getdefaultlocale', }) Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py Wed Nov 25 02:00:19 2009 @@ -21,8 +21,15 @@ includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] + + if sys.platform == 'darwin': + libraries = ['intl'] + else: + libraries = [] + _compilation_info_ = ExternalCompilationInfo( includes=includes, + libraries=libraries ) HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') lconv = platform.Struct("struct lconv", [ @@ -99,8 +106,11 @@ # should we support them? langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' - '_DATE_FMT'.split()) + 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' + 'NOEXPR'.split()) + _DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') + if _DATE_FMT: + langinfo_names.append('_DATE_FMT') for i in range(1, 8): langinfo_names.append("DAY_%d" % i) langinfo_names.append("ABDAY_%d" % i) @@ -455,5 +465,3 @@ finally: lltype.free(buf_lang, flavor='raw') lltype.free(buf_country, flavor='raw') -elif sys.platform == 'darwin': - raise NotImplementedError() Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py Wed Nov 25 02:00:19 2009 @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import py from pypy.conftest import gettestobjspace - +# from pypy.rpython.tool import rffi_platform as platform import sys class AppTestLocaleTrivia: @@ -65,11 +65,16 @@ # HAVE_LANGINFO + if sys.platform != 'win32': _LANGINFO_NAMES = ('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT ' 'T_FMT AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' - '_DATE_FMT').split() + 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' + 'NOEXPR').split() + # XXX don't know how to conditionally test this. + #_DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') + #if _DATE_FMT: + #langinfo_names.append('_DATE_FMT') for i in range(1, 8): _LANGINFO_NAMES.append("DAY_%d" % i) _LANGINFO_NAMES.append("ABDAY_%d" % i) From pedronis at codespeak.net Wed Nov 25 08:49:45 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 08:49:45 +0100 (CET) Subject: [pypy-svn] r69624 - pypy/trunk/pypy/interpreter Message-ID: <20091125074945.88001168042@codespeak.net> Author: pedronis Date: Wed Nov 25 08:49:44 2009 New Revision: 69624 Modified: pypy/trunk/pypy/interpreter/executioncontext.py Log: this was a local change that got mixed in a merge, yesterday was not a good day Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Wed Nov 25 08:49:44 2009 @@ -11,9 +11,6 @@ space.wrap(frame), space.wrap(event), w_arg) -class FrameChainMismatch(Exception): - pass - class ExecutionContext(object): """An ExecutionContext holds the state of an execution thread in the Python interpreter.""" @@ -145,11 +142,6 @@ #assert frame is self.gettopframe() --- slowish if self.some_frame is frame: self.some_frame = frame.f_back_some - if self.some_frame and self.some_frame.f_forward: - if self.some_frame.f_forward is frame: - # and anything that actually goes forward from there, - # but this is both inefficient and breaks tests - raise FrameChainMismatch() else: f_back = frame.f_back() if f_back is not None: From fijall at gmail.com Wed Nov 25 09:40:11 2009 From: fijall at gmail.com (Maciej Fijalkowski) Date: Wed, 25 Nov 2009 09:40:11 +0100 Subject: [pypy-svn] r69623 - in pypy/branch/force-arch-darwin: lib-python lib-python/2.5.2 lib-python/2.5.2/test lib-python/modified-2.5.2/test pypy/module/_locale pypy/module/_locale/test In-Reply-To: <20091125010020.4017C168027@codespeak.net> References: <20091125010020.4017C168027@codespeak.net> Message-ID: <693bc9ab0911250040u47a8dc60ha514818b41e4ac44@mail.gmail.com> Hi. This commit contains many things and some of them are wrong. Please revert. There is _locale module which requires _locale module obviously and _locale in lib which requires _rawffi. It's not enabled by default because it's unfinished and has not enough tests. Modification of locale.py is unnecessary and dangerous, please don't do that. Also please concentrate on writing new tests instead of trying to run CPython ones - they're not good to start with. Cheers, fijal On Wed, Nov 25, 2009 at 2:00 AM, wrote: > Author: santagada > Date: Wed Nov 25 02:00:19 2009 > New Revision: 69623 > > Added: > ? pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py > ? ? ?- copied, changed from r69562, pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py > Removed: > ? pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py > Modified: > ? pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py > ? pypy/branch/force-arch-darwin/lib-python/conftest.py > ? pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py > ? pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py > ? pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py > Log: > Adding support for locale (probably another 10.4+ only patch). > > Modified: pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py > ============================================================================== > --- pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py ? ?(original) > +++ pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py ? ?Wed Nov 25 02:00:19 2009 > @@ -30,7 +30,7 @@ > ? ? from _locale import * > > ?except ImportError: > - > + ? ?raise > ? ? # Locale emulation > > ? ? CHAR_MAX = 127 > > Modified: pypy/branch/force-arch-darwin/lib-python/conftest.py > ============================================================================== > --- pypy/branch/force-arch-darwin/lib-python/conftest.py ? ? ? ?(original) > +++ pypy/branch/force-arch-darwin/lib-python/conftest.py ? ? ? ?Wed Nov 25 02:00:19 2009 > @@ -282,7 +282,7 @@ > ? ? RegrTest('test_largefile.py'), > ? ? RegrTest('test_linuxaudiodev.py', skip="unsupported extension module"), > ? ? RegrTest('test_list.py', core=True), > - ? ?RegrTest('test_locale.py'), > + ? ?RegrTest('test_locale.py', usemodules='_rawffi'), > ? ? RegrTest('test_logging.py', usemodules='thread'), > ? ? RegrTest('test_long.py', core=True), > ? ? RegrTest('test_long_future.py', core=True), > > Copied: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py (from r69562, pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py) > ============================================================================== > --- pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py ?(original) > +++ pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py Wed Nov 25 02:00:19 2009 > @@ -2,8 +2,6 @@ > ?import locale > ?import sys > > -if sys.platform == 'darwin': > - ? ?raise TestSkipped("Locale support on MacOSX is minimal and cannot be tested") > ?oldlocale = locale.setlocale(locale.LC_NUMERIC) > > ?if sys.platform.startswith("win"): > > Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py > ============================================================================== > --- pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py ? ? ? (original) > +++ pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py ? ? ? Wed Nov 25 02:00:19 2009 > @@ -12,7 +12,7 @@ > ? ? ? ? ? ? 'strxfrm': ? ? ? ? ? ? ? ? ?'interp_locale.strxfrm', > ? ? ? ? ? ? } > > - ? ?if sys.platform in ('win32', 'darwin'): > + ? ?if sys.platform == 'win32': > ? ? ? ? interpleveldefs.update({ > ? ? ? ? ? ? '_getdefaultlocale': ? ? ? ?'interp_locale.getdefaultlocale', > ? ? ? ? ? ? }) > > Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py > ============================================================================== > --- pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py ?(original) > +++ pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py ?Wed Nov 25 02:00:19 2009 > @@ -21,8 +21,15 @@ > ? ? ? ? includes += ['libintl.h'] > ? ? if sys.platform == 'win32': > ? ? ? ? includes += ['windows.h'] > + > + ? ?if sys.platform == 'darwin': > + ? ? ? ?libraries = ['intl'] > + ? ?else: > + ? ? ? ?libraries = [] > + > ? ? _compilation_info_ = ExternalCompilationInfo( > ? ? ? ? includes=includes, > + ? ? ? ?libraries=libraries > ? ? ) > ? ? HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') > ? ? lconv = platform.Struct("struct lconv", [ > @@ -99,8 +106,11 @@ > ? ? # should we support them? > ? ? langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' > ? ? ? ? ? ? ? ? ? ? ? ? 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' > - ? ? ? ? ? ? ? ? ? ? ? ?'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' > - ? ? ? ? ? ? ? ? ? ? ? ?'_DATE_FMT'.split()) > + ? ? ? ? ? ? ? ? ? ? ? ?'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' > + ? ? ? ? ? ? ? ? ? ? ? ?'NOEXPR'.split()) > + ? ?_DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') > + ? ?if _DATE_FMT: > + ? ? ? ?langinfo_names.append('_DATE_FMT') > ? ? for i in range(1, 8): > ? ? ? ? langinfo_names.append("DAY_%d" % i) > ? ? ? ? langinfo_names.append("ABDAY_%d" % i) > @@ -455,5 +465,3 @@ > ? ? ? ? finally: > ? ? ? ? ? ? lltype.free(buf_lang, flavor='raw') > ? ? ? ? ? ? lltype.free(buf_country, flavor='raw') > -elif sys.platform == 'darwin': > - ? ?raise NotImplementedError() > > Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py > ============================================================================== > --- pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py ? ? ? (original) > +++ pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py ? ? ? Wed Nov 25 02:00:19 2009 > @@ -1,7 +1,7 @@ > ?# -*- coding: utf-8 -*- > ?import py > ?from pypy.conftest import gettestobjspace > - > +# from pypy.rpython.tool import rffi_platform as platform > ?import sys > > ?class AppTestLocaleTrivia: > @@ -65,11 +65,16 @@ > > > ? ? ? ? # HAVE_LANGINFO > + > ? ? ? ? if sys.platform != 'win32': > ? ? ? ? ? ? _LANGINFO_NAMES = ('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT ' > ? ? ? ? ? ? ? ? ? ? ? ? 'T_FMT AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' > - ? ? ? ? ? ? ? ? ? ? ? ?'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' > - ? ? ? ? ? ? ? ? ? ? ? ?'_DATE_FMT').split() > + ? ? ? ? ? ? ? ? ? ? ? ?'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' > + ? ? ? ? ? ? ? ? ? ? ? ?'NOEXPR').split() > + ? ? ? ? ? ?# XXX don't know how to conditionally test this. > + ? ? ? ? ? ?#_DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') > + ? ? ? ? ? ?#if _DATE_FMT: > + ? ? ? ? ? ?#langinfo_names.append('_DATE_FMT') > ? ? ? ? ? ? for i in range(1, 8): > ? ? ? ? ? ? ? ? _LANGINFO_NAMES.append("DAY_%d" % i) > ? ? ? ? ? ? ? ? _LANGINFO_NAMES.append("ABDAY_%d" % i) > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From pedronis at codespeak.net Wed Nov 25 11:45:36 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 11:45:36 +0100 (CET) Subject: [pypy-svn] r69626 - pypy/trunk/virtual-forcing Message-ID: <20091125104536.87AA61680AB@codespeak.net> Author: pedronis Date: Wed Nov 25 11:45:36 2009 New Revision: 69626 Added: pypy/trunk/virtual-forcing/ - copied from r69625, pypy/trunk/ Log: a branch in which to work finishing virtualizable first and then think about more general ways to store and possibly virtual objects into real ones From pedronis at codespeak.net Wed Nov 25 11:48:05 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 11:48:05 +0100 (CET) Subject: [pypy-svn] r69627 - in pypy/trunk/virtual-forcing/pypy/jit: backend backend/test metainterp Message-ID: <20091125104805.EE87C1680AB@codespeak.net> Author: pedronis Date: Wed Nov 25 11:48:05 2009 New Revision: 69627 Modified: pypy/trunk/virtual-forcing/pypy/jit/backend/model.py pypy/trunk/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/trunk/virtual-forcing/pypy/jit/metainterp/resoperation.py Log: (fijal, pedronis) start sketching a test for the backend interface required to implement forcing virtual(lizable) objects Modified: pypy/trunk/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/trunk/virtual-forcing/pypy/jit/backend/model.py Wed Nov 25 11:48:05 2009 @@ -205,6 +205,12 @@ def do_cast_ptr_to_int(self, ptrbox): raise NotImplementedError + def do_force_token(self): + raise NotImplementedError + + def do_call_may_force(self, args, calldescr): + raise NotImplementedError + # ootype specific operations # -------------------------- Modified: pypy/trunk/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/virtual-forcing/pypy/jit/backend/test/runner_test.py Wed Nov 25 11:48:05 2009 @@ -1221,6 +1221,47 @@ else: assert record == [] + def test_force_operations(self): + values = [] + def maybe_force(token, flag): + if flag: + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(1)) + + FUNC = self.FuncType([llmemory.Address, lltype.Signed], lltype.Void) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + tok = BoxInt() + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=BasicFailDescr(1)), + ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_int(0) == 20 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_int(1) == 10 + assert values == [1, 10] + + # pure do_ / descr features def test_do_operations(self): Modified: pypy/trunk/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/trunk/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/trunk/virtual-forcing/pypy/jit/metainterp/resoperation.py Wed Nov 25 11:48:05 2009 @@ -125,6 +125,7 @@ 'GUARD_EXCEPTION', 'GUARD_NO_OVERFLOW', 'GUARD_OVERFLOW', + 'GUARD_NOT_FORCED', '_GUARD_LAST', # ----- end of guard operations ----- '_NOSIDEEFFECT_FIRST', # ----- start of no_side_effect operations ----- @@ -220,9 +221,11 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only + 'FORCE_TOKEN', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', + 'CALL_MAY_FORCE', 'OOSEND', # ootype operation '_CANRAISE_LAST', # ----- end of can_raise operations ----- From pedronis at codespeak.net Wed Nov 25 11:52:07 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 11:52:07 +0100 (CET) Subject: [pypy-svn] r69628 - pypy/branch/virtual-forcing Message-ID: <20091125105207.11A621680AB@codespeak.net> Author: pedronis Date: Wed Nov 25 11:52:06 2009 New Revision: 69628 Added: pypy/branch/virtual-forcing/ - copied from r69627, pypy/trunk/virtual-forcing/ Log: move to sane location From pedronis at codespeak.net Wed Nov 25 11:52:22 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 11:52:22 +0100 (CET) Subject: [pypy-svn] r69629 - pypy/trunk/virtual-forcing Message-ID: <20091125105222.ADF7C1680AB@codespeak.net> Author: pedronis Date: Wed Nov 25 11:52:22 2009 New Revision: 69629 Removed: pypy/trunk/virtual-forcing/ Log: kill insane branch point From afa at codespeak.net Wed Nov 25 12:08:05 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 12:08:05 +0100 (CET) Subject: [pypy-svn] r69630 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125110805.508BC1680AF@codespeak.net> Author: afa Date: Wed Nov 25 12:08:04 2009 New Revision: 69630 Added: pypy/trunk/pypy/module/oracle/interp_lob.py pypy/trunk/pypy/module/oracle/test/test_lobvar.py Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: Start to implement BLOB variables Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Nov 25 12:08:04 2009 @@ -17,6 +17,7 @@ 'LONG_BINARY': 'interp_variable.VT_LongBinary', 'FIXED_CHAR': 'interp_variable.VT_FixedChar', 'CURSOR': 'interp_variable.VT_Cursor', + 'BLOB': 'interp_variable.VT_BLOB', 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', 'Date': 'interp_error.get(space).w_DateType', Added: pypy/trunk/pypy/module/oracle/interp_lob.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_lob.py Wed Nov 25 12:08:04 2009 @@ -0,0 +1,34 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import ObjSpace +from pypy.interpreter.gateway import interp2app + +class W_ExternalLob(Wrappable): + def __init__(self, var, pos): + self.lobVar = var + self.pos = pos + self.internalFetchNum = var.internalFetchNum + + def _verify(self, space): + if self.internalFetchNum != self.lobVar.internalFetchNum: + raise OperationError( + get(space).w_ProgrammingError, + space.wrap( + "LOB variable no longer valid after subsequent fetch")) + + def read(self, space, offset=-1, amount=-1): + self._verify(space) + return self.lobVar.read(space, self.pos, offset, amount) + read.unwrap_spec=['self', ObjSpace, int, int] + + def desc_str(self, space): + return self.read(space, offset=1, amount=-1) + desc_str.unwrap_spec=['self', ObjSpace] + +W_ExternalLob.typedef = TypeDef( + 'ExternalLob', + read = interp2app(W_ExternalLob.read, + unwrap_spec=W_ExternalLob.read.unwrap_spec), + __str__ = interp2app(W_ExternalLob.desc_str, + unwrap_spec=W_ExternalLob.desc_str.unwrap_spec), + ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 25 12:08:04 2009 @@ -8,7 +8,7 @@ from pypy.rlib.rarithmetic import ovfcheck import sys -from pypy.module.oracle import roci, config, transform +from pypy.module.oracle import roci, config, transform, interp_lob from pypy.module.oracle.interp_error import W_Error, get from pypy.module.oracle.config import string_w, StringBuffer @@ -914,18 +914,165 @@ self.environment.checkForError( status, "IntervalVar_SetValue()") +class W_LobVariable(W_VariableWithDescriptor): + descriptorType = roci.OCI_DTYPE_LOB + descriptionText = "LobVar" + temporaryLobType = roci.OCI_TEMP_CLOB -class VT_CLOB(W_Variable): - pass + def initialize(self, space, cursor): + super(W_LobVariable, self).initialize(space, cursor) + self.connection = cursor.connection -class VT_NCLOB(W_Variable): - pass + def ensureTemporary(self, space, pos): + # make sure we have a temporary LOB set up + temporaryptr = lltype.malloc(rffi.CArrayPtr(roci.boolean).TO, 1, flavor='raw') + try: + status = roci.OCILobIsTemporary( + self.environment.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + temporaryptr); + self.environment.checkForError( + status, + "LobVar_SetValue(): check temporary") + temporary = temporaryptr[0] + finally: + lltype.free(temporaryptr, flavor='raw') -class VT_BLOB(W_Variable): - pass + if temporary: + return -class VT_BFILE(W_Variable): - pass + status = roci.OCILobCreateTemporary( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + roci.OCI_DEFAULT, + roci.OCI_DEFAULT, + self.temporaryLobType, + False, + roci.OCI_DURATION_SESSION) + self.environment.checkForError( + status, + "LobVar_SetValue(): create temporary") + + + def setValueProc(self, space, pos, w_value): + self.ensureTemporary(space, pos) + + # trim the current value + status = roci.OCILobTrim( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + 0) + self.environment.checkForError( + status, + "LobVar_SetValue(): trim") + + # set the value + self.write(space, pos, w_value, 1) + + def getLength(self, space, pos): + "Return the size of the LOB variable for internal comsumption." + lengthptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') + try: + status = roci.OCILobGetLength( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + lengthptr) + self.environment.checkForError( + status, + "LobVar_GetLength()") + return int(lengthptr[0]) # XXX test overflow + finally: + lltype.free(lengthptr, flavor='raw') + + def read(self, space, pos, offset, amount): + # modify the arguments + if offset <= 0: + offset = 1 + if amount < 0: + amount = self.getLength(space, pos) - offset + 1 + if amount <= 0: + amount = 1 + + bufferSize = amount + raw_buffer, gc_buffer = rffi.alloc_buffer(bufferSize) + amountptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') + amountptr[0] = rffi.cast(roci.ub4, amount) + try: + status = roci.OCILobRead( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + amountptr, offset, + raw_buffer, bufferSize, + None, None, + config.CHARSETID, self.charsetForm) + self.environment.checkForError( + status, + "LobVar_Read()") + amount = int(amountptr[0]) # XXX test overflow + value = rffi.str_from_buffer(raw_buffer, gc_buffer, bufferSize, amount) + return space.wrap(value) + finally: + lltype.free(amountptr, flavor='raw') + rffi.keep_buffer_alive_until_here(raw_buffer, gc_buffer) + + def write(self, space, pos, w_value, offset): + databuf = config.StringBuffer() + databuf.fill(space, w_value) + amountptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') + amountptr[0] = rffi.cast(roci.ub4, databuf.size) + + try: + status = roci.OCILobWrite( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + amountptr, offset, + databuf.ptr, databuf.size, + roci.OCI_ONE_PIECE, + None, None, + config.CHARSETID, self.charsetForm) + self.environment.checkForError( + status, + "LobVar_Write()") + amount = amountptr[0] + finally: + lltype.free(amountptr, flavor='raw') + databuf.clear() + + return amount + + def getValueProc(self, space, pos): + return space.wrap(interp_lob.W_ExternalLob(self, pos)) + +class VT_CLOB(W_LobVariable): + oracleType = roci.SQLT_CLOB + +class VT_NCLOB(W_LobVariable): + oracleType = roci.SQLT_CLOB + +class VT_BLOB(W_LobVariable): + oracleType = roci.SQLT_BLOB + temporaryLobType = roci.OCI_TEMP_BLOB + +class VT_BFILE(W_LobVariable): + oracleType = roci.SQLT_BFILE + + def write(self, space, pos, w_value, offset): + raise OperationError( + space.w_TypeError, + space.wrap("BFILEs are read only")) + + def read(self, space, pos, offset, amount): + self.fileOpen() + try: + return W_LobVariable.read(self, space, pos, offset, amount) + finally: + self.fileClose() class VT_Cursor(W_Variable): oracleType = roci.SQLT_RSET Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 12:08:04 2009 @@ -31,6 +31,8 @@ sb4 = platform.SimpleType('sb4', rffi.INT) sword = platform.SimpleType('sword', rffi.INT) uword = platform.SimpleType('uword', rffi.UINT) + boolean = platform.SimpleType('boolean', rffi.UINT) + OCIDuration = platform.SimpleType('OCIDuration', rffi.UINT) OCINumber = platform.Struct('OCINumber', []) OCITime = platform.Struct('OCITime', @@ -50,7 +52,7 @@ OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV - OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS + OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT @@ -69,6 +71,7 @@ SQLT_TIMESTAMP_TZ SQLT_TIMESTAMP_LTZ SQLT_INTERVAL_DS SQLT_CLOB SQLT_CLOB SQLT_BLOB SQLT_BFILE SQLT_RSET SQLT_NTY SQLCS_IMPLICIT SQLCS_NCHAR + OCI_TEMP_CLOB OCI_TEMP_BLOB OCI_DURATION_SESSION OCI_ONE_PIECE OCI_NUMBER_SIGNED OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA '''.split() @@ -93,13 +96,14 @@ OCISnapshot = rffi.VOIDP OCIDateTime = rffi.VOIDP OCIInterval = rffi.VOIDP +OCILobLocator = rffi.VOIDP +Ptr = rffi.CArrayPtr void = lltype.Void dvoidp = rffi.VOIDP dvoidpp = rffi.VOIDPP size_t = rffi.SIZE_T oratext = rffi.CCHARP -Ptr = rffi.CArrayPtr def external(name, args, result): return rffi.llexternal(name, args, result, compilation_info=eci) @@ -318,6 +322,77 @@ ub4], # mode sword) +# LOB Functions + +OCILobCreateTemporary = external( + 'OCILobCreateTemporary', + [OCISvcCtx, # svchp + OCIError, # errhp + OCILobLocator, # locp + ub2, # csid + ub1, # csfrm + ub1, # lobtype + boolean, # cache + OCIDuration], # duration + sword) + +OCILobGetLength = external( + 'OCILobGetLength', + [OCIEnv, # envhp + OCIError, # errhp + OCILobLocator, # locp + Ptr(ub4)], # lenp + sword) + +OCILobIsTemporary = external( + 'OCILobIsTemporary', + [OCIEnv, # envhp + OCIError, # errhp + OCILobLocator, # locp + Ptr(boolean)], # is_temporary + sword) + +OCICallbackLobRead = dvoidp +OCILobRead = external( + 'OCILobRead', + [OCISvcCtx, # svchp + OCIError, # errhp + OCILobLocator, # locp + Ptr(ub4), # amtp + ub4, # offset + dvoidp, # bufp + ub4, # buflen + dvoidp, # ctxp + OCICallbackLobRead, # cbfp + ub2, # csid + ub1], # csfrm + sword) + +OCILobTrim = external( + 'OCILobTrim', + [OCISvcCtx, # svchp + OCIError, # errhp + OCILobLocator, # locp + ub4], # newlen + sword) + +OCICallbackLobWrite = dvoidp +OCILobWrite = external( + 'OCILobWrite', + [OCISvcCtx, # svchp + OCIError, # errhp + OCILobLocator, # locp + Ptr(ub4), # amtp + ub4, # offset + dvoidp, # bufp + ub4, # buflen + ub1, # piece + dvoidp, # ctxp + OCICallbackLobWrite, # cbfp + ub2, # csid + ub1], # csfrm + sword) + # Transaction Functions OCITransCommit = external( Added: pypy/trunk/pypy/module/oracle/test/test_lobvar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_lobvar.py Wed Nov 25 12:08:04 2009 @@ -0,0 +1,33 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class LobTests(object): + @classmethod + def setup_class(cls): + super(LobTests, cls).setup_class() + cls.w_lobType = cls.space.wrap(cls.lobType) + + def test_bind(self): + inputType = getattr(oracle, self.lobType) + + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_lobtable") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_lobtable " + "(lobcol %s)" % self.lobType) + + longString = "" + for i in range(2): + if i > 0: + longString += chr(65+i) * 25000 + + cur.setinputsizes(lob=inputType) + cur.execute("insert into pypy_temp_lobtable values (:lob)", + lob=longString) + cur.execute("select lobcol from pypy_temp_lobtable") + lob, = cur.fetchone() + assert lob.read() == longString + +class AppTestLob(LobTests, OracleTestBase): + lobType = "BLOB" From afa at codespeak.net Wed Nov 25 12:58:09 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 12:58:09 +0100 (CET) Subject: [pypy-svn] r69631 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125115809.BB1D41680AB@codespeak.net> Author: afa Date: Wed Nov 25 12:58:08 2009 New Revision: 69631 Modified: pypy/trunk/pypy/module/oracle/interp_lob.py pypy/trunk/pypy/module/oracle/test/test_lobvar.py Log: Add method ExternalLob.size() Modified: pypy/trunk/pypy/module/oracle/interp_lob.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_lob.py (original) +++ pypy/trunk/pypy/module/oracle/interp_lob.py Wed Nov 25 12:58:08 2009 @@ -21,6 +21,11 @@ return self.lobVar.read(space, self.pos, offset, amount) read.unwrap_spec=['self', ObjSpace, int, int] + def size(self, space): + self._verify(space) + return space.wrap(self.lobVar.getLength(space, self.pos)) + size.unwrap_spec=['self', ObjSpace] + def desc_str(self, space): return self.read(space, offset=1, amount=-1) desc_str.unwrap_spec=['self', ObjSpace] @@ -29,6 +34,8 @@ 'ExternalLob', read = interp2app(W_ExternalLob.read, unwrap_spec=W_ExternalLob.read.unwrap_spec), + size = interp2app(W_ExternalLob.size, + unwrap_spec=W_ExternalLob.size.unwrap_spec), __str__ = interp2app(W_ExternalLob.desc_str, unwrap_spec=W_ExternalLob.desc_str.unwrap_spec), ) Modified: pypy/trunk/pypy/module/oracle/test/test_lobvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_lobvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_lobvar.py Wed Nov 25 12:58:08 2009 @@ -27,6 +27,7 @@ lob=longString) cur.execute("select lobcol from pypy_temp_lobtable") lob, = cur.fetchone() + assert lob.size() == len(longString) assert lob.read() == longString class AppTestLob(LobTests, OracleTestBase): From afa at codespeak.net Wed Nov 25 13:24:07 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 13:24:07 +0100 (CET) Subject: [pypy-svn] r69632 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125122407.474651680F1@codespeak.net> Author: afa Date: Wed Nov 25 13:24:06 2009 New Revision: 69632 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_lob.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_lobvar.py Log: Implement ExternalLob.trim() + tests for CLOB Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Nov 25 13:24:06 2009 @@ -18,6 +18,7 @@ 'FIXED_CHAR': 'interp_variable.VT_FixedChar', 'CURSOR': 'interp_variable.VT_Cursor', 'BLOB': 'interp_variable.VT_BLOB', + 'CLOB': 'interp_variable.VT_CLOB', 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', 'Date': 'interp_error.get(space).w_DateType', Modified: pypy/trunk/pypy/module/oracle/interp_lob.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_lob.py (original) +++ pypy/trunk/pypy/module/oracle/interp_lob.py Wed Nov 25 13:24:06 2009 @@ -2,6 +2,9 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import ObjSpace from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError + +from pypy.module.oracle.interp_error import get class W_ExternalLob(Wrappable): def __init__(self, var, pos): @@ -26,6 +29,11 @@ return space.wrap(self.lobVar.getLength(space, self.pos)) size.unwrap_spec=['self', ObjSpace] + def trim(self, space, newSize=0): + self._verify(space) + self.lobVar.trim(space, self.pos, newSize) + trim.unwrap_spec=['self', ObjSpace, int] + def desc_str(self, space): return self.read(space, offset=1, amount=-1) desc_str.unwrap_spec=['self', ObjSpace] @@ -36,6 +44,8 @@ unwrap_spec=W_ExternalLob.read.unwrap_spec), size = interp2app(W_ExternalLob.size, unwrap_spec=W_ExternalLob.size.unwrap_spec), + trim = interp2app(W_ExternalLob.trim, + unwrap_spec=W_ExternalLob.trim.unwrap_spec), __str__ = interp2app(W_ExternalLob.desc_str, unwrap_spec=W_ExternalLob.desc_str.unwrap_spec), ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 25 13:24:06 2009 @@ -958,18 +958,7 @@ def setValueProc(self, space, pos, w_value): self.ensureTemporary(space, pos) - - # trim the current value - status = roci.OCILobTrim( - self.connection.handle, - self.environment.errorHandle, - self.getDataptr(pos)[0], - 0) - self.environment.checkForError( - status, - "LobVar_SetValue(): trim") - - # set the value + self.trim(space, pos, 0) self.write(space, pos, w_value, 1) def getLength(self, space, pos): @@ -988,6 +977,16 @@ finally: lltype.free(lengthptr, flavor='raw') + def trim(self, space, pos, newSize): + status = roci.OCILobTrim( + self.connection.handle, + self.environment.errorHandle, + self.getDataptr(pos)[0], + newSize) + self.environment.checkForError( + status, + "LobVar_Trim()") + def read(self, space, pos, offset, amount): # modify the arguments if offset <= 0: Modified: pypy/trunk/pypy/module/oracle/test/test_lobvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_lobvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_lobvar.py Wed Nov 25 13:24:06 2009 @@ -30,5 +30,31 @@ assert lob.size() == len(longString) assert lob.read() == longString -class AppTestLob(LobTests, OracleTestBase): + def test_trim(self): + inputType = getattr(oracle, self.lobType) + + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_temp_lobtable") + except oracle.DatabaseError: + pass + cur.execute("create table pypy_temp_lobtable " + "(lobcol %s)" % self.lobType) + + longString = "X" * 75000 + cur.setinputsizes(lob=inputType) + cur.execute("insert into pypy_temp_lobtable values (:lob)", + lob=longString) + cur.execute("select lobcol from pypy_temp_lobtable") + lob, = cur.fetchone() + assert lob.size() == 75000 + lob.trim(25000) + assert lob.size() == 25000 + lob.trim() + assert lob.size() == 0 + +class AppTestBlob(LobTests, OracleTestBase): lobType = "BLOB" + +class AppTestClob(LobTests, OracleTestBase): + lobType = "CLOB" From afa at codespeak.net Wed Nov 25 13:48:37 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 13:48:37 +0100 (CET) Subject: [pypy-svn] r69633 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125124837.1C8AC1680F1@codespeak.net> Author: afa Date: Wed Nov 25 13:48:36 2009 New Revision: 69633 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_lobvar.py Log: Bah. Fix the test, then fix the bug. Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 25 13:48:36 2009 @@ -1026,6 +1026,10 @@ amountptr[0] = rffi.cast(roci.ub4, databuf.size) try: + # nothing to do if no data to write + if databuf.size == 0: + return + status = roci.OCILobWrite( self.connection.handle, self.environment.errorHandle, Modified: pypy/trunk/pypy/module/oracle/test/test_lobvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_lobvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_lobvar.py Wed Nov 25 13:48:36 2009 @@ -19,16 +19,17 @@ longString = "" for i in range(2): + cur.execute("truncate table pypy_temp_lobtable") if i > 0: longString += chr(65+i) * 25000 - cur.setinputsizes(lob=inputType) - cur.execute("insert into pypy_temp_lobtable values (:lob)", - lob=longString) - cur.execute("select lobcol from pypy_temp_lobtable") - lob, = cur.fetchone() - assert lob.size() == len(longString) - assert lob.read() == longString + cur.setinputsizes(lob=inputType) + cur.execute("insert into pypy_temp_lobtable values (:lob)", + lob=longString) + cur.execute("select lobcol from pypy_temp_lobtable") + lob, = cur.fetchone() + assert lob.size() == len(longString) + assert lob.read() == longString def test_trim(self): inputType = getattr(oracle, self.lobType) From pedronis at codespeak.net Wed Nov 25 14:48:24 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 14:48:24 +0100 (CET) Subject: [pypy-svn] r69634 - pypy/branch/virtual-forcing/pypy/jit/backend/test Message-ID: <20091125134824.1A8221680F1@codespeak.net> Author: pedronis Date: Wed Nov 25 14:48:22 2009 New Revision: 69634 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Log: kill tabs Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Wed Nov 25 14:48:22 2009 @@ -1222,45 +1222,45 @@ assert record == [] def test_force_operations(self): - values = [] - def maybe_force(token, flag): + values = [] + def maybe_force(token, flag): if flag: - self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(0)) - values.append(self.cpu.get_latest_value_int(1)) + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(1)) FUNC = self.FuncType([llmemory.Address, lltype.Signed], lltype.Void) - func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() - calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - cpu = self.cpu - i0 = BoxInt() - i1 = BoxInt() - tok = BoxInt() - ops = [ - ResOperation(rop.FORCE_TOKEN, [], tok), - ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, descr=calldescr), - ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=BasicFailDescr(1)), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) - ] - ops[2].fail_args = [i1, i0] - looptoken = LoopToken() - self.cpu.compile_loop([i0, i1], ops, looptoken) - self.cpu.set_future_value_int(0, 20) - self.cpu.set_future_value_int(1, 0) + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + tok = BoxInt() + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=BasicFailDescr(1)), + ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) fail = self.cpu.execute_token(looptoken) - assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(0) == 20 - assert values == [] + assert fail.identifier == 0 + assert self.cpu.get_latest_value_int(0) == 20 + assert values == [] - self.cpu.set_future_value_int(0, 10) - self.cpu.set_future_value_int(1, 1) + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) fail = self.cpu.execute_token(looptoken) - assert fail.identifier == 1 - assert self.cpu.get_latest_value_int(0) == 1 - assert self.cpu.get_latest_value_int(1) == 10 - assert values == [1, 10] - + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_int(1) == 10 + assert values == [1, 10] # pure do_ / descr features From pedronis at codespeak.net Wed Nov 25 14:49:30 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Wed, 25 Nov 2009 14:49:30 +0100 (CET) Subject: [pypy-svn] r69635 - in pypy/branch/virtual-forcing/pypy/jit/backend: . llgraph Message-ID: <20091125134930.A9FC11680F1@codespeak.net> Author: pedronis Date: Wed Nov 25 14:49:29 2009 New Revision: 69635 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py Log: support passing test_force_operations with the llgraph backend, not completely pretty but not too bad Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Wed Nov 25 14:49:29 2009 @@ -145,6 +145,9 @@ 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), 'cast_ptr_to_int' : (('ref',), 'int'), 'debug_merge_point': (('ref',), None), + 'force_token' : ((), 'int'), + 'call_may_force' : (('ref', 'varargs'), 'intorptr'), + 'guard_not_forced': ((), None) #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), #'newlist' : (('void', 'varargs'), 'ref'), @@ -384,6 +387,9 @@ def __init__(self, memocast): self.verbose = False self.memocast = memocast + self.opindex = 1 + self._forced = False + self._may_force = -1 def getenv(self, v): if isinstance(v, Constant): @@ -391,6 +397,16 @@ else: return self.env[v] + def _populate_fail_args(self, op): + fail_args = [] + if op.fail_args: + for fail_arg in op.fail_args: + if fail_arg is None: + fail_args.append(None) + else: + fail_args.append(self.getenv(fail_arg)) + self.fail_args = fail_args + def execute(self): """Execute all operations in a loop, possibly following to other loops as well. @@ -401,6 +417,7 @@ operations = self.loop.operations opindex = 0 while True: + self.opindex = opindex op = operations[opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): @@ -419,18 +436,11 @@ opindex = 0 continue else: - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - else: - fail_args.append(self.getenv(fail_arg)) + self._populate_fail_args(op) # a non-patched guard if self.verbose: log.trace('failed: %s' % ( ', '.join(map(str, fail_args)),)) - self.fail_args = fail_args return op.fail_index #verbose = self.verbose assert (result is None) == (op.result is None) @@ -776,6 +786,21 @@ def op_uint_xor(self, descr, arg1, arg2): return arg1 ^ arg2 + def op_force_token(self, descr): + opaque_frame = _to_opaque(self) + return llmemory.cast_ptr_to_adr(opaque_frame) + + def op_call_may_force(self, calldescr, func, *args): + assert not self._forced + self._may_force = self.opindex + self.op_call(calldescr, func, *args) + + def op_guard_not_forced(self, descr): + forced = self._forced + self._forced = False + if forced: + raise GuardFailed + class OOFrame(Frame): @@ -1042,6 +1067,16 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, _get_error(ZeroDivisionError).args[1]) +def force(force_token): + opaque_frame = llmemory.cast_adr_to_ptr(force_token, + lltype.Ptr(_TO_OPAQUE[Frame])) + frame = _from_opaque(opaque_frame) + frame._forced = True + assert frame._may_force >= 0 + guard_op = frame.loop.operations[frame._may_force+1] + frame._populate_fail_args(guard_op) + return opaque_frame + class MemoCast(object): def __init__(self): self.addresses = [llmemory.NULL] @@ -1411,6 +1446,7 @@ setannotation(get_overflow_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(get_zero_division_error, annmodel.SomeAddress()) setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) +setannotation(force, s_Frame) setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Wed Nov 25 14:49:29 2009 @@ -497,6 +497,9 @@ def do_cast_ptr_to_int(self, ptrbox): return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) + def force(self, force_token): + frame = llimpl.force(force_token) + self.latest_frame = frame class OOtypeCPU(BaseCPU): is_oo = True Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Wed Nov 25 14:49:29 2009 @@ -206,10 +206,13 @@ raise NotImplementedError def do_force_token(self): - raise NotImplementedError + raise NotImplementedError def do_call_may_force(self, args, calldescr): - raise NotImplementedError + raise NotImplementedError + + def force(self, force_token): + raise NotImplementedError # ootype specific operations # -------------------------- From afa at codespeak.net Wed Nov 25 16:03:54 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 16:03:54 +0100 (CET) Subject: [pypy-svn] r69636 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125150354.5B66D1680F3@codespeak.net> Author: afa Date: Wed Nov 25 16:03:53 2009 New Revision: 69636 Added: pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/test/test_objectvar.py Modified: pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: Start to implment the OBJECT datatype. The Oracle API is really a complex beast. Added: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_object.py Wed Nov 25 16:03:53 2009 @@ -0,0 +1,115 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.oracle import roci + +class W_ObjectType(Wrappable): + def __init__(self, connection, param): + self.environment = connection.environment + self.isCollection = False + self.initialize(connection, param) + + def initialize(self, connection, param): + nameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, 1, + flavor='raw') + lenptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, + flavor='raw') + try: + # determine the schema of the type + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, nameptr), + lenptr, + roci.OCI_ATTR_SCHEMA_NAME, + self.environment.errorHandle) + self.environment.checkForError( + status, + "ObjectType_Initialize(): get schema name") + self.schema = rffi.charpsize2str(nameptr[0], lenptr[0]) + + # determine the name of the type + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, nameptr), + lenptr, + roci.OCI_ATTR_TYPE_NAME, + self.environment.errorHandle) + self.environment.checkForError( + status, + "ObjectType_Initialize(): get schema name") + self.name = rffi.charpsize2str(nameptr[0], lenptr[0]) + finally: + lltype.free(nameptr, flavor='raw') + lltype.free(lenptr, flavor='raw') + + # retrieve TDO (type descriptor object) of the parameter + tdorefptr = lltype.malloc(rffi.CArrayPtr(roci.OCIRef).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, tdorefptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_REF_TDO, + self.environment.errorHandle) + self.environment.checkForError( + status, + "ObjectType_Initialize(): get TDO reference") + tdoref = tdorefptr[0] + finally: + lltype.free(tdorefptr, flavor='raw') + + tdoptr = lltype.malloc(rffi.CArrayPtr(roci.dvoidp).TO, 1, + flavor='raw') + try: + status = roci.OCIObjectPin( + self.environment.handle, + self.environment.errorHandle, + tdoref, + None, roci.OCI_PIN_ANY, + roci.OCI_DURATION_SESSION, roci.OCI_LOCK_NONE, + tdoptr) + self.environment.checkForError( + status, + "ObjectType_Initialize(): pin TDO reference") + self.tdo = tdoptr[0] + finally: + lltype.free(tdoptr, flavor='raw') + + # acquire a describe handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIDescribe).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_DESCRIBE, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "ObjectType_Initialize(): allocate describe handle") + describeHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # describe the type + try: + pass + #self.describe(connection, describeHandle) + finally: + roci.OCIHandleFree(describeHandle, roci.OCI_HTYPE_DESCRIBE) +W_ObjectType.typedef = TypeDef( + 'ObjectType', + schema = interp_attrproperty('schema', W_ObjectType), + name = interp_attrproperty('name', W_ObjectType), + ) + +class W_ExternalObject(Wrappable): + def __init__(self, ref, type, instance, indicator, isIndependent=True): + self.ref = ref + self.type = type + self.instance = instance + self.indicator = indicator + self.isIndependent = isIndependent +W_ExternalObject.typedef = TypeDef( + 'ExternalObject', + type = interp_attrproperty('type', W_ExternalObject), + ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 25 16:03:53 2009 @@ -8,7 +8,8 @@ from pypy.rlib.rarithmetic import ovfcheck import sys -from pypy.module.oracle import roci, config, transform, interp_lob +from pypy.module.oracle import roci, config, transform +from pypy.module.oracle import interp_lob, interp_object from pypy.module.oracle.interp_error import W_Error, get from pypy.module.oracle.config import string_w, StringBuffer @@ -111,7 +112,7 @@ lltype.free(handleptr, flavor='raw') # call the procedure to set values after define - var.postDefine() + var.postDefine(param) return var @@ -233,7 +234,7 @@ def preDefine(cls, param, environment): return cls - def postDefine(self): + def postDefine(self, param): pass def bind(self, space, cursor, w_name, pos): @@ -1126,8 +1127,79 @@ class VT_Object(W_Variable): + oracleType = roci.SQLT_NTY + size = rffi.sizeof(roci.dvoidp) canBeInArray = False + objectIndicator = None + + def initialize(self, space, cursor): + self.connection = cursor.connection + self.objectType = None + self.objectIndicator = lltype.malloc( + rffi.CArrayPtr(roci.dvoidp).TO, + self.allocatedElements, + flavor='raw', zero=True) + + def finalize(self): + for i in range(self.allocatedElements): + data = rffi.cast(roci.Ptr(roci.dvoidp), self.data) + roci.OCIObjectFree( + self.environment.handle, + self.environment.errorHandle, + data[i], + roci.OCI_OBJECTFREE_FORCE) + if self.objectIndicator: + lltype.free(self.objectIndicator, flavor='raw') + + def postDefine(self, param): + # XXX this used to be in preDefine + self.objectType = interp_object.W_ObjectType(self.connection, param) + + data = rffi.cast(roci.Ptr(roci.dvoidp), self.data) + + status = roci.OCIDefineObject( + self.defineHandle, + self.environment.errorHandle, + self.objectType.tdo, + data, + 0, + self.objectIndicator, + 0) + self.environment.checkForError( + status, + "ObjectVar_PostDefine(): define object") + + def isNull(self, pos): + # look at our own indicator array + if not self.objectIndicator[pos]: + return True + return (rffi.cast(roci.Ptr(roci.OCIInd), self.objectIndicator[pos])[0] + == + rffi.cast(lltype.Signed, roci.OCI_IND_NULL)) + + def getValueProc(self, space, pos): + data = rffi.cast(roci.Ptr(roci.dvoidp), self.data) + # only allowed to get the value once (for now) + if not data[pos]: + raise OperationError( + get(space).w_ProgrammingError, + space.wrap("variable value can only be acquired once")) + + + # for collections, return the list rather than the object + if self.objectType.isCollection: + return interp_object.convertCollection( + self.environment, data[pos], self, self.objectType) + + # for objects, return a representation of the object + var = interp_object.W_ExternalObject( + self, self.objectType, data[pos], self.objectIndicator[pos]) + + data[pos] = lltype.nullptr(roci.dvoidp.TO) + self.objectIndicator[pos] = lltype.nullptr(roci.dvoidp.TO) + return space.wrap(var) + all_variable_types = [] for name, cls in globals().items(): if not name.startswith('VT_') or not isinstance(cls, type): Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 16:03:53 2009 @@ -33,6 +33,9 @@ uword = platform.SimpleType('uword', rffi.UINT) boolean = platform.SimpleType('boolean', rffi.UINT) OCIDuration = platform.SimpleType('OCIDuration', rffi.UINT) + OCIInd = platform.SimpleType('OCIInd', rffi.INT) + OCIPinOpt = platform.SimpleType('OCIPinOpt', rffi.INT) + OCILockOpt = platform.SimpleType('OCILockOpt', rffi.INT) OCINumber = platform.Struct('OCINumber', []) OCITime = platform.Struct('OCITime', @@ -57,12 +60,14 @@ OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL - OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_CHARSET_FORM - OCI_ATTR_ENV_CHARSET_ID + OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_REF_TDO + OCI_ATTR_SCHEMA_NAME OCI_ATTR_TYPE_NAME + OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL + OCI_PIN_ANY OCI_LOCK_NONE OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB @@ -84,6 +89,7 @@ OCI_IND_NOTNULL = rffi.cast(rffi.SHORT, OCI_IND_NOTNULL) OCI_IND_NULL = rffi.cast(rffi.SHORT, OCI_IND_NULL) +# Various pointers to incomplete structures OCISvcCtx = rffi.VOIDP OCIEnv = rffi.VOIDP OCIError = rffi.VOIDP @@ -93,10 +99,14 @@ OCIParam = rffi.VOIDP OCIBind = rffi.VOIDP OCIDefine = rffi.VOIDP +OCIDescribe = rffi.VOIDP OCISnapshot = rffi.VOIDP OCIDateTime = rffi.VOIDP OCIInterval = rffi.VOIDP OCILobLocator = rffi.VOIDP +OCIRef = rffi.VOIDP +OCIType = rffi.VOIDP +OCIComplexObject = rffi.VOIDP Ptr = rffi.CArrayPtr void = lltype.Void @@ -262,6 +272,17 @@ ub4], # mode sword) +OCIDefineObject = external( + 'OCIDefineObject', + [OCIDefine, # defnp + OCIError, # errhp + OCIType, # type + dvoidpp, # pgvpp + ub4, # pvszsp + dvoidpp, # indpp + ub4], # indszp + sword) + OCIStmtGetBindInfo = external( 'OCIStmtGetBindInfo', [OCIStmt, # stmtp @@ -422,6 +443,20 @@ ub4], # type sword) +# OCI Object Pin, Unpin, and Free Functions + +OCIObjectPin = external( + 'OCIObjectPin', + [OCIEnv, # env, + OCIError, # err + OCIRef, # object_ref + OCIComplexObject, # corhdl + OCIPinOpt, # pin_option + OCIDuration, # pin_duration + OCILockOpt, # lock_option + dvoidpp], # object + sword) + # OCI Date, Datetime, and Interval Functions OCIDateTimeCheck = external( Added: pypy/trunk/pypy/module/oracle/test/test_objectvar.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/test/test_objectvar.py Wed Nov 25 16:03:53 2009 @@ -0,0 +1,34 @@ +from pypy.module.oracle.test.test_connect import OracleTestBase + +class AppTestObjectVar(OracleTestBase): + def test_fetch(self): + cur = self.cnx.cursor() + try: + cur.execute("drop table pypy_test_objtable") + except oracle.DatabaseError: + pass + try: + cur.execute("drop type pypy_test_objtype") + except oracle.DatabaseError: + pass + cur.execute("""\ + create type pypy_test_objtype as object ( + numbercol number, + stringcol varchar2(60), + datecol date); + """) + cur.execute("""\ + create table pypy_test_objtable ( + objcol pypy_test_objtype) + """) + cur.execute("""\ + insert into pypy_test_objtable values ( + pypy_test_objtype(1, 'someText', + to_date(20070306, 'YYYYMMDD'))) + """) + + cur.execute("select objcol from pypy_test_objtable") + objValue, = cur.fetchone() + assert objValue.type.schema == self.cnx.username.upper() + assert objValue.type.name == "PYPY_TEST_OBJTYPE" + #assert objValue.type.attributes[0].name == "NUMBERVALUE" From afa at codespeak.net Wed Nov 25 17:12:35 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 17:12:35 +0100 (CET) Subject: [pypy-svn] r69637 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125161235.397C91680F1@codespeak.net> Author: afa Date: Wed Nov 25 17:12:33 2009 New Revision: 69637 Modified: pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_objectvar.py Log: Add ability to describe OBJECT types. (Still incomplete) Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Wed Nov 25 17:12:33 2009 @@ -1,5 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable -from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import interp_attrproperty from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.oracle import roci @@ -92,14 +93,188 @@ # describe the type try: - pass - #self.describe(connection, describeHandle) + self.describe(connection, describeHandle) finally: roci.OCIHandleFree(describeHandle, roci.OCI_HTYPE_DESCRIBE) + + def describe(self, connection, describeHandle): + "Describe the type and store information about it as needed" + + # describe the type + status = roci.OCIDescribeAny( + connection.handle, + self.environment.errorHandle, + self.tdo, 0, + roci.OCI_OTYPE_PTR, + roci.OCI_DEFAULT, + roci.OCI_PTYPE_TYPE, + describeHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): describe type") + + # get top level parameter descriptor + paramptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + describeHandle, roci.OCI_HTYPE_DESCRIBE, + rffi.cast(roci.dvoidp, paramptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_PARAM, + self.environment.errorHandle) + self.environment.checkForError( + status, + "ObjectType_Describe(): get top level parameter descriptor") + toplevelParam = paramptr[0] + finally: + lltype.free(paramptr, flavor='raw') + + # determine type of type + typecodeptr = lltype.malloc(roci.Ptr(roci.OCITypeCode).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + toplevelParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, typecodeptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_TYPECODE, + self.environment.errorHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): get type code") + typeCode = typecodeptr[0] + finally: + lltype.free(typecodeptr, flavor='raw') + + # if a collection, need to determine the sub type + if typeCode == roci.OCI_TYPECODE_NAMEDCOLLECTION: + self.isCollection = 1 + + # determine type of collection + XXX + + # acquire collection parameter descriptor + XXX + + # determine type of element + XXX + + # if element type is an object type get its type + XXX + + # determine the number of attributes + numptr = lltype.malloc(roci.Ptr(roci.ub2).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + toplevelParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, numptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_NUM_TYPE_ATTRS, + self.environment.errorHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): get number of attributes") + numAttributes = numptr[0] + finally: + lltype.free(numptr, flavor='raw') + + # allocate the attribute list and dictionary + self.attributes = [] + self.attributesByName = {} + + # acquire the list parameter descriptor + listptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + toplevelParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, listptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_LIST_TYPE_ATTRS, + self.environment.errorHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): get list parameter descriptor") + attributeListParam = listptr[0] + finally: + lltype.free(listptr, flavor='raw') + + # create attribute information for each attribute + for i in range(numAttributes): + paramptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, + 1, flavor='raw') + try: + status = roci.OCIParamGet( + attributeListParam, roci.OCI_DTYPE_PARAM, + self.environment.errorHandle, + paramptr, i + 1) + self.environment.checkForError( + status, + "ObjectType_Describe(): get attribute param descriptor") + attribute = W_ObjectAttribute(connection, paramptr[0]) + finally: + lltype.free(paramptr, flavor='raw') + + self.attributes.append(attribute) + self.attributesByName[attribute.name] = attribute + + def get_attributes(space, self): + return space.newlist([space.wrap(attr) for attr in self.attributes]) + W_ObjectType.typedef = TypeDef( 'ObjectType', schema = interp_attrproperty('schema', W_ObjectType), name = interp_attrproperty('name', W_ObjectType), + attributes = GetSetProperty(W_ObjectType.get_attributes), + ) + +class W_ObjectAttribute(Wrappable): + def __init__(self, connection, param): + self.initialize(connection, param) + + def initialize(self, connection, param): + # determine the name of the attribute + nameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, 1, + flavor='raw') + lenptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, + flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, nameptr), + lenptr, + roci.OCI_ATTR_NAME, + connection.environment.errorHandle) + connection.environment.checkForError( + status, + "ObjectAttribute_Initialize(): get name") + self.name = rffi.charpsize2str(nameptr[0], lenptr[0]) + finally: + lltype.free(nameptr, flavor='raw') + lltype.free(lenptr, flavor='raw') + + # determine the type of the attribute + typecodeptr = lltype.malloc(roci.Ptr(roci.OCITypeCode).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + param, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, typecodeptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_TYPECODE, + connection.environment.errorHandle) + connection.environment.checkForError( + status, "ObjectType_Describe(): get type code") + self.typeCode = typecodeptr[0] + finally: + lltype.free(typecodeptr, flavor='raw') + + # if the type of the attribute is object, recurse + if self.typeCode in (roci.OCI_TYPECODE_NAMEDCOLLECTION, + roci.OCI_TYPECODE_OBJECT): + self.subType = W_ObjectType(connection, param) + +W_ObjectAttribute.typedef = TypeDef( + 'ObjectAttribute', + name = interp_attrproperty('name', W_ObjectAttribute), ) class W_ExternalObject(Wrappable): Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 17:12:33 2009 @@ -36,6 +36,7 @@ OCIInd = platform.SimpleType('OCIInd', rffi.INT) OCIPinOpt = platform.SimpleType('OCIPinOpt', rffi.INT) OCILockOpt = platform.SimpleType('OCILockOpt', rffi.INT) + OCITypeCode = platform.SimpleType('OCITypeCode', rffi.UINT) OCINumber = platform.Struct('OCINumber', []) OCITime = platform.Struct('OCITime', @@ -58,16 +59,18 @@ OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD - OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT + OCI_ATTR_STMT_TYPE OCI_ATTR_PARAM OCI_ATTR_PARAM_COUNT OCI_ATTR_ROW_COUNT OCI_ATTR_NAME OCI_ATTR_SCALE OCI_ATTR_PRECISION OCI_ATTR_IS_NULL OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_REF_TDO - OCI_ATTR_SCHEMA_NAME OCI_ATTR_TYPE_NAME + OCI_ATTR_SCHEMA_NAME OCI_ATTR_TYPE_NAME OCI_ATTR_TYPECODE + OCI_ATTR_NUM_TYPE_ATTRS OCI_ATTR_LIST_TYPE_ATTRS OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL OCI_PIN_ANY OCI_LOCK_NONE + OCI_OTYPE_PTR OCI_PTYPE_TYPE OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE SQLT_CHR SQLT_LNG SQLT_AFC SQLT_RDD SQLT_BIN SQLT_LBI SQLT_LVC SQLT_LVB @@ -78,6 +81,7 @@ SQLCS_IMPLICIT SQLCS_NCHAR OCI_TEMP_CLOB OCI_TEMP_BLOB OCI_DURATION_SESSION OCI_ONE_PIECE OCI_NUMBER_SIGNED + OCI_TYPECODE_NAMEDCOLLECTION OCI_TYPECODE_OBJECT OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA '''.split() @@ -283,6 +287,18 @@ ub4], # indszp sword) +OCIDescribeAny = external( + 'OCIDescribeAny', + [OCISvcCtx, # svchp + OCIError, # errhp + dvoidp, # objptr + ub4, # objptr_len + ub1, # objptr_typ + ub1, # info_level + ub1, # objtyp + OCIDescribe], # dschp + sword) + OCIStmtGetBindInfo = external( 'OCIStmtGetBindInfo', [OCIStmt, # stmtp Modified: pypy/trunk/pypy/module/oracle/test/test_objectvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_objectvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_objectvar.py Wed Nov 25 17:12:33 2009 @@ -31,4 +31,4 @@ objValue, = cur.fetchone() assert objValue.type.schema == self.cnx.username.upper() assert objValue.type.name == "PYPY_TEST_OBJTYPE" - #assert objValue.type.attributes[0].name == "NUMBERVALUE" + assert objValue.type.attributes[0].name == "NUMBERCOL" From arigo at codespeak.net Wed Nov 25 17:20:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 25 Nov 2009 17:20:17 +0100 (CET) Subject: [pypy-svn] r69638 - pypy/branch/virtual-forcing/pypy/jit/metainterp/doc Message-ID: <20091125162017.DF85C1680F1@codespeak.net> Author: arigo Date: Wed Nov 25 17:20:17 2009 New Revision: 69638 Removed: pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/linking.txt pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/matching_rules.txt pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/virtualizables.txt Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/jitpl5.txt pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/loop.txt Log: Delete and simplify the "explanations" here. Should be killed at some point, and written down as real documentation. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/jitpl5.txt ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/jitpl5.txt (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/jitpl5.txt Wed Nov 25 17:20:17 2009 @@ -78,16 +78,11 @@ matches the real data -- but this is delicate because of the non-escaping flag. -Instead, this is done by doing tracing from the start of the loop again. -At the end, we don't do perfect specialization (for now), but simply -check that the already-computed specialization still applies, and then -jump to the already-compiled loop. (If it does not match, for now we -just cancel everything.) - -If the loop is not only executed but *entered* often enough, then after -this tracing, we generate a second copy of the loop (a "bridge") that -starts with all variables unspecialized, and ends with a jump to the -real loop. From this point on, we can just jump directly to the bridge +Instead, this is done by "entry bridges": we do tracing from +the start of the loop again, and at the end, we try to compile +the recorded trace as a "bridge" that comes from the +interpreter (i.e. with no virtuals at all) and goes to the old +loop. Later on, we can just jump directly to the entry bridge from the JUMP_ABSOLUTE bytecode. @@ -105,10 +100,7 @@ take the set of live values and put them back into boxes, and proceed with tracing for the rest of the loop. -For now, we just check at the end of the loop that it matches the -already-computed specialization. If not, we cancel creating the -compiled version of it (and mark the guard so that future failures -always fall back to interpretation). To do this, when we created the -original loop, at every guard, we needed to record the set of live -values (mostly in which register or stack location they are) as well as -an "escaped-so-far" flag for each pointer. +At the end of the loop, we check that it matches an already-computed +specialization. If not, we go on tracing. This might unroll the loop +once. (Note that there is a global limit on the length of the recorded +trace, to avoid tracing forever.) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/loop.txt ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/loop.txt (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/doc/loop.txt Wed Nov 25 17:20:17 2009 @@ -36,32 +36,11 @@ . VirtualSpec(cls, name1=spec1, ...) | - VirtualizableSpec(cls, name1=spec1, ...) - | - FixedClassSpec(cls) - | NotSpec -For (a simplified) example, ``VirtualizableSpec(PyFrame, x = -VirtualSpec(W_IntObject, value = NotSpec))`` describes the virtualizable -frame for a loop in which the only used variable is ``x``, which is a -virtual ``W_IntObject``. - -The intersection rules are: - -* the intersection of two ``VirtualSpec`` of the same ``cls`` is a - further ``VirtualSpec``, and we proceed with the intersection of - each field. - -* the intersection of two ``VirtualizableSpec`` of the same ``cls`` is - like the previous case, except that some names may be omitted - completely from a given ``VirtualizableSpec``; in the case a name is - present in only one of the ``VirtualizableSpec``, we just keep it - unmodified in the intersection. - -* in other cases, the result is ``FixedClassSpec`` if the two specnodes - have the same class, or ``NotSpec`` if any is a ``NotSpec`` or if the - two classes differ. +For example, ``VirtualSpec(W_IntObject, value = NotSpec))`` describes a +variable which is a virtual ``W_IntObject``, containing a value that is +a real integer. Overall Approach From afa at codespeak.net Wed Nov 25 17:33:37 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 17:33:37 +0100 (CET) Subject: [pypy-svn] r69639 - pypy/trunk/pypy/module/oracle Message-ID: <20091125163337.415F61680F7@codespeak.net> Author: afa Date: Wed Nov 25 17:33:36 2009 New Revision: 69639 Modified: pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/roci.py Log: Ensure object cleanup Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Wed Nov 25 17:33:36 2009 @@ -6,10 +6,18 @@ class W_ObjectType(Wrappable): def __init__(self, connection, param): + self.tdo = None self.environment = connection.environment self.isCollection = False self.initialize(connection, param) + def __del__(self): + if self.tdo: + roci.OCIObjectUnpin( + self.environment.handle, + self.environment.errorHandle, + self.tdo) + def initialize(self, connection, param): nameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, 1, flavor='raw') Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 17:33:36 2009 @@ -69,7 +69,7 @@ OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL - OCI_PIN_ANY OCI_LOCK_NONE + OCI_PIN_ANY OCI_LOCK_NONE OCI_OBJECTFREE_FORCE OCI_OTYPE_PTR OCI_PTYPE_TYPE OCI_STMT_SELECT OCI_STMT_CREATE OCI_STMT_DROP OCI_STMT_ALTER OCI_STMT_INSERT OCI_STMT_DELETE OCI_STMT_UPDATE @@ -461,6 +461,14 @@ # OCI Object Pin, Unpin, and Free Functions +OCIObjectFree = external( + 'OCIObjectFree', + [OCIEnv, # env, + OCIError, # err + dvoidp, # instance + ub2], # flags + sword) + OCIObjectPin = external( 'OCIObjectPin', [OCIEnv, # env, @@ -473,6 +481,13 @@ dvoidpp], # object sword) +OCIObjectUnpin = external( + 'OCIObjectUnpin', + [OCIEnv, # env, + OCIError, # err + dvoidp], # object + sword) + # OCI Date, Datetime, and Interval Functions OCIDateTimeCheck = external( From afa at codespeak.net Wed Nov 25 18:21:30 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 18:21:30 +0100 (CET) Subject: [pypy-svn] r69640 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125172130.2E77B1680F4@codespeak.net> Author: afa Date: Wed Nov 25 18:21:29 2009 New Revision: 69640 Modified: pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_objectvar.py Log: Implement description and retrieval of collections types. Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Wed Nov 25 18:21:29 2009 @@ -2,7 +2,8 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.oracle import roci + +from pypy.module.oracle import roci, transform class W_ObjectType(Wrappable): def __init__(self, connection, param): @@ -158,16 +159,59 @@ self.isCollection = 1 # determine type of collection - XXX + typecodeptr = lltype.malloc(roci.Ptr(roci.OCITypeCode).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + toplevelParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, typecodeptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_TYPECODE, + self.environment.errorHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): get collection type code") + self.collectionTypeCode = typecodeptr[0] + finally: + lltype.free(typecodeptr, flavor='raw') # acquire collection parameter descriptor - XXX + paramptr = lltype.malloc(roci.Ptr(roci.OCIParam).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + toplevelParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, paramptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_COLLECTION_ELEMENT, + self.environment.errorHandle) + self.environment.checkForError( + status, + "ObjectType_Describe(): get collection descriptor") + collectionParam = paramptr[0] + finally: + lltype.free(paramptr, flavor='raw') # determine type of element - XXX + typecodeptr = lltype.malloc(roci.Ptr(roci.OCITypeCode).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + collectionParam, roci.OCI_DTYPE_PARAM, + rffi.cast(roci.dvoidp, typecodeptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + roci.OCI_ATTR_TYPECODE, + self.environment.errorHandle) + self.environment.checkForError( + status, "ObjectType_Describe(): get element type code") + self.elementTypeCode = typecodeptr[0] + finally: + lltype.free(typecodeptr, flavor='raw') # if element type is an object type get its type - XXX + if self.elementTypeCode == roci.OCI_TYPECODE_OBJECT: + self.elementType = W_ObjectType(connection, collectionParam) + else: + self.elementType = None # determine the number of attributes numptr = lltype.malloc(roci.Ptr(roci.ub2).TO, @@ -296,3 +340,98 @@ 'ExternalObject', type = interp_attrproperty('type', W_ExternalObject), ) + +def convertToPython(space, environment, typeCode, + value, indicator, var, subtype): + # null values returned as None + if rffi.cast(roci.Ptr(roci.OCIInd), indicator)[0] == roci.OCI_IND_NULL: + return space.w_None + + if typeCode in (roci.OCI_TYPECODE_CHAR, + roci.OCI_TYPECODE_VARCHAR, + roci.OCI_TYPECODE_VARCHAR2): + strValue = value + stringValue = roci.OCIStringPtr(environment.handle, strValue) + stringSize = roci.OCIStringPtr(environment.handle, strValue) + return config.w_string(space, stringValue, stringSize) + elif typeCode == roci.OCI_TYPECODE_NUMBER: + return transform.OracleNumberToPythonFloat( + environment, + rffi.cast(roci.Ptr(roci.OCINumber), value)) + elif typeCode == roci.OCI_TYPECODE_DATE: + return transform.OracleDateToPythonDate(environment, value) + elif typeCode == roci.OCI_TYPECODE_TIMESTAMP: + return transform.OracleTimestampToPythonDate(environment, value) + elif typeCode == roci.OCI_TYPECODE_OBJECT: + return space.wrap(W_ExternalObject(var, subType, value, indicator, + isIndependent=False)) + elif typeCode == roci.OCI_TYPECODE_NAMEDCOLLECTION: + return convertCollection(space, environment, value, var, subType) + + raise OperationError( + get(space).w_NotSupportedError, + space.wrap( + "ExternalObjectVar_GetAttributeValue(): unhandled data type %d" % ( + typeCode,))) + + +def convertCollection(space, environment, value, var, objectType): + "Convert a collection to a Python list" + + result_w = [] + + iterptr = lltype.malloc(rffi.CArrayPtr(roci.OCIIter).TO, 1, flavor='raw') + try: + # create the iterator + status = roci.OCIIterCreate( + environment.handle, + environment.errorHandle, + value, + iterptr) + environment.checkForError( + status, "ExternalObjectVar_ConvertCollection(): creating iterator") + + try: + # create the result list + valueptr = lltype.malloc(rffi.CArrayPtr(roci.dvoidp).TO, + 1, flavor='raw') + indicatorptr = lltype.malloc(rffi.CArrayPtr(roci.dvoidp).TO, + 1, flavor='raw') + eofptr = lltype.malloc(rffi.CArrayPtr(roci.boolean).TO, + 1, flavor='raw') + try: + while True: + status = roci.OCIIterNext( + environment.handle, + environment.errorHandle, + iterptr[0], + valueptr, + indicatorptr, + eofptr) + environment.checkForError( + status, + "ExternalObjectVar_ConvertCollection(): get next") + + if eofptr[0]: + break + element = convertToPython( + space, environment, + objectType.elementTypeCode, + valueptr[0], indicatorptr[0], + var, objectType.elementType) + result_w.append(element) + finally: + lltype.free(valueptr, flavor='raw') + lltype.free(indicatorptr, flavor='raw') + lltype.free(eofptr, flavor='raw') + + finally: + roci.OCIIterDelete( + environment.handle, + environment.errorHandle, + iterptr) + finally: + lltype.free(iterptr, flavor='raw') + + return space.newlist(result_w) + Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Wed Nov 25 18:21:29 2009 @@ -1190,7 +1190,7 @@ # for collections, return the list rather than the object if self.objectType.isCollection: return interp_object.convertCollection( - self.environment, data[pos], self, self.objectType) + space, self.environment, data[pos], self, self.objectType) # for objects, return a representation of the object var = interp_object.W_ExternalObject( Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 18:21:29 2009 @@ -64,6 +64,7 @@ OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_REF_TDO OCI_ATTR_SCHEMA_NAME OCI_ATTR_TYPE_NAME OCI_ATTR_TYPECODE OCI_ATTR_NUM_TYPE_ATTRS OCI_ATTR_LIST_TYPE_ATTRS + OCI_ATTR_COLLECTION_ELEMENT OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS @@ -81,6 +82,8 @@ SQLCS_IMPLICIT SQLCS_NCHAR OCI_TEMP_CLOB OCI_TEMP_BLOB OCI_DURATION_SESSION OCI_ONE_PIECE OCI_NUMBER_SIGNED + OCI_TYPECODE_CHAR OCI_TYPECODE_VARCHAR OCI_TYPECODE_VARCHAR2 + OCI_TYPECODE_NUMBER OCI_TYPECODE_DATE OCI_TYPECODE_TIMESTAMP OCI_TYPECODE_NAMEDCOLLECTION OCI_TYPECODE_OBJECT OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA '''.split() @@ -111,6 +114,8 @@ OCIRef = rffi.VOIDP OCIType = rffi.VOIDP OCIComplexObject = rffi.VOIDP +OCIColl = rffi.VOIDP +OCIIter = rffi.VOIDP Ptr = rffi.CArrayPtr void = lltype.Void @@ -488,6 +493,32 @@ dvoidp], # object sword) +# OCI Collection and Iterator Functions +OCIIterCreate = external( + 'OCIIterCreate', + [OCIEnv, # env, + OCIError, # err + OCIColl, # coll + Ptr(OCIIter)], # itr + sword) + +OCIIterDelete = external( + 'OCIIterDelete', + [OCIEnv, # env, + OCIError, # err + Ptr(OCIIter)], # itr + sword) + +OCIIterNext = external( + 'OCIIterNext', + [OCIEnv, # env, + OCIError, # err + OCIIter, # itr + dvoidpp, # elem + dvoidpp, # elemind + Ptr(boolean)], # eoc + sword) + # OCI Date, Datetime, and Interval Functions OCIDateTimeCheck = external( Modified: pypy/trunk/pypy/module/oracle/test/test_objectvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_objectvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_objectvar.py Wed Nov 25 18:21:29 2009 @@ -1,7 +1,7 @@ from pypy.module.oracle.test.test_connect import OracleTestBase class AppTestObjectVar(OracleTestBase): - def test_fetch(self): + def test_fetch_object(self): cur = self.cnx.cursor() try: cur.execute("drop table pypy_test_objtable") @@ -11,6 +11,10 @@ cur.execute("drop type pypy_test_objtype") except oracle.DatabaseError: pass + try: + cur.execute("drop type pypy_test_arraytype") + except oracle.DatabaseError: + pass cur.execute("""\ create type pypy_test_objtype as object ( numbercol number, @@ -18,17 +22,24 @@ datecol date); """) cur.execute("""\ + create type pypy_test_arraytype as varray(10) of number; + """) + cur.execute("""\ create table pypy_test_objtable ( - objcol pypy_test_objtype) + objcol pypy_test_objtype, + arraycol pypy_test_arraytype) """) cur.execute("""\ insert into pypy_test_objtable values ( pypy_test_objtype(1, 'someText', - to_date(20070306, 'YYYYMMDD'))) + to_date(20070306, 'YYYYMMDD')), + pypy_test_arraytype(5, 10, null, 20)) """) - cur.execute("select objcol from pypy_test_objtable") - objValue, = cur.fetchone() + cur.execute("select objcol, arraycol from pypy_test_objtable") + objValue, arrayValue = cur.fetchone() assert objValue.type.schema == self.cnx.username.upper() assert objValue.type.name == "PYPY_TEST_OBJTYPE" assert objValue.type.attributes[0].name == "NUMBERCOL" + assert isinstance(arrayValue, list) + assert arrayValue == [5, 10, None, 20] From afa at codespeak.net Wed Nov 25 19:20:02 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 25 Nov 2009 19:20:02 +0100 (CET) Subject: [pypy-svn] r69641 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091125182002.0EACA1680F9@codespeak.net> Author: afa Date: Wed Nov 25 19:20:01 2009 New Revision: 69641 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_objectvar.py Log: Implement attribute access on OBJECT data Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Wed Nov 25 19:20:01 2009 @@ -19,6 +19,7 @@ 'CURSOR': 'interp_variable.VT_Cursor', 'BLOB': 'interp_variable.VT_BLOB', 'CLOB': 'interp_variable.VT_CLOB', + 'OBJECT': 'interp_variable.VT_Object', 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', 'Date': 'interp_error.get(space).w_DateType', Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Wed Nov 25 19:20:01 2009 @@ -1,9 +1,12 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty +from pypy.interpreter.gateway import ObjSpace +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.oracle import roci, transform +from pypy.module.oracle import roci, config, transform class W_ObjectType(Wrappable): def __init__(self, connection, param): @@ -323,6 +326,8 @@ if self.typeCode in (roci.OCI_TYPECODE_NAMEDCOLLECTION, roci.OCI_TYPECODE_OBJECT): self.subType = W_ObjectType(connection, param) + else: + self.subType = None W_ObjectAttribute.typedef = TypeDef( 'ObjectAttribute', @@ -330,15 +335,79 @@ ) class W_ExternalObject(Wrappable): - def __init__(self, ref, type, instance, indicator, isIndependent=True): + def __init__(self, ref, objectType, instance, indicator, isIndependent=True): self.ref = ref - self.type = type + self.objectType = objectType self.instance = instance self.indicator = indicator self.isIndependent = isIndependent + + def getattr(self, space, attr): + try: + attribute = self.objectType.attributesByName[attr] + except KeyError: + msg = "ExternalObject has no attribute '%s'" %(attr,) + raise OperationError(space.w_AttributeError, space.wrap(msg)) + + environment = self.objectType.environment + + scalarvalueindicatorptr = lltype.malloc(rffi.CArrayPtr(roci.OCIInd).TO, + 1, flavor='raw') + valueindicatorptr = lltype.malloc(rffi.CArrayPtr(roci.dvoidp).TO, + 1, flavor='raw') + valueptr = lltype.malloc(rffi.CArrayPtr(roci.dvoidp).TO, + 1, flavor='raw') + tdoptr = lltype.malloc(rffi.CArrayPtr(roci.OCIType).TO, + 1, flavor='raw') + nameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, + 1, flavor='raw') + nameptr[0] = rffi.str2charp(attr) + namelenptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + namelenptr[0] = rffi.cast(roci.ub4, len(attr)) + + try: + status = roci.OCIObjectGetAttr( + environment.handle, + environment.errorHandle, + self.instance, + self.indicator, + self.objectType.tdo, + nameptr, namelenptr, 1, + lltype.nullptr(roci.Ptr(roci.ub4).TO), 0, + scalarvalueindicatorptr, + valueindicatorptr, + valueptr, + tdoptr) + environment.checkForError( + status, "ExternalObject_GetAttributeValue(): getting value") + + # determine the proper null indicator + valueIndicator = valueindicatorptr[0] + if not valueIndicator: + valueIndicator = scalarvalueindicatorptr + value = valueptr[0] + + return convertToPython( + space, environment, + attribute.typeCode, + value, valueIndicator, + self, attribute.subType) + finally: + lltype.free(scalarvalueindicatorptr, flavor='raw') + lltype.free(valueindicatorptr, flavor='raw') + lltype.free(valueptr, flavor='raw') + lltype.free(tdoptr, flavor='raw') + rffi.free_charp(nameptr[0]) + lltype.free(nameptr, flavor='raw') + lltype.free(namelenptr, flavor='raw') + + getattr.unwrap_spec = ['self', ObjSpace, str] + W_ExternalObject.typedef = TypeDef( 'ExternalObject', - type = interp_attrproperty('type', W_ExternalObject), + type = interp_attrproperty('objectType', W_ExternalObject), + __getattr__ = interp2app(W_ExternalObject.getattr), ) def convertToPython(space, environment, typeCode, Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Wed Nov 25 19:20:01 2009 @@ -464,6 +464,27 @@ ub4], # type sword) +# OCI Miscellaneous Object Functions + +OCIObjectGetAttr = external( + 'OCIObjectGetAttr', + [OCIEnv, # env + OCIError, # err + dvoidp, # instance + dvoidp, # null_struct + OCIType, # tdo + Ptr(oratext), # names + Ptr(ub4), # lengths + ub4, # name_count + Ptr(ub4), # indexes + ub4, # index_count + Ptr(OCIInd), # attr_null_status + dvoidpp, # attr_null_struct + dvoidpp, # attr_value + dvoidpp], # attr_tdo + sword) + + # OCI Object Pin, Unpin, and Free Functions OCIObjectFree = external( Modified: pypy/trunk/pypy/module/oracle/test/test_objectvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_objectvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_objectvar.py Wed Nov 25 19:20:01 2009 @@ -43,3 +43,5 @@ assert objValue.type.attributes[0].name == "NUMBERCOL" assert isinstance(arrayValue, list) assert arrayValue == [5, 10, None, 20] + assert objValue.NUMBERCOL == 1 + raises(AttributeError, getattr, objValue, 'OTHERCOL') From santagada at codespeak.net Thu Nov 26 00:57:20 2009 From: santagada at codespeak.net (santagada at codespeak.net) Date: Thu, 26 Nov 2009 00:57:20 +0100 (CET) Subject: [pypy-svn] r69643 - in pypy/branch/force-arch-darwin: lib-python lib-python/2.5.2 lib-python/2.5.2/test lib-python/modified-2.5.2/test pypy/module/_locale pypy/module/_locale/test Message-ID: <20091125235720.410C41683CE@codespeak.net> Author: santagada Date: Thu Nov 26 00:57:18 2009 New Revision: 69643 Added: pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py - copied unchanged from r69622, pypy/branch/force-arch-darwin/lib-python/2.5.2/test/test_locale.py Removed: pypy/branch/force-arch-darwin/lib-python/modified-2.5.2/test/test_locale.py Modified: pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py pypy/branch/force-arch-darwin/lib-python/conftest.py pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py Log: reverted 69623 Modified: pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py (original) +++ pypy/branch/force-arch-darwin/lib-python/2.5.2/locale.py Thu Nov 26 00:57:18 2009 @@ -30,7 +30,7 @@ from _locale import * except ImportError: - raise + # Locale emulation CHAR_MAX = 127 Modified: pypy/branch/force-arch-darwin/lib-python/conftest.py ============================================================================== --- pypy/branch/force-arch-darwin/lib-python/conftest.py (original) +++ pypy/branch/force-arch-darwin/lib-python/conftest.py Thu Nov 26 00:57:18 2009 @@ -282,7 +282,7 @@ RegrTest('test_largefile.py'), RegrTest('test_linuxaudiodev.py', skip="unsupported extension module"), RegrTest('test_list.py', core=True), - RegrTest('test_locale.py', usemodules='_rawffi'), + RegrTest('test_locale.py'), RegrTest('test_logging.py', usemodules='thread'), RegrTest('test_long.py', core=True), RegrTest('test_long_future.py', core=True), Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/__init__.py Thu Nov 26 00:57:18 2009 @@ -12,7 +12,7 @@ 'strxfrm': 'interp_locale.strxfrm', } - if sys.platform == 'win32': + if sys.platform in ('win32', 'darwin'): interpleveldefs.update({ '_getdefaultlocale': 'interp_locale.getdefaultlocale', }) Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/interp_locale.py Thu Nov 26 00:57:18 2009 @@ -21,15 +21,8 @@ includes += ['libintl.h'] if sys.platform == 'win32': includes += ['windows.h'] - - if sys.platform == 'darwin': - libraries = ['intl'] - else: - libraries = [] - _compilation_info_ = ExternalCompilationInfo( includes=includes, - libraries=libraries ) HAVE_BIND_TEXTDOMAIN_CODESET = platform.Has('bind_textdomain_codeset') lconv = platform.Struct("struct lconv", [ @@ -106,11 +99,8 @@ # should we support them? langinfo_names.extend('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT T_FMT ' 'AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' - 'NOEXPR'.split()) - _DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') - if _DATE_FMT: - langinfo_names.append('_DATE_FMT') + 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' + '_DATE_FMT'.split()) for i in range(1, 8): langinfo_names.append("DAY_%d" % i) langinfo_names.append("ABDAY_%d" % i) @@ -465,3 +455,5 @@ finally: lltype.free(buf_lang, flavor='raw') lltype.free(buf_country, flavor='raw') +elif sys.platform == 'darwin': + raise NotImplementedError() Modified: pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py ============================================================================== --- pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py (original) +++ pypy/branch/force-arch-darwin/pypy/module/_locale/test/test_locale.py Thu Nov 26 00:57:18 2009 @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import py from pypy.conftest import gettestobjspace -# from pypy.rpython.tool import rffi_platform as platform + import sys class AppTestLocaleTrivia: @@ -65,16 +65,11 @@ # HAVE_LANGINFO - if sys.platform != 'win32': _LANGINFO_NAMES = ('RADIXCHAR THOUSEP CRNCYSTR D_T_FMT D_FMT ' 'T_FMT AM_STR PM_STR CODESET T_FMT_AMPM ERA ERA_D_FMT ' - 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR ' - 'NOEXPR').split() - # XXX don't know how to conditionally test this. - #_DATE_FMT = platform.SimpleType('_DATE_FMT', ifdef='_DATE_FMT') - #if _DATE_FMT: - #langinfo_names.append('_DATE_FMT') + 'ERA_D_T_FMT ERA_T_FMT ALT_DIGITS YESEXPR NOEXPR ' + '_DATE_FMT').split() for i in range(1, 8): _LANGINFO_NAMES.append("DAY_%d" % i) _LANGINFO_NAMES.append("ABDAY_%d" % i) From fijal at codespeak.net Thu Nov 26 00:58:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 00:58:18 +0100 (CET) Subject: [pypy-svn] r69644 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091125235818.C5B5E1683CE@codespeak.net> Author: fijal Date: Thu Nov 26 00:58:18 2009 New Revision: 69644 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: Use 2.6 features to get errno in platform-independent way. Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Thu Nov 26 00:58:18 2009 @@ -6,6 +6,11 @@ except ImportError: ctypes = None +if sys.version >= (2, 6): + load_library_kwargs = {'use_errno': True} +else: + load_library_kwargs = {} + import os from pypy import conftest from pypy.rpython.lltypesystem import lltype, llmemory @@ -808,7 +813,8 @@ return ctypes.util.find_library('c') libc_name = get_libc_name() # Make sure the name is determined during import, not at runtime - standard_c_lib = ctypes.cdll.LoadLibrary(get_libc_name()) + standard_c_lib = ctypes.cdll.LoadLibrary(get_libc_name(), + **load_library_kwargs) # ____________________________________________ @@ -897,7 +903,7 @@ # on ie slackware there was need for RTLD_GLOBAL here. # this breaks a lot of things, since passing RTLD_GLOBAL # creates symbol conflicts on C level. - clib = dllclass.LoadLibrary(libpath) + clib = dllclass.LoadLibrary(libpath, **load_library_kwargs) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break @@ -1206,17 +1212,28 @@ # helpers to save/restore the C-level errno -- platform-specific because # ctypes doesn't just do the right thing and expose it directly :-( -def _where_is_errno(): - raise NotImplementedError("don't know how to get the C-level errno!") -def _save_c_errno(): - errno_p = _where_is_errno() - TLS.errno = errno_p.contents.value - errno_p.contents.value = 0 - -def _restore_c_errno(): - if hasattr(TLS, 'errno'): - _where_is_errno().contents.value = TLS.errno +# on 2.6 ctypes does it right, use it + +if sys.version >= (2, 6): + def _save_c_errno(): + TLS.errno = ctypes.get_errno() + + def _restore_c_errno(): + pass + +else: + def _where_is_errno(): + raise NotImplementedError("don't know how to get the C-level errno!") + + def _save_c_errno(): + errno_p = _where_is_errno() + TLS.errno = errno_p.contents.value + errno_p.contents.value = 0 + + def _restore_c_errno(): + if hasattr(TLS, 'errno'): + _where_is_errno().contents.value = TLS.errno if ctypes: if sys.platform == 'win32': From fijal at codespeak.net Thu Nov 26 01:17:12 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 01:17:12 +0100 (CET) Subject: [pypy-svn] r69645 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20091126001712.29A351683CE@codespeak.net> Author: fijal Date: Thu Nov 26 01:17:11 2009 New Revision: 69645 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: * User version_info instead of version, bah * Happy ctypes mess. LoadLibrary does not accept kwargs of CDLL, hack hack hack Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Thu Nov 26 01:17:11 2009 @@ -6,7 +6,7 @@ except ImportError: ctypes = None -if sys.version >= (2, 6): +if sys.version_info >= (2, 6): load_library_kwargs = {'use_errno': True} else: load_library_kwargs = {} @@ -813,8 +813,8 @@ return ctypes.util.find_library('c') libc_name = get_libc_name() # Make sure the name is determined during import, not at runtime - standard_c_lib = ctypes.cdll.LoadLibrary(get_libc_name(), - **load_library_kwargs) + # XXX is this always correct??? + standard_c_lib = ctypes.CDLL(get_libc_name(), **load_library_kwargs) # ____________________________________________ @@ -903,7 +903,7 @@ # on ie slackware there was need for RTLD_GLOBAL here. # this breaks a lot of things, since passing RTLD_GLOBAL # creates symbol conflicts on C level. - clib = dllclass.LoadLibrary(libpath, **load_library_kwargs) + clib = dllclass._dlltype(libpath, **load_library_kwargs) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break @@ -1215,7 +1215,7 @@ # on 2.6 ctypes does it right, use it -if sys.version >= (2, 6): +if sys.version_info >= (2, 6): def _save_c_errno(): TLS.errno = ctypes.get_errno() @@ -1235,18 +1235,18 @@ if hasattr(TLS, 'errno'): _where_is_errno().contents.value = TLS.errno -if ctypes: - if sys.platform == 'win32': - standard_c_lib._errno.restype = ctypes.POINTER(ctypes.c_int) - def _where_is_errno(): - return standard_c_lib._errno() - - elif sys.platform in ('linux2', 'freebsd6'): - standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int) - def _where_is_errno(): - return standard_c_lib.__errno_location() - - elif sys.platform in ('darwin', 'freebsd7'): - standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) - def _where_is_errno(): - return standard_c_lib.__error() + if ctypes: + if sys.platform == 'win32': + standard_c_lib._errno.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib._errno() + + elif sys.platform in ('linux2', 'freebsd6'): + standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib.__errno_location() + + elif sys.platform in ('darwin', 'freebsd7'): + standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) + def _where_is_errno(): + return standard_c_lib.__error() From afa at codespeak.net Thu Nov 26 10:45:20 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 10:45:20 +0100 (CET) Subject: [pypy-svn] r69646 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126094520.18830168030@codespeak.net> Author: afa Date: Thu Nov 26 10:45:19 2009 New Revision: 69646 Modified: pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_objectvar.py Log: Test and fix in getting OBJECT attributes Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Thu Nov 26 10:45:19 2009 @@ -335,8 +335,9 @@ ) class W_ExternalObject(Wrappable): - def __init__(self, ref, objectType, instance, indicator, isIndependent=True): - self.ref = ref + def __init__(self, var, objectType, instance, indicator, + isIndependent=True): + self.var = var # keepalive self.objectType = objectType self.instance = instance self.indicator = indicator @@ -388,7 +389,7 @@ valueIndicator = scalarvalueindicatorptr value = valueptr[0] - return convertToPython( + return convertObject( space, environment, attribute.typeCode, value, valueIndicator, @@ -410,8 +411,8 @@ __getattr__ = interp2app(W_ExternalObject.getattr), ) -def convertToPython(space, environment, typeCode, - value, indicator, var, subtype): +def convertObject(space, environment, typeCode, + value, indicator, var, subtype): # null values returned as None if rffi.cast(roci.Ptr(roci.OCIInd), indicator)[0] == roci.OCI_IND_NULL: return space.w_None @@ -419,18 +420,20 @@ if typeCode in (roci.OCI_TYPECODE_CHAR, roci.OCI_TYPECODE_VARCHAR, roci.OCI_TYPECODE_VARCHAR2): - strValue = value - stringValue = roci.OCIStringPtr(environment.handle, strValue) - stringSize = roci.OCIStringPtr(environment.handle, strValue) - return config.w_string(space, stringValue, stringSize) + strValue = rffi.cast(roci.Ptr(roci.OCIString), value)[0] + ptr = roci.OCIStringPtr(environment.handle, strValue) + size = roci.OCIStringSize(environment.handle, strValue) + return config.w_string(space, ptr, size) elif typeCode == roci.OCI_TYPECODE_NUMBER: return transform.OracleNumberToPythonFloat( environment, rffi.cast(roci.Ptr(roci.OCINumber), value)) elif typeCode == roci.OCI_TYPECODE_DATE: - return transform.OracleDateToPythonDate(environment, value) + dateValue = rffi.cast(roci.Ptr(roci.OCIDate), value) + return transform.OracleDateToPythonDateTime(environment, dateValue) elif typeCode == roci.OCI_TYPECODE_TIMESTAMP: - return transform.OracleTimestampToPythonDate(environment, value) + dateValue = rffi.cast(roci.Ptr(roci.OCIDateTime), value) + return transform.OracleTimestampToPythonDate(environment, dateValue) elif typeCode == roci.OCI_TYPECODE_OBJECT: return space.wrap(W_ExternalObject(var, subType, value, indicator, isIndependent=False)) @@ -483,7 +486,7 @@ if eofptr[0]: break - element = convertToPython( + element = convertObject( space, environment, objectType.elementTypeCode, valueptr[0], indicatorptr[0], Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 26 10:45:19 2009 @@ -108,6 +108,7 @@ OCIDefine = rffi.VOIDP OCIDescribe = rffi.VOIDP OCISnapshot = rffi.VOIDP +OCIString = rffi.VOIDP OCIDateTime = rffi.VOIDP OCIInterval = rffi.VOIDP OCILobLocator = rffi.VOIDP @@ -671,6 +672,20 @@ oratext], # buf sword) +# OCI String Functions + +OCIStringPtr = external( + 'OCIStringPtr', + [OCIEnv, # envhp + OCIString], # vs + oratext) + +OCIStringSize = external( + 'OCIStringSize', + [OCIEnv, # envhp + OCIString], # vs + ub4) + # OCI Locale Functions OCINlsCharSetIdToName = external( Modified: pypy/trunk/pypy/module/oracle/test/test_objectvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_objectvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_objectvar.py Thu Nov 26 10:45:19 2009 @@ -2,6 +2,7 @@ class AppTestObjectVar(OracleTestBase): def test_fetch_object(self): + import datetime cur = self.cnx.cursor() try: cur.execute("drop table pypy_test_objtable") @@ -44,4 +45,6 @@ assert isinstance(arrayValue, list) assert arrayValue == [5, 10, None, 20] assert objValue.NUMBERCOL == 1 - raises(AttributeError, getattr, objValue, 'OTHERCOL') + assert objValue.STRINGCOL == "someText" + assert objValue.DATECOL == datetime.datetime(2007, 03, 06) + raises(AttributeError, getattr, objValue, 'OTHER') From fijal at codespeak.net Thu Nov 26 11:29:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 11:29:07 +0100 (CET) Subject: [pypy-svn] r69647 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20091126102907.CCE911683DD@codespeak.net> Author: fijal Date: Thu Nov 26 11:29:07 2009 New Revision: 69647 Modified: pypy/trunk/pypy/module/posix/__init__.py pypy/trunk/pypy/module/posix/app_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: (Ademan) Implement os.wait using waitpid Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Thu Nov 26 11:29:07 2009 @@ -26,6 +26,9 @@ 'popen3' : 'app_posix.popen3', 'popen4' : 'app_posix.popen4', }) + + if hasattr(os, 'wait'): + appleveldefs['wait'] = 'app_posix.wait' interpleveldefs = { 'open' : 'interp_posix.open', Modified: pypy/trunk/pypy/module/posix/app_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/app_posix.py (original) +++ pypy/trunk/pypy/module/posix/app_posix.py Thu Nov 26 11:29:07 2009 @@ -153,6 +153,9 @@ try_close(read_end) raise Exception, e # bare 'raise' does not work here :-( + def wait(): + return posix.waitpid(-1, 0) + else: # Windows implementations @@ -170,6 +173,7 @@ univ_nl = ('b' not in mode) import subprocess + if mode.startswith('r'): proc = subprocess.Popen(cmd, shell=True, @@ -194,7 +198,6 @@ raise ValueError("invalid mode %r" % (mode,)) import subprocess - p = subprocess.Popen(cmd, shell=True, bufsize=bufsize, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -210,7 +213,6 @@ raise ValueError("invalid mode %r" % (mode,)) import subprocess - p = subprocess.Popen(cmd, shell=True, bufsize=bufsize, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -228,7 +230,6 @@ raise ValueError("invalid mode %r" % (mode,)) import subprocess - p = subprocess.Popen(cmd, shell=True, bufsize=bufsize, stdin=subprocess.PIPE, stdout=subprocess.PIPE, Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Thu Nov 26 11:29:07 2009 @@ -290,6 +290,7 @@ for i in range(5): stream = os.popen('echo 1') assert stream.read() == '1\n' + stream.close() if hasattr(__import__(os.name), '_getfullpathname'): def test__getfullpathname(self): @@ -393,6 +394,22 @@ def test_os_sysconf_error(self): os = self.posix raises(ValueError, os.sysconf, "!@#$%!#$!@#") + + if hasattr(os, 'wait'): + def test_os_wait(self): + os = self.posix + exit_status = 0x33 + + if not hasattr(os, "fork"): + skip("Need fork() to test wait()") + child = os.fork() + if child == 0: # in child + os._exit(exit_status) + else: + pid, status = os.wait() + assert child == pid + assert os.WIFEXITED(status) + assert os.WEXITSTATUS(status) == exit_status if hasattr(os, 'fsync'): def test_fsync(self): From fijal at codespeak.net Thu Nov 26 11:31:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 11:31:08 +0100 (CET) Subject: [pypy-svn] r69648 - in pypy/branch/virtual-forcing/pypy/jit/backend: llgraph llgraph/test test Message-ID: <20091126103108.0CFFF1683DD@codespeak.net> Author: fijal Date: Thu Nov 26 11:31:07 2009 New Revision: 69648 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/test/test_llgraph.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Log: force_token is now an into instead of address (it's a raw pointer in the x86 backend) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Thu Nov 26 11:31:07 2009 @@ -146,7 +146,7 @@ 'cast_ptr_to_int' : (('ref',), 'int'), 'debug_merge_point': (('ref',), None), 'force_token' : ((), 'int'), - 'call_may_force' : (('ref', 'varargs'), 'intorptr'), + 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None) #'getitem' : (('void', 'ref', 'int'), 'int'), #'setitem' : (('void', 'ref', 'int', 'int'), None), Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Thu Nov 26 11:31:07 2009 @@ -498,7 +498,7 @@ return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) def force(self, force_token): - frame = llimpl.force(force_token) + frame = llimpl.force(self.cast_int_to_adr(force_token)) self.latest_frame = frame class OOtypeCPU(BaseCPU): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/test/test_llgraph.py Thu Nov 26 11:31:07 2009 @@ -10,6 +10,9 @@ from pypy.jit.backend.test.runner_test import LLtypeBackendTest class TestLLTypeLLGraph(LLtypeBackendTest): + # for individual tests see: + # ====> ../../test/runner_test.py + from pypy.jit.backend.llgraph.runner import LLtypeCPU as cpu_type def setup_method(self, _): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Thu Nov 26 11:31:07 2009 @@ -1229,7 +1229,7 @@ values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) - FUNC = self.FuncType([llmemory.Address, lltype.Signed], lltype.Void) + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) From fijal at codespeak.net Thu Nov 26 11:40:11 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 11:40:11 +0100 (CET) Subject: [pypy-svn] r69650 - pypy/trunk/pypy/module/posix Message-ID: <20091126104011.77A581683DD@codespeak.net> Author: fijal Date: Thu Nov 26 11:40:11 2009 New Revision: 69650 Modified: pypy/trunk/pypy/module/posix/app_posix.py pypy/trunk/pypy/module/posix/interp_posix.py Log: put a bit of docstrings Modified: pypy/trunk/pypy/module/posix/app_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/app_posix.py (original) +++ pypy/trunk/pypy/module/posix/app_posix.py Thu Nov 26 11:40:11 2009 @@ -154,6 +154,10 @@ raise Exception, e # bare 'raise' does not work here :-( def wait(): + """ wait() -> (pid, status) + + Wait for completion of a child process. + """ return posix.waitpid(-1, 0) else: Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Nov 26 11:40:11 2009 @@ -500,6 +500,7 @@ readlink.unwrap_spec = [ObjSpace, str] def fork(space): + try: pid = os.fork() except OSError, e: @@ -507,6 +508,10 @@ return space.wrap(pid) def waitpid(space, pid, options): + """ waitpid(pid, options) -> (pid, status) + + Wait for completion of a given child process. + """ try: pid, status = os.waitpid(pid, options) except OSError, e: @@ -811,6 +816,7 @@ else: def WSTAR(space, status): return space.newbool(getattr(os, name)(status)) + WSTAR.__doc__ = getattr(os, name).__doc__ WSTAR.unwrap_spec = [ObjSpace, int] WSTAR.func_name = name return WSTAR From afa at codespeak.net Thu Nov 26 11:59:40 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 11:59:40 +0100 (CET) Subject: [pypy-svn] r69651 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126105940.A33341683DD@codespeak.net> Author: afa Date: Thu Nov 26 11:59:39 2009 New Revision: 69651 Added: pypy/trunk/pypy/module/oracle/interp_pool.py Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py Log: Start implementing oracle.SessionPool Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 26 11:59:39 2009 @@ -23,6 +23,7 @@ 'Variable': 'interp_variable.W_Variable', 'Timestamp': 'interp_error.get(space).w_DateTimeType', 'Date': 'interp_error.get(space).w_DateType', + 'SessionPool': 'interp_pool.W_SessionPool', } appleveldefs = { Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Thu Nov 26 11:59:39 2009 @@ -12,7 +12,7 @@ from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS from pypy.module.oracle.interp_environ import Environment from pypy.module.oracle.interp_cursor import W_Cursor -#from pypy.module.oracle.interp_pool import W_Pool +from pypy.module.oracle.interp_pool import W_Pool from pypy.module.oracle.interp_variable import VT_String class W_Connection(Wrappable): @@ -43,7 +43,7 @@ W_Connection.__init__(self) # set up the environment - if 0 and w_pool: # XXX + if w_pool: pool = space.instance_w(W_Pool, w_pool) self.environment = pool.environment.clone() else: Added: pypy/trunk/pypy/module/oracle/interp_pool.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/oracle/interp_pool.py Thu Nov 26 11:59:39 2009 @@ -0,0 +1,144 @@ +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.rpython.lltypesystem import rffi, lltype + +Null = NoneNotWrapped + +from pypy.module.oracle import roci, config, interp_error, interp_environ + + +class W_SessionPool(Wrappable): + def __init__(self): + self.environment = None + + def descr_new(space, w_subtype, + w_user, w_password, w_dsn, + min, max, increment, + w_connectiontype=Null, + threaded=False, + getmode=roci.OCI_SPOOL_ATTRVAL_NOWAIT, + events=False, + homogeneous=True): + self = space.allocate_instance(W_SessionPool, w_subtype) + W_SessionPool.__init__(self) + + if w_connectiontype is not None: + if not space.is_true(space.issubtype(w_value, + interp_connect.W_Connection)): + raise OperationError( + interp_error.get(space).w_ProgrammingError, + space.wrap( + "connectiontype must be a subclass of Connection")) + self.w_username = w_user + self.w_password = w_password + self.w_tnsentry = w_dsn + + self.w_connectionType = w_connectiontype + self.minSessions = min + self.maxSessions = max + self.sessionIncrement = increment + self.homogeneous = homogeneous + + # set up the environment + self.environment = interp_environ.Environment(space, threaded, events) + + # create the session pool handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIServer).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, roci.OCI_HTYPE_SPOOL, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "SessionPool_New(): allocate handle") + self.handle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + # prepare pool mode + poolMode = roci.OCI_SPC_STMTCACHE + if self.homogeneous: + poolMode |= roci.OCI_SPC_HOMOGENEOUS + + # create the session pool + user_buf = config.StringBuffer() + user_buf.fill(space, self.w_username) + password_buf = config.StringBuffer() + password_buf.fill(space, self.w_password) + dsn_buf = config.StringBuffer() + dsn_buf.fill(space, self.w_tnsentry) + poolnameptr = lltype.malloc(rffi.CArrayPtr(roci.oratext).TO, 1, + flavor='raw') + poolnamelenptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, + flavor='raw') + + try: + status = roci.OCISessionPoolCreate( + self.environment.handle, + self.environment.errorHandle, + self.handle, + poolnameptr, poolnamelenptr, + dsn_buf.ptr, dsn_buf.size, + min, max, increment, + user_buf.ptr, user_buf.size, + password_buf.ptr, password_buf.size, + poolMode) + self.environment.checkForError( + status, "SessionPool_New(): create pool") + + self.w_name = config.w_string(space, poolnameptr[0], + poolnamelenptr[0]) + finally: + user_buf.clear() + password_buf.clear() + dsn_buf.clear() + + return space.wrap(self) + descr_new.unwrap_spec = [ObjSpace, W_Root, + W_Root, W_Root, W_Root, + int, int, int, + W_Root, + bool, int, bool, bool] + + def checkConnected(self, space): + if not self.handle: + raise OperationError( + get(space).w_InterfaceError, + space.wrap("not connected")) + +def computedProperty(oci_attr_code, oci_value_type): + def fget(space, self): + self.checkConnected(space) + + valueptr = lltype.malloc(rffi.CArrayPtr(oci_value_type).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrGet( + self.handle, roci.OCI_HTYPE_SPOOL, + rffi.cast(roci.dvoidp, valueptr), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + oci_attr_code, + self.environment.errorHandle) + return space.wrap(valueptr[0]) + finally: + lltype.free(valueptr, flavor='raw') + return GetSetProperty(fget) + +W_SessionPool.typedef = TypeDef( + "SessionPool", + __new__ = interp2app(W_SessionPool.descr_new.im_func), + username = interp_attrproperty_w('w_username', W_SessionPool), + password = interp_attrproperty_w('w_password', W_SessionPool), + tnsentry = interp_attrproperty_w('w_tnsentry', W_SessionPool), + min = interp_attrproperty('minSessions', W_SessionPool), + max = interp_attrproperty('maxSessions', W_SessionPool), + increment = interp_attrproperty('sessionIncrement', W_SessionPool), + opened = computedProperty(roci.OCI_ATTR_SPOOL_OPEN_COUNT, roci.ub4), + busy = computedProperty(roci.OCI_ATTR_SPOOL_BUSY_COUNT, roci.ub4), + timeout = computedProperty(roci.OCI_ATTR_SPOOL_TIMEOUT, roci.ub4), + getmode = computedProperty(roci.OCI_ATTR_SPOOL_GETMODE, roci.ub1), + ) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 26 11:59:39 2009 @@ -55,7 +55,7 @@ OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION - OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD @@ -67,6 +67,8 @@ OCI_ATTR_COLLECTION_ELEMENT OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET + OCI_ATTR_SPOOL_OPEN_COUNT OCI_ATTR_SPOOL_BUSY_COUNT OCI_ATTR_SPOOL_TIMEOUT + OCI_ATTR_SPOOL_GETMODE OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL @@ -86,6 +88,7 @@ OCI_TYPECODE_NUMBER OCI_TYPECODE_DATE OCI_TYPECODE_TIMESTAMP OCI_TYPECODE_NAMEDCOLLECTION OCI_TYPECODE_OBJECT OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA + OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS '''.split() for c in constants: @@ -102,6 +105,7 @@ OCIError = rffi.VOIDP OCIServer = rffi.VOIDP OCISession = rffi.VOIDP +OCISPool = rffi.VOIDP OCIStmt = rffi.VOIDP OCIParam = rffi.VOIDP OCIBind = rffi.VOIDP @@ -169,6 +173,25 @@ ub4], # mode sword) +OCISessionPoolCreate = external( + 'OCISessionPoolCreate', + [OCISvcCtx, # svchp + OCIError, # errhp + OCISPool, # spoolhp + Ptr(oratext), # poolName + Ptr(ub4), # poolNameLen + oratext, # connStr + ub4, # connStrLen + ub4, # sessMin + ub4, # sessMax + ub4, # sessIncr + oratext, # userid + ub4, # useridLen + oratext, # password + ub4, # passwordLen + ub4], # mode + sword) + # Handle and Descriptor Functions OCIAttrGet = external( Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Thu Nov 26 11:59:39 2009 @@ -116,3 +116,16 @@ assert encoding != "" +class AppTestPool(OracleNotConnectedTestBase): + def test_pool(self): + pool = oracle.SessionPool(self.username, self.password, + self.tnsentry, + 2, 8, 3) + assert pool.username == self.username + assert pool.password == self.password + assert pool.tnsentry == self.tnsentry + assert pool.max == 8 + assert pool.min == 2 + assert pool.increment == 3 + assert pool.opened == 2 + assert pool.busy == 0 From fijal at codespeak.net Thu Nov 26 12:52:37 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 12:52:37 +0100 (CET) Subject: [pypy-svn] r69652 - pypy/trunk/pypy/translator/platform Message-ID: <20091126115237.C14B316807A@codespeak.net> Author: fijal Date: Thu Nov 26 12:52:37 2009 New Revision: 69652 Modified: pypy/trunk/pypy/translator/platform/distutils_platform.py Log: set this attribute on class as well Modified: pypy/trunk/pypy/translator/platform/distutils_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/distutils_platform.py (original) +++ pypy/trunk/pypy/translator/platform/distutils_platform.py Thu Nov 26 12:52:37 2009 @@ -21,6 +21,7 @@ point soon completely """ name = "distutils" + so_ext = so_ext def __init__(self, cc=None): self.cc = cc From pedronis at codespeak.net Thu Nov 26 16:05:29 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 26 Nov 2009 16:05:29 +0100 (CET) Subject: [pypy-svn] r69653 - pypy/build/bot2/pypybuildbot Message-ID: <20091126150529.8DC1C1680F9@codespeak.net> Author: pedronis Date: Thu Nov 26 16:05:28 2009 New Revision: 69653 Modified: pypy/build/bot2/pypybuildbot/builds.py pypy/build/bot2/pypybuildbot/master.py Log: simplify own build factory Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Thu Nov 26 16:05:28 2009 @@ -87,12 +87,10 @@ workdir=workdir)) -class PyPyOwnTestFactory(factory.BuildFactory): +class Own(factory.BuildFactory): - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - cherrypick = kw.pop('cherrypick', '') - factory.BuildFactory.__init__(self, *a, **kw) + def __init__(self, platform='linux', cherrypick='', extra_cfgs=[]): + factory.BuildFactory.__init__(self) setup_steps(platform, self) @@ -102,7 +100,8 @@ "--logfile=testrun.log", "--config=pypy/testrunner_cfg.py", "--config=~/machine_cfg.py", - "--root=pypy", "--timeout=10800"], + "--root=pypy", "--timeout=10800" + ] + ["--config=%s" % cfg for cfg in extra_cfgs], logfiles={'pytestLog': 'testrun.log'}, timeout = 4000, env={"PYTHONPATH": ['.'], Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Nov 26 16:05:28 2009 @@ -44,10 +44,9 @@ pypybuilds = load('pypybuildbot.builds') -pypyOwnTestFactory = pypybuilds.PyPyOwnTestFactory() -pypyOwnTestFactoryWin = pypybuilds.PyPyOwnTestFactory(platform="win32") - -pypyJitOnlyOwnTestFactory = pypybuilds.PyPyOwnTestFactory(cherrypick="jit") +pypyOwnTestFactory = pypybuilds.Own() +pypyOwnTestFactoryWin = pypybuilds.Own(platform="win32") +pypyJitOnlyOwnTestFactory = pypybuilds.Own(cherrypick="jit") pypyTranslatedLibPythonTestFactory = pypybuilds.PyPyTranslatedLibPythonTestFactory() pypyTranslatedLibPythonTestFactoryWin = pypybuilds.PyPyTranslatedLibPythonTestFactory(platform="win32") From afa at codespeak.net Thu Nov 26 16:32:45 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 16:32:45 +0100 (CET) Subject: [pypy-svn] r69654 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126153245.946EB1680C1@codespeak.net> Author: afa Date: Thu Nov 26 16:32:44 2009 New Revision: 69654 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_error.py pypy/trunk/pypy/module/oracle/interp_pool.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py Log: Implement SessionPool.acquire() and release() Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 26 16:32:44 2009 @@ -5,6 +5,7 @@ interpleveldefs = { 'connect': 'interp_connect.W_Connection', + 'Connection': 'interp_connect.W_Connection', 'UNICODE': 'interp_variable.VT_NationalCharString', 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Thu Nov 26 16:32:44 2009 @@ -12,7 +12,7 @@ from pypy.module.oracle.config import string_w, StringBuffer, MAX_STRING_CHARS from pypy.module.oracle.interp_environ import Environment from pypy.module.oracle.interp_cursor import W_Cursor -from pypy.module.oracle.interp_pool import W_Pool +from pypy.module.oracle.interp_pool import W_SessionPool from pypy.module.oracle.interp_variable import VT_String class W_Connection(Wrappable): @@ -21,10 +21,13 @@ self.environment = None self.autocommit = False + self.sessionHandle = None + self.w_inputTypeHandler = None self.w_outputTypeHandler = None self.w_version = None + self.release = False def descr_new(space, w_subtype, w_user=NoneNotWrapped, @@ -44,10 +47,11 @@ # set up the environment if w_pool: - pool = space.instance_w(W_Pool, w_pool) + pool = space.interp_w(W_SessionPool, w_pool) self.environment = pool.environment.clone() else: - self.environment = Environment(space, threaded, events) + pool = None + self.environment = Environment.create(space, threaded, events) self.w_username = w_user self.w_password = w_password @@ -66,7 +70,10 @@ space.call_method(self.w_password, 'split', space.wrap('@'), space.wrap(1))) - self.connect(space, mode, twophase) + if pool or w_cclass: + self.getConnection(space, pool, w_cclass, purity) + else: + self.connect(space, mode, twophase) return space.wrap(self) descr_new.unwrap_spec = [ObjSpace, W_Root, @@ -77,7 +84,28 @@ W_Root, bool, W_Root] - + + def __del__(self): + if self.release: + roci.OCITransRollback( + self.handle, self.environment.errorHandle, + roci.OCI_DEFAULT) + roci.OCISessionRelease( + self.handle, self.environment.errorHandle, + None, 0, roci.OCI_DEFAULT) + else: + if self.sessionHandle: + roci.OCITransRollback( + self.handle, self.environment.errorHandle, + roci.OCI_DEFAULT) + roci.OCISessionEnd( + self.handle, self.environment.errorHandle, + self.sessionHandle, roci.OCI_DEFAULT) + if self.serverHandle: + roci.OCIServerDetach( + self.serverHandle, self.environment.errorHandle, + roci.OCI_DEFAULT) + def connect(self, space, mode, twophase): stringBuffer = StringBuffer() @@ -107,7 +135,7 @@ status, "Connection_Connect(): server attach") finally: stringBuffer.clear() - + # allocate the service context handle handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCISvcCtx).TO, 1, flavor='raw') @@ -131,7 +159,7 @@ self.environment.errorHandle) self.environment.checkForError( status, "Connection_Connect(): set server handle") - + # set the internal and external names; these are needed for global # transactions but are limited in terms of the lengths of the strings @@ -207,6 +235,145 @@ self.sessionHandle = lltype.nullptr(roci.OCISession.TO) raise + def getConnection(self, space, pool, w_cclass, purity): + """Get a connection using the OCISessionGet() interface + rather than using the low level interface for connecting.""" + + proxyCredentials = False + authInfo = None + + if pool: + w_dbname = pool.w_name + mode = roci.OCI_SESSGET_SPOOL + if not pool.homogeneous and pool.w_username and self.w_username: + proxyCredentials = space.ne(pool.w_username, self.w_username) + mode |= roci.OCI_SESSGET_CREDPROXY + else: + w_dbname = self.w_tnsentry + mode = roci.OCI_SESSGET_STMTCACHE + + stringBuffer = StringBuffer() + + # set up authorization handle, if needed + if not pool or w_cclass or proxyCredentials: + # create authorization handle + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, + roci.HTYPE_AUTHINFO, + 0, None) + self.environment.checkForError( + status, "Connection_GetConnection(): allocate handle") + + externalCredentials = True + + # set the user name, if applicable + stringBuffer.fill(space, self.w_username) + try: + if stringBuffer.size > 0: + externalCredentials = False + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_PASSWORD, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetConnection(): set user name") + finally: + stringBuffer.clear() + + # set the password, if applicable + stringBuffer.fill(space, self.w_password) + try: + if stringBuffer.size > 0: + externalCredentials = False + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_USERNAME, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetConnection(): set password") + finally: + stringBuffer.clear() + + # if no user name or password are set, using external credentials + if not pool and externalCredentials: + mode |= roci.OCI_SESSGET_CREDEXT + + # set the connection class, if applicable + stringBuffer.fill(space, w_cclass) + try: + if stringBuffer.size > 0: + externalCredentials = False + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + stringBuffer.ptr, stringBuffer.size, + roci.OCI_ATTR_CONNECTION_CLASS, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetConnection(): set connection class") + finally: + stringBuffer.clear() + + # set the purity, if applicable + purityptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + try: + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + purityptr, rffi.sizeof(roci.ub4), + roci.OCI_ATTR_PURITY, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetConnection(): set purity") + finally: + lltype.free(purityptr, flavor='raw') + + # acquire the new session + stringBuffer.fill(space, w_dbname) + foundptr = lltype.malloc(rffi.CArrayPtr(roci.boolean).TO, + 1, flavor='raw') + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCISvcCtx).TO, + 1, flavor='raw') + try: + status = roci.OCISessionGet( + self.environment.handle, + self.environment.errorHandle, + handleptr, + authInfo, + stringBuffer.ptr, stringBuffer.size, + None, 0, + lltype.nullptr(roci.Ptr(roci.oratext).TO), + lltype.nullptr(roci.Ptr(roci.ub4).TO), + foundptr, + mode) + self.environment.checkForError( + status, "Connection_GetConnection(): get connection") + + self.handle = handleptr[0] + finally: + stringBuffer.clear() + lltype.free(foundptr, flavor='raw') + + # eliminate the authorization handle immediately, if applicable + if authInfo: + roci.OCIHandleFree(authInfo, roci.OCI_HTYPE_AUTHINFO) + + # copy members in the case where a pool is being used + if pool: + if not proxyCredentials: + self.w_username = pool.w_username + self.w_password = pool.w_password + self.w_tnsentry = pool.w_tnsentry + self.sessionPool = pool + + self.release = True + def _checkConnected(self, space): if not self.handle: raise OperationError( Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Thu Nov 26 16:32:44 2009 @@ -4,9 +4,52 @@ from pypy.module.oracle.interp_error import W_Error, get -class Environment: - def __init__(self, space, threaded, events): +class Environment(object): + def __init__(self, space, handle): self.space = space + self.handle = handle + + # create the error handle + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIError).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.handle, + handleptr, roci.OCI_HTYPE_ERROR, 0, + lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.checkForError( + status, "Environment_New(): create error handle") + self.errorHandle = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') + + + def checkForError(self, status, context): + if status in (roci.OCI_SUCCESS, roci.OCI_SUCCESS_WITH_INFO): + return + + if status != roci.OCI_INVALID_HANDLE: + # At this point it is assumed that the Oracle + # environment is fully initialized + error = W_Error(self.space, self, context, 1) + if error.code in (1, 1400, 2290, 2291, 2292): + w_type = get(self.space).w_IntegrityError + elif error.code in (1012, 1033, 1034, 1089, 3113, 3114, + 12203, 12500, 12571): + w_type = get(self.space).w_OperationalError + else: + w_type = get(self.space).w_DatabaseError + raise OperationError(w_type, self.space.wrap(error)) + + error = W_Error(self.space, self, context, 0) + error.code = 0 + error.w_message = self.space.wrap("Invalid handle!") + raise OperationError(get(self.space).w_DatabaseError, + self.space.wrap(error)) + + @classmethod + def create(cls, space, threaded, events): + "Create a new environment object from scratch" mode = roci.OCI_OBJECT if threaded: mode |= roci.OCI_THREADED @@ -40,48 +83,25 @@ self.space.wrap( "Unable to acquire Oracle environment handle")) - self.handle = handleptr[0] + handle = handleptr[0] finally: lltype.free(handleptr, flavor='raw') - - self.maxBytesPerCharacter = config.BYTES_PER_CHAR - self.maxStringBytes = config.BYTES_PER_CHAR * config.MAX_STRING_CHARS - - # create the error handle - handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIError).TO, - 1, flavor='raw') try: - status = roci.OCIHandleAlloc( - self.handle, - handleptr, roci.OCI_HTYPE_ERROR, 0, - lltype.nullptr(rffi.CArray(roci.dvoidp))) - self.checkForError( - status, "Environment_New(): create error handle") - self.errorHandle = handleptr[0] - finally: - lltype.free(handleptr, flavor='raw') + newenv = cls(space, handle) + except: + roci.OCIHandleFree(handle, roci.OCI_HTYPE_ENV) + raise + + newenv.maxBytesPerCharacter = config.BYTES_PER_CHAR + newenv.maxStringBytes = config.BYTES_PER_CHAR * config.MAX_STRING_CHARS + return newenv + + def clone(self): + """Clone an existing environment. + used when acquiring a connection from a session pool, for example.""" + newenv = type(self)(self.space, self.handle) + newenv.maxBytesPerCharacter = self.maxBytesPerCharacter + newenv.maxStringBytes = self.maxStringBytes + return newenv - - def checkForError(self, status, context): - if status in (roci.OCI_SUCCESS, roci.OCI_SUCCESS_WITH_INFO): - return - - if status != roci.OCI_INVALID_HANDLE: - # At this point it is assumed that the Oracle - # environment is fully initialized - error = W_Error(self.space, self, context, 1) - if error.code in (1, 1400, 2290, 2291, 2292): - w_type = get(self.space).w_IntegrityError - elif error.code in (1012, 1033, 1034, 1089, 3113, 3114, - 12203, 12500, 12571): - w_type = get(self.space).w_OperationalError - else: - w_type = get(self.space).w_DatabaseError - raise OperationError(w_type, self.space.wrap(error)) - - error = W_Error(self.space, self, context, 0) - error.code = 0 - error.w_message = self.space.wrap("Invalid handle!") - raise OperationError(get(self.space).w_DatabaseError, - self.space.wrap(error)) Modified: pypy/trunk/pypy/module/oracle/interp_error.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_error.py (original) +++ pypy/trunk/pypy/module/oracle/interp_error.py Thu Nov 26 16:32:44 2009 @@ -22,6 +22,7 @@ self.w_InternalError = get('InternalError') self.w_DataError = get('DataError') self.w_Variable = get('Variable') + self.w_Connection = get('Connection') w_import = space.builtin.get('__import__') w_decimal = space.call(w_import, space.newlist( Modified: pypy/trunk/pypy/module/oracle/interp_pool.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_pool.py (original) +++ pypy/trunk/pypy/module/oracle/interp_pool.py Thu Nov 26 16:32:44 2009 @@ -1,4 +1,5 @@ from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.argument import Arguments, Signature from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty @@ -7,8 +8,9 @@ Null = NoneNotWrapped -from pypy.module.oracle import roci, config, interp_error, interp_environ - +from pypy.module.oracle import roci, config +from pypy.module.oracle import interp_error, interp_environ +from pypy.module.oracle.interp_error import get class W_SessionPool(Wrappable): def __init__(self): @@ -36,14 +38,16 @@ self.w_password = w_password self.w_tnsentry = w_dsn - self.w_connectionType = w_connectiontype + from pypy.module.oracle.interp_connect import W_Connection + self.w_connectionType = w_connectiontype or get(space).w_Connection self.minSessions = min self.maxSessions = max self.sessionIncrement = increment self.homogeneous = homogeneous # set up the environment - self.environment = interp_environ.Environment(space, threaded, events) + self.environment = interp_environ.Environment.create( + space, threaded, events) # create the session pool handle handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIServer).TO, @@ -110,6 +114,66 @@ get(space).w_InterfaceError, space.wrap("not connected")) + def acquire(self, space, __args__): + (w_user, w_password, w_cclass, w_purity + ) = __args__.parse_obj( + None, "acquire", + Signature(["user", "password", "cclass", "purity"]), + defaults_w=[None, None, None, space.w_False]) + if self.homogeneous and (w_user or w_password): + raise OperationError( + get(space).w_ProgrammingError, + space.wrap("pool is homogeneous. " + "Proxy authentication is not possible.")) + + self.checkConnected(space) + + newargs = Arguments(space, + __args__.arguments_w, + __args__.keywords + ["pool"], + __args__.keywords_w + [space.wrap(self)]) + return space.call_args(self.w_connectionType, newargs) + acquire.unwrap_spec = ['self', ObjSpace, Arguments] + + def release(self, space, w_connection): + self._release(space, w_connection, roci.OCI_DEFAULT) + release.unwrap_spec = ['self', ObjSpace, W_Root] + + def drop(self, space, w_connection): + self._release(space, w_connection, roci.OCI_SESSRLS_DROPSESS) + drop.unwrap_spec = ['self', ObjSpace, W_Root] + + def _release(self, space, w_connection, mode): + from pypy.module.oracle.interp_connect import W_Connection + connection = space.interp_w(W_Connection, w_connection) + + self.checkConnected(space) + + if connection.sessionPool is not self: + raise OperationError( + get(space).w_ProgrammingError, + space.wrap("connection not acquired with this session pool")) + + # attempt a rollback + status = roci.OCITransRollback( + connection.handle, connection.environment.errorHandle, + roci.OCI_DEFAULT) + # if dropping the connection from the pool, ignore the error + if mode != roci.OCI_SESSRLS_DROPSESS: + self.environment.checkForError( + status, "SessionPool_Release(): rollback") + + # release the connection + status = roci.OCISessionRelease( + connection.handle, connection.environment.errorHandle, + None, 0, mode) + self.environment.checkForError( + status, "SessionPool_Release(): release session") + + # ensure that the connection behaves as closed + connection.sessionPool = None + connection.handle = None + def computedProperty(oci_attr_code, oci_value_type): def fget(space, self): self.checkConnected(space) @@ -131,6 +195,10 @@ W_SessionPool.typedef = TypeDef( "SessionPool", __new__ = interp2app(W_SessionPool.descr_new.im_func), + acquire = interp2app(W_SessionPool.acquire), + release = interp2app(W_SessionPool.release), + drop = interp2app(W_SessionPool.drop), + username = interp_attrproperty_w('w_username', W_SessionPool), password = interp_attrproperty_w('w_password', W_SessionPool), tnsentry = interp_attrproperty_w('w_tnsentry', W_SessionPool), Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 26 16:32:44 2009 @@ -89,6 +89,8 @@ OCI_TYPECODE_NAMEDCOLLECTION OCI_TYPECODE_OBJECT OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS + OCI_SESSGET_SPOOL OCI_SESSGET_CREDPROXY OCI_SESSGET_STMTCACHE + OCI_SESSRLS_DROPSESS '''.split() for c in constants: @@ -105,6 +107,7 @@ OCIError = rffi.VOIDP OCIServer = rffi.VOIDP OCISession = rffi.VOIDP +OCIAuthInfo = rffi.VOIDP OCISPool = rffi.VOIDP OCIStmt = rffi.VOIDP OCIParam = rffi.VOIDP @@ -160,6 +163,11 @@ ub4], # mode sword) +OCIServerDetach = external( + 'OCIServerDetach', + [OCIServer, OCIError, ub4], + sword) + OCISessionBegin = external( 'OCISessionBegin', [OCISvcCtx, OCIError, OCISession, ub4, ub4], @@ -173,6 +181,22 @@ ub4], # mode sword) +OCISessionGet = external( + 'OCISessionGet', + [OCIEnv, # envhp + OCIError, # errhp + Ptr(OCISvcCtx), # svchp + OCIAuthInfo, # authInfop, + oratext, # dbName + ub4, # dbName_len + oratext, # tagInfo + ub4, # tagInfo_len + Ptr(oratext), # retTagInfo + Ptr(ub4), # retTagInfo_len + Ptr(boolean), # found + ub4], # mode + sword) + OCISessionPoolCreate = external( 'OCISessionPoolCreate', [OCISvcCtx, # svchp @@ -192,6 +216,15 @@ ub4], # mode sword) +OCISessionRelease = external( + 'OCISessionRelease', + [OCISvcCtx, # svchp + OCIError, # errhp + oratext, # tag + ub4, # tag_len + ub4], # mode + sword) + # Handle and Descriptor Functions OCIAttrGet = external( Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Thu Nov 26 16:32:44 2009 @@ -117,7 +117,7 @@ class AppTestPool(OracleNotConnectedTestBase): - def test_pool(self): + def test_pool_basicattributes(self): pool = oracle.SessionPool(self.username, self.password, self.tnsentry, 2, 8, 3) @@ -129,3 +129,20 @@ assert pool.increment == 3 assert pool.opened == 2 assert pool.busy == 0 + + def test_pool_acquire(self): + pool = oracle.SessionPool(self.username, self.password, + self.tnsentry, + 2, 8, 3) + assert (pool.busy, pool.opened) == (0, 2) + c1 = pool.acquire() + assert (pool.busy, pool.opened) == (1, 2) + c2 = pool.acquire() + assert (pool.busy, pool.opened) == (2, 2) + c3 = pool.acquire() + assert (pool.busy, pool.opened) == (3, 5) + pool.release(c3) + assert pool.busy == 2 + del c2 + import gc; gc.collect() + assert pool.busy == 1 From pedronis at codespeak.net Thu Nov 26 16:41:02 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Thu, 26 Nov 2009 16:41:02 +0100 (CET) Subject: [pypy-svn] r69656 - pypy/build/bot2/pypybuildbot Message-ID: <20091126154102.9EEB616807A@codespeak.net> Author: pedronis Date: Thu Nov 26 16:41:01 2009 New Revision: 69656 Modified: pypy/build/bot2/pypybuildbot/builds.py pypy/build/bot2/pypybuildbot/master.py Log: restructure things, now we compile pypy-c only once for app-level+lib python tests, use just one configurable factory for the all lot of translated pypy tests may have broken stuff, we'll see Modified: pypy/build/bot2/pypybuildbot/builds.py ============================================================================== --- pypy/build/bot2/pypybuildbot/builds.py (original) +++ pypy/build/bot2/pypybuildbot/builds.py Thu Nov 26 16:41:01 2009 @@ -107,67 +107,70 @@ env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick})) -class PyPyTranslatedLibPythonTestFactory(factory.BuildFactory): +class Translated(factory.BuildFactory): - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - factory.BuildFactory.__init__(self, *a, **kw) + def __init__(self, platform='linux', + translationArgs=['-O2'], targetArgs=[], + app_tests=False, + lib_python=False, + pypyjit=False + ): + factory.BuildFactory.__init__(self) setup_steps(platform, self) - self.addStep(Translate(["-O2"], [])) + self.addStep(Translate(translationArgs, targetArgs)) - self.addStep(ShellCmd( - description="lib-python test", - command=["python", "pypy/test_all.py", - "--pypy=pypy/translator/goal/pypy-c", - "--resultlog=cpython.log", "lib-python"], - logfiles={'pytestLog': 'cpython.log'})) - -class PyPyTranslatedAppLevelTestFactory(factory.BuildFactory): - - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - factory.BuildFactory.__init__(self, *a, **kw) + if app_tests: + if app_tests == True: + app_tests = [] + self.addStep(ShellCmd( + description="app-level (-A) test", + command=["python", "testrunner/runner.py", + "--logfile=pytest-A.log", + "--config=pypy/pytest-A.cfg", + "--root=pypy", "--timeout=1800" + ] + ["--config=%s" % cfg for cfg in app_tests], + logfiles={'pytestLog': 'pytest-A.log'}, + timeout = 4000, + env={"PYTHONPATH": ['.']})) + + if lib_python: + self.addStep(ShellCmd( + description="lib-python test", + command=["python", "pypy/test_all.py", + "--pypy=pypy/translator/goal/pypy-c", + "--resultlog=cpython.log", "lib-python"], + logfiles={'pytestLog': 'cpython.log'})) + + if pypyjit: + self.addStep(ShellCmd( + description="pypyjit tests", + command=["python", "pypy/test_all.py", + "--pypy=pypy/translator/goal/pypy-c", + "--resultlog=pypyjit.log", + "pypy/module/pypyjit/test/test_pypy_c.py"], + logfiles={'pytestLog': 'pypyjit.log'})) + + +class JITBenchmark(factory.BuildFactory): + def __init__(self, platform='linux'): + factory.BuildFactory.__init__(self) setup_steps(platform, self) - self.addStep(Translate(["-O2"], [])) - + self.addStep(Translate(['-Ojit', '--gc=hybrid', + '--gcrootfinder=asmgcc'], + ['--withoutmod-thread'])) self.addStep(ShellCmd( - description="app-level (-A) test", - command=["python", "testrunner/runner.py", - "--logfile=pytest-A.log", - "--config=pypy/pytest-A.cfg", - "--root=pypy", "--timeout=1800"], - logfiles={'pytestLog': 'pytest-A.log'}, - timeout = 4000, - env={"PYTHONPATH": ['.']})) - - -class PyPyStacklessTranslatedAppLevelTestFactory(factory.BuildFactory): - - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - factory.BuildFactory.__init__(self, *a, **kw) - - setup_steps(platform, self) - - self.addStep(Translate(["-O2", "--stackless"], [])) + descritpion="run richards & upload results", + command=["python", "pypy/translator/benchmark/jitbench.py", + "pypy/translator/goal/pypy-c"])) - self.addStep(ShellCmd( - description="app-level (-A) test", - command=["python", "testrunner/runner.py", - "--logfile=pytest-A.log", - "--config=pypy/pytest-A.cfg", - "--config=pypy/pytest-A-stackless.cfg", - "--root=pypy", "--timeout=1800"], - logfiles={'pytestLog': 'pytest-A.log'}, - timeout = 4000, - env={"PYTHONPATH": ['.']})) -class PyPyTranslatedScratchboxTestFactory(factory.BuildFactory): +# xxx keep style +class TranslatedScratchbox(factory.BuildFactory): def __init__(self, *a, **kw): USERNAME = 'buildbot' WORKDIR = '/scratchbox/users/%s/home/%s/build' % (USERNAME, USERNAME) @@ -193,46 +196,3 @@ self.addStep(ShellCmd( description="copy build", command=["scp", "pypy-c", "fijal at codespeak.net:builds/pypy-c-scratchbox"], workdir = workdir)) - -class PyPyJITTranslatedTestFactory(factory.BuildFactory): - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - factory.BuildFactory.__init__(self, *a, **kw) - - setup_steps(platform, self) - - self.addStep(Translate(['-Ojit', '--gc=hybrid', - '--gcrootfinder=asmgcc', - '--jit-debug=steps'], - ['--withoutmod-thread'])) - - self.addStep(ShellCmd( - description="lib-python test", - command=["python", "pypy/test_all.py", - "--pypy=pypy/translator/goal/pypy-c", - "--resultlog=cpython.log", "lib-python"], - logfiles={'pytestLog': 'cpython.log'})) - - self.addStep(ShellCmd( - description="pypyjit tests", - command=["python", "pypy/test_all.py", - "--pypy=pypy/translator/goal/pypy-c", - "--resultlog=pypyjit.log", - "pypy/module/pypyjit/test/test_pypy_c.py"], - logfiles={'pytestLog': 'pypyjit.log'})) - -class PyPyJITBenchmarkFactory(factory.BuildFactory): - def __init__(self, *a, **kw): - platform = kw.pop('platform', 'linux') - factory.BuildFactory.__init__(self, *a, **kw) - - setup_steps(platform, self) - - self.addStep(Translate(['-Ojit', '--gc=hybrid', - '--gcrootfinder=asmgcc'], - ['--withoutmod-thread'])) - self.addStep(ShellCmd( - descritpion="run richards & upload results", - command=["python", "pypy/translator/benchmark/jitbench.py", - "pypy/translator/goal/pypy-c"])) - Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Nov 26 16:41:01 2009 @@ -48,36 +48,56 @@ pypyOwnTestFactoryWin = pypybuilds.Own(platform="win32") pypyJitOnlyOwnTestFactory = pypybuilds.Own(cherrypick="jit") -pypyTranslatedLibPythonTestFactory = pypybuilds.PyPyTranslatedLibPythonTestFactory() -pypyTranslatedLibPythonTestFactoryWin = pypybuilds.PyPyTranslatedLibPythonTestFactory(platform="win32") -pypyTranslatedLibPythonMaemoTestFactory = pypybuilds.PyPyTranslatedScratchboxTestFactory() - -pypyTranslatedAppLevelTestFactory = pypybuilds.PyPyTranslatedAppLevelTestFactory() - -pypyStacklessTranslatedAppLevelTestFactory = pypybuilds.PyPyStacklessTranslatedAppLevelTestFactory() -pypyJITTranslatedTestFactory = pypybuilds.PyPyJITTranslatedTestFactory() -pypyJITBenchmarkFactory = pypybuilds.PyPyJITBenchmarkFactory() +pypyTranslatedAppLevelTestFactory = pypybuilds.Translated(lib_python=True, + app_tests=True) + +pypyStacklessTranslatedAppLevelTestFactory = pypybuilds.Translated( + translationArgs=["-O2", "--stackless"], targetArgs=[], + lib_python=False, + app_tests = ["pypy/pytest-A-stackless.cfg"] +) + +pypyTranslatedAppLevelTestFactoryWin = pypybuilds.Translated( + platform="win32", + lib_python=True, + app_tests=True) + +pypyJITTranslatedTestFactory = pypybuilds.Translated( + translationArgs=['-Ojit', '--gc=hybrid', + '--gcrootfinder=asmgcc', + '--jit-debug=steps'], + targetArgs = ['--withoutmod-thread'], + lib_python=True, + pypyjit=True + ) + +pypyJITBenchmarkFactory = pypybuilds.JITBenchmark() + +pypyTranslatedLibPythonMaemoTestFactory = pypybuilds.TranslatedScratchbox() + LINUX32 = "own-linux-x86-32" MACOSX32 = "own-macosx-x86-32" -CPYLINUX32 = "pypy-c-lib-python-linux-x86-32" -CPYWIN32 = "pypy-c-lib-python-win-32" -CPYLINUX32_VM = 'pypy-c-lib-python-linux-x86-32vm' -BUILDMAEMO = "pypy-c-maemo-build" APPLVLLINUX32 = "pypy-c-app-level-linux-x86-32" STACKLESSAPPLVLLINUX32 = "pypy-c-stackless-app-level-linux-x86-32" -CPYFREEBSD64 = 'pypy-c-lib-python-freebsd-7-x86-64' -JITCPYLINUX32 = "pypy-c-jit-lib-python-linux-x86-32" + +APPLVLWIN32 = "pypy-c-app-level-win-32" +APPLVLFREEBSD64 = 'pypy-c-app-level-freebsd-7-x86-64' + +JITLINUX32 = "pypy-c-jit-linux-x86-32" + JITONLYLINUX32 = "jitonly-own-linux-x86-32" JITBENCH = "jit-benchmark-linux-x86-32" +BUILDMAEMO = "pypy-c-maemo-build" + BuildmasterConfig = { 'slavePortnum': slavePortnum, 'change_source': [], 'schedulers': [ - Nightly("nightly", [LINUX32, CPYLINUX32, APPLVLLINUX32, CPYWIN32, - STACKLESSAPPLVLLINUX32, JITCPYLINUX32, + Nightly("nightly", [LINUX32, APPLVLLINUX32, APPLVLWIN32, + STACKLESSAPPLVLLINUX32, JITLINUX32, MACOSX32], hour=4, minute=45), Nightly("nightly-benchmark", [JITBENCH], @@ -102,12 +122,6 @@ "factory": pypyOwnTestFactory, "category": 'mac' }, - {"name": CPYLINUX32, - "slavenames": ["wyvern", "cobra"], - "builddir": CPYLINUX32, - "factory": pypyTranslatedLibPythonTestFactory, - "category": 'lib-python' - }, {"name": APPLVLLINUX32, "slavenames": ["wyvern", "cobra"], "builddir": APPLVLLINUX32, @@ -120,16 +134,10 @@ "factory": pypyStacklessTranslatedAppLevelTestFactory, "category": 'stackless' }, - {"name" : CPYLINUX32_VM, - "slavenames": ['bigdogvm1'], - "builddir": CPYLINUX32_VM, - "factory": pypyTranslatedLibPythonTestFactory, - "category": 'lib-python' - }, - {"name": CPYWIN32, + {"name": APPLVLWIN32, "slavenames": ["bigboard"], - "builddir": CPYWIN32, - "factory": pypyTranslatedLibPythonTestFactoryWin, + "builddir": APPLVLWIN32, + "factory": pypyTranslatedAppLevelTestFactoryWin, "category": "windows" }, {"name" : BUILDMAEMO, @@ -138,15 +146,15 @@ "factory": pypyTranslatedLibPythonMaemoTestFactory, "category": 'maemo' }, - {"name" : CPYFREEBSD64, + {"name" : APPLVLFREEBSD64, "slavenames": ['headless'], - 'builddir' : CPYFREEBSD64, - 'factory' : pypyTranslatedLibPythonTestFactory, + 'builddir' : APPLVLFREEBSD64, + 'factory' : pypyTranslatedAppLevelTestFactory, "category": 'other' }, - {"name" : JITCPYLINUX32, + {"name" : JITLINUX32, "slavenames": ["bigdogvm1"], - 'builddir' : JITCPYLINUX32, + 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'jit', }, From fijal at codespeak.net Thu Nov 26 16:49:39 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 16:49:39 +0100 (CET) Subject: [pypy-svn] r69657 - in pypy/branch/virtual-forcing/pypy/jit/backend/llsupport: . test Message-ID: <20091126154939.A8B7D16807A@codespeak.net> Author: fijal Date: Thu Nov 26 16:49:39 2009 New Revision: 69657 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/test/test_regalloc.py Log: support for saving all registers Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py Thu Nov 26 16:49:39 2009 @@ -304,7 +304,7 @@ self.assembler.regalloc_mov(reg, to) # otherwise it's clean - def before_call(self, force_store=[]): + def before_call(self, force_store=[], save_all_regs=False): """ Spill registers before a call, as described by 'self.save_around_call_regs'. Registers are not spilled if they don't survive past the current operation, unless they @@ -316,8 +316,8 @@ del self.reg_bindings[v] self.free_regs.append(reg) continue - if reg not in self.save_around_call_regs: - # we don't need to + if not save_all_regs and reg not in self.save_around_call_regs: + # we don't have to continue self._sync_var(v) del self.reg_bindings[v] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/test/test_regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/test/test_regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/test/test_regalloc.py Thu Nov 26 16:49:39 2009 @@ -278,6 +278,30 @@ assert len(rm.reg_bindings) == 3 rm._check_invariants() + def test_call_support_save_all_regs(self): + class XRegisterManager(RegisterManager): + save_around_call_regs = [r1, r2] + + def call_result_location(self, v): + return r1 + + sm = TStackManager() + asm = MockAsm() + boxes, longevity = boxes_and_longevity(5) + rm = XRegisterManager(longevity, stack_manager=sm, + assembler=asm) + for b in boxes[:-1]: + rm.force_allocate_reg(b) + rm.before_call(save_all_regs=True) + assert len(rm.reg_bindings) == 0 + assert sm.stack_depth == 4 + assert len(asm.moves) == 4 + rm._check_invariants() + rm.after_call(boxes[-1]) + assert len(rm.reg_bindings) == 1 + rm._check_invariants() + + def test_different_stack_width(self): class XRegisterManager(RegisterManager): reg_width = 2 From afa at codespeak.net Thu Nov 26 16:55:22 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 16:55:22 +0100 (CET) Subject: [pypy-svn] r69658 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126155522.C80B616807A@codespeak.net> Author: afa Date: Thu Nov 26 16:55:22 2009 New Revision: 69658 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_pool.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py Log: Test and fix for proxy connections Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Thu Nov 26 16:55:22 2009 @@ -22,6 +22,7 @@ self.autocommit = False self.sessionHandle = None + self.serverHandle = None self.w_inputTypeHandler = None self.w_outputTypeHandler = None @@ -70,7 +71,7 @@ space.call_method(self.w_password, 'split', space.wrap('@'), space.wrap(1))) - if pool or w_cclass: + if pool or w_cclass is not None: self.getConnection(space, pool, w_cclass, purity) else: self.connect(space, mode, twophase) @@ -246,7 +247,7 @@ w_dbname = pool.w_name mode = roci.OCI_SESSGET_SPOOL if not pool.homogeneous and pool.w_username and self.w_username: - proxyCredentials = space.ne(pool.w_username, self.w_username) + proxyCredentials = space.is_true(space.ne(pool.w_username, self.w_username)) mode |= roci.OCI_SESSGET_CREDPROXY else: w_dbname = self.w_tnsentry @@ -257,13 +258,20 @@ # set up authorization handle, if needed if not pool or w_cclass or proxyCredentials: # create authorization handle - status = roci.OCIHandleAlloc( - self.environment.handle, - handleptr, - roci.HTYPE_AUTHINFO, - 0, None) - self.environment.checkForError( - status, "Connection_GetConnection(): allocate handle") + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIServer).TO, + 1, flavor='raw') + try: + status = roci.OCIHandleAlloc( + self.environment.handle, + handleptr, + roci.OCI_HTYPE_AUTHINFO, + 0, lltype.nullptr(rffi.CArray(roci.dvoidp))) + self.environment.checkForError( + status, "Connection_GetConnection(): allocate handle") + + authInfo = handleptr[0] + finally: + lltype.free(handleptr, flavor='raw') externalCredentials = True @@ -320,19 +328,21 @@ stringBuffer.clear() # set the purity, if applicable - purityptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, - 1, flavor='raw') - try: - status = roci.OCIAttrSet( - authInfo, - roci.OCI_HTYPE_AUTHINFO, - purityptr, rffi.sizeof(roci.ub4), - roci.OCI_ATTR_PURITY, - self.environment.errorHandle) - self.environment.checkForError( - status, "Connection_GetConnection(): set purity") - finally: - lltype.free(purityptr, flavor='raw') + if purity != roci.OCI_ATTR_PURITY_DEFAULT: + purityptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, + 1, flavor='raw') + purityptr[0] = purity + try: + status = roci.OCIAttrSet( + authInfo, + roci.OCI_HTYPE_AUTHINFO, + purityptr, rffi.sizeof(roci.ub4), + roci.OCI_ATTR_PURITY, + self.environment.errorHandle) + self.environment.checkForError( + status, "Connection_GetConnection(): set purity") + finally: + lltype.free(purityptr, flavor='raw') # acquire the new session stringBuffer.fill(space, w_dbname) Modified: pypy/trunk/pypy/module/oracle/interp_pool.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_pool.py (original) +++ pypy/trunk/pypy/module/oracle/interp_pool.py Thu Nov 26 16:55:22 2009 @@ -4,6 +4,7 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w +from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype Null = NoneNotWrapped @@ -205,6 +206,7 @@ min = interp_attrproperty('minSessions', W_SessionPool), max = interp_attrproperty('maxSessions', W_SessionPool), increment = interp_attrproperty('sessionIncrement', W_SessionPool), + homogeneous = interp_attrproperty('homogeneous', W_SessionPool), opened = computedProperty(roci.OCI_ATTR_SPOOL_OPEN_COUNT, roci.ub4), busy = computedProperty(roci.OCI_ATTR_SPOOL_BUSY_COUNT, roci.ub4), timeout = computedProperty(roci.OCI_ATTR_SPOOL_TIMEOUT, roci.ub4), Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 26 16:55:22 2009 @@ -56,6 +56,7 @@ OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_HTYPE_SPOOL + OCI_HTYPE_AUTHINFO OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD @@ -68,7 +69,7 @@ OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_ATTR_SPOOL_OPEN_COUNT OCI_ATTR_SPOOL_BUSY_COUNT OCI_ATTR_SPOOL_TIMEOUT - OCI_ATTR_SPOOL_GETMODE + OCI_ATTR_SPOOL_GETMODE OCI_ATTR_PURITY OCI_NTV_SYNTAX OCI_COMMIT_ON_SUCCESS OCI_FETCH_NEXT OCI_IND_NULL OCI_IND_NOTNULL @@ -90,7 +91,7 @@ OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS OCI_SESSGET_SPOOL OCI_SESSGET_CREDPROXY OCI_SESSGET_STMTCACHE - OCI_SESSRLS_DROPSESS + OCI_SESSRLS_DROPSESS OCI_ATTR_PURITY_DEFAULT '''.split() for c in constants: Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Thu Nov 26 16:55:22 2009 @@ -146,3 +146,17 @@ del c2 import gc; gc.collect() assert pool.busy == 1 + + def test_proxy_auth(self): + pool = oracle.SessionPool(self.username, self.password, + self.tnsentry, + 2, 8, 3) + assert pool.homogeneous is True + raises(oracle.ProgrammingError, pool.acquire, user="proxyuser") + pool = oracle.SessionPool(self.username, self.password, + self.tnsentry, + 2, 8, 3, homogeneous=False) + assert pool.homogeneous is False + e = raises(oracle.DatabaseError, pool.acquire, user="proxyuser") + # ORA-28150: proxy not authorized to connect as client + assert e.value[0].code == 28150 From afa at codespeak.net Thu Nov 26 18:18:14 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 18:18:14 +0100 (CET) Subject: [pypy-svn] r69661 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126171814.72E811680B8@codespeak.net> Author: afa Date: Thu Nov 26 18:18:13 2009 New Revision: 69661 Modified: pypy/trunk/pypy/module/oracle/config.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Support for binding and fetching unicode data Modified: pypy/trunk/pypy/module/oracle/config.py ============================================================================== --- pypy/trunk/pypy/module/oracle/config.py (original) +++ pypy/trunk/pypy/module/oracle/config.py Thu Nov 26 18:18:13 2009 @@ -1,4 +1,5 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.module.oracle import roci WITH_UNICODE = False @@ -13,6 +14,7 @@ else: def string_w(space, w_obj): return space.str_w(w_obj) + def w_string(space, buf, len=-1): if len < 0: return space.wrap(rffi.charp2str(buf)) @@ -22,6 +24,7 @@ BYTES_PER_CHAR = 1 class StringBuffer: + "Fill a char* buffer with data, suitable to pass to Oracle functions" def __init__(self): pass @@ -32,6 +35,15 @@ self.ptr = string_w(space, w_string) self.size = len(self.ptr) + def fill_with_unicode(self, space, w_unicode): + if w_unicode is None or space.is_w(w_unicode, space.w_None): + self.clear() + else: + # XXX ucs2 only probably + unistr = space.unicode_w(w_unicode) + self.ptr = rffi.cast(roci.oratext, rffi.unicode2wcharp(unistr)) + self.size = len(unistr) * 2 + def clear(self): self.ptr = None self.size = 0 Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 26 18:18:13 2009 @@ -293,6 +293,52 @@ finally: lltype.free(bindHandlePtr, flavor='raw') + # set the charset form and id if applicable + if self.charsetForm != roci.SQLCS_IMPLICIT: + charsetformptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, + zero=True, flavor='raw') + charsetformptr[0] = rffi.cast(roci.ub1, self.charsetForm) + try: + status = roci.OCIAttrSet( + self.bindHandle, roci.OCI_HTYPE_BIND, + rffi.cast(roci.dvoidp, charsetformptr), 0, + roci.OCI_ATTR_CHARSET_FORM, + self.environment.errorHandle) + self.environment.checkForError( + status, "NumberVar_InternalBind(): set charset form") + finally: + lltype.free(charsetformptr, flavor='raw') + + charsetidptr = lltype.malloc(roci.Ptr(roci.ub2).TO, 1, + zero=True, flavor='raw') + charsetidptr[0] = rffi.cast(roci.ub2, roci.OCI_UTF16ID) + try: + status = roci.OCIAttrSet( + self.bindHandle, roci.OCI_HTYPE_BIND, + rffi.cast(roci.dvoidp, charsetidptr), 0, + roci.OCI_ATTR_CHARSET_ID, + self.environment.errorHandle) + self.environment.checkForError( + status, "NumberVar_InternalBind(): set charset Id") + finally: + lltype.free(charsetidptr, flavor='raw') + + # set the max data size for strings + buffersizeptr = lltype.malloc(roci.Ptr(roci.ub4).TO, 1, + zero=True, flavor='raw') + buffersizeptr[0] = rffi.cast(roci.ub4, self.size) + try: + status = roci.OCIAttrSet( + self.bindHandle, roci.OCI_HTYPE_BIND, + rffi.cast(roci.dvoidp, buffersizeptr), 0, + roci.OCI_ATTR_MAXDATA_SIZE, + self.environment.errorHandle) + self.environment.checkForError( + status, "NumberVar_InternalBind(): set max data size") + finally: + lltype.free(buffersizeptr, flavor='raw') + + def isNull(self, pos): return (rffi.cast(lltype.Signed, self.indicator[pos]) == @@ -439,8 +485,8 @@ return space.wrap(''.join(l)) else: while i < length: - l.append(unichr((ord(self.data[offset + i]) << 8) + - ord(self.data[offset + i + 1]))) + l.append(unichr((ord(self.data[offset + i + 1]) << 8) + + ord(self.data[offset + i]))) i += 2 return space.wrap(u''.join(l)) else: @@ -451,8 +497,8 @@ return space.wrap(''.join(l)) else: while i < length: - l.append(unichr((ord(self.data[offset + i]) << 8) + - ord(self.data[offset + i + 1]))) + l.append(unichr((ord(self.data[offset + i + 1]) << 8) + + ord(self.data[offset + i]))) i += 2 return space.wrap(u''.join(l)) @@ -474,7 +520,7 @@ else: if space.is_true(space.isinstance(w_value, space.w_unicode)): buf = config.StringBuffer() - buf.fill(space, w_value) + buf.fill_with_unicode(space, w_value) size = buf.size else: raise OperationError( @@ -503,8 +549,42 @@ oracleType = roci.SQLT_AFC size = 2000 -class VT_NationalCharString(W_Variable): - pass +class VT_NationalCharString(VT_String): + charsetForm = roci.SQLCS_NCHAR + + def postDefine(self, param): + charsetformptr = lltype.malloc(roci.Ptr(roci.ub1).TO, 1, + zero=True, flavor='raw') + charsetformptr[0] = rffi.cast(roci.ub1, self.charsetForm) + try: + status = roci.OCIAttrSet( + self.defineHandle, roci.OCI_HTYPE_DEFINE, + rffi.cast(roci.dvoidp, charsetformptr), 0, + roci.OCI_ATTR_CHARSET_FORM, + self.environment.errorHandle) + self.environment.checkForError( + status, "StringVar_PostDefine(): set charset form") + finally: + lltype.free(charsetformptr, flavor='raw') + + charsetidptr = lltype.malloc(roci.Ptr(roci.ub2).TO, 1, + zero=True, flavor='raw') + charsetidptr[0] = rffi.cast(roci.ub2, roci.OCI_UTF16ID) + try: + status = roci.OCIAttrSet( + self.defineHandle, roci.OCI_HTYPE_DEFINE, + rffi.cast(roci.dvoidp, charsetidptr), 0, + roci.OCI_ATTR_CHARSET_ID, + self.environment.errorHandle) + self.environment.checkForError( + status, "StringVar_PostDefine(): set charset Id") + finally: + lltype.free(charsetidptr, flavor='raw') + + +class VT_FixedNationalChar(VT_NationalCharString): + oracleType = roci.SQLT_AFC + size = 2000 class VT_LongString(W_Variable): oracleType = roci.SQLT_LVC @@ -538,9 +618,6 @@ finally: buf.clear() -class VT_FixedNationalChar(W_Variable): - pass - class VT_Rowid(VT_String): oracleType = roci.SQLT_CHR size = 18 @@ -1329,7 +1406,9 @@ else: return VT_String, size, numElements - # XXX Unicode + if space.is_true(space.isinstance(w_value, space.w_unicode)): + size = space.int_w(space.len(w_value)) + return VT_NationalCharString, size, numElements if space.is_true(space.isinstance(w_value, space.w_int)): return VT_Integer, 0, numElements Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Thu Nov 26 18:18:13 2009 @@ -55,8 +55,8 @@ OCI_DEFAULT OCI_OBJECT OCI_THREADED OCI_EVENTS OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION - OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_ENV OCI_HTYPE_SPOOL - OCI_HTYPE_AUTHINFO + OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_BIND OCI_HTYPE_DEFINE + OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_HTYPE_AUTHINFO OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD @@ -65,8 +65,8 @@ OCI_ATTR_DATA_SIZE OCI_ATTR_DATA_TYPE OCI_ATTR_REF_TDO OCI_ATTR_SCHEMA_NAME OCI_ATTR_TYPE_NAME OCI_ATTR_TYPECODE OCI_ATTR_NUM_TYPE_ATTRS OCI_ATTR_LIST_TYPE_ATTRS - OCI_ATTR_COLLECTION_ELEMENT - OCI_ATTR_CHARSET_FORM OCI_ATTR_ENV_CHARSET_ID + OCI_ATTR_COLLECTION_ELEMENT OCI_ATTR_MAXDATA_SIZE + OCI_ATTR_CHARSET_FORM OCI_ATTR_CHARSET_ID OCI_ATTR_ENV_CHARSET_ID OCI_ATTR_PARSE_ERROR_OFFSET OCI_ATTR_SPOOL_OPEN_COUNT OCI_ATTR_SPOOL_BUSY_COUNT OCI_ATTR_SPOOL_TIMEOUT OCI_ATTR_SPOOL_GETMODE OCI_ATTR_PURITY @@ -88,7 +88,7 @@ OCI_TYPECODE_CHAR OCI_TYPECODE_VARCHAR OCI_TYPECODE_VARCHAR2 OCI_TYPECODE_NUMBER OCI_TYPECODE_DATE OCI_TYPECODE_TIMESTAMP OCI_TYPECODE_NAMEDCOLLECTION OCI_TYPECODE_OBJECT - OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA + OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA OCI_UTF16ID OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS OCI_SESSGET_SPOOL OCI_SESSGET_CREDPROXY OCI_SESSGET_STMTCACHE OCI_SESSRLS_DROPSESS OCI_ATTR_PURITY_DEFAULT Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Thu Nov 26 18:18:13 2009 @@ -147,3 +147,11 @@ data = cur.fetchall() assert data == [("long string",)] +class AppTestUnicode(OracleTestBase): + def test_bind(self): + cur = self.cnx.cursor() + value = u"Unicode(\u3042)" + cur.execute("select :value from dual", value=value) + data, = cur.fetchone() + assert data == value + From afa at codespeak.net Thu Nov 26 18:35:19 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 26 Nov 2009 18:35:19 +0100 (CET) Subject: [pypy-svn] r69662 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091126173519.17F1E16807A@codespeak.net> Author: afa Date: Thu Nov 26 18:35:18 2009 New Revision: 69662 Modified: pypy/trunk/pypy/module/oracle/__init__.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/test/test_stringvar.py Log: Fix check of maximum allowed size for unicode values Modified: pypy/trunk/pypy/module/oracle/__init__.py ============================================================================== --- pypy/trunk/pypy/module/oracle/__init__.py (original) +++ pypy/trunk/pypy/module/oracle/__init__.py Thu Nov 26 18:35:18 2009 @@ -6,9 +6,9 @@ interpleveldefs = { 'connect': 'interp_connect.W_Connection', 'Connection': 'interp_connect.W_Connection', - 'UNICODE': 'interp_variable.VT_NationalCharString', 'NUMBER': 'interp_variable.VT_Float', 'STRING': 'interp_variable.VT_String', + 'UNICODE': 'interp_variable.VT_NationalCharString', 'DATETIME': 'interp_variable.VT_DateTime', 'DATE': 'interp_variable.VT_Date', 'TIMESTAMP': 'interp_variable.VT_Timestamp', @@ -17,6 +17,7 @@ 'LONG_STRING': 'interp_variable.VT_LongString', 'LONG_BINARY': 'interp_variable.VT_LongBinary', 'FIXED_CHAR': 'interp_variable.VT_FixedChar', + 'FIXED_UNICODE': 'interp_variable.VT_FixedNationalChar', 'CURSOR': 'interp_variable.VT_Cursor', 'BLOB': 'interp_variable.VT_BLOB', 'CLOB': 'interp_variable.VT_CLOB', Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Thu Nov 26 18:35:18 2009 @@ -528,10 +528,16 @@ space.wrap("expecting unicode data")) try: - if buf.size > self.environment.maxStringBytes: - raise OperationError( - space.w_ValueError, - space.wrap("string data too large")) + if wantBytes: + if buf.size > self.environment.maxStringBytes: + raise OperationError( + space.w_ValueError, + space.wrap("string data too large")) + else: + if buf.size > config.MAX_STRING_CHARS * 2: + raise OperationError( + space.w_ValueError, + space.wrap("unicode data too large")) # ensure that the buffer is large enough if buf.size > self.bufferSize: Modified: pypy/trunk/pypy/module/oracle/test/test_stringvar.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_stringvar.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_stringvar.py Thu Nov 26 18:35:18 2009 @@ -155,3 +155,11 @@ data, = cur.fetchone() assert data == value + def test_large_unicode(self): + cur = self.cnx.cursor() + var = cur.var(oracle.UNICODE) + value = u"1234567890" * 400 + var.setvalue(0, value) + assert var.getvalue() == value + value += "X" + raises(ValueError, var.setvalue, 0, value) From fijal at codespeak.net Thu Nov 26 19:03:43 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 19:03:43 +0100 (CET) Subject: [pypy-svn] r69663 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091126180343.C9C541680AF@codespeak.net> Author: fijal Date: Thu Nov 26 19:03:41 2009 New Revision: 69663 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Log: update comment Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Thu Nov 26 19:03:41 2009 @@ -211,7 +211,7 @@ self.mc.PUSH(esi) self.mc.PUSH(edi) # NB. exactly 4 pushes above; if this changes, fix stack_pos(). - # You must also keep _get_callshape() in sync. + # You must also keep get_basic_shape() in sync. adr_stackadjust = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] From fijal at codespeak.net Thu Nov 26 19:23:20 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 19:23:20 +0100 (CET) Subject: [pypy-svn] r69667 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091126182320.E35491680AF@codespeak.net> Author: fijal Date: Thu Nov 26 19:23:20 2009 New Revision: 69667 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: Start supporting forcing of things. Does not force anything right now, but at least keeps flag about things being forced (not supported by guard_not_forced yet) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Thu Nov 26 19:23:20 2009 @@ -210,6 +210,8 @@ self.mc.PUSH(ebx) self.mc.PUSH(esi) self.mc.PUSH(edi) + # could be really PUSH(0), but that way is safer + self.mc.MOV(mem(ebp, self.cpu.virtualizable_ofs), imm(0)) # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep get_basic_shape() in sync. adr_stackadjust = self._patchable_stackadjust() @@ -1104,6 +1106,7 @@ self.mc.AND(eax, imm(0xffff)) genop_call_pure = genop_call + genop_call_may_force = genop_call def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Thu Nov 26 19:23:20 2009 @@ -130,6 +130,9 @@ def _prepare(self, inputargs, operations): self.sm = X86StackManager() + # a bit of a hack - always grab one position at the beginning + loc = self.sm.loc(TempBox(), 1) + self.assembler.cpu.virtualizable_ofs = loc.ofs_relative_to_ebp() cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables @@ -395,6 +398,9 @@ consider_guard_nonnull = _consider_guard consider_guard_isnull = _consider_guard + def consider_guard_not_forced(self, op, ignored): + pass + def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] locs_are_ref = [v.type == REF for v in op.args] @@ -604,9 +610,9 @@ self.Perform(op, [loc0], loc1) self.rm.possibly_free_var(op.args[0]) - def _call(self, op, arglocs, force_store=[]): - self.rm.before_call(force_store) - self.xrm.before_call(force_store) + def _call(self, op, arglocs, force_store=[], save_all_regs=False): + self.rm.before_call(force_store, save_all_regs=save_all_regs) + self.xrm.before_call(force_store, save_all_regs=save_all_regs) self.Perform(op, arglocs, eax) if op.result is not None: if op.result.type == FLOAT: @@ -621,6 +627,15 @@ size = calldescr.get_result_size(self.translate_support_code) self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args]) + def consider_call_may_force(self, op, ignored): + calldescr = op.descr + assert isinstance(calldescr, BaseCallDescr) + assert len(calldescr.arg_classes) == len(op.args) - 1 + size = calldescr.get_result_size(self.translate_support_code) + self._call(op, [imm(size), ebp] + + [self.loc(arg) for arg in op.args], + save_all_regs=True) + consider_call_pure = consider_call def consider_cond_call_gc_wb(self, op, ignored): @@ -927,6 +942,9 @@ assert reg is eax # ok to ignore this one return gcrootmap.compress_callshape(shape) + def consider_force_token(self, op, ignored): + pass + def not_implemented_op(self, op, ignored): msg = "[regalloc] Not implemented operation: %s" % op.getopname() print msg Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Thu Nov 26 19:23:20 2009 @@ -87,6 +87,10 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) + def force(self, stack_base): + TP = rffi.CArrayPtr(lltype.Signed) + rffi.cast(TP, stack_base + self.virtualizable_ofs)[0] = 1 + # force stuff actually... class CPU386_NO_SSE2(CPU386): supports_floats = False From fijal at codespeak.net Thu Nov 26 20:21:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 26 Nov 2009 20:21:03 +0100 (CET) Subject: [pypy-svn] r69669 - in pypy/branch/virtual-forcing/pypy/jit/backend: llsupport x86 Message-ID: <20091126192103.6CE291680AF@codespeak.net> Author: fijal Date: Thu Nov 26 20:21:02 2009 New Revision: 69669 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Log: Don't hardcode ebp as argument to call_may_force. Use ebp as storage for a result box for now. Disable check_invariants for now, as it's not prepared for people storing stuff on ebp. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py Thu Nov 26 20:21:02 2009 @@ -82,15 +82,15 @@ self.possibly_free_var(v) def _check_invariants(self): - if not we_are_translated(): - # make sure no duplicates - assert len(dict.fromkeys(self.reg_bindings.values())) == len(self.reg_bindings) - rev_regs = dict.fromkeys(self.reg_bindings.values()) - for reg in self.free_regs: - assert reg not in rev_regs - assert len(rev_regs) + len(self.free_regs) == len(self.all_regs) - else: - assert len(self.reg_bindings) + len(self.free_regs) == len(self.all_regs) + #if not we_are_translated(): + # # make sure no duplicates + # assert len(dict.fromkeys(self.reg_bindings.values())) == len(self.reg_bindings) + # rev_regs = dict.fromkeys(self.reg_bindings.values()) + # for reg in self.free_regs: + # assert reg not in rev_regs + # assert len(rev_regs) + len(self.free_regs) == len(self.all_regs) + #else: + # assert len(self.reg_bindings) + len(self.free_regs) == len(self.all_regs) if self.longevity: for v in self.reg_bindings: assert self.longevity[v][1] > self.position Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Thu Nov 26 20:21:02 2009 @@ -632,7 +632,7 @@ assert isinstance(calldescr, BaseCallDescr) assert len(calldescr.arg_classes) == len(op.args) - 1 size = calldescr.get_result_size(self.translate_support_code) - self._call(op, [imm(size), ebp] + + self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args], save_all_regs=True) @@ -943,7 +943,7 @@ return gcrootmap.compress_callshape(shape) def consider_force_token(self, op, ignored): - pass + self.rm.reg_bindings[op.result] = ebp def not_implemented_op(self, op, ignored): msg = "[regalloc] Not implemented operation: %s" % op.getopname() From fijal at codespeak.net Fri Nov 27 01:27:02 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 27 Nov 2009 01:27:02 +0100 (CET) Subject: [pypy-svn] r69673 - in pypy/branch/virtual-forcing/pypy/jit/backend: . llgraph test x86 Message-ID: <20091127002702.CB2CA1680F9@codespeak.net> Author: fijal Date: Fri Nov 27 01:27:01 2009 New Revision: 69673 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: Pass descr and live args as argument to force, up to discussion. Enough to implement this on x86 backend. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Fri Nov 27 01:27:01 2009 @@ -497,7 +497,7 @@ def do_cast_ptr_to_int(self, ptrbox): return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) - def force(self, force_token): + def force(self, force_token, descr, args): frame = llimpl.force(self.cast_int_to_adr(force_token)) self.latest_frame = frame Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Fri Nov 27 01:27:01 2009 @@ -211,7 +211,7 @@ def do_call_may_force(self, args, calldescr): raise NotImplementedError - def force(self, force_token): + def force(self, force_token, descr, args): raise NotImplementedError # ootype specific operations Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 01:27:01 2009 @@ -1225,7 +1225,7 @@ values = [] def maybe_force(token, flag): if flag: - self.cpu.force(token) + self.cpu.force(token, faildescr, [i1, i0]) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) @@ -1237,11 +1237,12 @@ i0 = BoxInt() i1 = BoxInt() tok = BoxInt() + faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, descr=calldescr), - ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=BasicFailDescr(1)), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) ] ops[2].fail_args = [i1, i0] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 01:27:01 2009 @@ -684,6 +684,11 @@ self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) + def genop_guard_guard_not_forced(self, ign_1, guard_op, addr, + locs, ign_2): + self.mc.CMP(mem(ebp, self.cpu.virtualizable_ofs), imm(0)) + return self.implement_guard(addr, self.mc.JNZ) + def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): loc = locs[0] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 01:27:01 2009 @@ -399,7 +399,7 @@ consider_guard_isnull = _consider_guard def consider_guard_not_forced(self, op, ignored): - pass + self.perform_guard(op, [], None) def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 01:27:01 2009 @@ -8,7 +8,6 @@ from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU - class CPU386(AbstractLLCPU): debug = True supports_floats = True @@ -87,10 +86,26 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) - def force(self, stack_base): + def force(self, stack_base, descr, args): + # args parameter is there only for types TP = rffi.CArrayPtr(lltype.Signed) rffi.cast(TP, stack_base + self.virtualizable_ofs)[0] = 1 - # force stuff actually... + # move things to latest values + arglocs = self.assembler.rebuild_faillocs_from_descr( + descr._x86_failure_recovery_bytecode) + assert len(arglocs) == len(args) + for i in range(len(arglocs)): + arg = args[i] + argloc = arglocs[i] + if arg.type == history.FLOAT: + xxx + elif arg.type == history.REF: + xxx + elif arg.type == history.INT: + pos = stack_base + argloc.ofs_relative_to_ebp() + self.assembler.fail_boxes_int.setitem(i, rffi.cast(TP, pos)[0]) + else: + raise NotImplementedError class CPU386_NO_SSE2(CPU386): supports_floats = False From fijal at codespeak.net Fri Nov 27 10:49:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 27 Nov 2009 10:49:05 +0100 (CET) Subject: [pypy-svn] r69674 - pypy/branch/virtual-forcing/pypy/jit/backend/llsupport Message-ID: <20091127094905.90540168101@codespeak.net> Author: fijal Date: Fri Nov 27 10:49:04 2009 New Revision: 69674 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py Log: Cleanup - revert commenting out invariants check Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py Fri Nov 27 10:49:04 2009 @@ -82,15 +82,15 @@ self.possibly_free_var(v) def _check_invariants(self): - #if not we_are_translated(): - # # make sure no duplicates - # assert len(dict.fromkeys(self.reg_bindings.values())) == len(self.reg_bindings) - # rev_regs = dict.fromkeys(self.reg_bindings.values()) - # for reg in self.free_regs: - # assert reg not in rev_regs - # assert len(rev_regs) + len(self.free_regs) == len(self.all_regs) - #else: - # assert len(self.reg_bindings) + len(self.free_regs) == len(self.all_regs) + if not we_are_translated(): + # make sure no duplicates + assert len(dict.fromkeys(self.reg_bindings.values())) == len(self.reg_bindings) + rev_regs = dict.fromkeys(self.reg_bindings.values()) + for reg in self.free_regs: + assert reg not in rev_regs + assert len(rev_regs) + len(self.free_regs) == len(self.all_regs) + else: + assert len(self.reg_bindings) + len(self.free_regs) == len(self.all_regs) if self.longevity: for v in self.reg_bindings: assert self.longevity[v][1] > self.position From fijal at codespeak.net Fri Nov 27 10:49:25 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 27 Nov 2009 10:49:25 +0100 (CET) Subject: [pypy-svn] r69675 - in pypy/branch/virtual-forcing/pypy/jit/backend: llgraph test Message-ID: <20091127094925.C2EDE168104@codespeak.net> Author: fijal Date: Fri Nov 27 10:49:25 2009 New Revision: 69675 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Log: Kill extra args to force Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Fri Nov 27 10:49:25 2009 @@ -497,7 +497,7 @@ def do_cast_ptr_to_int(self, ptrbox): return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) - def force(self, force_token, descr, args): + def force(self, force_token): frame = llimpl.force(self.cast_int_to_adr(force_token)) self.latest_frame = frame Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 10:49:25 2009 @@ -1225,7 +1225,7 @@ values = [] def maybe_force(token, flag): if flag: - self.cpu.force(token, faildescr, [i1, i0]) + self.cpu.force(token) values.append(self.cpu.get_latest_value_int(0)) values.append(self.cpu.get_latest_value_int(1)) From fijal at codespeak.net Fri Nov 27 10:50:26 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 27 Nov 2009 10:50:26 +0100 (CET) Subject: [pypy-svn] r69676 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091127095026.720C0168101@codespeak.net> Author: fijal Date: Fri Nov 27 10:50:25 2009 New Revision: 69676 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: * Use regular regalloc interface, no special support for ebp now * Instead of virtualizable offset, store vable_loc on regalloc, returned by force token * Not finished, XXX force() in runner Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 10:50:25 2009 @@ -210,8 +210,7 @@ self.mc.PUSH(ebx) self.mc.PUSH(esi) self.mc.PUSH(edi) - # could be really PUSH(0), but that way is safer - self.mc.MOV(mem(ebp, self.cpu.virtualizable_ofs), imm(0)) + self.mc.PUSH(imm(0)) # the virtualizable flag # NB. exactly 4 pushes above; if this changes, fix stack_pos(). # You must also keep get_basic_shape() in sync. adr_stackadjust = self._patchable_stackadjust() @@ -686,7 +685,7 @@ def genop_guard_guard_not_forced(self, ign_1, guard_op, addr, locs, ign_2): - self.mc.CMP(mem(ebp, self.cpu.virtualizable_ofs), imm(0)) + self.mc.CMP(locs[0], imm(0)) return self.implement_guard(addr, self.mc.JNZ) def genop_guard_guard_exception(self, ign_1, guard_op, addr, @@ -1142,6 +1141,9 @@ assert 0 < offset <= 127 mc.overwrite(jz_location-1, [chr(offset)]) + def genop_force_token(self, op, arglocs, resloc): + self.mc.LEA(resloc, arglocs[0]) + def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() print msg Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 10:50:25 2009 @@ -131,8 +131,7 @@ def _prepare(self, inputargs, operations): self.sm = X86StackManager() # a bit of a hack - always grab one position at the beginning - loc = self.sm.loc(TempBox(), 1) - self.assembler.cpu.virtualizable_ofs = loc.ofs_relative_to_ebp() + self.vable_loc = self.sm.loc(TempBox(), 1) cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables @@ -399,7 +398,7 @@ consider_guard_isnull = _consider_guard def consider_guard_not_forced(self, op, ignored): - self.perform_guard(op, [], None) + self.perform_guard(op, [self.vable_loc], None) def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] @@ -943,7 +942,8 @@ return gcrootmap.compress_callshape(shape) def consider_force_token(self, op, ignored): - self.rm.reg_bindings[op.result] = ebp + loc = self.rm.force_allocate_reg(op.result) + self.Perform(op, [self.vable_loc], loc) def not_implemented_op(self, op, ignored): msg = "[regalloc] Not implemented operation: %s" % op.getopname() Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 10:50:25 2009 @@ -86,8 +86,9 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) - def force(self, stack_base, descr, args): + def force(self, stack_base): # args parameter is there only for types + XXX # rewrite, kill TP = rffi.CArrayPtr(lltype.Signed) rffi.cast(TP, stack_base + self.virtualizable_ofs)[0] = 1 # move things to latest values From afa at codespeak.net Fri Nov 27 11:34:11 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 11:34:11 +0100 (CET) Subject: [pypy-svn] r69677 - pypy/trunk/pypy/module/oracle Message-ID: <20091127103411.EA0931680F8@codespeak.net> Author: afa Date: Fri Nov 27 11:34:11 2009 New Revision: 69677 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py Log: Add a marker for code that I did not translate from cx_Oracle. cx_Oracle has no test about it :-( Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 27 11:34:11 2009 @@ -163,7 +163,10 @@ # set the internal and external names; these are needed for global # transactions but are limited in terms of the lengths of the strings - + if twophase: + raise OperationError( + interp_error.get(space).w_NotSupportedError + space.wrap("XXX write me")) # allocate the session handle handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCISession).TO, From afa at codespeak.net Fri Nov 27 11:36:56 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 11:36:56 +0100 (CET) Subject: [pypy-svn] r69678 - pypy/trunk/pypy/module/oracle Message-ID: <20091127103656.6C2C41680F8@codespeak.net> Author: afa Date: Fri Nov 27 11:36:54 2009 New Revision: 69678 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py Log: Typo Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 27 11:36:54 2009 @@ -165,7 +165,7 @@ # transactions but are limited in terms of the lengths of the strings if twophase: raise OperationError( - interp_error.get(space).w_NotSupportedError + interp_error.get(space).w_NotSupportedError, space.wrap("XXX write me")) # allocate the session handle From afa at codespeak.net Fri Nov 27 11:40:54 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 11:40:54 +0100 (CET) Subject: [pypy-svn] r69679 - pypy/trunk/pypy/module/oracle Message-ID: <20091127104054.E151C1680F8@codespeak.net> Author: afa Date: Fri Nov 27 11:40:54 2009 New Revision: 69679 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_cursor.py pypy/trunk/pypy/module/oracle/interp_lob.py pypy/trunk/pypy/module/oracle/interp_variable.py Log: Remove this unwrap_spec stuff, the gateway is intelligent enough to find it in the function attributes. Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 27 11:40:54 2009 @@ -526,16 +526,12 @@ username = interp_attrproperty_w('w_username', W_Connection), password = interp_attrproperty_w('w_password', W_Connection), tnsentry = interp_attrproperty_w('w_tnsentry', W_Connection), - - close = interp2app(W_Connection.close, - unwrap_spec=W_Connection.close.unwrap_spec), - commit = interp2app(W_Connection.commit, - unwrap_spec=W_Connection.commit.unwrap_spec), - rollback = interp2app(W_Connection.rollback, - unwrap_spec=W_Connection.rollback.unwrap_spec), - cursor = interp2app(W_Connection.newCursor, - unwrap_spec=W_Connection.newCursor.unwrap_spec), + close = interp2app(W_Connection.close), + commit = interp2app(W_Connection.commit), + rollback = interp2app(W_Connection.rollback), + + cursor = interp2app(W_Connection.newCursor), encoding = GetSetProperty(W_Connection.get_encoding), nationalencoding = GetSetProperty(W_Connection.get_nationalencoding), Modified: pypy/trunk/pypy/module/oracle/interp_cursor.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_cursor.py (original) +++ pypy/trunk/pypy/module/oracle/interp_cursor.py Fri Nov 27 11:40:54 2009 @@ -1079,34 +1079,20 @@ W_Cursor.typedef = TypeDef( 'Cursor', - execute = interp2app(W_Cursor.execute, - unwrap_spec=W_Cursor.execute.unwrap_spec), - executemany = interp2app(W_Cursor.executemany, - unwrap_spec=W_Cursor.executemany.unwrap_spec), - prepare = interp2app(W_Cursor.prepare, - unwrap_spec=W_Cursor.prepare.unwrap_spec), - fetchone = interp2app(W_Cursor.fetchone, - unwrap_spec=W_Cursor.fetchone.unwrap_spec), - fetchmany = interp2app(W_Cursor.fetchmany, - unwrap_spec=W_Cursor.fetchmany.unwrap_spec), - fetchall = interp2app(W_Cursor.fetchall, - unwrap_spec=W_Cursor.fetchall.unwrap_spec), - close = interp2app(W_Cursor.close, - unwrap_spec=W_Cursor.close.unwrap_spec), - bindnames = interp2app(W_Cursor.bindnames, - unwrap_spec=W_Cursor.bindnames.unwrap_spec), - callfunc = interp2app(W_Cursor.callfunc, - unwrap_spec=W_Cursor.callfunc.unwrap_spec), - callproc = interp2app(W_Cursor.callproc, - unwrap_spec=W_Cursor.callproc.unwrap_spec), - var = interp2app(W_Cursor.var, - unwrap_spec=W_Cursor.var.unwrap_spec), - arrayvar = interp2app(W_Cursor.arrayvar, - unwrap_spec=W_Cursor.arrayvar.unwrap_spec), - setinputsizes = interp2app(W_Cursor.setinputsizes, - unwrap_spec=W_Cursor.setinputsizes.unwrap_spec), - setoutputsize = interp2app(W_Cursor.setoutputsize, - unwrap_spec=W_Cursor.setoutputsize.unwrap_spec), + execute = interp2app(W_Cursor.execute), + executemany = interp2app(W_Cursor.executemany), + prepare = interp2app(W_Cursor.prepare), + fetchone = interp2app(W_Cursor.fetchone), + fetchmany = interp2app(W_Cursor.fetchmany), + fetchall = interp2app(W_Cursor.fetchall), + close = interp2app(W_Cursor.close), + bindnames = interp2app(W_Cursor.bindnames), + callfunc = interp2app(W_Cursor.callfunc), + callproc = interp2app(W_Cursor.callproc), + var = interp2app(W_Cursor.var), + arrayvar = interp2app(W_Cursor.arrayvar), + setinputsizes = interp2app(W_Cursor.setinputsizes), + setoutputsize = interp2app(W_Cursor.setoutputsize), __iter__ = interp2app(W_Cursor.descr_iter), next = interp2app(W_Cursor.descr_next), Modified: pypy/trunk/pypy/module/oracle/interp_lob.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_lob.py (original) +++ pypy/trunk/pypy/module/oracle/interp_lob.py Fri Nov 27 11:40:54 2009 @@ -22,30 +22,26 @@ def read(self, space, offset=-1, amount=-1): self._verify(space) return self.lobVar.read(space, self.pos, offset, amount) - read.unwrap_spec=['self', ObjSpace, int, int] + read.unwrap_spec = ['self', ObjSpace, int, int] def size(self, space): self._verify(space) return space.wrap(self.lobVar.getLength(space, self.pos)) - size.unwrap_spec=['self', ObjSpace] + size.unwrap_spec = ['self', ObjSpace] def trim(self, space, newSize=0): self._verify(space) self.lobVar.trim(space, self.pos, newSize) - trim.unwrap_spec=['self', ObjSpace, int] + trim.unwrap_spec = ['self', ObjSpace, int] def desc_str(self, space): return self.read(space, offset=1, amount=-1) - desc_str.unwrap_spec=['self', ObjSpace] + desc_str.unwrap_spec = ['self', ObjSpace] W_ExternalLob.typedef = TypeDef( 'ExternalLob', - read = interp2app(W_ExternalLob.read, - unwrap_spec=W_ExternalLob.read.unwrap_spec), - size = interp2app(W_ExternalLob.size, - unwrap_spec=W_ExternalLob.size.unwrap_spec), - trim = interp2app(W_ExternalLob.trim, - unwrap_spec=W_ExternalLob.trim.unwrap_spec), - __str__ = interp2app(W_ExternalLob.desc_str, - unwrap_spec=W_ExternalLob.desc_str.unwrap_spec), + read = interp2app(W_ExternalLob.read), + size = interp2app(W_ExternalLob.size), + trim = interp2app(W_ExternalLob.trim), + __str__ = interp2app(W_ExternalLob.desc_str), ) Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Fri Nov 27 11:40:54 2009 @@ -442,10 +442,8 @@ W_Variable.typedef = TypeDef( 'Variable', - getvalue = interp2app(W_Variable.getValue, - unwrap_spec=W_Variable.getValue.unwrap_spec), - setvalue = interp2app(W_Variable.setValue, - unwrap_spec=W_Variable.setValue.unwrap_spec), + getvalue = interp2app(W_Variable.getValue), + setvalue = interp2app(W_Variable.setValue), maxlength = interp_attrproperty('bufferSize', W_Variable), bufferSize = interp_attrproperty('bufferSize', W_Variable), From arigo at codespeak.net Fri Nov 27 12:33:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 12:33:42 +0100 (CET) Subject: [pypy-svn] r69680 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091127113342.D80C41680F9@codespeak.net> Author: arigo Date: Fri Nov 27 12:33:42 2009 New Revision: 69680 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Log: (pedronis, fijal partly, arigo) Kill RET_BP and replace it with a FRAME_FIXED_SIZE constant with a hopefully clearer meaning. Use it instead of hard-coding the value 4 in get_ebp_ofs(). Also kill the parts of force_token and call_may_force and guard_not_forced that we plan to change. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 12:33:42 2009 @@ -9,7 +9,8 @@ from pypy.rpython.annlowlevel import llhelper from pypy.tool.uid import fixid from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, lower_byte,\ - X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs + X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, FRAME_FIXED_SIZE,\ + FORCE_INDEX_OFS from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import codebuf from pypy.jit.backend.x86.ri386 import * @@ -22,8 +23,6 @@ # our calling convention - we pass first 6 args in registers # and the rest stays on the stack -RET_BP = 5 # ret ip + bp + bx + esi + edi = 5 words - if sys.platform == 'darwin': # darwin requires the stack to be 16 bytes aligned on calls CALL_ALIGN = 4 @@ -200,7 +199,11 @@ # patch stack adjustment LEA # possibly align, e.g. for Mac OS X mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - mc.write(packimm32(-(stack_depth + RET_BP - 2) * WORD)) + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + # Given that [EBP] is where we saved EBP, i.e. in the last word + # of our fixed frame, then the 'words' value is: + words = (FRAME_FIXED_SIZE - 1) + stack_depth + mc.write(packimm32(-WORD * words)) mc.done() def _assemble_bootstrap_code(self, inputargs, arglocs): @@ -210,9 +213,8 @@ self.mc.PUSH(ebx) self.mc.PUSH(esi) self.mc.PUSH(edi) - self.mc.PUSH(imm(0)) # the virtualizable flag - # NB. exactly 4 pushes above; if this changes, fix stack_pos(). - # You must also keep get_basic_shape() in sync. + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. adr_stackadjust = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] @@ -683,11 +685,6 @@ self.mc.CMP(heap(self.cpu.pos_exception()), imm(0)) return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_not_forced(self, ign_1, guard_op, addr, - locs, ign_2): - self.mc.CMP(locs[0], imm(0)) - return self.implement_guard(addr, self.mc.JNZ) - def genop_guard_guard_exception(self, ign_1, guard_op, addr, locs, resloc): loc = locs[0] @@ -1003,11 +1000,11 @@ # now we return from the complete frame, which starts from # _assemble_bootstrap_code(). The LEA below throws away most # of the frame, including all the PUSHes that we did just above. - mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) - mc.POP(edi) - mc.POP(esi) - mc.POP(ebx) - mc.POP(ebp) + mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() self.mc2.done() self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr @@ -1048,14 +1045,14 @@ addr = self.cpu.get_on_leave_jitted_int(save_exception=exc) mc.CALL(rel32(addr)) - # don't break the following code sequence! + # don't break the following code sequence! xxx no reason any more? mc = mc._mc - mc.LEA(esp, addr_add(ebp, imm((-RET_BP + 2) * WORD))) + mc.LEA(esp, addr_add(ebp, imm(-3 * WORD))) mc.MOV(eax, imm(fail_index)) - mc.POP(edi) - mc.POP(esi) - mc.POP(ebx) - mc.POP(ebp) + mc.POP(edi) # [ebp-12] + mc.POP(esi) # [ebp-8] + mc.POP(ebx) # [ebp-4] + mc.POP(ebp) # [ebp] mc.RET() @specialize.arg(2) @@ -1110,7 +1107,10 @@ self.mc.AND(eax, imm(0xffff)) genop_call_pure = genop_call - genop_call_may_force = genop_call + + def genop_guard_call_may_force(self, op, guard_op, addr, + arglocs, result_loc): + xxx #... def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1142,7 +1142,7 @@ mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - self.mc.LEA(resloc, arglocs[0]) + xxx #self.mc.LEA(resloc, ...) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 12:33:42 2009 @@ -19,6 +19,8 @@ TempBox WORD = 4 +FRAME_FIXED_SIZE = 5 # ebp + ebx + esi + edi + force_index = 5 words +FORCE_INDEX_OFS = -4*WORD width_of_type = { INT : 1, @@ -130,8 +132,6 @@ def _prepare(self, inputargs, operations): self.sm = X86StackManager() - # a bit of a hack - always grab one position at the beginning - self.vable_loc = self.sm.loc(TempBox(), 1) cpu = self.assembler.cpu cpu.gc_ll_descr.rewrite_assembler(cpu, operations) # compute longevity of variables @@ -310,7 +310,10 @@ self.assembler.dump('%s(%s)' % (op, arglocs)) self.assembler.regalloc_perform_discard(op, arglocs) - def can_optimize_cmp_op(self, op, i, operations): + def can_merge_with_next_guard(self, op, i, operations): + if op.opnum == rop.CALL_MAY_FORCE: + assert operations[i + 1].opnum == rop.GUARD_NOT_FORCED + return True if not op.is_comparison(): return False if (operations[i + 1].opnum != rop.GUARD_TRUE and @@ -334,7 +337,7 @@ i += 1 self.possibly_free_vars(op.args) continue - if self.can_optimize_cmp_op(op, i, operations): + if self.can_merge_with_next_guard(op, i, operations): oplist[op.opnum](self, op, operations[i + 1]) i += 1 else: @@ -397,9 +400,6 @@ consider_guard_nonnull = _consider_guard consider_guard_isnull = _consider_guard - def consider_guard_not_forced(self, op, ignored): - self.perform_guard(op, [self.vable_loc], None) - def consider_finish(self, op, ignored): locs = [self.loc(v) for v in op.args] locs_are_ref = [v.type == REF for v in op.args] @@ -609,34 +609,36 @@ self.Perform(op, [loc0], loc1) self.rm.possibly_free_var(op.args[0]) - def _call(self, op, arglocs, force_store=[], save_all_regs=False): + def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None): + save_all_regs = guard_not_forced_op is not None self.rm.before_call(force_store, save_all_regs=save_all_regs) self.xrm.before_call(force_store, save_all_regs=save_all_regs) - self.Perform(op, arglocs, eax) + if guard_not_forced_op is not None: + self.perform_with_guard(op, guard_not_forced_op, arglocs, eax) + else: + self.Perform(op, arglocs, eax) if op.result is not None: if op.result.type == FLOAT: self.xrm.after_call(op.result) else: self.rm.after_call(op.result) - def consider_call(self, op, ignored): - calldescr = op.descr - assert isinstance(calldescr, BaseCallDescr) - assert len(calldescr.arg_classes) == len(op.args) - 1 - size = calldescr.get_result_size(self.translate_support_code) - self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args]) - - def consider_call_may_force(self, op, ignored): + def _consider_call(self, op, guard_not_forced_op=None): calldescr = op.descr assert isinstance(calldescr, BaseCallDescr) assert len(calldescr.arg_classes) == len(op.args) - 1 size = calldescr.get_result_size(self.translate_support_code) - self._call(op, [imm(size)] + - [self.loc(arg) for arg in op.args], - save_all_regs=True) + self._call(op, [imm(size)] + [self.loc(arg) for arg in op.args], + guard_not_forced_op=guard_not_forced_op) + def consider_call(self, op, ignored): + self._consider_call(op) consider_call_pure = consider_call + def consider_call_may_force(self, op, guard_op): + assert guard_op is not None + self._consider_call(op, guard_op) + def consider_cond_call_gc_wb(self, op, ignored): assert op.result is None arglocs = [self.loc(arg) for arg in op.args] @@ -943,7 +945,7 @@ def consider_force_token(self, op, ignored): loc = self.rm.force_allocate_reg(op.result) - self.Perform(op, [self.vable_loc], loc) + self.Perform(op, [], loc) def not_implemented_op(self, op, ignored): msg = "[regalloc] Not implemented operation: %s" % op.getopname() @@ -960,10 +962,9 @@ def get_ebp_ofs(position): # Argument is a stack position (0, 1, 2...). - # Returns (ebp-16), (ebp-20), (ebp-24)... - # This depends on the fact that our function prologue contains - # exactly 4 PUSHes. - return -WORD * (4 + position) + # Returns (ebp-20), (ebp-24), (ebp-28)... + # i.e. the n'th word beyond the fixed frame size. + return -WORD * (FRAME_FIXED_SIZE + position) def lower_byte(reg): # argh, kill, use lowest8bits instead From arigo at codespeak.net Fri Nov 27 13:31:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 13:31:43 +0100 (CET) Subject: [pypy-svn] r69687 - in pypy/branch/virtual-forcing/pypy: jit/backend/x86 rpython rpython/lltypesystem Message-ID: <20091127123143.97CF81680F9@codespeak.net> Author: arigo Date: Fri Nov 27 13:31:43 2009 New Revision: 69687 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py pypy/branch/virtual-forcing/pypy/rpython/llinterp.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Log: (pedronis, fijal, arigo) Implement cpu.force() by calling the existing guard recovery code. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 13:31:43 2009 @@ -879,6 +879,69 @@ arglocs.append(loc) return arglocs[:] + def grab_frame_values(self, bytecode, frame_addr, + registers=lltype.nullptr(rffi.LONGP.TO)): + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode + code = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + if code >= 4*self.DESCR_FROMSTACK: + if code > 0x7F: + shift = 7 + code &= 0x7F + while True: + nextcode = rffi.cast(lltype.Signed, bytecode[0]) + bytecode = rffi.ptradd(bytecode, 1) + code |= (nextcode & 0x7F) << shift + shift += 7 + if nextcode <= 0x7F: + break + # load the value from the stack + kind = code & 3 + code = (code >> 2) - self.DESCR_FROMSTACK + stackloc = frame_addr + get_ebp_ofs(code) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == self.DESCR_FLOAT: + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - 4)[0] + else: + # 'code' identifies a register: load its value + kind = code & 3 + if kind == self.DESCR_SPECIAL: + if code == self.DESCR_HOLE: + num += 1 + continue + assert code == self.DESCR_STOP + break + assert registers # it's NULL when called from cpu.force() + code >>= 2 + if kind == self.DESCR_FLOAT: + xmmregisters = rffi.ptradd(registers, -16) + value = xmmregisters[2*code] + value_hi = xmmregisters[2*code + 1] + else: + value = registers[code] + + # store the loaded value into fail_boxes_ + if kind == self.DESCR_INT: + tgt = self.fail_boxes_int.get_addr_for_num(num) + elif kind == self.DESCR_REF: + tgt = self.fail_boxes_ptr.get_addr_for_num(num) + elif kind == self.DESCR_FLOAT: + tgt = self.fail_boxes_float.get_addr_for_num(num) + rffi.cast(rffi.LONGP, tgt)[1] = value_hi + else: + assert 0, "bogus kind" + rffi.cast(rffi.LONGP, tgt)[0] = value + num += 1 + # + if not we_are_translated(): + assert bytecode[4] == 0xCC + fail_index = rffi.cast(rffi.LONGP, bytecode)[0] + return fail_index + def setup_failure_recovery(self): def failure_recovery_func(registers): @@ -889,65 +952,7 @@ # recovery bytecode. See _build_failure_recovery() for details. stack_at_ebp = registers[ebp.op] bytecode = rffi.cast(rffi.UCHARP, registers[8]) - num = 0 - value_hi = 0 - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= 4*self.DESCR_FROMSTACK: - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - # load the value from the stack - kind = code & 3 - code = (code >> 2) - self.DESCR_FROMSTACK - stackloc = stack_at_ebp + get_ebp_ofs(code) - value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == self.DESCR_FLOAT: - value_hi = value - value = rffi.cast(rffi.LONGP, stackloc - 4)[0] - else: - # 'code' identifies a register: load its value - kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.DESCR_HOLE: - num += 1 - continue - assert code == self.DESCR_STOP - break - code >>= 2 - if kind == self.DESCR_FLOAT: - xmmregisters = rffi.ptradd(registers, -16) - value = xmmregisters[2*code] - value_hi = xmmregisters[2*code + 1] - else: - value = registers[code] - - # store the loaded value into fail_boxes_ - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - elif kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGP, tgt)[1] = value_hi - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value - num += 1 - # - if not we_are_translated(): - assert bytecode[4] == 0xCC - fail_index = rffi.cast(rffi.LONGP, bytecode)[0] - return fail_index + return self.grab_frame_values(bytecode, stack_at_ebp, registers) self.failure_recovery_func = failure_recovery_func self.failure_recovery_code = [0, 0, 0, 0] @@ -1110,7 +1115,12 @@ def genop_guard_call_may_force(self, op, guard_op, addr, arglocs, result_loc): - xxx #... + faildescr = guard_op.descr + fail_index = self.cpu.get_fail_descr_number(faildescr) + self.mc.MOV(mem(ebp, FORCE_INDEX_OFS), imm(fail_index)) + self.genop_call(op, arglocs, result_loc) + self.mc.CMP(mem(ebp, FORCE_INDEX_OFS), imm(0)) + return self.implement_guard(addr, self.mc.JL) def genop_discard_cond_call_gc_wb(self, op, arglocs): # use 'mc._mc' directly instead of 'mc', to avoid @@ -1142,7 +1152,7 @@ mc.overwrite(jz_location-1, [chr(offset)]) def genop_force_token(self, op, arglocs, resloc): - xxx #self.mc.LEA(resloc, ...) + self.mc.LEA(resloc, mem(ebp, FORCE_INDEX_OFS)) def not_implemented_op_discard(self, op, arglocs): msg = "not implemented operation: %s" % op.getopname() Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 13:31:43 2009 @@ -289,7 +289,8 @@ self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, self.sm.stack_depth) - self.rm.possibly_free_var(op.result) + if op.result is not None: + self.rm.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) def perform_guard(self, guard_op, arglocs, result_loc): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 13:31:43 2009 @@ -6,6 +6,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history from pypy.jit.backend.x86.assembler import Assembler386 +from pypy.jit.backend.x86.regalloc import FORCE_INDEX_OFS from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU class CPU386(AbstractLLCPU): @@ -86,27 +87,24 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) - def force(self, stack_base): - # args parameter is there only for types - XXX # rewrite, kill + def force(self, addr_of_force_index): TP = rffi.CArrayPtr(lltype.Signed) - rffi.cast(TP, stack_base + self.virtualizable_ofs)[0] = 1 - # move things to latest values - arglocs = self.assembler.rebuild_faillocs_from_descr( - descr._x86_failure_recovery_bytecode) - assert len(arglocs) == len(args) - for i in range(len(arglocs)): - arg = args[i] - argloc = arglocs[i] - if arg.type == history.FLOAT: - xxx - elif arg.type == history.REF: - xxx - elif arg.type == history.INT: - pos = stack_base + argloc.ofs_relative_to_ebp() - self.assembler.fail_boxes_int.setitem(i, rffi.cast(TP, pos)[0]) - else: - raise NotImplementedError + fail_index = rffi.cast(TP, addr_of_force_index)[0] + if fail_index < 0: + xxx # write a test and kill this line + return # already forced + faildescr = self.get_fail_descr_from_number(fail_index) + rffi.cast(TP, addr_of_force_index)[0] = -1 + bytecode = rffi.cast(rffi.UCHARP, + faildescr._x86_failure_recovery_bytecode) + # start of "no gc operation!" block + fail_index_2 = self.assembler.grab_frame_values( + bytecode, + addr_of_force_index - FORCE_INDEX_OFS) + self.assembler.leave_jitted_hook() + # end of "no gc operation!" block + assert fail_index == fail_index_2 + class CPU386_NO_SSE2(CPU386): supports_floats = False Modified: pypy/branch/virtual-forcing/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/llinterp.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/llinterp.py Fri Nov 27 13:31:43 2009 @@ -807,9 +807,6 @@ def op_gc__collect(self, *gen): self.heap.collect(*gen) - def op_gc_assume_young_pointers(self, addr): - raise NotImplementedError - def op_gc_heap_stats(self): raise NotImplementedError Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/lloperation.py Fri Nov 27 13:31:43 2009 @@ -460,7 +460,7 @@ # allocating non-GC structures only 'gc_thread_run' : LLOp(), 'gc_thread_die' : LLOp(), - 'gc_assume_young_pointers': LLOp(), + 'gc_assume_young_pointers': LLOp(canrun=True), 'gc_heap_stats' : LLOp(canunwindgc=True), # ------- JIT & GC interaction, only for some GCs ---------- Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/opimpl.py Fri Nov 27 13:31:43 2009 @@ -486,6 +486,9 @@ def op_get_member_index(memberoffset): raise NotImplementedError +def op_gc_assume_young_pointers(addr): + pass + # ____________________________________________________________ def get_op_impl(opname): From fijal at codespeak.net Fri Nov 27 13:38:13 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 27 Nov 2009 13:38:13 +0100 (CET) Subject: [pypy-svn] r69689 - in pypy/branch/virtual-forcing/pypy/jit/backend: test x86 Message-ID: <20091127123813.6A89A1680F9@codespeak.net> Author: fijal Date: Fri Nov 27 13:38:12 2009 New Revision: 69689 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: Write a missing test and kill xxx Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 13:38:12 2009 @@ -1263,6 +1263,38 @@ assert self.cpu.get_latest_value_int(1) == 10 assert values == [1, 10] + def test_double_force(self): + values = [] + def double_force(token): + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(1)) + + FUNC = self.FuncType([lltype.Signed], lltype.Void) + func_ptr = llhelper(lltype.Ptr(FUNC), double_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok], None, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert values == [0, 20] + # pure do_ / descr features def test_do_operations(self): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 13:38:12 2009 @@ -91,7 +91,6 @@ TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] if fail_index < 0: - xxx # write a test and kill this line return # already forced faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 From arigo at codespeak.net Fri Nov 27 15:02:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 15:02:14 +0100 (CET) Subject: [pypy-svn] r69700 - in pypy/branch/virtual-forcing/pypy/jit/backend: test x86 Message-ID: <20091127140214.CCE64168102@codespeak.net> Author: arigo Date: Fri Nov 27 15:02:13 2009 New Revision: 69700 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: (pedronis, arigo) It actually makes no sense for cpu.force() to be a no-op if already forced. Kill the feature and the test. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 15:02:13 2009 @@ -1263,38 +1263,6 @@ assert self.cpu.get_latest_value_int(1) == 10 assert values == [1, 10] - def test_double_force(self): - values = [] - def double_force(token): - self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(0)) - self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(1)) - - FUNC = self.FuncType([lltype.Signed], lltype.Void) - func_ptr = llhelper(lltype.Ptr(FUNC), double_force) - funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() - calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) - cpu = self.cpu - i0 = BoxInt() - i1 = BoxInt() - tok = BoxInt() - faildescr = BasicFailDescr(1) - ops = [ - ResOperation(rop.FORCE_TOKEN, [], tok), - ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok], None, - descr=calldescr), - ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) - ] - ops[2].fail_args = [i1, i0] - looptoken = LoopToken() - self.cpu.compile_loop([i0, i1], ops, looptoken) - self.cpu.set_future_value_int(0, 20) - self.cpu.set_future_value_int(1, 0) - fail = self.cpu.execute_token(looptoken) - assert values == [0, 20] - # pure do_ / descr features def test_do_operations(self): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 15:02:13 2009 @@ -90,8 +90,7 @@ def force(self, addr_of_force_index): TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] - if fail_index < 0: - return # already forced + assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 bytecode = rffi.cast(rffi.UCHARP, From arigo at codespeak.net Fri Nov 27 16:04:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 16:04:38 +0100 (CET) Subject: [pypy-svn] r69702 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091127150438.3DB1D168102@codespeak.net> Author: arigo Date: Fri Nov 27 16:04:37 2009 New Revision: 69702 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: (pedronis, fijal?, arigo) Simplify a bit virtualizables: when it escapes, then we abort tracing instead of generating a trace with the escaped virtualizable. Such a trace is horribly horribly bad anyway. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Nov 27 16:04:37 2009 @@ -1318,8 +1318,8 @@ # residual calls require attention to keep virtualizables in-sync. # CALL_PURE doesn't need it because so far 'promote_virtualizable' # as an operation is enough to make the called function non-pure. - require_attention = (opnum == rop.CALL or opnum == rop.OOSEND) - if require_attention and not self.is_blackholing(): + is_a_call = (opnum == rop.CALL or opnum == rop.OOSEND) + if is_a_call: self.before_residual_call() # execute the operation profiler = self.staticdata.profiler @@ -1328,17 +1328,14 @@ if self.is_blackholing(): profiler.count_ops(opnum, BLACKHOLED_OPS) else: - if require_attention: - require_attention = self.after_residual_call() # check if the operation can be constant-folded away argboxes = list(argboxes) if rop._ALWAYS_PURE_FIRST <= opnum <= rop._ALWAYS_PURE_LAST: resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes) - # if we are blackholing require_attention has the initial meaning - if require_attention: - self.after_generate_residual_call() + if is_a_call: + self.after_residual_call() return resbox def _record_helper_pure(self, opnum, resbox, descr, *argboxes): @@ -1725,6 +1722,8 @@ vinfo.clear_vable_rti(virtualizable) def before_residual_call(self): + if self.is_blackholing(): + return vinfo = self.staticdata.virtualizable_info if vinfo is not None: virtualizable_box = self.virtualizable_boxes[-1] @@ -1732,26 +1731,21 @@ vinfo.tracing_before_residual_call(virtualizable) def after_residual_call(self): - vinfo = self.staticdata.virtualizable_info - if vinfo is not None: - virtualizable_box = self.virtualizable_boxes[-1] - virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - if vinfo.tracing_after_residual_call(virtualizable): - # This is after the residual call is done, but before it - # is actually generated. We first generate a store- - # everything-back, *without actually performing it now* - # as it contains the old values (before the call)! - self.gen_store_back_in_virtualizable_no_perform() - return True # must call after_generate_residual_call() - # otherwise, don't call after_generate_residual_call() - return False - - def after_generate_residual_call(self): - # Called after generating a residual call, and only if - # after_residual_call() returned True, i.e. if code in the residual - # call causes the virtualizable to escape. Reload the modified - # fields of the virtualizable. - self.gen_load_fields_from_virtualizable() + if self.is_blackholing(): + vable_escapes = True + else: + vable_escapes = False + vinfo = self.staticdata.virtualizable_info + if vinfo is not None: + virtualizable_box = self.virtualizable_boxes[-1] + virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + if vinfo.tracing_after_residual_call(virtualizable): + # We just did the residual call, and it shows that the + # virtualizable escapes. + self.switch_to_blackhole() + vable_escapes = True + if vable_escapes: + self.load_fields_from_virtualizable() def handle_exception(self): etype = self.cpu.get_exception() @@ -1809,27 +1803,17 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.write_boxes(virtualizable, self.virtualizable_boxes) - def gen_load_fields_from_virtualizable(self): + def load_fields_from_virtualizable(self): + # Force a reload of the virtualizable fields into the local + # boxes (called only in escaping cases) + assert self.is_blackholing() vinfo = self.staticdata.virtualizable_info if vinfo is not None: - vbox = self.virtualizable_boxes[-1] - for i in range(vinfo.num_static_extra_boxes): - descr = vinfo.static_field_descrs[i] - fieldbox = self.execute_and_record(rop.GETFIELD_GC, descr, - vbox) - self.virtualizable_boxes[i] = fieldbox - i = vinfo.num_static_extra_boxes - virtualizable = vinfo.unwrap_virtualizable_box(vbox) - for k in range(vinfo.num_arrays): - descr = vinfo.array_field_descrs[k] - abox = self.execute_and_record(rop.GETFIELD_GC, descr, vbox) - descr = vinfo.array_descrs[k] - for j in range(vinfo.get_array_length(virtualizable, k)): - itembox = self.execute_and_record(rop.GETARRAYITEM_GC, - descr, abox, ConstInt(j)) - self.virtualizable_boxes[i] = itembox - i += 1 - assert i + 1 == len(self.virtualizable_boxes) + virtualizable_box = self.virtualizable_boxes[-1] + virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) + self.virtualizable_boxes = vinfo.read_boxes(self.cpu, + virtualizable) + self.virtualizable_boxes.append(virtualizable_box) def gen_store_back_in_virtualizable(self): vinfo = self.staticdata.virtualizable_info @@ -1853,29 +1837,6 @@ abox, ConstInt(j), itembox) assert i + 1 == len(self.virtualizable_boxes) - def gen_store_back_in_virtualizable_no_perform(self): - vinfo = self.staticdata.virtualizable_info - # xxx only write back the fields really modified - vbox = self.virtualizable_boxes[-1] - for i in range(vinfo.num_static_extra_boxes): - fieldbox = self.virtualizable_boxes[i] - self.history.record(rop.SETFIELD_GC, [vbox, fieldbox], None, - descr=vinfo.static_field_descrs[i]) - i = vinfo.num_static_extra_boxes - virtualizable = vinfo.unwrap_virtualizable_box(vbox) - for k in range(vinfo.num_arrays): - abox = vinfo.BoxArray() - self.history.record(rop.GETFIELD_GC, [vbox], abox, - descr=vinfo.array_field_descrs[k]) - for j in range(vinfo.get_array_length(virtualizable, k)): - itembox = self.virtualizable_boxes[i] - i += 1 - self.history.record(rop.SETARRAYITEM_GC, - [abox, ConstInt(j), itembox], - None, - descr=vinfo.array_descrs[k]) - assert i + 1 == len(self.virtualizable_boxes) - def replace_box(self, oldbox, newbox): for frame in self.framestack: boxes = frame.env Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py Fri Nov 27 16:04:37 2009 @@ -50,8 +50,8 @@ assert get_stats().enter_count <= count def check_jumps(self, maxcount): assert get_stats().exec_jumps <= maxcount - def check_aborted_count(self, maxcount): - assert get_stats().aborted_count == maxcount + def check_aborted_count(self, count): + assert get_stats().aborted_count == count def meta_interp(self, *args, **kwds): kwds['CPUClass'] = self.CPUClass Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Fri Nov 27 16:04:37 2009 @@ -200,72 +200,6 @@ assert res == 134 self.check_loops(getfield_gc=1, setfield_gc=1) - def test_external_read_while_tracing(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'xy'], - virtualizables = ['xy']) - class Outer: - pass - outer = Outer() - def ext(): - xy = outer.xy - promote_virtualizable(xy, 'inst_x') - return xy.inst_x + 2 - def f(n): - xy = self.setup() - xy.inst_x = 10 - outer.xy = xy - m = 0 - while n > 0: - myjitdriver.can_enter_jit(xy=xy, n=n, m=m) - myjitdriver.jit_merge_point(xy=xy, n=n, m=m) - promote_virtualizable(xy, 'inst_x') - xy.inst_x = n + 9998 # virtualized away - m += ext() # 2x setfield_gc, 2x getfield_gc - promote_virtualizable(xy, 'inst_x') - xy.inst_x = 10 # virtualized away - n -= 1 - return m - assert f(20) == 10000*20 + (20*21)/2 - res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) - assert res == 10000*20 + (20*21)/2 - # there are no getfields because the optimizer gets rid of them - self.check_loops(call=1, getfield_gc=0, setfield_gc=2) - # xxx for now a call that forces the virtualizable during tracing - # is supposed to always force it later too. - - def test_external_write_while_tracing(self): - myjitdriver = JitDriver(greens = [], reds = ['n', 'm', 'xy'], - virtualizables = ['xy']) - class Outer: - pass - outer = Outer() - def ext(): - xy = outer.xy - promote_virtualizable(xy, 'inst_x') - xy.inst_x += 2 - def f(n): - xy = self.setup() - xy.inst_x = 10 - outer.xy = xy - m = 0 - while n > 0: - myjitdriver.can_enter_jit(xy=xy, n=n, m=m) - myjitdriver.jit_merge_point(xy=xy, n=n, m=m) - promote_virtualizable(xy, 'inst_x') - xy.inst_x = n + 9998 # virtualized away - ext() # 2x setfield_gc, 2x getfield_gc - promote_virtualizable(xy, 'inst_x') - m += xy.inst_x # virtualized away - n -= 1 - return m - res = self.meta_interp(f, [20], policy=StopAtXPolicy(ext)) - assert res == f(20) - # the getfield_gc of inst_node is optimized away, because ext does not - # write to it - self.check_loops(call=1, getfield_gc=1, setfield_gc=2) - # xxx for now a call that forces the virtualizable during tracing - # is supposed to always force it later too. - # ------------------------------ XY2 = lltype.GcStruct( @@ -649,7 +583,8 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) - + self.check_aborted_count(2) + self.check_tree_loop_count(0) def test_external_write(self): jitdriver = JitDriver(greens = [], reds = ['frame'], @@ -680,6 +615,8 @@ res = self.meta_interp(f, [240], policy=StopAtXPolicy(g)) assert res == f(240) + self.check_aborted_count(3) + self.check_tree_loop_count(0) def test_external_read_sometimes(self): py.test.skip("known bug: access the frame in a residual call but" From afa at codespeak.net Fri Nov 27 16:05:00 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 16:05:00 +0100 (CET) Subject: [pypy-svn] r69703 - pypy/trunk/pypy/module/oracle Message-ID: <20091127150500.39DE1168102@codespeak.net> Author: afa Date: Fri Nov 27 16:04:59 2009 New Revision: 69703 Modified: pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py Log: Fix test failures on debian 64bit, except for a unicode issue related to ucs-4 encoding. Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Fri Nov 27 16:04:59 2009 @@ -64,23 +64,15 @@ status = roci.OCIEnvNlsCreate( handleptr, mode, None, - rffi.cast(rffi.CCallback( # malocfp - (roci.dvoidp, roci.size_t), roci.dvoidp), - 0), - rffi.cast(rffi.CCallback( # ralocfp - (roci.dvoidp, roci.dvoidp, roci.size_t), roci.dvoidp), - 0), - rffi.cast(rffi.CCallback( # mfreefp - (roci.dvoidp, roci.dvoidp), lltype.Void), - 0), + None, None, None, 0, lltype.nullptr(rffi.CArray(roci.dvoidp)), config.CHARSETID, config.CHARSETID) if not handleptr[0] or status not in (roci.OCI_SUCCESS, roci.OCI_SUCCESS_WITH_INFO): raise OperationError( - get(self.space).w_InterfaceError, - self.space.wrap( + get(space).w_InterfaceError, + space.wrap( "Unable to acquire Oracle environment handle")) handle = handleptr[0] Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Fri Nov 27 16:04:59 2009 @@ -190,6 +190,11 @@ try: dataLength = ovfcheck(self.allocatedElements * self.bufferSize) except OverflowError: + too_large = True + else: + too_large = False + + if too_large or dataLength >= roci.INT_MAX: raise OperationError( space.w_ValueError, space.wrap("array size too large")) @@ -428,7 +433,7 @@ space.wrap("Variable_SetArrayValue: array size exceeded")) # set all of the values - self.actualElementsPtr[0] = rffi.cast(lltype.Unsigned, len(elements_w)) + self.actualElementsPtr[0] = rffi.cast(roci.ub4, len(elements_w)) for i in range(len(elements_w)): self.setSingleValue(space, i, elements_w[i]) @@ -714,7 +719,7 @@ format_buf.fill(space, space.wrap("TM9")) sizeptr = lltype.malloc(rffi.CArray(roci.ub4), 1, flavor='raw') BUFSIZE = 200 - sizeptr[0] = rffi.cast(lltype.Unsigned, BUFSIZE) + sizeptr[0] = rffi.cast(roci.ub4, BUFSIZE) textbuf, text = rffi.alloc_buffer(BUFSIZE) try: status = roci.OCINumberToText( Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Fri Nov 27 16:04:59 2009 @@ -2,7 +2,7 @@ from pypy.rpython.tool import rffi_platform as platform from pypy.rpython.lltypesystem import rffi, lltype from pypy.conftest import option -import os +import sys, os import py oracle_home = getattr(option, 'oracle_home', @@ -13,12 +13,20 @@ raise ImportError( "Please set ORACLE_HOME to the root of an Oracle client installation") -eci = ExternalCompilationInfo( - includes = ['oci.h'], - include_dirs = [str(ORACLE_HOME.join('OCI', 'include'))], - libraries = ['oci'], - library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))], - ) +if sys.platform == 'win32': + eci = ExternalCompilationInfo( + includes = ['oci.h'], + include_dirs = [str(ORACLE_HOME.join('OCI', 'include'))], + libraries = ['oci'], + library_dirs = [str(ORACLE_HOME.join('OCI', 'lib', 'MSVC'))], + ) +else: + eci = ExternalCompilationInfo( + includes = ['oci.h'], + include_dirs = [str(ORACLE_HOME.join('sdk', 'include'))], + libraries = ['clntsh'], + library_dirs = [str(ORACLE_HOME.join('lib'))], + ) class CConfig: _compilation_info_ = eci @@ -97,6 +105,9 @@ for c in constants: locals()[c] = platform.ConstantInteger(c) + INT_MAX = platform.ConstantInteger('INT_MAX') + + globals().update(platform.configure(CConfig)) OCI_IND_NOTNULL = rffi.cast(rffi.SHORT, OCI_IND_NOTNULL) @@ -143,12 +154,9 @@ [rffi.CArrayPtr(OCIEnv), # envhpp ub4, # mode dvoidp, # ctxp - rffi.CCallback( # malocfp - [dvoidp, size_t], dvoidp), - rffi.CCallback( # ralocfp - [dvoidp, dvoidp, size_t], dvoidp), - rffi.CCallback( # mfreefp - [dvoidp, dvoidp], lltype.Void), + dvoidp, # malocfp + dvoidp, # ralocfp + dvoidp, # mfreefp size_t, # xtramemsz dvoidpp, # usermempp ub2, # charset From arigo at codespeak.net Fri Nov 27 16:49:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 16:49:57 +0100 (CET) Subject: [pypy-svn] r69704 - pypy/branch/virtual-forcing/pypy/translator/backendopt Message-ID: <20091127154957.6298C1680FD@codespeak.net> Author: arigo Date: Fri Nov 27 16:49:56 2009 New Revision: 69704 Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py Log: This reload() makes no sense (at least any more). Kill. Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py Fri Nov 27 16:49:56 2009 @@ -1,6 +1,5 @@ from pypy.translator.backendopt import graphanalyze from pypy.rpython.ootypesystem import ootype -reload(graphanalyze) top_set = object() empty_set = frozenset() From arigo at codespeak.net Fri Nov 27 17:40:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 17:40:33 +0100 (CET) Subject: [pypy-svn] r69709 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091127164033.8F7CA1680FE@codespeak.net> Author: arigo Date: Fri Nov 27 17:40:33 2009 New Revision: 69709 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Log: (pedronis, arigo) Store in the EffectInfo attached to calldescrs a flag that tells us if the residual call can possibly access a virtualizable's virtualized fields. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Fri Nov 27 17:40:33 2009 @@ -14,6 +14,7 @@ from pypy.translator.backendopt.writeanalyze import WriteAnalyzer from pypy.jit.metainterp.typesystem import deref, arrayItem, fieldType from pypy.jit.metainterp.effectinfo import effectinfo_from_writeanalyze +from pypy.jit.metainterp.effectinfo import VirtualizableAnalyzer import py, sys from pypy.tool.ansi_print import ansi_log @@ -182,8 +183,10 @@ self.metainterp_sd = metainterp_sd self.cpu = metainterp_sd.cpu self.portal_runner_ptr = portal_runner_ptr - self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) - self.write_analyzer = WriteAnalyzer(self.rtyper.annotator.translator) + translator = self.rtyper.annotator.translator + self.raise_analyzer = RaiseAnalyzer(translator) + self.write_analyzer = WriteAnalyzer(translator) + self.virtualizable_analyzer = VirtualizableAnalyzer(translator) def make_portal_bytecode(self, graph): log.info("making JitCodes...") @@ -323,7 +326,9 @@ # ok if consider_effects_of is not None: effectinfo = effectinfo_from_writeanalyze( - self.write_analyzer.analyze(consider_effects_of), self.cpu) + self.write_analyzer.analyze(consider_effects_of), + self.cpu, + self.virtualizable_analyzer.analyze(consider_effects_of)) calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) else: calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py Fri Nov 27 17:40:33 2009 @@ -2,21 +2,25 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype +from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer class EffectInfo(object): _cache = {} - def __new__(cls, write_descrs_fields, write_descrs_arrays): - key = frozenset(write_descrs_fields), frozenset(write_descrs_arrays) + def __new__(cls, write_descrs_fields, write_descrs_arrays, + promotes_virtualizables): + key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), + promotes_virtualizables) if key in cls._cache: return cls._cache[key] result = object.__new__(cls) result.write_descrs_fields = write_descrs_fields result.write_descrs_arrays = write_descrs_arrays + result.promotes_virtualizables = promotes_virtualizables cls._cache[key] = result return result -def effectinfo_from_writeanalyze(effects, cpu): +def effectinfo_from_writeanalyze(effects, cpu, promotes_virtualizables=False): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set: return None @@ -39,7 +43,8 @@ write_descrs_arrays.append(descr) else: assert 0 - return EffectInfo(write_descrs_fields, write_descrs_arrays) + return EffectInfo(write_descrs_fields, write_descrs_arrays, + promotes_virtualizables) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: @@ -55,7 +60,6 @@ return False return True - def consider_array(ARRAY): if arrayItem(ARRAY) is lltype.Void: return False @@ -64,3 +68,9 @@ if not isinstance(ARRAY, lltype.GcArray): # can be a non-GC-array return False return True + +# ____________________________________________________________ + +class VirtualizableAnalyzer(BoolGraphAnalyzer): + def analyze_simple_operation(self, op): + return op.opname == 'promote_virtualizable' Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Fri Nov 27 17:40:33 2009 @@ -121,8 +121,8 @@ supports_floats = False def fielddescrof(self, STRUCT, fieldname): return ('fielddescr', STRUCT, fieldname) - def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, stuff=None): - return ('calldescr', FUNC, NON_VOID_ARGS, RESULT) + def calldescrof(self, FUNC, NON_VOID_ARGS, RESULT, effectinfo=None): + return ('calldescr', FUNC, NON_VOID_ARGS, RESULT, effectinfo) def typedescrof(self, CLASS): return ('typedescr', CLASS) def methdescrof(self, CLASS, methname): @@ -386,6 +386,47 @@ assert cw.list_of_addr2name[0][1].endswith('.A1') assert cw.list_of_addr2name[1][1] == 'A1.g' + def test_promote_virtualizable_effectinfo(self): + class Frame(object): + _virtualizable2_ = ['x'] + + def __init__(self, x, y): + self.x = x + self.y = y + + def g1(f): + f.x += 1 + + def g2(f): + return f.x + + def h(f): + f.y -= 1 + + def f(n): + f_inst = Frame(n+1, n+2) + g1(f_inst) + r = g2(f_inst) + h(f_inst) + return r + + graphs = self.make_graphs(f, [5]) + cw = CodeWriter(self.rtyper) + cw.candidate_graphs = [graphs[0]] + cw._start(self.metainterp_sd, None) + jitcode = cw.make_one_bytecode((graphs[0], None), False) + calldescrs = [calldescr for calldescr in jitcode.constants + if isinstance(calldescr, tuple) and + calldescr[0] == 'calldescr'] + assert len(calldescrs) == 4 # for __init__, g1, g2, h. + effectinfo_g1 = calldescrs[1][4] + effectinfo_g2 = calldescrs[2][4] + effectinfo_h = calldescrs[3][4] + assert effectinfo_g1.promotes_virtualizables + assert effectinfo_g2.promotes_virtualizables + assert not effectinfo_h.promotes_virtualizables + + class ImmutableFieldsTests: def test_fields(self): From afa at codespeak.net Fri Nov 27 19:30:29 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 19:30:29 +0100 (CET) Subject: [pypy-svn] r69710 - in pypy/trunk/pypy/module/oracle: . test Message-ID: <20091127183029.6A3E1168101@codespeak.net> Author: afa Date: Fri Nov 27 19:30:28 2009 New Revision: 69710 Modified: pypy/trunk/pypy/module/oracle/config.py pypy/trunk/pypy/module/oracle/interp_connect.py pypy/trunk/pypy/module/oracle/interp_environ.py pypy/trunk/pypy/module/oracle/interp_object.py pypy/trunk/pypy/module/oracle/interp_pool.py pypy/trunk/pypy/module/oracle/interp_variable.py pypy/trunk/pypy/module/oracle/roci.py pypy/trunk/pypy/module/oracle/test/test_connect.py Log: Now the cx_Oracle module translates, compiles, and works! Tested on Windows. Support for ucs4 builds of pypy is likely incomplete. Likewise, the "full unicode" mode of cx_Oracle is not implemented. And the cx_Oracle test suite itself is not complete... The module works well enough for most usages, though. Modified: pypy/trunk/pypy/module/oracle/config.py ============================================================================== --- pypy/trunk/pypy/module/oracle/config.py (original) +++ pypy/trunk/pypy/module/oracle/config.py Fri Nov 27 19:30:28 2009 @@ -26,24 +26,28 @@ class StringBuffer: "Fill a char* buffer with data, suitable to pass to Oracle functions" def __init__(self): - pass + self.ptr = lltype.nullptr(roci.oratext.TO) + self.size = 0 - def fill(self, space, w_string): - if w_string is None or space.is_w(w_string, space.w_None): + def fill(self, space, w_value): + if w_value is None or space.is_w(w_value, space.w_None): self.clear() else: - self.ptr = string_w(space, w_string) - self.size = len(self.ptr) + strvalue = space.str_w(w_value) + self.ptr = rffi.str2charp(strvalue) + self.size = len(strvalue) - def fill_with_unicode(self, space, w_unicode): - if w_unicode is None or space.is_w(w_unicode, space.w_None): + def fill_with_unicode(self, space, w_value): + if w_value is None or space.is_w(w_value, space.w_None): self.clear() else: # XXX ucs2 only probably - unistr = space.unicode_w(w_unicode) - self.ptr = rffi.cast(roci.oratext, rffi.unicode2wcharp(unistr)) - self.size = len(unistr) * 2 + univalue = space.unicode_w(w_value) + self.ptr = rffi.cast(roci.oratext, rffi.unicode2wcharp(univalue)) + self.size = len(univalue) * 2 def clear(self): - self.ptr = None + if self.ptr: + rffi.free_charp(self.ptr) + self.ptr = lltype.nullptr(roci.oratext.TO) self.size = 0 Modified: pypy/trunk/pypy/module/oracle/interp_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_connect.py (original) +++ pypy/trunk/pypy/module/oracle/interp_connect.py Fri Nov 27 19:30:28 2009 @@ -21,8 +21,8 @@ self.environment = None self.autocommit = False - self.sessionHandle = None - self.serverHandle = None + self.sessionHandle = lltype.nullptr(roci.OCISession.TO) + self.serverHandle = lltype.nullptr(roci.OCIServer.TO) self.w_inputTypeHandler = None self.w_outputTypeHandler = None @@ -41,7 +41,7 @@ twophase=False, events=False, w_cclass=Null, - purity=False, + purity=0, w_newpassword=Null): self = space.allocate_instance(W_Connection, w_subtype) W_Connection.__init__(self) @@ -83,7 +83,7 @@ W_Root, bool, bool, bool, W_Root, - bool, + int, W_Root] def __del__(self): @@ -244,7 +244,7 @@ rather than using the low level interface for connecting.""" proxyCredentials = False - authInfo = None + authInfo = lltype.nullptr(roci.OCIAuthInfo.TO) if pool: w_dbname = pool.w_name @@ -261,7 +261,7 @@ # set up authorization handle, if needed if not pool or w_cclass or proxyCredentials: # create authorization handle - handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIServer).TO, + handleptr = lltype.malloc(rffi.CArrayPtr(roci.OCIAuthInfo).TO, 1, flavor='raw') try: status = roci.OCIHandleAlloc( @@ -287,7 +287,7 @@ authInfo, roci.OCI_HTYPE_AUTHINFO, stringBuffer.ptr, stringBuffer.size, - roci.OCI_ATTR_PASSWORD, + roci.OCI_ATTR_USERNAME, self.environment.errorHandle) self.environment.checkForError( status, "Connection_GetConnection(): set user name") @@ -303,7 +303,7 @@ authInfo, roci.OCI_HTYPE_AUTHINFO, stringBuffer.ptr, stringBuffer.size, - roci.OCI_ATTR_USERNAME, + roci.OCI_ATTR_PASSWORD, self.environment.errorHandle) self.environment.checkForError( status, "Connection_GetConnection(): set password") @@ -334,12 +334,13 @@ if purity != roci.OCI_ATTR_PURITY_DEFAULT: purityptr = lltype.malloc(rffi.CArrayPtr(roci.ub4).TO, 1, flavor='raw') - purityptr[0] = purity + purityptr[0] = rffi.cast(roci.ub4, purity) try: status = roci.OCIAttrSet( authInfo, roci.OCI_HTYPE_AUTHINFO, - purityptr, rffi.sizeof(roci.ub4), + rffi.cast(roci.dvoidp, purityptr), + rffi.sizeof(roci.ub4), roci.OCI_ATTR_PURITY, self.environment.errorHandle) self.environment.checkForError( Modified: pypy/trunk/pypy/module/oracle/interp_environ.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_environ.py (original) +++ pypy/trunk/pypy/module/oracle/interp_environ.py Fri Nov 27 19:30:28 2009 @@ -47,8 +47,8 @@ raise OperationError(get(self.space).w_DatabaseError, self.space.wrap(error)) - @classmethod - def create(cls, space, threaded, events): + @staticmethod + def create(space, threaded, events): "Create a new environment object from scratch" mode = roci.OCI_OBJECT if threaded: @@ -80,7 +80,7 @@ lltype.free(handleptr, flavor='raw') try: - newenv = cls(space, handle) + newenv = Environment(space, handle) except: roci.OCIHandleFree(handle, roci.OCI_HTYPE_ENV) raise @@ -92,7 +92,7 @@ def clone(self): """Clone an existing environment. used when acquiring a connection from a session pool, for example.""" - newenv = type(self)(self.space, self.handle) + newenv = Environment(self.space, self.handle) newenv.maxBytesPerCharacter = self.maxBytesPerCharacter newenv.maxStringBytes = self.maxStringBytes return newenv Modified: pypy/trunk/pypy/module/oracle/interp_object.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_object.py (original) +++ pypy/trunk/pypy/module/oracle/interp_object.py Fri Nov 27 19:30:28 2009 @@ -7,10 +7,11 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.oracle import roci, config, transform +from pypy.module.oracle.interp_error import get class W_ObjectType(Wrappable): def __init__(self, connection, param): - self.tdo = None + self.tdo = lltype.nullptr(roci.dvoidp.TO) self.environment = connection.environment self.isCollection = False self.initialize(connection, param) @@ -153,7 +154,7 @@ self.environment.errorHandle) self.environment.checkForError( status, "ObjectType_Describe(): get type code") - typeCode = typecodeptr[0] + typeCode = rffi.cast(lltype.Signed, typecodeptr[0]) finally: lltype.free(typecodeptr, flavor='raw') @@ -206,7 +207,7 @@ self.environment.errorHandle) self.environment.checkForError( status, "ObjectType_Describe(): get element type code") - self.elementTypeCode = typecodeptr[0] + self.elementTypeCode = rffi.cast(lltype.Signed, typecodeptr[0]) finally: lltype.free(typecodeptr, flavor='raw') @@ -318,7 +319,7 @@ connection.environment.errorHandle) connection.environment.checkForError( status, "ObjectType_Describe(): get type code") - self.typeCode = typecodeptr[0] + self.typeCode = rffi.cast(lltype.Signed, typecodeptr[0]) finally: lltype.free(typecodeptr, flavor='raw') @@ -386,7 +387,8 @@ # determine the proper null indicator valueIndicator = valueindicatorptr[0] if not valueIndicator: - valueIndicator = scalarvalueindicatorptr + valueIndicator = rffi.cast(roci.dvoidp, + scalarvalueindicatorptr) value = valueptr[0] return convertObject( @@ -413,8 +415,13 @@ def convertObject(space, environment, typeCode, value, indicator, var, subtype): + # null values returned as None - if rffi.cast(roci.Ptr(roci.OCIInd), indicator)[0] == roci.OCI_IND_NULL: + if (rffi.cast(lltype.Signed, + rffi.cast(roci.Ptr(roci.OCIInd), + indicator)[0]) + == + rffi.cast(lltype.Signed, roci.OCI_IND_NULL)): return space.w_None if typeCode in (roci.OCI_TYPECODE_CHAR, @@ -435,10 +442,10 @@ dateValue = rffi.cast(roci.Ptr(roci.OCIDateTime), value) return transform.OracleTimestampToPythonDate(environment, dateValue) elif typeCode == roci.OCI_TYPECODE_OBJECT: - return space.wrap(W_ExternalObject(var, subType, value, indicator, + return space.wrap(W_ExternalObject(var, subtype, value, indicator, isIndependent=False)) elif typeCode == roci.OCI_TYPECODE_NAMEDCOLLECTION: - return convertCollection(space, environment, value, var, subType) + return convertCollection(space, environment, value, var, subtype) raise OperationError( get(space).w_NotSupportedError, Modified: pypy/trunk/pypy/module/oracle/interp_pool.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_pool.py (original) +++ pypy/trunk/pypy/module/oracle/interp_pool.py Fri Nov 27 19:30:28 2009 @@ -29,18 +29,20 @@ W_SessionPool.__init__(self) if w_connectiontype is not None: - if not space.is_true(space.issubtype(w_value, - interp_connect.W_Connection)): + if not space.is_true(space.issubtype(w_connectiontype, + get(space).w_Connection)): raise OperationError( interp_error.get(space).w_ProgrammingError, space.wrap( "connectiontype must be a subclass of Connection")) + self.w_connectionType = w_connectiontype + else: + self.w_connectionType = get(space).w_Connection + self.w_username = w_user self.w_password = w_password self.w_tnsentry = w_dsn - from pypy.module.oracle.interp_connect import W_Connection - self.w_connectionType = w_connectiontype or get(space).w_Connection self.minSessions = min self.maxSessions = max self.sessionIncrement = increment @@ -173,7 +175,7 @@ # ensure that the connection behaves as closed connection.sessionPool = None - connection.handle = None + connection.handle = lltype.nullptr(roci.OCISvcCtx.TO) def computedProperty(oci_attr_code, oci_value_type): def fget(space, self): @@ -191,7 +193,7 @@ return space.wrap(valueptr[0]) finally: lltype.free(valueptr, flavor='raw') - return GetSetProperty(fget) + return GetSetProperty(fget, cls=W_SessionPool) W_SessionPool.typedef = TypeDef( "SessionPool", Modified: pypy/trunk/pypy/module/oracle/interp_variable.py ============================================================================== --- pypy/trunk/pypy/module/oracle/interp_variable.py (original) +++ pypy/trunk/pypy/module/oracle/interp_variable.py Fri Nov 27 19:30:28 2009 @@ -476,32 +476,28 @@ def getValueProc(self, space, pos): offset = pos * self.bufferSize + dataptr = rffi.ptradd(self.data, offset) length = rffi.cast(lltype.Signed, self.actualLength[pos]) - l = [] i = 0 if config.WITH_UNICODE: if isinstance(self, VT_Binary): - while i < length: - l.append(self.data[offset + i]) - i += 1 - return space.wrap(''.join(l)) + return space.wrap(rffi.charpsize2str(dataptr, length)) else: + l = [] while i < length: - l.append(unichr((ord(self.data[offset + i + 1]) << 8) + - ord(self.data[offset + i]))) + l.append(unichr((ord(dataptr[i + 1]) << 8) + + ord(dataptr[i]))) i += 2 return space.wrap(u''.join(l)) else: if self.charsetForm == roci.SQLCS_IMPLICIT: - while i < length: - l.append(self.data[offset + i]) - i += 1 - return space.wrap(''.join(l)) + return space.wrap(rffi.charpsize2str(dataptr, length)) else: + l = [] while i < length: - l.append(unichr((ord(self.data[offset + i + 1]) << 8) + - ord(self.data[offset + i]))) + l.append(unichr((ord(dataptr[i + 1]) << 8) + + ord(dataptr[i]))) i += 2 return space.wrap(u''.join(l)) @@ -1007,7 +1003,7 @@ temporaryLobType = roci.OCI_TEMP_CLOB def initialize(self, space, cursor): - super(W_LobVariable, self).initialize(space, cursor) + W_VariableWithDescriptor.initialize(self, space, cursor) self.connection = cursor.connection def ensureTemporary(self, space, pos): @@ -1060,7 +1056,8 @@ self.environment.checkForError( status, "LobVar_GetLength()") - return int(lengthptr[0]) # XXX test overflow + return rffi.cast(lltype.Signed, + lengthptr[0]) # XXX test overflow finally: lltype.free(lengthptr, flavor='raw') @@ -1099,7 +1096,8 @@ self.environment.checkForError( status, "LobVar_Read()") - amount = int(amountptr[0]) # XXX test overflow + amount = rffi.cast(lltype.Signed, + amountptr[0]) # XXX test overflow value = rffi.str_from_buffer(raw_buffer, gc_buffer, bufferSize, amount) return space.wrap(value) finally: @@ -1115,7 +1113,7 @@ try: # nothing to do if no data to write if databuf.size == 0: - return + return 0 status = roci.OCILobWrite( self.connection.handle, @@ -1158,11 +1156,17 @@ space.wrap("BFILEs are read only")) def read(self, space, pos, offset, amount): - self.fileOpen() + self.openFile() try: return W_LobVariable.read(self, space, pos, offset, amount) finally: - self.fileClose() + self.closeFile() + + def openFile(self): + pass # XXX + + def closeFile(self): + pass # XXX class VT_Cursor(W_Variable): oracleType = roci.SQLT_RSET @@ -1217,7 +1221,7 @@ size = rffi.sizeof(roci.dvoidp) canBeInArray = False - objectIndicator = None + objectIndicator = lltype.nullptr(rffi.CArrayPtr(roci.dvoidp).TO) def initialize(self, space, cursor): self.connection = cursor.connection @@ -1260,7 +1264,9 @@ # look at our own indicator array if not self.objectIndicator[pos]: return True - return (rffi.cast(roci.Ptr(roci.OCIInd), self.objectIndicator[pos])[0] + return (rffi.cast(lltype.Signed, + rffi.cast(roci.Ptr(roci.OCIInd), + self.objectIndicator[pos])[0]) == rffi.cast(lltype.Signed, roci.OCI_IND_NULL)) Modified: pypy/trunk/pypy/module/oracle/roci.py ============================================================================== --- pypy/trunk/pypy/module/oracle/roci.py (original) +++ pypy/trunk/pypy/module/oracle/roci.py Fri Nov 27 19:30:28 2009 @@ -64,7 +64,7 @@ OCI_SUCCESS OCI_SUCCESS_WITH_INFO OCI_INVALID_HANDLE OCI_NO_DATA OCI_HTYPE_ERROR OCI_HTYPE_SVCCTX OCI_HTYPE_SERVER OCI_HTYPE_SESSION OCI_HTYPE_STMT OCI_HTYPE_DESCRIBE OCI_HTYPE_BIND OCI_HTYPE_DEFINE - OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_HTYPE_AUTHINFO + OCI_HTYPE_ENV OCI_HTYPE_SPOOL OCI_HTYPE_AUTHINFO OCI_ATTR_CONNECTION_CLASS OCI_DTYPE_PARAM OCI_DTYPE_TIMESTAMP OCI_DTYPE_INTERVAL_DS OCI_DTYPE_LOB OCI_CRED_RDBMS OCI_CRED_EXT OCI_SPOOL_ATTRVAL_NOWAIT OCI_ATTR_SERVER OCI_ATTR_SESSION OCI_ATTR_USERNAME OCI_ATTR_PASSWORD @@ -99,7 +99,7 @@ OCI_NLS_MAXBUFSZ OCI_NLS_CS_ORA_TO_IANA OCI_UTF16ID OCI_SPC_STMTCACHE OCI_SPC_HOMOGENEOUS OCI_SESSGET_SPOOL OCI_SESSGET_CREDPROXY OCI_SESSGET_STMTCACHE - OCI_SESSRLS_DROPSESS OCI_ATTR_PURITY_DEFAULT + OCI_SESSGET_CREDEXT OCI_SESSRLS_DROPSESS OCI_ATTR_PURITY_DEFAULT '''.split() for c in constants: Modified: pypy/trunk/pypy/module/oracle/test/test_connect.py ============================================================================== --- pypy/trunk/pypy/module/oracle/test/test_connect.py (original) +++ pypy/trunk/pypy/module/oracle/test/test_connect.py Fri Nov 27 19:30:28 2009 @@ -158,5 +158,6 @@ 2, 8, 3, homogeneous=False) assert pool.homogeneous is False e = raises(oracle.DatabaseError, pool.acquire, user="proxyuser") + # ORA-01017: invalid username/password; logon denied # ORA-28150: proxy not authorized to connect as client - assert e.value[0].code == 28150 + assert e.value[0].code in (1017, 28150) From afa at codespeak.net Fri Nov 27 19:35:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 27 Nov 2009 19:35:43 +0100 (CET) Subject: [pypy-svn] r69711 - pypy/trunk/pypy/rlib Message-ID: <20091127183543.22B87168101@codespeak.net> Author: afa Date: Fri Nov 27 19:35:42 2009 New Revision: 69711 Modified: pypy/trunk/pypy/rlib/rwin32.py Log: Attempt to fix the windows buildbot, which uses Visual Studio 7: To generate the map between Windows error codes and errno, we compile and run an external program, statically linked with the CRT. (Visual Studio 7 provides the function _dosmaperr only in the static C library) As a bonus, loading the module is now faster: it saves 65000 calls through ll2ctypes... Modified: pypy/trunk/pypy/rlib/rwin32.py ============================================================================== --- pypy/trunk/pypy/rlib/rwin32.py (original) +++ pypy/trunk/pypy/rlib/rwin32.py Fri Nov 27 19:35:42 2009 @@ -3,6 +3,7 @@ """ from pypy.rpython.tool import rffi_platform +from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rarithmetic import intmask @@ -14,13 +15,8 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'errno.h'], + includes = ['windows.h'], libraries = ['kernel32'], - separate_module_sources = [""" - long WINAPI pypy_dosmaperr(long winerror) - { _dosmaperr(winerror); return errno; } - """], - export_symbols = ["pypy_dosmaperr"], ) else: eci = ExternalCompilationInfo() @@ -97,20 +93,40 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) - dosmaperr = winexternal('pypy_dosmaperr', [rffi.LONG], rffi.LONG) - - def build_winerror_to_errno(): """Build a dictionary mapping windows error numbers to POSIX errno. The function returns the dict, and the default value for codes not in the dict.""" - default = errno.EINVAL - errors = {} - for i in range(1, 65000): - error = dosmaperr(i) - if error != default: - errors[i] = error - return errors, default + # Prior to Visual Studio 8, the MSVCRT dll doesn't export the + # _dosmaperr() function, which is available only when compiled + # against the static CRT library. + from pypy.translator.platform import platform, Windows + static_platform = Windows() + if static_platform.name == 'msvc': + static_platform.cflags = ['/MT'] # static CRT + static_platform.version = 0 # no manifest + cfile = udir.join('dosmaperr.c') + cfile.write(r''' + #include + int main() + { + int i; + for(i=1; i < 65000; i++) { + _dosmaperr(i); + if (errno == EINVAL) + continue; + printf("%d\t%d\n", i, errno); + } + return 0; + }''') + exename = static_platform.compile( + [cfile], ExternalCompilationInfo(), + outputfilename = "dosmaperr", + standalone=True) + output = os.popen(str(exename)) + errors = dict(map(int, line.split()) + for line in output) + return errors, errno.EINVAL # A bit like strerror... def FormatError(code): From arigo at codespeak.net Fri Nov 27 21:50:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 21:50:32 +0100 (CET) Subject: [pypy-svn] r69712 - in pypy/branch/virtual-forcing/pypy/jit/backend: . llgraph test x86 Message-ID: <20091127205032.C4E3B168105@codespeak.net> Author: arigo Date: Fri Nov 27 21:50:30 2009 New Revision: 69712 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: (pedronis, arigo) Two fixes for cpu.force(): - ignores the result of the call in the failargs (the call did not return yet!). - returns the faildescr. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Fri Nov 27 21:50:30 2009 @@ -397,15 +397,16 @@ else: return self.env[v] - def _populate_fail_args(self, op): + def _populate_fail_args(self, op, skip=None): fail_args = [] if op.fail_args: for fail_arg in op.fail_args: - if fail_arg is None: + if fail_arg is None or fail_arg is skip: fail_args.append(None) else: fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args + self.fail_args = fail_args + self.fail_index = op.fail_index def execute(self): """Execute all operations in a loop, @@ -793,7 +794,7 @@ def op_call_may_force(self, calldescr, func, *args): assert not self._forced self._may_force = self.opindex - self.op_call(calldescr, func, *args) + return self.op_call(calldescr, func, *args) def op_guard_not_forced(self, descr): forced = self._forced @@ -1071,12 +1072,19 @@ opaque_frame = llmemory.cast_adr_to_ptr(force_token, lltype.Ptr(_TO_OPAQUE[Frame])) frame = _from_opaque(opaque_frame) + assert not frame._forced frame._forced = True assert frame._may_force >= 0 + call_op = frame.loop.operations[frame._may_force] guard_op = frame.loop.operations[frame._may_force+1] - frame._populate_fail_args(guard_op) + frame._populate_fail_args(guard_op, skip=call_op.result) return opaque_frame +def get_forced_fail_index(opaque_frame): + frame = _from_opaque(opaque_frame) + assert frame._forced + return frame.fail_index + class MemoCast(object): def __init__(self): self.addresses = [llmemory.NULL] @@ -1447,6 +1455,7 @@ setannotation(get_zero_division_error, annmodel.SomeAddress()) setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(force, s_Frame) +setannotation(get_forced_fail_index, annmodel.SomeInteger()) setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Fri Nov 27 21:50:30 2009 @@ -497,9 +497,13 @@ def do_cast_ptr_to_int(self, ptrbox): return history.BoxInt(llimpl.cast_to_int(ptrbox.getref_base(), self.memo_cast)) + def force(self, force_token): frame = llimpl.force(self.cast_int_to_adr(force_token)) self.latest_frame = frame + fail_index = llimpl.get_forced_fail_index(frame) + return self.get_fail_descr_from_number(fail_index) + class OOtypeCPU(BaseCPU): is_oo = True Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Fri Nov 27 21:50:30 2009 @@ -209,9 +209,9 @@ raise NotImplementedError def do_call_may_force(self, args, calldescr): - raise NotImplementedError + return self.do_call(args, calldescr) - def force(self, force_token, descr, args): + def force(self, force_token): raise NotImplementedError # ootype specific operations Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 21:50:30 2009 @@ -1221,7 +1221,7 @@ else: assert record == [] - def test_force_operations(self): + def test_force_operations_returning_void(self): values = [] def maybe_force(token, flag): if flag: @@ -1263,6 +1263,51 @@ assert self.cpu.get_latest_value_int(1) == 10 assert values == [1, 10] + def test_force_operations_returning_int(self): + values = [] + def maybe_force(token, flag): + if flag: + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(2)) + return 42 + + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, i2, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_int(0) == 42 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_int(1) == 42 + assert self.cpu.get_latest_value_int(2) == 10 + assert values == [1, 10] + # pure do_ / descr features def test_do_operations(self): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 21:50:30 2009 @@ -879,8 +879,7 @@ arglocs.append(loc) return arglocs[:] - def grab_frame_values(self, bytecode, frame_addr, - registers=lltype.nullptr(rffi.LONGP.TO)): + def grab_frame_values(self, bytecode, frame_addr, registers): num = 0 value_hi = 0 while 1: @@ -915,7 +914,6 @@ continue assert code == self.DESCR_STOP break - assert registers # it's NULL when called from cpu.force() code >>= 2 if kind == self.DESCR_FLOAT: xmmregisters = rffi.ptradd(registers, -16) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 21:50:30 2009 @@ -615,6 +615,8 @@ self.rm.before_call(force_store, save_all_regs=save_all_regs) self.xrm.before_call(force_store, save_all_regs=save_all_regs) if guard_not_forced_op is not None: + if op.result is not None: + self.force_allocate_reg(op.result, selected_reg=eax) # XXX!!! self.perform_with_guard(op, guard_not_forced_op, arglocs, eax) else: self.Perform(op, arglocs, eax) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 21:50:30 2009 @@ -87,6 +87,9 @@ adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) + all_null_registers = lltype.malloc(rffi.LONGP.TO, 24, + flavor='raw', zero=True) + def force(self, addr_of_force_index): TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] @@ -98,7 +101,8 @@ # start of "no gc operation!" block fail_index_2 = self.assembler.grab_frame_values( bytecode, - addr_of_force_index - FORCE_INDEX_OFS) + addr_of_force_index - FORCE_INDEX_OFS, + rffi.ptradd(self.all_null_registers, 16)) self.assembler.leave_jitted_hook() # end of "no gc operation!" block assert fail_index == fail_index_2 From arigo at codespeak.net Fri Nov 27 22:23:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 22:23:13 +0100 (CET) Subject: [pypy-svn] r69713 - in pypy/branch/virtual-forcing/pypy/jit/backend: llsupport test x86 Message-ID: <20091127212313.E248F16810C@codespeak.net> Author: arigo Date: Fri Nov 27 22:23:13 2009 New Revision: 69713 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Log: (pedronis, arigo) Fix CALL_MAY_FORCE for returning floats, by sanitizing the interface a bit. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llsupport/regalloc.py Fri Nov 27 22:23:13 2009 @@ -327,12 +327,12 @@ """ Adjust registers according to the result of the call, which is in variable v. """ - if v is not None: - self._check_type(v) - r = self.call_result_location(v) - self.reg_bindings[v] = r - self.free_regs = [fr for fr in self.free_regs if fr is not r] - + self._check_type(v) + r = self.call_result_location(v) + self.reg_bindings[v] = r + self.free_regs = [fr for fr in self.free_regs if fr is not r] + return r + # abstract methods, override def convert_to_imm(self, c): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 22:23:13 2009 @@ -1308,6 +1308,51 @@ assert self.cpu.get_latest_value_int(2) == 10 assert values == [1, 10] + def test_force_operations_returning_float(self): + values = [] + def maybe_force(token, flag): + if flag: + self.cpu.force(token) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(2)) + return 42.5 + + FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) + func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) + funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() + calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT) + cpu = self.cpu + i0 = BoxInt() + i1 = BoxInt() + f2 = BoxFloat() + tok = BoxInt() + faildescr = BasicFailDescr(1) + ops = [ + ResOperation(rop.FORCE_TOKEN, [], tok), + ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2, + descr=calldescr), + ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), + ResOperation(rop.FINISH, [f2], None, descr=BasicFailDescr(0)) + ] + ops[2].fail_args = [i1, f2, i0] + looptoken = LoopToken() + self.cpu.compile_loop([i0, i1], ops, looptoken) + self.cpu.set_future_value_int(0, 20) + self.cpu.set_future_value_int(1, 0) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 0 + assert self.cpu.get_latest_value_float(0) == 42.5 + assert values == [] + + self.cpu.set_future_value_int(0, 10) + self.cpu.set_future_value_int(1, 1) + fail = self.cpu.execute_token(looptoken) + assert fail.identifier == 1 + assert self.cpu.get_latest_value_int(0) == 1 + assert self.cpu.get_latest_value_float(1) == 42.5 + assert self.cpu.get_latest_value_int(2) == 10 + assert values == [1, 10] + # pure do_ / descr features def test_do_operations(self): Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Fri Nov 27 22:23:13 2009 @@ -269,9 +269,6 @@ regalloc_mov = mov # legacy interface - def regalloc_fstp(self, loc): - self.mc.FSTP(loc) - def regalloc_push(self, loc): if isinstance(loc, XMMREG): self.mc.SUB(esp, imm(2*WORD)) @@ -1104,7 +1101,9 @@ self.mc.CALL(x) self.mark_gc_roots() self.mc.ADD(esp, imm(extra_on_stack)) - if size == 1: + if isinstance(resloc, MODRM64): + self.mc.FSTP(resloc) + elif size == 1: self.mc.AND(eax, imm(0xff)) elif size == 2: self.mc.AND(eax, imm(0xffff)) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/regalloc.py Fri Nov 27 22:23:13 2009 @@ -100,10 +100,9 @@ def after_call(self, v): # the result is stored in st0, but we don't have this around, - # so we move it to some stack location - if v is not None: - loc = self.stack_manager.loc(v, 2) - self.assembler.regalloc_fstp(loc) + # so genop_call will move it to some stack location immediately + # after the call + return self.stack_manager.loc(v, 2) class X86StackManager(StackManager): @@ -290,7 +289,7 @@ arglocs, result_loc, self.sm.stack_depth) if op.result is not None: - self.rm.possibly_free_var(op.result) + self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.fail_args) def perform_guard(self, guard_op, arglocs, result_loc): @@ -614,17 +613,17 @@ save_all_regs = guard_not_forced_op is not None self.rm.before_call(force_store, save_all_regs=save_all_regs) self.xrm.before_call(force_store, save_all_regs=save_all_regs) - if guard_not_forced_op is not None: - if op.result is not None: - self.force_allocate_reg(op.result, selected_reg=eax) # XXX!!! - self.perform_with_guard(op, guard_not_forced_op, arglocs, eax) - else: - self.Perform(op, arglocs, eax) if op.result is not None: if op.result.type == FLOAT: - self.xrm.after_call(op.result) + resloc = self.xrm.after_call(op.result) else: - self.rm.after_call(op.result) + resloc = self.rm.after_call(op.result) + else: + resloc = None + if guard_not_forced_op is not None: + self.perform_with_guard(op, guard_not_forced_op, arglocs, resloc) + else: + self.Perform(op, arglocs, resloc) def _consider_call(self, op, guard_not_forced_op=None): calldescr = op.descr From arigo at codespeak.net Fri Nov 27 22:40:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 22:40:47 +0100 (CET) Subject: [pypy-svn] r69714 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph backend/test backend/x86 metainterp metainterp/test Message-ID: <20091127214047.D818E1683B7@codespeak.net> Author: arigo Date: Fri Nov 27 22:40:46 2009 New Revision: 69714 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/typesystem.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/warmstate.py Log: (pedronis, arigo) Progress. The tests pass, but the work is not done. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Fri Nov 27 22:40:46 2009 @@ -401,8 +401,10 @@ fail_args = [] if op.fail_args: for fail_arg in op.fail_args: - if fail_arg is None or fail_arg is skip: + if fail_arg is None: fail_args.append(None) + elif fail_arg is skip: + fail_args.append(fail_arg.concretetype._defl()) else: fail_args.append(self.getenv(fail_arg)) self.fail_args = fail_args Modified: pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/test/runner_test.py Fri Nov 27 22:40:46 2009 @@ -1225,9 +1225,10 @@ values = [] def maybe_force(token, flag): if flag: - self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(0)) - values.append(self.cpu.get_latest_value_int(1)) + descr = self.cpu.force(token) + values.append(descr) + values.append(self.cpu.get_latest_value_int(0)) + values.append(self.cpu.get_latest_value_int(1)) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) @@ -1261,7 +1262,7 @@ assert fail.identifier == 1 assert self.cpu.get_latest_value_int(0) == 1 assert self.cpu.get_latest_value_int(1) == 10 - assert values == [1, 10] + assert values == [faildescr, 1, 10] def test_force_operations_returning_int(self): values = [] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Fri Nov 27 22:40:46 2009 @@ -106,6 +106,7 @@ self.assembler.leave_jitted_hook() # end of "no gc operation!" block assert fail_index == fail_index_2 + return faildescr class CPU386_NO_SSE2(CPU386): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Fri Nov 27 22:40:46 2009 @@ -1214,6 +1214,7 @@ canraise = True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise if pure: + assert not calldescr.effectinfo.promotes_virtualizables self.emit('residual_call_pure') elif canraise: self.emit('residual_call') Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Fri Nov 27 22:40:46 2009 @@ -221,6 +221,7 @@ if box: fail_arg_types[i] = box.type self.fail_arg_types = fail_arg_types + # XXX ^^^ kill this attribute def handle_fail(self, metainterp_sd): from pypy.jit.metainterp.pyjitpl import MetaInterp @@ -236,6 +237,17 @@ send_bridge_to_backend(metainterp.staticdata, self, inputargs, new_loop.operations) + def force_virtualizable(self, vinfo, virtualizable): + from pypy.jit.metainterp.pyjitpl import MetaInterp + from pypy.jit.metainterp.resume import force_from_resumedata + metainterp = MetaInterp(self.metainterp_sd) + metainterp.history = None # blackholing + liveboxes = metainterp.load_values_from_failure(self) + virtualizable_boxes = force_from_resumedata(metainterp, + liveboxes, self) + vinfo.write_boxes(virtualizable, virtualizable_boxes) + + class ResumeFromInterpDescr(ResumeDescr): def __init__(self, original_greenkey, redkey): ResumeDescr.__init__(self, original_greenkey) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/effectinfo.py Fri Nov 27 22:40:46 2009 @@ -8,7 +8,7 @@ _cache = {} def __new__(cls, write_descrs_fields, write_descrs_arrays, - promotes_virtualizables): + promotes_virtualizables=False): key = (frozenset(write_descrs_fields), frozenset(write_descrs_arrays), promotes_virtualizables) if key in cls._cache: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Fri Nov 27 22:40:46 2009 @@ -632,6 +632,7 @@ varargs = [jitcode.cfnptr] + varargs res = self.execute_varargs(rop.CALL, varargs, descr=jitcode.calldescr, exc=True) + self.metainterp.load_fields_from_virtualizable() else: # for oosends (ootype only): calldescr is a MethDescr res = self.execute_varargs(rop.OOSEND, varargs, @@ -651,7 +652,7 @@ @arguments("descr", "varargs") def opimpl_residual_call(self, calldescr, varargs): - return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) + return self.do_residual_call(varargs, descr=calldescr, exc=True) @arguments("varargs") def opimpl_recursion_leave_prep(self, varargs): @@ -675,11 +676,11 @@ greenkey = varargs[1:num_green_args + 1] if warmrunnerstate.can_inline_callable(greenkey): return self.perform_call(portal_code, varargs[1:], greenkey) - return self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=True) + return self.do_residual_call(varargs, descr=calldescr, exc=True) @arguments("descr", "varargs") def opimpl_residual_call_noexception(self, calldescr, varargs): - self.execute_varargs(rop.CALL, varargs, descr=calldescr, exc=False) + self.do_residual_call(varargs, descr=calldescr, exc=False) @arguments("descr", "varargs") def opimpl_residual_call_pure(self, calldescr, varargs): @@ -696,8 +697,8 @@ return self.perform_call(jitcode, varargs) else: # but we should not follow calls to that graph - return self.execute_varargs(rop.CALL, [box] + varargs, - descr=calldescr, exc=True) + return self.do_residual_call([box] + varargs, + descr=calldescr, exc=True) @arguments("orgpc", "methdescr", "varargs") def opimpl_oosend(self, pc, methdescr, varargs): @@ -980,6 +981,24 @@ return self.metainterp.handle_exception() return False + def do_residual_call(self, argboxes, descr, exc): + effectinfo = descr.get_extra_info() + if effectinfo is None or effectinfo.promotes_virtualizables: + # residual calls require attention to keep virtualizables in-sync + self.metainterp.vable_before_residual_call() + # xxx do something about code duplication + resbox = self.metainterp.execute_and_record_varargs( + rop.CALL_MAY_FORCE, argboxes, descr=descr) + self.metainterp.vable_after_residual_call() + if resbox is not None: + self.make_result_box(resbox) + self.generate_guard(self.pc, rop.GUARD_NOT_FORCED, None, []) + if exc: + return self.metainterp.handle_exception() + return False + else: + return self.execute_varargs(rop.CALL, argboxes, descr, exc) + # ____________________________________________________________ class MetaInterpStaticData(object): @@ -1298,7 +1317,8 @@ @specialize.arg(1) def execute_and_record(self, opnum, descr, *argboxes): history.check_descr(descr) - assert opnum != rop.CALL and opnum != rop.OOSEND + assert (opnum != rop.CALL and opnum != rop.CALL_MAY_FORCE + and opnum != rop.OOSEND) # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) @@ -1315,12 +1335,6 @@ @specialize.arg(1) def execute_and_record_varargs(self, opnum, argboxes, descr=None): history.check_descr(descr) - # residual calls require attention to keep virtualizables in-sync. - # CALL_PURE doesn't need it because so far 'promote_virtualizable' - # as an operation is enough to make the called function non-pure. - is_a_call = (opnum == rop.CALL or opnum == rop.OOSEND) - if is_a_call: - self.before_residual_call() # execute the operation profiler = self.staticdata.profiler profiler.count_ops(opnum) @@ -1334,8 +1348,6 @@ resbox = self._record_helper_pure_varargs(opnum, resbox, descr, argboxes) else: resbox = self._record_helper_nonpure_varargs(opnum, resbox, descr, argboxes) - if is_a_call: - self.after_residual_call() return resbox def _record_helper_pure(self, opnum, resbox, descr, *argboxes): @@ -1719,9 +1731,9 @@ vinfo = self.staticdata.virtualizable_info virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - vinfo.clear_vable_rti(virtualizable) + vinfo.clear_vable_token(virtualizable) - def before_residual_call(self): + def vable_before_residual_call(self): if self.is_blackholing(): return vinfo = self.staticdata.virtualizable_info @@ -1729,8 +1741,14 @@ virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.tracing_before_residual_call(virtualizable) + # + force_token_box = history.BoxInt() + self.history.record(rop.FORCE_TOKEN, [], force_token_box) + self.history.record(rop.SETFIELD_GC, [virtualizable_box, + force_token_box], + None, descr=vinfo.vable_token_descr) - def after_residual_call(self): + def vable_after_residual_call(self): if self.is_blackholing(): vable_escapes = True else: @@ -1787,7 +1805,7 @@ # is and stays NULL. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) - assert not virtualizable.vable_rti + assert not virtualizable.vable_token self.synchronize_virtualizable() def check_synchronized_virtualizable(self): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Fri Nov 27 22:40:46 2009 @@ -455,6 +455,10 @@ metainterp.framestack.reverse() return virtualizable_boxes +def force_from_resumedata(metainterp, newboxes, storage): + resumereader = ResumeDataReader(storage, newboxes, metainterp) + return resumereader.consume_boxes() + class ResumeDataReader(object): virtuals = None Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Fri Nov 27 22:40:46 2009 @@ -7,7 +7,6 @@ from pypy.rlib.jit import OPTIMIZER_SIMPLE, OPTIMIZER_FULL from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.rpython.lltypesystem.rvirtualizable2 import VABLERTIPTR from pypy.rpython.rclass import FieldListAccessor from pypy.jit.metainterp.warmspot import get_stats, get_translator from pypy.jit.metainterp import history, heaptracker @@ -41,8 +40,7 @@ XY = lltype.GcStruct( 'XY', ('parent', rclass.OBJECT), - ('vable_base', llmemory.Address), - ('vable_rti', VABLERTIPTR), + ('vable_token', lltype.Signed), ('inst_x', lltype.Signed), ('inst_node', lltype.Ptr(LLtypeMixin.NODE)), hints = {'virtualizable2_accessor': FieldListAccessor()}) @@ -57,7 +55,7 @@ def setup(self): xy = lltype.malloc(self.XY) - xy.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy.vable_token = 0 xy.parent.typeptr = self.xy_vtable return xy @@ -205,8 +203,7 @@ XY2 = lltype.GcStruct( 'XY2', ('parent', rclass.OBJECT), - ('vable_base', llmemory.Address), - ('vable_rti', VABLERTIPTR), + ('vable_token', lltype.Signed), ('inst_x', lltype.Signed), ('inst_l1', lltype.Ptr(lltype.GcArray(lltype.Signed))), ('inst_l2', lltype.Ptr(lltype.GcArray(lltype.Signed))), @@ -219,7 +216,7 @@ def setup2(self): xy2 = lltype.malloc(self.XY2) - xy2.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy2.vable_token = 0 xy2.parent.typeptr = self.xy2_vtable return xy2 @@ -392,7 +389,7 @@ def setup2sub(self): xy2 = lltype.malloc(self.XY2SUB) - xy2.parent.vable_rti = lltype.nullptr(VABLERTIPTR.TO) + xy2.parent.vable_token = 0 xy2.parent.parent.typeptr = self.xy2_vtable return xy2 @@ -619,8 +616,8 @@ self.check_tree_loop_count(0) def test_external_read_sometimes(self): - py.test.skip("known bug: access the frame in a residual call but" - " only sometimes, so that it's not seen during tracing") + #py.test.skip("known bug: access the frame in a residual call but" + # " only sometimes, so that it's not seen during tracing") jitdriver = JitDriver(greens = [], reds = ['frame'], virtualizables = ['frame']) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/typesystem.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/typesystem.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/typesystem.py Fri Nov 27 22:40:46 2009 @@ -49,10 +49,6 @@ CONST_NULL = history.ConstPtr(history.ConstPtr.value) CVAL_NULLREF = None # patched by optimizeopt.py - def get_VABLERTI(self): - from pypy.rpython.lltypesystem.rvirtualizable2 import VABLERTIPTR - return VABLERTIPTR - def new_ConstRef(self, x): ptrval = lltype.cast_opaque_ptr(llmemory.GCREF, x) return history.ConstPtr(ptrval) @@ -159,10 +155,6 @@ loops_done_with_this_frame_ref = None # patched by compile.py CONST_NULL = history.ConstObj(history.ConstObj.value) CVAL_NULLREF = None # patched by optimizeopt.py - - def get_VABLERTI(self): - from pypy.rpython.ootypesystem.rvirtualizable2 import VABLERTI - return VABLERTI def new_ConstRef(self, x): obj = ootype.cast_to_object(x) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Fri Nov 27 22:40:46 2009 @@ -4,19 +4,24 @@ from pypy.rpython import rvirtualizable2 from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.nonconst import NonConstant from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.jit.metainterp import history from pypy.jit.metainterp.warmstate import wrap, unwrap class VirtualizableInfo: + token_none = 0 + token_tracing = -1 + def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc jitdriver = warmrunnerdesc.jitdriver cpu = warmrunnerdesc.cpu + if cpu.ts.name == 'ootype': + import py + py.test.skip("ootype: fix virtualizables") self.cpu = cpu - self.VABLERTI = cpu.ts.get_VABLERTI() - self.null_vable_rti = cpu.ts.nullptr(deref(self.VABLERTI)) self.BoxArray = cpu.ts.BoxRef # assert len(jitdriver.virtualizables) == 1 # for now @@ -29,6 +34,7 @@ self.VTYPEPTR = VTYPEPTR self.VTYPE = VTYPE = deref(VTYPEPTR) self.null_vable = cpu.ts.nullptr(VTYPE) + self.vable_token_descr = cpu.fielddescrof(VTYPE, 'vable_token') # accessor = VTYPE._hints['virtualizable2_accessor'] all_fields = accessor.fields @@ -148,7 +154,7 @@ def finish(self): # def force_if_necessary(virtualizable): - if virtualizable.vable_rti: + if virtualizable.vable_token: self.force_now(virtualizable) force_if_necessary._always_inline_ = True # @@ -169,72 +175,55 @@ def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) - def cast_instance_to_base_ptr(self, vable_rti): - if we_are_translated(): - return self.cpu.ts.cast_instance_to_base_ref(vable_rti) - else: - vable_rti._TYPE = self.VABLERTI # hack for non-translated mode - return vable_rti + def reset_vable_token(self, virtualizable): + virtualizable.vable_token = self.token_none - def clear_vable_rti(self, virtualizable): - if virtualizable.vable_rti: + def clear_vable_token(self, virtualizable): + if virtualizable.vable_token: self.force_now(virtualizable) - assert not virtualizable.vable_rti + assert not virtualizable.vable_token def tracing_before_residual_call(self, virtualizable): - assert not virtualizable.vable_rti - ptr = self.cast_instance_to_base_ptr(tracing_vable_rti) - virtualizable.vable_rti = ptr + assert not virtualizable.vable_token + virtualizable.vable_token = self.token_tracing def tracing_after_residual_call(self, virtualizable): - if virtualizable.vable_rti: + if virtualizable.vable_token: # not modified by the residual call; assert that it is still # set to 'tracing_vable_rti' and clear it. - ptr = self.cast_instance_to_base_ptr(tracing_vable_rti) - assert virtualizable.vable_rti == ptr - virtualizable.vable_rti = self.null_vable_rti + assert virtualizable.vable_token == self.token_tracing + virtualizable.vable_token = self.token_none return False else: # marker "modified during residual call" set. return True def force_now(self, virtualizable): - rti = virtualizable.vable_rti - virtualizable.vable_rti = self.null_vable_rti - if we_are_translated(): - rti = cast_base_ptr_to_instance(AbstractVableRti, rti) - rti.force_now(virtualizable) + token = virtualizable.vable_token + virtualizable.vable_token = self.token_none + if token == self.token_tracing: + # The values in the virtualizable are always correct during + # tracing. We only need to reset vable_token to token_none + # as a marker for the tracing, to tell it that this + # virtualizable escapes. + pass + else: + faildescr = self.cpu.force(token) + faildescr.force_virtualizable(self, virtualizable) force_now._dont_inline_ = True # ____________________________________________________________ # -# The 'vable_rti' field of a virtualizable is either NULL or points -# to an instance of the following classes. It is: +# The 'vable_token' field of a virtualizable is either 0, -1, or points +# into the CPU stack to a particular field in the current frame. It is: # -# 1. NULL if not in the JIT at all, except as described below. +# 1. 0 (token_none) if not in the JIT at all, except as described below. # -# 2. always NULL when tracing is in progress. +# 2. equal to 0 when tracing is in progress; except: # -# 3. 'tracing_vable_rti' during tracing when we do a residual call, +# 3. equal to -1 (token_tracing) during tracing when we do a residual call, # calling random unknown other parts of the interpreter; it is -# reset to NULL as soon as something occurs to the virtualizable. +# reset to 0 as soon as something occurs to the virtualizable. # -# 4. NULL for now when running the machine code with a virtualizable; -# later it will be a RunningVableRti(). - - -class AbstractVableRti(object): - - def force_now(self, virtualizable): - raise NotImplementedError - - -class TracingVableRti(AbstractVableRti): - - def force_now(self, virtualizable): - # The values if the virtualizable are always correct during tracing. - # We only need to set a marker to tell that forcing occurred. - # As the caller resets vable_rti to NULL, it plays the role of marker. - pass - -tracing_vable_rti = TracingVableRti() +# 4. when running the machine code with a virtualizable, it is set +# to the address in the CPU stack by the FORCE_TOKEN operation. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/warmstate.py Fri Nov 27 22:40:46 2009 @@ -213,6 +213,8 @@ virtualizable = vinfo.cast_to_vtype(virtualizable) assert virtualizable != globaldata.blackhole_virtualizable, ( "reentering same frame via blackhole") + else: + virtualizable = None # look for the cell corresponding to the current greenargs greenargs = args[:num_green_args] @@ -247,6 +249,8 @@ fail_descr = metainterp_sd.cpu.execute_token(loop_token) debug_stop("jit-running") metainterp_sd.profiler.end_running() + if vinfo is not None: + vinfo.reset_vable_token(virtualizable) loop_token = fail_descr.handle_fail(metainterp_sd) maybe_compile_and_run._dont_inline_ = True From arigo at codespeak.net Fri Nov 27 23:15:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 27 Nov 2009 23:15:16 +0100 (CET) Subject: [pypy-svn] r69715 - in pypy/branch/virtual-forcing/pypy/rpython: lltypesystem ootypesystem test Message-ID: <20091127221516.9532E1683B7@codespeak.net> Author: arigo Date: Fri Nov 27 23:15:16 2009 New Revision: 69715 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rvirtualizable2.py pypy/branch/virtual-forcing/pypy/rpython/ootypesystem/rvirtualizable2.py pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py Log: Oups. Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rvirtualizable2.py Fri Nov 27 23:15:16 2009 @@ -3,23 +3,21 @@ from pypy.rpython.lltypesystem.rclass import InstanceRepr, OBJECTPTR from pypy.rpython.rvirtualizable2 import AbstractVirtualizable2InstanceRepr -VABLERTIPTR = OBJECTPTR class Virtualizable2InstanceRepr(AbstractVirtualizable2InstanceRepr, InstanceRepr): def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_base', llmemory.Address)) - llfields.append(('vable_rti', VABLERTIPTR)) + llfields.append(('vable_token', lltype.Signed)) return llfields def set_vable(self, llops, vinst, force_cast=False): if self.top_of_virtualizable_hierarchy: if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - cname = inputconst(lltype.Void, 'vable_rti') - vvalue = inputconst(VABLERTIPTR, lltype.nullptr(VABLERTIPTR.TO)) - llops.genop('setfield', [vinst, cname, vvalue]) + cname = inputconst(lltype.Void, 'vable_token') + cvalue = inputconst(lltype.Signed, 0) + llops.genop('setfield', [vinst, cname, cvalue]) else: self.rbase.set_vable(llops, vinst, force_cast=True) Modified: pypy/branch/virtual-forcing/pypy/rpython/ootypesystem/rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/ootypesystem/rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/ootypesystem/rvirtualizable2.py Fri Nov 27 23:15:16 2009 @@ -11,7 +11,7 @@ def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_rti', VABLERTI)) + llfields.append(('vable_token', VABLERTI)) return llfields def set_vable(self, llops, vinst, force_cast=False): Modified: pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/test/test_rvirtualizable2.py Fri Nov 27 23:15:16 2009 @@ -361,7 +361,7 @@ assert res.item1 == 42 res = lltype.normalizeptr(res.item0) assert res.inst_v == 42 - assert not res.vable_rti + assert res.vable_token == 0 class TestOOtype(OORtypeMixin, BaseTest): prefix = 'o' From arigo at codespeak.net Sun Nov 29 09:31:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 09:31:43 +0100 (CET) Subject: [pypy-svn] r69723 - pypy/extradoc/sprintinfo/leysin-winter-2010 Message-ID: <20091129083143.57F931683F4@codespeak.net> Author: arigo Date: Sun Nov 29 09:31:39 2009 New Revision: 69723 Added: pypy/extradoc/sprintinfo/leysin-winter-2010/ - copied from r69385, pypy/extradoc/sprintinfo/leysin-winter-2009/ Removed: pypy/extradoc/sprintinfo/leysin-winter-2010/planning.txt Modified: pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt pypy/extradoc/sprintinfo/leysin-winter-2010/people.txt Log: Announcement. Modified: pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2009/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2010/announcement.txt Sun Nov 29 09:31:39 2009 @@ -1,34 +1,33 @@ ===================================================================== - PyPy Leysin Winter Sprint (14-21th April 2009) + PyPy Leysin Winter Sprint (23-30th January 2010) ===================================================================== .. image:: http://www.ermina.ch/002.JPG The next PyPy sprint will be in Leysin, Switzerland, for the -sixth time. This sprint will take place immediately after -Easter. This is a fully public sprint: newcomers and topics +seventh time. This is a fully public sprint: newcomers and topics other than those proposed below are welcome. ------------------------------ Goals and topics of the sprint ------------------------------ -* The overall idea of the sprint is to continue working on making PyPy ready - for general use. There are a few tasks left in there. In parallel, we - will continue the work on the JIT, if there is general interest. And as - usual, we are ready to add any other task -- please mention on the mailing +* The main idea of the sprint is to prepare the upcoming release of PyPy. + This means mostly JIT or JIT-related work. It is a bit hard to be more + precise right now, because it depends on how much is left to do in the + core JIT. So work will be focused either on the JIT itself or on "make + this or that feature of PyPy more JIT-friendly". Of course, as usual, + we are ready to add any other task -- please mention on the mailing list what you would like to work on; the list of task is not really fixed. * And as usual, the main side goal is to have fun in winter sports :-) - We can take a day off for ski until Sunday, the 19th; afterwards, the - installations close. (There was quite a lot of snow this winter, so - there should be some left even though it's relatively late in the season.) + We can take a day off for ski. ----------- Exact times ----------- -Officially, 14-21 April 2009. The idea is that the 14th should be the arrival -day, so we won't work too much until the 15th. +Officially, 23-30 January 2010. The idea is that the 23rd should be the +arrival day, so we won't work too much until the 24th. ----------------------- Location & Accomodation @@ -52,7 +51,7 @@ Please register by svn: - http://codespeak.net/svn/pypy/extradoc/sprintinfo/leysin-winter-2009/people.txt + http://codespeak.net/svn/pypy/extradoc/sprintinfo/leysin-winter-2010/people.txt or on the pypy-sprint mailing list if you do not yet have check-in rights: Modified: pypy/extradoc/sprintinfo/leysin-winter-2010/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2009/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2010/people.txt Sun Nov 29 09:31:39 2009 @@ -1,5 +1,5 @@ -People coming to the Leysin sprint Winter 2009 +People coming to the Leysin sprint Winter 2010 ================================================== People who have a ``?`` in their arrive/depart or accomodation @@ -11,29 +11,23 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo -- private -Carl Friedrich Bolz 14th-21st Ermina -Antonio Cuni 15th-21st Ermina -Samuele Pedroni 14th-22nd Ermina -Anders Hammarquist 14th-22nd Ermina -Christian Tismer 14th-22nd Ermina -Niko Matsakis 14th-19th Ermina ==================== ============== ======================= -Samuele and Anders Hammarquist would like to share a two persons room. - -Niko, Carl and Antonio plan to share a room. - -Christian is sorry but he needs a single room. People on the following list were present at previous sprints: ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== +Carl Friedrich Bolz ? ? +Antonio Cuni ? ? +Samuele Pedroni ? ? +Anders Hammarquist ? ? +Christian Tismer ? ? +Niko Matsakis ? ? Maciej Fijalkowski ? ? Toby Watson ? ? Paul deGrandis ? ? -Christian Tismer ? ? Michael Hudson ? ? Anders Lehmann ? ? Niklaus Haldimann ? ? @@ -49,7 +43,6 @@ Stephan Diehl ? ? Alexander Schremmer ? ? Anders Chrigstroem ? ? -Samuele Pedroni ? ? Eric van Riet Paap ? ? Jacob Hallen ? ? Laura Creighton ? ? From arigo at codespeak.net Sun Nov 29 11:24:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 11:24:42 +0100 (CET) Subject: [pypy-svn] r69724 - in pypy/branch/virtual-forcing/pypy/jit: backend backend/llgraph metainterp metainterp/test Message-ID: <20091129102442.8B4C21683F0@codespeak.net> Author: arigo Date: Sun Nov 29 11:24:41 2009 New Revision: 69724 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Log: Handle virtuals correctly around forcing, instead of allocating them twice (once when forcing, once when returning and seeing the guard_not_forced fail). A bit hackish, but not obvious how to best organize it. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Sun Nov 29 11:24:41 2009 @@ -1070,9 +1070,7 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, _get_error(ZeroDivisionError).args[1]) -def force(force_token): - opaque_frame = llmemory.cast_adr_to_ptr(force_token, - lltype.Ptr(_TO_OPAQUE[Frame])) +def force(opaque_frame): frame = _from_opaque(opaque_frame) assert not frame._forced frame._forced = True @@ -1080,13 +1078,13 @@ call_op = frame.loop.operations[frame._may_force] guard_op = frame.loop.operations[frame._may_force+1] frame._populate_fail_args(guard_op, skip=call_op.result) - return opaque_frame - -def get_forced_fail_index(opaque_frame): - frame = _from_opaque(opaque_frame) - assert frame._forced return frame.fail_index +def get_forced_token_frame(force_token): + opaque_frame = llmemory.cast_adr_to_ptr(force_token, + lltype.Ptr(_TO_OPAQUE[Frame])) + return opaque_frame + class MemoCast(object): def __init__(self): self.addresses = [llmemory.NULL] @@ -1456,8 +1454,8 @@ setannotation(get_overflow_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(get_zero_division_error, annmodel.SomeAddress()) setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) -setannotation(force, s_Frame) -setannotation(get_forced_fail_index, annmodel.SomeInteger()) +setannotation(force, annmodel.SomeInteger()) +setannotation(get_forced_token_frame, s_Frame) setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Sun Nov 29 11:24:41 2009 @@ -4,6 +4,7 @@ import sys from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython.llinterp import LLInterpreter @@ -233,6 +234,13 @@ def get_latest_value_float(self, index): return llimpl.frame_float_getvalue(self.latest_frame, index) + def get_latest_force_token(self): + token = self.latest_force_token + if not we_are_translated(): + frame = llimpl.get_forced_token_frame(token) + assert frame._obj.externalobj is self.latest_frame._obj.externalobj + return self.cast_adr_to_int(token) + # ---------- def get_exception(self): @@ -499,9 +507,11 @@ self.memo_cast)) def force(self, force_token): - frame = llimpl.force(self.cast_int_to_adr(force_token)) + token = self.cast_int_to_adr(force_token) + frame = llimpl.get_forced_token_frame(token) + fail_index = llimpl.force(frame) + self.latest_force_token = token self.latest_frame = frame - fail_index = llimpl.get_forced_fail_index(frame) return self.get_fail_descr_from_number(fail_index) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Sun Nov 29 11:24:41 2009 @@ -78,6 +78,11 @@ or from 'args' if it was a FINISH). Returns a ptr or an obj.""" raise NotImplementedError + def get_latest_force_token(self): + """After a GUARD_NOT_FORCED fails, this function returns the + same FORCE_TOKEN result as the one in the just-failed loop.""" + raise NotImplementedError + def get_exception(self): raise NotImplementedError Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Sun Nov 29 11:24:41 2009 @@ -237,15 +237,35 @@ send_bridge_to_backend(metainterp.staticdata, self, inputargs, new_loop.operations) - def force_virtualizable(self, vinfo, virtualizable): + +class ResumeGuardForcedDescr(ResumeGuardDescr): + + def handle_fail(self, metainterp_sd): + from pypy.jit.metainterp.pyjitpl import MetaInterp + metainterp = MetaInterp(metainterp_sd) + token = metainterp_sd.cpu.get_latest_force_token() + metainterp._already_allocated_resume_virtuals = self.fetch_data(token) + return metainterp.handle_guard_failure(self) + + def force_virtualizable(self, vinfo, virtualizable, force_token): from pypy.jit.metainterp.pyjitpl import MetaInterp from pypy.jit.metainterp.resume import force_from_resumedata metainterp = MetaInterp(self.metainterp_sd) metainterp.history = None # blackholing liveboxes = metainterp.load_values_from_failure(self) - virtualizable_boxes = force_from_resumedata(metainterp, - liveboxes, self) + virtualizable_boxes, data = force_from_resumedata(metainterp, + liveboxes, self) vinfo.write_boxes(virtualizable, virtualizable_boxes) + self.save_data(force_token, data) + + def save_data(self, key, value): + globaldata = self.metainterp_sd.globaldata + assert key not in globaldata.resume_virtuals + globaldata.resume_virtuals[key] = value + + def fetch_data(self, key): + globaldata = self.metainterp_sd.globaldata + return globaldata.resume_virtuals.pop(key) class ResumeFromInterpDescr(ResumeDescr): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Nov 29 11:24:41 2009 @@ -935,7 +935,12 @@ else: moreargs = list(extraargs) original_greenkey = metainterp.resumekey.original_greenkey - resumedescr = compile.ResumeGuardDescr(metainterp_sd, original_greenkey) + if opnum == rop.GUARD_NOT_FORCED: + resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, + original_greenkey) + else: + resumedescr = compile.ResumeGuardDescr(metainterp_sd, + original_greenkey) guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -1161,6 +1166,7 @@ self.indirectcall_dict = None self.addr2name = None self.loopnumbering = 0 + self.resume_virtuals = {} # state = staticdata.state if state is not None: @@ -1187,6 +1193,8 @@ class MetaInterp(object): in_recursion = 0 + _already_allocated_resume_virtuals = None + def __init__(self, staticdata): self.staticdata = staticdata self.cpu = staticdata.cpu Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Sun Nov 29 11:24:41 2009 @@ -457,7 +457,7 @@ def force_from_resumedata(metainterp, newboxes, storage): resumereader = ResumeDataReader(storage, newboxes, metainterp) - return resumereader.consume_boxes() + return resumereader.consume_boxes(), resumereader.virtuals class ResumeDataReader(object): @@ -472,11 +472,14 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - self.virtuals = [None] * len(virtuals) - for i in range(len(virtuals)): - vinfo = virtuals[i] - if vinfo is not None: - self.virtuals[i] = vinfo.allocate(metainterp) + if metainterp._already_allocated_resume_virtuals is None: + self.virtuals = [None] * len(virtuals) + for i in range(len(virtuals)): + vinfo = virtuals[i] + if vinfo is not None: + self.virtuals[i] = vinfo.allocate(metainterp) + else: + self.virtuals = metainterp._already_allocated_resume_virtuals for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 29 11:24:41 2009 @@ -616,8 +616,6 @@ self.check_tree_loop_count(0) def test_external_read_sometimes(self): - #py.test.skip("known bug: access the frame in a residual call but" - # " only sometimes, so that it's not seen during tracing") jitdriver = JitDriver(greens = [], reds = ['frame'], virtualizables = ['frame']) @@ -653,6 +651,49 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_external_read_sometimes_with_virtuals(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class Y: + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + y = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read') + else: + y = None + return y + + def f(n): + frame = Frame() + frame.x = n + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.y = y = Y() + result = g() + if frame.y is not y: + return -660 + if result: + if result is not y: + return -661 + frame.y = None + frame.x -= 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/virtualizable.py Sun Nov 29 11:24:41 2009 @@ -208,8 +208,10 @@ # virtualizable escapes. pass else: + from pypy.jit.metainterp.compile import ResumeGuardForcedDescr faildescr = self.cpu.force(token) - faildescr.force_virtualizable(self, virtualizable) + assert isinstance(faildescr, ResumeGuardForcedDescr) + faildescr.force_virtualizable(self, virtualizable, token) force_now._dont_inline_ = True # ____________________________________________________________ From arigo at codespeak.net Sun Nov 29 11:33:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 11:33:40 +0100 (CET) Subject: [pypy-svn] r69725 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091129103340.2E71B1683F0@codespeak.net> Author: arigo Date: Sun Nov 29 11:33:39 2009 New Revision: 69725 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Log: Implement get_latest_force_token() in the x86 backend. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Sun Nov 29 11:33:39 2009 @@ -89,6 +89,7 @@ self.fail_boxes_int = NonmovableGrowableArraySigned() self.fail_boxes_ptr = NonmovableGrowableArrayGCREF() self.fail_boxes_float = NonmovableGrowableArrayFloat() + self.fail_ebp = 0 self.setup_failure_recovery() def leave_jitted_hook(self): @@ -877,6 +878,7 @@ return arglocs[:] def grab_frame_values(self, bytecode, frame_addr, registers): + self.fail_ebp = registers[ebp.op] num = 0 value_hi = 0 while 1: Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Sun Nov 29 11:33:39 2009 @@ -59,6 +59,9 @@ llmemory.GCREF.TO)) return ptrvalue + def get_latest_force_token(self): + return self.assembler.fail_ebp + FORCE_INDEX_OFS + def execute_token(self, executable_token): addr = executable_token._x86_bootstrap_code func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr) From arigo at codespeak.net Sun Nov 29 11:39:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 11:39:30 +0100 (CET) Subject: [pypy-svn] r69726 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091129103930.41C2D1683F0@codespeak.net> Author: arigo Date: Sun Nov 29 11:39:28 2009 New Revision: 69726 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Handle possible exceptions on guard_not_forced. A big test for a one-liner fix. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Nov 29 11:39:28 2009 @@ -1589,7 +1589,8 @@ self.framestack[-1].follow_jump() elif opnum == rop.GUARD_FALSE: # a goto_if_not that stops jumping self.framestack[-1].dont_follow_jump() - elif opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION: + elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION + or opnum == rop.GUARD_NOT_FORCED): self.handle_exception() elif opnum == rop.GUARD_NO_OVERFLOW: # an overflow now detected self.raise_overflow_error() Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 29 11:39:28 2009 @@ -694,6 +694,48 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_external_read_sometimes_with_exception(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class FooBarError(Exception): + pass + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + result = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', result) + assert result == 79 + raise FooBarError + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + try: + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + except FooBarError: + pass + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) From arigo at codespeak.net Sun Nov 29 12:28:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 12:28:47 +0100 (CET) Subject: [pypy-svn] r69727 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20091129112847.925231683F5@codespeak.net> Author: arigo Date: Sun Nov 29 12:28:46 2009 New Revision: 69727 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Minor fixes with a new test: - prevent GUARD_NOT_FORCED from ever being compiled. - in the llgraph backend, make sure that frame.loop is kept up-to-date by the JUMP operation! :-( Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Sun Nov 29 12:28:46 2009 @@ -466,7 +466,8 @@ if op.opnum == rop.JUMP: assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) - operations = op.jump_target.operations + self.loop = op.jump_target + operations = self.loop.operations opindex = 0 _stats.exec_jumps += 1 elif op.opnum == rop.FINISH: @@ -497,7 +498,7 @@ try: res = ophandler(self, descr, *values) finally: - if verbose: + if 0: # if verbose: argtypes, restype = TYPES[opname] if res is None: resdata = '' @@ -506,9 +507,9 @@ else: resdata = '-> ' + repr1(res, restype, self.memocast) # fish the types - #log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes, - # self.memocast), - # resdata)) + log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes, + self.memocast), + resdata)) return res def as_int(self, x): @@ -796,7 +797,10 @@ def op_call_may_force(self, calldescr, func, *args): assert not self._forced self._may_force = self.opindex - return self.op_call(calldescr, func, *args) + try: + return self.op_call(calldescr, func, *args) + finally: + self._may_force = -1 def op_guard_not_forced(self, descr): forced = self._forced @@ -1077,6 +1081,7 @@ assert frame._may_force >= 0 call_op = frame.loop.operations[frame._may_force] guard_op = frame.loop.operations[frame._may_force+1] + assert call_op.opnum == rop.CALL_MAY_FORCE frame._populate_fail_args(guard_op, skip=call_op.result) return frame.fail_index Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Sun Nov 29 12:28:46 2009 @@ -245,6 +245,7 @@ metainterp = MetaInterp(metainterp_sd) token = metainterp_sd.cpu.get_latest_force_token() metainterp._already_allocated_resume_virtuals = self.fetch_data(token) + self.counter = -2 # never compile return metainterp.handle_guard_failure(self) def force_virtualizable(self, vinfo, virtualizable, force_token): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/pyjitpl.py Sun Nov 29 12:28:46 2009 @@ -925,7 +925,6 @@ if isinstance(box, Const): # no need for a guard return metainterp = self.metainterp - metainterp_sd = metainterp.staticdata if metainterp.is_blackholing(): return saved_pc = self.pc @@ -934,6 +933,7 @@ moreargs = [box] + extraargs else: moreargs = list(extraargs) + metainterp_sd = metainterp.staticdata original_greenkey = metainterp.resumekey.original_greenkey if opnum == rop.GUARD_NOT_FORCED: resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 29 12:28:46 2009 @@ -736,6 +736,42 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) assert res == f(123) + def test_external_read_sometimes_dont_compile_guard(self): + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + result = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', result) + assert result == 79 + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + return frame.x + + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g), repeat=7) + assert res == f(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) From arigo at codespeak.net Sun Nov 29 12:45:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 12:45:00 +0100 (CET) Subject: [pypy-svn] r69728 - pypy/branch/virtual-forcing/pypy/jit/backend/x86 Message-ID: <20091129114500.EA6701683F5@codespeak.net> Author: arigo Date: Sun Nov 29 12:45:00 2009 New Revision: 69728 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Log: Fix. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Sun Nov 29 12:45:00 2009 @@ -759,7 +759,8 @@ def implement_guard_recovery(self, guard_opnum, faildescr, failargs, fail_locs): exc = (guard_opnum == rop.GUARD_EXCEPTION or - guard_opnum == rop.GUARD_NO_EXCEPTION) + guard_opnum == rop.GUARD_NO_EXCEPTION or + guard_opnum == rop.GUARD_NOT_FORCED) return self.generate_quick_failure(faildescr, failargs, fail_locs, exc) def generate_quick_failure(self, faildescr, failargs, fail_locs, exc): From arigo at codespeak.net Sun Nov 29 12:50:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 12:50:42 +0100 (CET) Subject: [pypy-svn] r69729 - in pypy/branch: shorter-guard-path shorter-guard-path-2 Message-ID: <20091129115042.C0BE51683F5@codespeak.net> Author: arigo Date: Sun Nov 29 12:50:42 2009 New Revision: 69729 Removed: pypy/branch/shorter-guard-path/ pypy/branch/shorter-guard-path-2/ Log: Remove the branch, merged some days ago. From arigo at codespeak.net Sun Nov 29 13:10:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 13:10:50 +0100 (CET) Subject: [pypy-svn] r69730 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091129121050.5B2FE1683F1@codespeak.net> Author: arigo Date: Sun Nov 29 13:10:49 2009 New Revision: 69730 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Kill lots of whitespace at the end of a range of lines. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 29 13:10:49 2009 @@ -882,26 +882,26 @@ assert res == 55 self.check_loops(new_with_vtable=0) - def test_check_for_nonstandardness_only_once(self): - myjitdriver = JitDriver(greens = [], reds = ['frame'], - virtualizables = ['frame']) - - class Frame(object): - _virtualizable2_ = ['x', 'y', 'z'] - - def __init__(self, x, y, z=1): - self = hint(self, access_directly=True) - self.x = x - self.y = y - self.z = z - - class SomewhereElse: - pass - somewhere_else = SomewhereElse() - - def f(n): - frame = Frame(n, 0) - somewhere_else.top_frame = frame # escapes + def test_check_for_nonstandardness_only_once(self): + myjitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y', 'z'] + + def __init__(self, x, y, z=1): + self = hint(self, access_directly=True) + self.x = x + self.y = y + self.z = z + + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def f(n): + frame = Frame(n, 0) + somewhere_else.top_frame = frame # escapes frame = hint(frame, access_directly=True) while frame.x > 0: myjitdriver.can_enter_jit(frame=frame) From arigo at codespeak.net Sun Nov 29 13:16:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 13:16:16 +0100 (CET) Subject: [pypy-svn] r69731 - in pypy/branch/virtual-forcing/pypy/jit: backend backend/llgraph metainterp Message-ID: <20091129121616.8EDBE1683F4@codespeak.net> Author: arigo Date: Sun Nov 29 13:16:14 2009 New Revision: 69731 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/backend/model.py pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Log: Translation fixes. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Sun Nov 29 13:16:14 2009 @@ -100,6 +100,7 @@ llimpl._stats = self.stats llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] + self.latest_force_token = llmemory.NULL def _freeze_(self): assert self.translate_support_code Modified: pypy/branch/virtual-forcing/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/model.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/model.py Sun Nov 29 13:16:14 2009 @@ -211,6 +211,7 @@ raise NotImplementedError def do_force_token(self): + # this should not be implemented at all by the backends raise NotImplementedError def do_call_may_force(self, args, calldescr): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/compile.py Sun Nov 29 13:16:14 2009 @@ -266,7 +266,10 @@ def fetch_data(self, key): globaldata = self.metainterp_sd.globaldata - return globaldata.resume_virtuals.pop(key) + assert key in globaldata.resume_virtuals + data = globaldata.resume_virtuals[key] + del globaldata.resume_virtuals[key] + return data class ResumeFromInterpDescr(ResumeDescr): Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resoperation.py Sun Nov 29 13:16:14 2009 @@ -221,7 +221,7 @@ 'COND_CALL_GC_MALLOC', # [a, b, if_(a<=b)_result, if_(a>b)_call, args...] # => result (for mallocs) 'DEBUG_MERGE_POINT/1', # debugging only - 'FORCE_TOKEN', + 'FORCE_TOKEN/0', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- 'CALL', From arigo at codespeak.net Sun Nov 29 13:46:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 13:46:31 +0100 (CET) Subject: [pypy-svn] r69733 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph metainterp/test Message-ID: <20091129124631.7B1A11683F4@codespeak.net> Author: arigo Date: Sun Nov 29 13:46:30 2009 New Revision: 69733 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Log: Test and fix: - kill 'latest_force_token', because it can get out of date - use 'get_frame_forced_token' instead - two opaque ptrs from the same object don't compare as equal, which was causing addresses of the same frame object to be different. Fix in _to_opaque(). Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/llimpl.py Sun Nov 29 13:46:30 2009 @@ -35,8 +35,13 @@ _TO_OPAQUE = {} def _to_opaque(value): - return lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) + try: + return value._the_opaque_pointer + except AttributeError: + op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', + externalobj=value) + value._the_opaque_pointer = op + return op def from_opaque_string(s): if isinstance(s, str): @@ -1090,6 +1095,9 @@ lltype.Ptr(_TO_OPAQUE[Frame])) return opaque_frame +def get_frame_forced_token(opaque_frame): + return llmemory.cast_ptr_to_adr(opaque_frame) + class MemoCast(object): def __init__(self): self.addresses = [llmemory.NULL] @@ -1461,6 +1469,7 @@ setannotation(get_zero_division_error_value, annmodel.SomePtr(llmemory.GCREF)) setannotation(force, annmodel.SomeInteger()) setannotation(get_forced_token_frame, s_Frame) +setannotation(get_frame_forced_token, annmodel.SomeAddress()) setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Sun Nov 29 13:46:30 2009 @@ -100,7 +100,6 @@ llimpl._stats = self.stats llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] - self.latest_force_token = llmemory.NULL def _freeze_(self): assert self.translate_support_code @@ -236,10 +235,7 @@ return llimpl.frame_float_getvalue(self.latest_frame, index) def get_latest_force_token(self): - token = self.latest_force_token - if not we_are_translated(): - frame = llimpl.get_forced_token_frame(token) - assert frame._obj.externalobj is self.latest_frame._obj.externalobj + token = llimpl.get_frame_forced_token(self.latest_frame) return self.cast_adr_to_int(token) # ---------- @@ -511,7 +507,6 @@ token = self.cast_int_to_adr(force_token) frame = llimpl.get_forced_token_frame(token) fail_index = llimpl.force(frame) - self.latest_force_token = token self.latest_frame = frame return self.get_fail_descr_from_number(fail_index) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_virtualizable.py Sun Nov 29 13:46:30 2009 @@ -772,6 +772,57 @@ res = self.meta_interp(f, [123], policy=StopAtXPolicy(g), repeat=7) assert res == f(123) + def test_external_read_sometimes_recursive(self): + jitdriver = JitDriver(greens = [], reds = ['frame', 'rec'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(rec): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + frame = somewhere_else.top_frame + result1 = frame.y # external read + result2 = frame.back.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', + result1, result2) + assert result1 == 13 + assert result2 == 1023 + result = 2 + elif rec: + res = f(4, False) + assert res == 0 or res == -1 + result = 1 + else: + result = 1 + return result + + def f(n, rec): + frame = Frame() + frame.x = n + frame.y = 10 + 1000 * rec + frame.back = somewhere_else.top_frame + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame, rec=rec) + jitdriver.jit_merge_point(frame=frame, rec=rec) + frame.x -= g(rec) + frame.y += 1 + somewhere_else.top_frame = frame.back + return frame.x + + def main(n): + somewhere_else.counter = 0 + somewhere_else.top_frame = None + return f(n, True) + + res = self.meta_interp(main, [123], policy=StopAtXPolicy(g)) + assert res == main(123) + def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], virtualizables = ['frame']) From arigo at codespeak.net Sun Nov 29 13:58:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 13:58:33 +0100 (CET) Subject: [pypy-svn] r69734 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091129125833.637371683F4@codespeak.net> Author: arigo Date: Sun Nov 29 13:58:32 2009 New Revision: 69734 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Log: Oups. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Sun Nov 29 13:58:32 2009 @@ -1214,7 +1214,9 @@ canraise = True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise if pure: - assert not calldescr.effectinfo.promotes_virtualizables + effectinfo = calldescr.get_extra_info() + assert (effectinfo is not None and + not effectinfo.promotes_virtualizables) self.emit('residual_call_pure') elif canraise: self.emit('residual_call') From arigo at codespeak.net Sun Nov 29 16:38:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 16:38:50 +0100 (CET) Subject: [pypy-svn] r69735 - in pypy/branch/virtual-forcing/pypy: annotation rpython/lltypesystem rpython/lltypesystem/test translator/backendopt translator/backendopt/test translator/stackless Message-ID: <20091129153850.A55411683F6@codespeak.net> Author: arigo Date: Sun Nov 29 16:38:49 2009 New Revision: 69735 Modified: pypy/branch/virtual-forcing/pypy/annotation/description.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rffi.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/virtual-forcing/pypy/translator/backendopt/canraise.py pypy/branch/virtual-forcing/pypy/translator/backendopt/graphanalyze.py pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_canraise.py pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py pypy/branch/virtual-forcing/pypy/translator/stackless/transform.py Log: issue487 testing (pedronis, arigo) Fix, mostly for the WriteAnalyzer and the VirtualizableAnalyzer: record exactly which RPython callback functions a given llexternal function can invoke, and use this information in the GraphAnalyzer. Modified: pypy/branch/virtual-forcing/pypy/annotation/description.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/annotation/description.py (original) +++ pypy/branch/virtual-forcing/pypy/annotation/description.py Sun Nov 29 16:38:49 2009 @@ -202,6 +202,9 @@ graph.name = alt_name return graph + def getgraphs(self): + return self._cache.values() + def getuniquegraph(self): if len(self._cache) != 1: raise NoStandardGraph(self) Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/rffi.py Sun Nov 29 16:38:49 2009 @@ -57,7 +57,7 @@ def llexternal(name, args, result, _callable=None, compilation_info=ExternalCompilationInfo(), sandboxsafe=False, threadsafe='auto', - canraise=False, _nowrapper=False, calling_conv='c', + _nowrapper=False, calling_conv='c', oo_primitive=None, pure_function=False): """Build an external function that will invoke the C function 'name' with the given 'args' types and 'result' type. @@ -68,6 +68,10 @@ pointing to a read-only null-terminated character of arrays, as usual for C. + The C function can have callbacks, but they must be specified explicitly + as constant RPython functions. We don't support yet C functions that + invoke callbacks passed otherwise (e.g. set by a previous C call). + threadsafe: whether it's ok to release the GIL around the call. Default is yes, unless sandboxsafe is set, in which case we consider that the function is really short-running and @@ -84,12 +88,22 @@ kwds = {} if oo_primitive: kwds['oo_primitive'] = oo_primitive + + has_callback = False + for ARG in args: + if _isfunctype(ARG): + has_callback = True + if has_callback: + kwds['_callbacks'] = callbackholder = CallbackHolder() + else: + callbackholder = None + funcptr = lltype.functionptr(ext_type, name, external='C', compilation_info=compilation_info, _callable=_callable, _safe_not_sandboxed=sandboxsafe, _debugexc=True, # on top of llinterp - canraise=canraise, + canraise=False, **kwds) if isinstance(_callable, ll2ctypes.LL2CtypesCallable): _callable.funcptr = funcptr @@ -170,9 +184,11 @@ # XXX pass additional arguments if invoke_around_handlers: arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, + callbackholder, aroundstate)) else: - arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg)) + arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, + callbackholder)) else: SOURCE = lltype.typeOf(arg) if SOURCE != TARGET: @@ -202,7 +218,11 @@ return func_with_new_name(wrapper, name) -def _make_wrapper_for(TP, callable, aroundstate=None): +class CallbackHolder: + def __init__(self): + self.callbacks = {} + +def _make_wrapper_for(TP, callable, callbackholder, aroundstate=None): """ Function creating wrappers for callbacks. Note that this is cheating as we assume constant callbacks and we just memoize wrappers """ @@ -213,6 +233,7 @@ else: errorcode = TP.TO.RESULT._example() callable_name = getattr(callable, '__name__', '?') + callbackholder.callbacks[callable] = True args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" def wrapper(%s): # no *args - no GIL for mallocing the tuple Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/test/test_rffi.py Sun Nov 29 16:38:49 2009 @@ -387,6 +387,7 @@ fn = self.compile(f, []) assert fn() == 6 + assert eating_callback._ptr._obj._callbacks.callbacks == {g: True} def test_double_callback(self): eating_callback = self.eating_callback() @@ -406,6 +407,8 @@ fn = self.compile(f, [int]) assert fn(4) == 4 assert fn(1) == 3 + assert eating_callback._ptr._obj._callbacks.callbacks == {one: True, + two: True} def test_exception_callback(self): eating_callback = self.eating_callback() Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/canraise.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/canraise.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/canraise.py Sun Nov 29 16:38:49 2009 @@ -17,7 +17,7 @@ log.WARNING("Unknown operation: %s" % op.opname) return True - def analyze_external_call(self, op): + def analyze_external_call(self, op, seen=None): fnobj = get_funcobj(op.args[0].value) return getattr(fnobj, 'canraise', True) Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/graphanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/graphanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/graphanalyze.py Sun Nov 29 16:38:49 2009 @@ -1,4 +1,4 @@ -from pypy.translator.simplify import get_graph +from pypy.translator.simplify import get_graph, get_funcobj from pypy.rpython.lltypesystem.lloperation import llop, LL_OPERATIONS from pypy.rpython.lltypesystem import lltype @@ -38,8 +38,17 @@ def analyze_startblock(self, block, seen=None): return self.bottom_result() - def analyze_external_call(self, op): - return self.top_result() + def analyze_external_call(self, op, seen=None): + funcobj = get_funcobj(op.args[0].value) + result = self.bottom_result() + if hasattr(funcobj, '_callbacks'): + bk = self.translator.annotator.bookkeeper + for function in funcobj._callbacks.callbacks: + desc = bk.getdesc(function) + for graph in desc.getgraphs(): + result = self.join_two_results( + result, self.analyze_direct_call(graph, seen)) + return result def analyze_external_method(self, op, TYPE, meth): return self.top_result() @@ -59,7 +68,7 @@ if op.opname == "direct_call": graph = get_graph(op.args[0], self.translator) if graph is None: - return self.analyze_external_call(op) + return self.analyze_external_call(op, seen) return self.analyze_direct_call(graph, seen) elif op.opname == "indirect_call": if op.args[-1].value is None: Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_canraise.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_canraise.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_canraise.py Sun Nov 29 16:38:49 2009 @@ -189,7 +189,8 @@ result = ra.can_raise(fgraph.startblock.operations[0]) assert not result - z = llexternal('z', [lltype.Signed], lltype.Signed, canraise=True) + z = lltype.functionptr(lltype.FuncType([lltype.Signed], lltype.Signed), + 'foobar') def g(x): return z(x) t, ra = self.translate(g, [int]) Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/test/test_writeanalyze.py Sun Nov 29 16:38:49 2009 @@ -178,6 +178,31 @@ assert name == "length" assert S1 is S2 + def test_llexternal_with_callback(self): + from pypy.rpython.lltypesystem.rffi import llexternal + from pypy.rpython.lltypesystem import lltype + + class Abc: + pass + abc = Abc() + + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + z = llexternal('z', [lltype.Ptr(FUNC)], lltype.Signed) + def g(n): + abc.foobar = n + return n + 1 + def f(x): + return z(g) + t, wa = self.translate(f, [int]) + fgraph = graphof(t, f) + backend_optimizations(t) + assert fgraph.startblock.operations[0].opname == 'direct_call' + + result = wa.analyze(fgraph.startblock.operations[0]) + assert len(result) == 1 + (struct, T, name), = result + assert struct == "struct" + assert name.endswith("foobar") class TestOOtype(BaseTestCanRaise): Modified: pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/backendopt/writeanalyze.py Sun Nov 29 16:38:49 2009 @@ -37,9 +37,6 @@ def _array_result(self, TYPE): return frozenset([("array", TYPE)]) - def analyze_external_call(self, op): - return self.bottom_result() # an external call cannot change anything - def analyze_external_method(self, op, TYPE, meth): if isinstance(TYPE, ootype.Array): methname = op.args[0].value Modified: pypy/branch/virtual-forcing/pypy/translator/stackless/transform.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/translator/stackless/transform.py (original) +++ pypy/branch/virtual-forcing/pypy/translator/stackless/transform.py Sun Nov 29 16:38:49 2009 @@ -260,7 +260,7 @@ return LL_OPERATIONS[op.opname].canunwindgc return False - def analyze_external_call(self, op): + def analyze_external_call(self, op, seen=None): # An external call cannot cause a stack unwind # Note that this is essential to get good performance in framework GCs # because there is a pseudo-external call to ROUND_UP_FOR_ALLOCATION From arigo at codespeak.net Sun Nov 29 19:14:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 19:14:01 +0100 (CET) Subject: [pypy-svn] r69738 - in pypy/branch/virtual-forcing/pypy: jit/metainterp rlib Message-ID: <20091129181401.C57B3168407@codespeak.net> Author: arigo Date: Sun Nov 29 19:14:00 2009 New Revision: 69738 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/rlib/jit.py Log: Translation fixes. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Sun Nov 29 19:14:00 2009 @@ -1208,15 +1208,19 @@ if op.opname == "direct_call": func = getattr(get_funcobj(op.args[0].value), '_callable', None) pure = getattr(func, "_pure_function_", False) + all_promoted_args = getattr(func, + "_pure_function_with_all_promoted_args_", False) + if pure and not all_promoted_args: + effectinfo = calldescr.get_extra_info() + assert (effectinfo is not None and + not effectinfo.promotes_virtualizables) try: canraise = self.codewriter.raise_analyzer.can_raise(op) except lltype.DelayedPointer: canraise = True # if we need to look into the delayed ptr that is # the portal, then it's certainly going to raise if pure: - effectinfo = calldescr.get_extra_info() - assert (effectinfo is not None and - not effectinfo.promotes_virtualizables) + # XXX check what to do about exceptions (also MemoryError?) self.emit('residual_call_pure') elif canraise: self.emit('residual_call') Modified: pypy/branch/virtual-forcing/pypy/rlib/jit.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rlib/jit.py (original) +++ pypy/branch/virtual-forcing/pypy/rlib/jit.py Sun Nov 29 19:14:00 2009 @@ -22,6 +22,7 @@ def purefunction_promote(func): import inspect purefunction(func) + func._pure_function_with_all_promoted_args_ = True args, varargs, varkw, defaults = inspect.getargspec(func) args = ["v%s" % (i, ) for i in range(len(args))] assert varargs is None and varkw is None From arigo at codespeak.net Sun Nov 29 19:28:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 29 Nov 2009 19:28:25 +0100 (CET) Subject: [pypy-svn] r69739 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20091129182825.8DB36168407@codespeak.net> Author: arigo Date: Sun Nov 29 19:28:25 2009 New Revision: 69739 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py Log: Two more instances of r68747. Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_strptime.py Sun Nov 29 19:28:25 2009 @@ -487,9 +487,9 @@ _strptime.strptime("10", "%d") _strptime.strptime("2005", "%Y") _strptime._TimeRE_cache.locale_time.lang = "Ni" - original_time_re = id(_strptime._TimeRE_cache) + original_time_re = _strptime._TimeRE_cache _strptime.strptime("10", "%d") - self.failIfEqual(original_time_re, id(_strptime._TimeRE_cache)) + self.assert_(original_time_re is not _strptime._TimeRE_cache) self.failUnlessEqual(len(_strptime._regex_cache), 1) def test_regex_cleanup(self): @@ -508,11 +508,11 @@ def test_new_localetime(self): # A new LocaleTime instance should be created when a new TimeRE object # is created. - locale_time_id = id(_strptime._TimeRE_cache.locale_time) + original_locale_time = _strptime._TimeRE_cache.locale_time _strptime._TimeRE_cache.locale_time.lang = "Ni" _strptime.strptime("10", "%d") - self.failIfEqual(locale_time_id, - id(_strptime._TimeRE_cache.locale_time)) + self.assert_(original_locale_time + is not _strptime._TimeRE_cache.locale_time) def test_TimeRE_recreation(self): # The TimeRE instance should be recreated upon changing the locale. From afa at codespeak.net Mon Nov 30 09:17:41 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 30 Nov 2009 09:17:41 +0100 (CET) Subject: [pypy-svn] r69745 - pypy/trunk/pypy/module/posix Message-ID: <20091130081741.1966D1683E5@codespeak.net> Author: afa Date: Mon Nov 30 09:17:40 2009 New Revision: 69745 Modified: pypy/trunk/pypy/module/posix/interp_posix.py Log: Fix translation on Windows, which was broken by the unconditional access to os.WCOREDUMP.__doc__ Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Mon Nov 30 09:17:40 2009 @@ -822,8 +822,9 @@ return WSTAR for name in RegisterOs.w_star: - func = declare_new_w_star(name) - globals()[name] = func + if hasattr(os, name): + func = declare_new_w_star(name) + globals()[name] = func def ttyname(space, fd): try: From arigo at codespeak.net Mon Nov 30 10:30:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 10:30:25 +0100 (CET) Subject: [pypy-svn] r69746 - in pypy/branch/virtual-forcing/pypy/jit: backend/cli/test metainterp/test Message-ID: <20091130093025.AB85C168016@codespeak.net> Author: arigo Date: Mon Nov 30 10:30:23 2009 New Revision: 69746 Added: pypy/branch/virtual-forcing/pypy/jit/backend/cli/test/conftest.py - copied, changed from r69745, pypy/branch/virtual-forcing/pypy/jit/backend/llvm/test/conftest.py Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py Log: Skip the ootype and the CLI tests for now. Copied: pypy/branch/virtual-forcing/pypy/jit/backend/cli/test/conftest.py (from r69745, pypy/branch/virtual-forcing/pypy/jit/backend/llvm/test/conftest.py) ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llvm/test/conftest.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/cli/test/conftest.py Mon Nov 30 10:30:23 2009 @@ -2,4 +2,4 @@ class Directory(py.test.collect.Directory): def collect(self): - py.test.skip("llvm backend tests skipped for now") + py.test.skip("CLI backend tests skipped for now") Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_basic.py Mon Nov 30 10:30:23 2009 @@ -148,6 +148,9 @@ type_system = 'ootype' CPUClass = runner.OOtypeCPU + def setup_class(cls): + py.test.skip("ootype tests skipped for now") + @staticmethod def Ptr(T): return T From arigo at codespeak.net Mon Nov 30 10:55:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 10:55:38 +0100 (CET) Subject: [pypy-svn] r69747 - in pypy/branch/virtual-forcing/pypy/jit/metainterp: . test Message-ID: <20091130095538.0B77716801B@codespeak.net> Author: arigo Date: Mon Nov 30 10:55:38 2009 New Revision: 69747 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Log: Pass 'consider_effects_of' to getcalldescr() in case of indirect calls too. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/codewriter.py Mon Nov 30 10:55:38 2009 @@ -1248,9 +1248,8 @@ def handle_regular_indirect_call(self, op): self.codewriter.register_indirect_call_targets(op) args = op.args[1:-1] - calldescr, non_void_args = self.codewriter.getcalldescr(op.args[0], - args, - op.result) + calldescr, non_void_args = self.codewriter.getcalldescr( + op.args[0], args, op.result, consider_effects_of=op) self.minimize_variables() self.emit('indirect_call') self.emit(self.get_position(calldescr)) Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_codewriter.py Mon Nov 30 10:55:38 2009 @@ -273,9 +273,17 @@ cw._start(self.metainterp_sd, None) jitcode = cw.make_one_bytecode((graphs[0], None), False) assert len(self.metainterp_sd.indirectcalls) == 1 - names = [jitcode.name for (fnaddress, jitcode) + names = [jitcode1.name for (fnaddress, jitcode1) in self.metainterp_sd.indirectcalls] assert dict.fromkeys(names) == {'g': None} + calldescrs = [calldescr for calldescr in jitcode.constants + if isinstance(calldescr, tuple) and + calldescr[0] == 'calldescr'] + assert len(calldescrs) == 1 + assert calldescrs[0][4] is not None + assert not calldescrs[0][4].write_descrs_fields + assert not calldescrs[0][4].write_descrs_arrays + assert not calldescrs[0][4].promotes_virtualizables def test_oosend_look_inside_only_one(self): class A: From arigo at codespeak.net Mon Nov 30 10:56:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 10:56:47 +0100 (CET) Subject: [pypy-svn] r69748 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091130095647.D64DB16801B@codespeak.net> Author: arigo Date: Mon Nov 30 10:56:47 2009 New Revision: 69748 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py Log: This will not be fixed right now. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py Mon Nov 30 10:56:47 2009 @@ -78,7 +78,9 @@ return 1 res = self.meta_interp(f, [20], optimizer=OPTIMIZER_SIMPLE) assert res == 1 - self.check_loops(call=1) # for the case B(), but not for the case A() + self.check_loops(call_may_force=1) + # for the case B(), but not for the case A() + # XXX it should really be 'call', not 'call_may_force'. class TestLLtype(DelTests, LLJitMixin): From arigo at codespeak.net Mon Nov 30 10:58:26 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 10:58:26 +0100 (CET) Subject: [pypy-svn] r69749 - pypy/branch/virtual-forcing/pypy/jit/metainterp Message-ID: <20091130095826.7B86816801B@codespeak.net> Author: arigo Date: Mon Nov 30 10:58:25 2009 New Revision: 69749 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Log: Fix for test_optimizeopt: allow _already_allocated_resume_virtuals to be absent. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/resume.py Mon Nov 30 10:58:25 2009 @@ -472,14 +472,18 @@ def _prepare_virtuals(self, metainterp, virtuals): if virtuals: - if metainterp._already_allocated_resume_virtuals is None: + try: + v = metainterp._already_allocated_resume_virtuals + except AttributeError: + v = None + if v is None: self.virtuals = [None] * len(virtuals) for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: self.virtuals[i] = vinfo.allocate(metainterp) else: - self.virtuals = metainterp._already_allocated_resume_virtuals + self.virtuals = v for i in range(len(virtuals)): vinfo = virtuals[i] if vinfo is not None: From arigo at codespeak.net Mon Nov 30 11:05:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 11:05:49 +0100 (CET) Subject: [pypy-svn] r69750 - pypy/branch/virtual-forcing/pypy/jit/metainterp/test Message-ID: <20091130100549.8F889168016@codespeak.net> Author: arigo Date: Mon Nov 30 11:05:48 2009 New Revision: 69750 Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_recursive.py Log: Fix test: because of recursion, we use CALL_MAY_FORCE systematically. Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_recursive.py Mon Nov 30 11:05:48 2009 @@ -144,10 +144,11 @@ f = self.get_interpreter(codes) assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE) == 42 - self.check_loops(int_add = 1, call = 1) + self.check_loops(int_add = 1, call_may_force = 1, call = 0) assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(int_add = 2, call = 0, guard_no_exception = 0) + self.check_loops(int_add = 2, call_may_force = 0, call = 0, + guard_no_exception = 0) def test_inline_jitdriver_check(self): code = "021" @@ -158,7 +159,7 @@ assert self.meta_interp(f, [0, 0, 0], optimizer=OPTIMIZER_SIMPLE, inline=True) == 42 - self.check_loops(call = 1) + self.check_loops(call_may_force = 1, call = 0) def test_inline_faulty_can_inline(self): code = "021" @@ -488,10 +489,10 @@ return loop(100) res = self.meta_interp(main, [0], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) - self.check_loops(call=1) + self.check_loops(call_may_force=1, call=0) res = self.meta_interp(main, [1], optimizer=OPTIMIZER_SIMPLE, trace_limit=TRACE_LIMIT) - self.check_loops(call=0) + self.check_loops(call_may_force=0, call=0) def test_leave_jit_hook(self): from pypy.rpython.annlowlevel import hlstr @@ -645,7 +646,7 @@ result += f('-c-----------l-', i+100) self.meta_interp(g, [10], backendopt=True) self.check_aborted_count(1) - self.check_history(call=1) + self.check_history(call_may_force=1, call=0) self.check_tree_loop_count(3) From arigo at codespeak.net Mon Nov 30 11:07:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 11:07:58 +0100 (CET) Subject: [pypy-svn] r69751 - pypy/branch/virtual-forcing/pypy/jit/backend/x86/test Message-ID: <20091130100758.11323168016@codespeak.net> Author: arigo Date: Mon Nov 30 11:07:56 2009 New Revision: 69751 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_gc_integration.py Log: Fix this test to allow a result based on FRAME_FIXED_SIZE. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/test/test_gc_integration.py Mon Nov 30 11:07:56 2009 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.x86.runner import CPU -from pypy.jit.backend.x86.regalloc import RegAlloc, WORD +from pypy.jit.backend.x86.regalloc import RegAlloc, WORD, FRAME_FIXED_SIZE from pypy.jit.metainterp.test.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.annlowlevel import llhelper @@ -83,7 +83,8 @@ # mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) assert mark[0] == 'compressed' - expected = ['ebx', 'esi', 'edi', -16, -20, -24] + base = -WORD * FRAME_FIXED_SIZE + expected = ['ebx', 'esi', 'edi', base, base-4, base-8] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) class TestRegallocGcIntegration(BaseTestRegalloc): From cfbolz at codespeak.net Mon Nov 30 11:51:14 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 11:51:14 +0100 (CET) Subject: [pypy-svn] r69752 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20091130105114.E358216801B@codespeak.net> Author: cfbolz Date: Mon Nov 30 11:51:13 2009 New Revision: 69752 Modified: pypy/trunk/pypy/objspace/std/inlinedict.py pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Log: Fix the bug Maciej found. It was not about slots at all, but the old .__dict__ that was at some point being read out of an object no longer being valid if .__dict__ is written to. Kill the apptest and write a unit-test for the problem. Modified: pypy/trunk/pypy/objspace/std/inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/inlinedict.py Mon Nov 30 11:51:13 2009 @@ -118,6 +118,10 @@ return True def setdict(self, space, w_dict): + # if somebody asked for the __dict__, and it did not devolve, it + # needs to stay valid even if we set a new __dict__ on this object + if self.w__dict__ is not None and self._inlined_dict_valid(): + make_rdict(self) self._clear_fields() # invalidate attributes on self self.w__dict__ = check_new_dictionary(space, w_dict) Modified: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Mon Nov 30 11:51:13 2009 @@ -56,6 +56,21 @@ w_dict1 = obj1.getdict() obj2 = self.make_obj() w_dict2 = obj2.getdict() + obj2.setdict(self.space, w_dict1) + assert obj2.getdictvalue(self.fakespace, "hello") == 1 + assert obj2.getdictvalue(self.fakespace, "world") == 2 + obj1.setdictvalue(self.fakespace, "hello", 4) + obj1.setdictvalue(self.fakespace, "world", 5) + assert obj2.getdictvalue(self.fakespace, "hello") == 4 + assert obj2.getdictvalue(self.fakespace, "world") == 5 + assert w_dict2.getitem("hello") == 1 + assert w_dict2.getitem("world") == 2 + + def test_setdict_devolves_existing_dict(self): + obj1 = self.make_obj() + w_dict1 = obj1.getdict() + obj2 = self.make_obj() + w_dict2 = obj2.getdict() w_dict2.setitem(4, 1) # devolve dict w_dict2.setitem(5, 2) obj2.setdict(self.space, w_dict1) @@ -67,6 +82,7 @@ assert obj2.getdictvalue(self.fakespace, "world") == 5 + def test_dict_devolves_via_dict(self): obj = self.make_obj() w_dict = obj.getdict() @@ -82,6 +98,7 @@ obj.deldictvalue(self.fakespace, "hello") assert obj.getdictvalue(self.fakespace, "hello") is None + class TestMixinShared(TestMixin): Mixin = make_inlinedict_mixin(SharedDictImplementation, "structure") class FakeObject(Mixin): @@ -125,22 +142,3 @@ assert w_a.w__dict__ is None assert self.space.int_w(w_a.content['x']) == 12 assert self.space.int_w(w_a.content['y']) == 13 - -class AppTestSharingDict(object): - def setup_class(cls): - cls.space = gettestobjspace(withsharingdict=True, withinlineddict=True) - - @py.test.mark.xfail() - def test_bug(self): - class X(object): - __slots__ = 'a', 'b', 'c' - - class Y(X): - def __setattr__(self, name, value): - d = object.__getattribute__(self, '__dict__') - object.__setattr__(self, '__dict__', d) - return object.__setattr__(self, name, value) - - x = Y() - x.number = 42 - assert x.number == 42 From antocuni at codespeak.net Mon Nov 30 11:51:57 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 30 Nov 2009 11:51:57 +0100 (CET) Subject: [pypy-svn] r69753 - in pypy/trunk/pypy/jit/backend/cli: . test Message-ID: <20091130105157.BCFDB16801B@codespeak.net> Author: antocuni Date: Mon Nov 30 11:51:57 2009 New Revision: 69753 Modified: pypy/trunk/pypy/jit/backend/cli/method.py pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Log: - fix test_runner.test_compile_bridge_with_holes - skip few test_basic tests that has no chance to work without translation Modified: pypy/trunk/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/method.py (original) +++ pypy/trunk/pypy/jit/backend/cli/method.py Mon Nov 30 11:51:57 2009 @@ -259,6 +259,7 @@ return v def match_var_fox_boxes(self, failargs, inputargs): + failargs = [arg for arg in failargs if arg is not None] assert len(failargs) == len(inputargs) for i in range(len(failargs)): v = self.boxes[failargs[i]] Modified: pypy/trunk/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/trunk/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/trunk/pypy/jit/backend/cli/test/test_basic.py Mon Nov 30 11:51:57 2009 @@ -48,7 +48,12 @@ test_oosend_look_inside_only_one = skip test_residual_external_call = skip test_merge_guardclass_guardvalue = skip + test_merge_guardnonnull_guardclass = skip + test_merge_guardnonnull_guardvalue = skip + test_merge_guardnonnull_guardvalue_2 = skip + test_merge_guardnonnull_guardclass_guardvalue = skip test_residual_call_doesnt_lose_info = skip test_oohash = skip test_identityhash = skip test_guard_isnull_nonnull = skip + test_r_dict = skip From cfbolz at codespeak.net Mon Nov 30 12:01:44 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 12:01:44 +0100 (CET) Subject: [pypy-svn] r69755 - pypy/trunk/pypy/objspace/std/test Message-ID: <20091130110144.4AA3A16801B@codespeak.net> Author: cfbolz Date: Mon Nov 30 12:01:43 2009 New Revision: 69755 Modified: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Log: Cover anther case that was also fixed by r69752 Modified: pypy/trunk/pypy/objspace/std/test/test_inlinedict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_inlinedict.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_inlinedict.py Mon Nov 30 12:01:43 2009 @@ -66,7 +66,7 @@ assert w_dict2.getitem("hello") == 1 assert w_dict2.getitem("world") == 2 - def test_setdict_devolves_existing_dict(self): + def test_setdict_keeps_previous_dict_working(self): obj1 = self.make_obj() w_dict1 = obj1.getdict() obj2 = self.make_obj() @@ -81,7 +81,22 @@ assert obj2.getdictvalue(self.fakespace, "hello") == 4 assert obj2.getdictvalue(self.fakespace, "world") == 5 - + def test_setdict_devolves_existing_dict(self): + obj1 = self.make_obj() + w_dict1 = obj1.getdict() + obj2 = self.make_obj() + obj2.setdictvalue(self.fakespace, "hello", 6) + obj2.setdictvalue(self.fakespace, "world", 7) + w_dict2 = obj2.getdict() + obj2.setdict(self.space, w_dict1) + assert w_dict2.getitem("hello") == 6 + assert w_dict2.getitem("world") == 7 + assert obj2.getdictvalue(self.fakespace, "hello") == 1 + assert obj2.getdictvalue(self.fakespace, "world") == 2 + obj1.setdictvalue(self.fakespace, "hello", 4) + obj1.setdictvalue(self.fakespace, "world", 5) + assert obj2.getdictvalue(self.fakespace, "hello") == 4 + assert obj2.getdictvalue(self.fakespace, "world") == 5 def test_dict_devolves_via_dict(self): obj = self.make_obj() From xoraxax at codespeak.net Mon Nov 30 12:03:20 2009 From: xoraxax at codespeak.net (xoraxax at codespeak.net) Date: Mon, 30 Nov 2009 12:03:20 +0100 (CET) Subject: [pypy-svn] r69756 - pypy/trunk/pypy/lib Message-ID: <20091130110320.22B8916801B@codespeak.net> Author: xoraxax Date: Mon Nov 30 12:03:19 2009 New Revision: 69756 Added: pypy/trunk/pypy/lib/PyQt4.py pypy/trunk/pypy/lib/_rpyc_support.py pypy/trunk/pypy/lib/sip.py Log: Added RPyC support code and the necessary proxy modules to run PyQt4 applications on PyPy. Added: pypy/trunk/pypy/lib/PyQt4.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/PyQt4.py Mon Nov 30 12:03:19 2009 @@ -0,0 +1,9 @@ +from _rpyc_support import proxy_sub_module, remote_eval + + +for name in ("QtCore", "QtGui", "QtWebKit"): + proxy_sub_module(globals(), name) + +s = "__import__('PyQt4').QtGui.QDialogButtonBox." +QtGui.QDialogButtonBox.Cancel = remote_eval("%sCancel | %sCancel" % (s, s)) +QtGui.QDialogButtonBox.Ok = remote_eval("%sOk | %sOk" % (s, s)) Added: pypy/trunk/pypy/lib/_rpyc_support.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/_rpyc_support.py Mon Nov 30 12:03:19 2009 @@ -0,0 +1,24 @@ +import sys +import socket + +from rpyc import connect, SlaveService +from rpyc.utils.classic import DEFAULT_SERVER_PORT + +try: + conn = connect("localhost", DEFAULT_SERVER_PORT, SlaveService, + config=dict(call_by_value_for_builtin_mutable_types=True)) +except socket.error, e: + raise ImportError("Error while connecting: " + str(e)) + + +remote_eval = conn.eval + + +def proxy_module(globals): + module = getattr(conn.modules, globals["__name__"]) + for name in module.__dict__.keys(): + globals[name] = getattr(module, name) + +def proxy_sub_module(globals, name): + fullname = globals["__name__"] + "." + name + sys.modules[fullname] = globals[name] = conn.modules[fullname] Added: pypy/trunk/pypy/lib/sip.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/lib/sip.py Mon Nov 30 12:03:19 2009 @@ -0,0 +1,4 @@ +from _rpyc_support import proxy_module + +proxy_module(globals()) +del proxy_module From arigo at codespeak.net Mon Nov 30 12:06:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 12:06:48 +0100 (CET) Subject: [pypy-svn] r69757 - in pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform: . test Message-ID: <20091130110648.5112316801B@codespeak.net> Author: arigo Date: Mon Nov 30 12:06:47 2009 New Revision: 69757 Modified: pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py Log: Test and fix: the CollectAnalyzer must say "yes" when seeing an external call to a function that can release the GIL, because around such calls we must write the stack push/pops too. Fixes pypy/module/thread/test/test_gil.py. Modified: pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/framework.py Mon Nov 30 12:06:47 2009 @@ -29,12 +29,15 @@ def analyze_direct_call(self, graph, seen=None): try: func = graph.func + except AttributeError: + pass + else: if func is rstack.stack_check: return self.translator.config.translation.stackless - if func._gctransformer_hint_cannot_collect_: + if getattr(func, '_gctransformer_hint_cannot_collect_', False): return False - except AttributeError: - pass + if getattr(func, '_gctransformer_hint_close_stack_', False): + return True return graphanalyze.GraphAnalyzer.analyze_direct_call(self, graph, seen) Modified: pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/memory/gctransform/test/test_framework.py Mon Nov 30 12:06:47 2009 @@ -6,7 +6,7 @@ from pypy.rpython.memory.gctransform.transform import GcHighLevelOp from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer, \ CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.rtyper import LowLevelOpList from pypy.translator.c.gc import FrameworkGcPolicy from pypy.translator.translator import TranslationContext, graphof @@ -87,6 +87,33 @@ can_collect = CollectAnalyzer(t).analyze_direct_call(with_check_graph) assert can_collect +def test_cancollect_external(): + fext1 = rffi.llexternal('fext1', [], lltype.Void, threadsafe=False) + def g(): + fext1() + t = rtype(g, []) + gg = graphof(t, g) + assert not CollectAnalyzer(t).analyze_direct_call(gg) + + fext2 = rffi.llexternal('fext2', [], lltype.Void, threadsafe=True) + def g(): + fext2() + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + + S = lltype.GcStruct('S', ('x', lltype.Signed)) + FUNC = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) + fext3 = rffi.llexternal('fext3', [FUNC], lltype.Void, threadsafe=False) + def h(x): + lltype.malloc(S, zero=True) + def g(): + fext3(h) + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + + class WriteBarrierTransformer(FrameworkGCTransformer): clean_sets = {} GC_PARAMS = {} From afa at codespeak.net Mon Nov 30 12:10:35 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 30 Nov 2009 12:10:35 +0100 (CET) Subject: [pypy-svn] r69759 - pypy/trunk/pypy/translator/c/src Message-ID: <20091130111035.DDEA816801B@codespeak.net> Author: afa Date: Mon Nov 30 12:10:35 2009 New Revision: 69759 Modified: pypy/trunk/pypy/translator/c/src/debug.h pypy/trunk/pypy/translator/c/src/instrument.h Log: WIN32 is not defined on all windows compilers. Use #ifdef _WIN32 instead. Modified: pypy/trunk/pypy/translator/c/src/debug.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/debug.h (original) +++ pypy/trunk/pypy/translator/c/src/debug.h Mon Nov 30 12:10:35 2009 @@ -115,7 +115,7 @@ /* asm_xxx.h may contain a specific implementation of READ_TIMESTAMP. * This is the default generic timestamp implementation. */ -# ifdef WIN32 +# ifdef _WIN32 # define READ_TIMESTAMP(val) QueryPerformanceCounter(&(val)) # else # include Modified: pypy/trunk/pypy/translator/c/src/instrument.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/instrument.h (original) +++ pypy/trunk/pypy/translator/c/src/instrument.h Mon Nov 30 12:10:35 2009 @@ -9,7 +9,7 @@ #include #include #include -#ifndef WIN32 +#ifndef _WIN32 #include #include #include @@ -26,7 +26,7 @@ char *fname = getenv("_INSTRUMENT_COUNTERS"); if (fname) { int fd; -#ifdef WIN32 +#ifdef _WIN32 HANDLE map_handle; HANDLE file_handle; #endif @@ -36,7 +36,7 @@ if (sz > 0) { lseek(fd, sz-1, SEEK_SET); write(fd, "", 1); -#ifndef WIN32 +#ifndef _WIN32 buf = mmap(NULL, sz, PROT_WRITE|PROT_READ, MAP_SHARED, fd, 0); if (buf == MAP_FAILED) { From antocuni at codespeak.net Mon Nov 30 12:29:27 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 30 Nov 2009 12:29:27 +0100 (CET) Subject: [pypy-svn] r69763 - pypy/branch/cli-jit Message-ID: <20091130112927.C6E53168023@codespeak.net> Author: antocuni Date: Mon Nov 30 12:29:27 2009 New Revision: 69763 Added: pypy/branch/cli-jit/ - copied from r69762, pypy/trunk/ Log: a branch to continue my work on the CLI JIT backend, in the meantime while it's broken on trunk From cfbolz at codespeak.net Mon Nov 30 14:28:56 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 14:28:56 +0100 (CET) Subject: [pypy-svn] r69764 - pypy/trunk/pypy/objspace/std Message-ID: <20091130132856.AC2BF168023@codespeak.net> Author: cfbolz Date: Mon Nov 30 14:28:55 2009 New Revision: 69764 Modified: pypy/trunk/pypy/objspace/std/intobject.py pypy/trunk/pypy/objspace/std/smallintobject.py Log: Vacation playing around: Clean up small integers a tiny bit: - kill outdated comments - share comparison code with intobject.py - get rid of a number of overflow checks. Many overflows cannot occur, because the .intvalue of a W_SmallIntObject fits into 31/63 bits, so the result of e.g. an addition will always fit into 32/64 bits. Modified: pypy/trunk/pypy/objspace/std/intobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/intobject.py (original) +++ pypy/trunk/pypy/objspace/std/intobject.py Mon Nov 30 14:28:55 2009 @@ -53,7 +53,8 @@ str__Int = repr__Int -def declare_new_int_comparison(opname): +def declare_new_int_comparison(opname, clsname): + # also used by smallintobject.py import operator from pypy.tool.sourcetools import func_with_new_name op = getattr(operator, opname) @@ -61,11 +62,11 @@ i = w_int1.intval j = w_int2.intval return space.newbool(op(i, j)) - name = opname + "__Int_Int" + name = "%s__%s_%s" % (opname, clsname, clsname) return func_with_new_name(f, name), name for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op) + func, name = declare_new_int_comparison(op, "Int") globals()[name] = func def hash__Int(space, w_int1): Modified: pypy/trunk/pypy/objspace/std/smallintobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/smallintobject.py (original) +++ pypy/trunk/pypy/objspace/std/smallintobject.py Mon Nov 30 14:28:55 2009 @@ -6,7 +6,7 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint from pypy.objspace.std.inttype import wrapint -from pypy.objspace.std.intobject import W_IntObject +from pypy.objspace.std.intobject import W_IntObject, declare_new_int_comparison from pypy.rlib.objectmodel import UnboxedValue # XXX this is a complete copy of intobject.py. Find a better but still @@ -37,14 +37,6 @@ return space.newcomplex(float(w_small.intval), 0.0) -""" -XXX not implemented: -free list -FromString -FromUnicode -print -""" - def int_w__SmallInt(space, w_int1): return int(w_int1.intval) @@ -63,20 +55,8 @@ str__SmallInt = repr__SmallInt - -def declare_new_int_comparison(opname): - import operator - from pypy.tool.sourcetools import func_with_new_name - op = getattr(operator, opname) - def f(space, w_int1, w_int2): - i = w_int1.intval - j = w_int2.intval - return space.newbool(op(i, j)) - name = opname + "__SmallInt_SmallInt" - return func_with_new_name(f, name), name - for op in ['lt', 'le', 'eq', 'ne', 'gt', 'ge']: - func, name = declare_new_int_comparison(op) + func, name = declare_new_int_comparison(op, "SmallInt") globals()[name] = func def hash__SmallInt(space, w_int1): @@ -93,22 +73,17 @@ def add__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval - try: - z = ovfcheck(x + y) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer addition")) - return wrapint(space, z) + # note that no overflow checking is necessary here: x and y fit into 31 + # bits (or 63 bits respectively), so their sum fits into 32 (or 64) bits. + # wrapint then makes sure that either a tagged int or a normal int is + # created + return wrapint(space, x + y) def sub__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval - try: - z = ovfcheck(x - y) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer substraction")) - return wrapint(space, z) + # see comment in add__SmallInt_SmallInt + return wrapint(space, x - y) def mul__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval @@ -123,15 +98,11 @@ def div__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval - try: - z = ovfcheck(x // y) - except ZeroDivisionError: + if y == 0: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer division by zero")) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer division")) - return wrapint(space, z) + # no overflow possible + return wrapint(space, x // y) floordiv__SmallInt_SmallInt = div__SmallInt_SmallInt @@ -145,28 +116,20 @@ def mod__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval - try: - z = ovfcheck(x % y) - except ZeroDivisionError: + if y == 0: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer modulo by zero")) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer modulo")) - return wrapint(space, z) + # no overflow possible + return wrapint(space, x % y) def divmod__SmallInt_SmallInt(space, w_int1, w_int2): x = w_int1.intval y = w_int2.intval - try: - z = ovfcheck(x // y) - except ZeroDivisionError: + if y == 0: raise OperationError(space.w_ZeroDivisionError, space.wrap("integer divmod by zero")) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer modulo")) # no overflow possible + z = x // y m = x % y return space.newtuple([space.wrap(z), space.wrap(m)]) @@ -188,12 +151,8 @@ def neg__SmallInt(space, w_int1): a = w_int1.intval - try: - x = ovfcheck(-a) - except OverflowError: - raise FailedToImplement(space.w_OverflowError, - space.wrap("integer negation")) - return wrapint(space, x) + # no overflow possible since a fits into 31/63 bits + return wrapint(space, -a) def abs__SmallInt(space, w_int1): @@ -221,20 +180,8 @@ if b >= LONG_BIT: raise FailedToImplement(space.w_OverflowError, space.wrap("integer left shift")) - ## - ## XXX please! have a look into pyport.h and see how to implement - ## the overflow checking, using macro Py_ARITHMETIC_RIGHT_SHIFT - ## we *assume* that the overflow checking is done correctly - ## in the code generator, which is not trivial! - - ## XXX also note that Python 2.3 returns a long and never raises - ## OverflowError. try: c = ovfcheck_lshift(a, b) - ## the test in C code is - ## if (a != Py_ARITHMETIC_RIGHT_SHIFT(long, c, b)) { - ## if (PyErr_Warn(PyExc_FutureWarning, - # and so on except OverflowError: raise FailedToImplement(space.w_OverflowError, space.wrap("integer left shift")) @@ -254,8 +201,6 @@ else: a = 0 else: - ## please look into pyport.h, how >> should be implemented! - ## a = Py_ARITHMETIC_RIGHT_SHIFT(long, a, b); a = a >> b return wrapint(space, a) @@ -277,19 +222,6 @@ res = a | b return wrapint(space, res) -# coerce is not wanted -## -##static int -##coerce__Int(PyObject **pv, PyObject **pw) -##{ -## if (PyInt_Check(*pw)) { -## Py_INCREF(*pv); -## Py_INCREF(*pw); -## return 0; -## } -## return 1; /* Can't do it */ -##} - # int__SmallInt is supposed to do nothing, unless it has # a derived integer object, where it should return # an exact one. @@ -297,17 +229,9 @@ if space.is_w(space.type(w_int1), space.w_int): return w_int1 a = w_int1.intval - return wrapint(space, a) + return W_SmallIntObject(a) pos__SmallInt = int__SmallInt -""" -# Not registered -def long__SmallInt(space, w_int1): - a = w_int1.intval - x = long(a) ## XXX should this really be done so? - return space.newlong(x) -""" - def float__SmallInt(space, w_int1): a = w_int1.intval x = float(a) From arigo at codespeak.net Mon Nov 30 14:40:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 14:40:55 +0100 (CET) Subject: [pypy-svn] r69765 - in pypy/branch/virtual-forcing/pypy/jit: backend/llgraph metainterp/test Message-ID: <20091130134055.7CBED168023@codespeak.net> Author: arigo Date: Mon Nov 30 14:40:54 2009 New Revision: 69765 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py Log: Someone forgot that the llgraph.Descr has custom __eq__ and __hash__ functions... Argh, took a while. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Mon Nov 30 14:40:54 2009 @@ -34,17 +34,19 @@ return self.extrainfo def __hash__(self): - return hash((self.ofs, self.typeinfo)) + return hash((self.ofs, self.typeinfo, self.extrainfo)) def __eq__(self, other): if not isinstance(other, Descr): return NotImplemented - return self.ofs == other.ofs and self.typeinfo == other.typeinfo + return (self.ofs == other.ofs and self.typeinfo == other.typeinfo + and self.extrainfo == other.extrainfo) def __ne__(self, other): if not isinstance(other, Descr): return NotImplemented - return self.ofs != other.ofs or self.typeinfo != other.typeinfo + return (self.ofs != other.ofs or self.typeinfo != other.typeinfo + or self.extrainfo != other.extrainfo) def sort_key(self): return self.ofs Modified: pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/metainterp/test/test_del.py Mon Nov 30 14:40:54 2009 @@ -78,9 +78,7 @@ return 1 res = self.meta_interp(f, [20], optimizer=OPTIMIZER_SIMPLE) assert res == 1 - self.check_loops(call_may_force=1) - # for the case B(), but not for the case A() - # XXX it should really be 'call', not 'call_may_force'. + self.check_loops(call=1) # for the case B(), but not for the case A() class TestLLtype(DelTests, LLJitMixin): From arigo at codespeak.net Mon Nov 30 14:44:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 14:44:56 +0100 (CET) Subject: [pypy-svn] r69766 - pypy/branch/virtual-forcing/pypy/jit/backend/llgraph Message-ID: <20091130134456.12317168023@codespeak.net> Author: arigo Date: Mon Nov 30 14:44:56 2009 New Revision: 69766 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Log: Mention it explicitly in a comment. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Mon Nov 30 14:44:56 2009 @@ -29,6 +29,8 @@ self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo + # Note: this class has a custom __eq__, so don't forget + # to fix it if you add more fields. def get_extra_info(self): return self.extrainfo From antocuni at codespeak.net Mon Nov 30 14:54:36 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 30 Nov 2009 14:54:36 +0100 (CET) Subject: [pypy-svn] r69767 - pypy/trunk/pypy/lib Message-ID: <20091130135436.A8E9A168023@codespeak.net> Author: antocuni Date: Mon Nov 30 14:54:36 2009 New Revision: 69767 Modified: pypy/trunk/pypy/lib/_pypy_irc_topic.py Log: update irc topics Modified: pypy/trunk/pypy/lib/_pypy_irc_topic.py ============================================================================== --- pypy/trunk/pypy/lib/_pypy_irc_topic.py (original) +++ pypy/trunk/pypy/lib/_pypy_irc_topic.py Mon Nov 30 14:54:36 2009 @@ -135,6 +135,50 @@ Gur rkgen oynax yvarf gung clcl cevagf haqre jvaqbjf pbzr sebz qvfghgvyf gung qbrf abg svaq Ivfhny Fghqvb 6 ClCl 1.1.0orgn eryrnfrq: uggc://pbqrfcrnx.arg/clcl/qvfg/clcl/qbp/eryrnfr-1.1.0.ugzy "gurer fubhyq or bar naq bayl bar boivbhf jnl gb qb vg". ClCl inevnag: "gurer pna or A unys-ohttl jnlf gb qb vg" +1.1 svany eryrnfrq: uggc://pbqrfcrnx.arg/clcl/qvfg/clcl/qbp/eryrnfr-1.1.0.ugzy +1.1 svany eryrnfrq | nzq64 naq ccp ner bayl ninvynoyr va ragrecevfr irefvba +Vf gurer n clcl gvzr? - vs lbh pna srry vg (?) gura gurer vf + ab, abezny jbex vf fhpu zhpu yrff gvevat guna inpngvbaf + ab, abezny jbex vf fb zhpu yrff gvevat guna inpngvbaf +SVEFG gurl vtaber lbh, gura gurl ynhtu ng lbh, gura gurl svtug lbh, gura lbh jva. +vg'f Fhaqnl, znlor +vg'f Fhaqnl, ntnva +"3 + 3 = 8" Nagb va gur WVG gnyx +RPBBC vf unccravat +RPBBC vf svavfurq +cflpb rngf bar oenva cre vapu bs cebterff +"Fbzrjurer vafvqr gur ynetr yhzorevat oybo bs ClCl gurer vf na ryrtnag pber gelvat gb trg bhg." - psobym + gur NV fgngr vf vaqrrq pybfr +gur yytencu onpxraq vf n PCH gbb +jung ner n srj snyfr cbfvgvirf orgjrra sevraqf +abg vaperqvoyl ryrtnag, ohg pbhyq jbex +Jura lbhe unzzre vf P++, rirelguvat ortvaf gb ybbx yvxr n guhzo. +Gurersber, fcrpvsvp vasbezngvba, V jnf va na vqrny pbagrkg, V unq gb ernyvmr gur snvgu +ClCl qrirybczrag: gur neg bs jnvgvat +ubj gb pbafgehpg gur oynpxubyr vagrecergre: jr erhfr gur genpvat bar, nqq ybgf bs vsf naq cenl +clcl vf zber fgnoyr guna qrovna +gur jbeyq qbrfa'g jnag hf gb xabj +guvatf jbexrq avpryl gvyy gbqnl whfg ol punapr +enaqbz ohtf va enaqbz (abg n shaal wbxr) +ab enaqbz ohtf va enaqbz, nsgre nyy +vg'f ornhgvshy: irel erpgnathyne naq lryybj +V jnag n terra gevnathyne lryybj cbal +Vf vg n pnpghf oht be ceboyrz jvgu zl jne? +vg'f qhpg gncr nyy gur jnl qbja ohg abg gur jnl hc +"ribyhgrq" = havba bs "ribyirq" naq "pbaibyhgrq" +vg'f abafrafr nyy gur jnl qbja, naq nyfb nyy gur jnl hc +vg'f abg pbzcyrgryl fnar, ohg gung unf arire fgbccrq hf +gur shgher unf whfg ortha +jung jr npuvrirq gbqnl: jr qvfphffrq gur cbalarff bs gvzr-znpuvarf +Gur gvzr qrpernfrf! +"Jul qb jr raq hc va n aba pbzzba pnfr va gur pbzzba pnfr?" (svwny) +"nu, whfg va gvzr qbphzragngvba" (__nc__) +ClCl vf abg n erny IZ: ab frtsnhyg unaqyref gb qb gur ener pnfrf +lbh pna'g unir obgu pbairavrapr naq fcrrq +gur WVG qbrfa'g jbex ba BF/K (abi'09) +ab fhccbeg sbe BF/K evtug abj! (abi'09) +fyvccref urvtug pna or zrnfherq va k86 ertvfgref +clcl vf n enpr orgjrra gur vaqhfgel gelvat gb ohvyq znpuvarf jvgu zber naq zber erfbheprf, naq gur clcl qrirybcref gelvat gb rng nyy bs gurz. Fb sne, gur jvaare vf fgvyy hapyrne """ def some_topic(): From arigo at codespeak.net Mon Nov 30 15:10:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 15:10:46 +0100 (CET) Subject: [pypy-svn] r69768 - pypy/branch/virtual-forcing/pypy/jit/backend/llgraph Message-ID: <20091130141046.E8B86168023@codespeak.net> Author: arigo Date: Mon Nov 30 15:10:46 2009 New Revision: 69768 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Log: Modernize the descr construction and caching. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/llgraph/runner.py Mon Nov 30 15:10:46 2009 @@ -21,35 +21,16 @@ class Descr(history.AbstractDescr): - name = None - ofs = -1 - typeinfo = '?' - - def __init__(self, ofs, typeinfo='?', extrainfo=None): + + def __init__(self, ofs, typeinfo, extrainfo=None, name=None): self.ofs = ofs self.typeinfo = typeinfo self.extrainfo = extrainfo - # Note: this class has a custom __eq__, so don't forget - # to fix it if you add more fields. + self.name = name def get_extra_info(self): return self.extrainfo - def __hash__(self): - return hash((self.ofs, self.typeinfo, self.extrainfo)) - - def __eq__(self, other): - if not isinstance(other, Descr): - return NotImplemented - return (self.ofs == other.ofs and self.typeinfo == other.typeinfo - and self.extrainfo == other.extrainfo) - - def __ne__(self, other): - if not isinstance(other, Descr): - return NotImplemented - return (self.ofs != other.ofs or self.typeinfo != other.typeinfo - or self.extrainfo != other.extrainfo) - def sort_key(self): return self.ofs @@ -80,9 +61,12 @@ raise TypeError("cannot use comparison on Descrs") def __repr__(self): + args = [repr(self.ofs), repr(self.typeinfo)] if self.name is not None: - return '' % (self.ofs, self.typeinfo, self.name) - return '' % (self.ofs, self.typeinfo) + args.append(repr(self.name)) + if self.extrainfo is not None: + args.append('E') + return '' % (', '.join(args),) history.TreeLoop._compiled_version = lltype.nullptr(llimpl.COMPILEDLOOP.TO) @@ -104,11 +88,21 @@ llimpl._stats = self.stats llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] + self._descrs = {} def _freeze_(self): assert self.translate_support_code return False + def getdescr(self, ofs, typeinfo='?', extrainfo=None, name=None): + key = (ofs, typeinfo, extrainfo, name) + try: + return self._descrs[key] + except KeyError: + descr = Descr(ofs, typeinfo, extrainfo, name) + self._descrs[key] = descr + return descr + def set_class_sizes(self, class_sizes): self.class_sizes = class_sizes for vtable, size in class_sizes.items(): @@ -261,16 +255,9 @@ return (self.cast_adr_to_int(llimpl.get_zero_division_error()), llimpl.get_zero_division_error_value()) - @staticmethod - def sizeof(S): + def sizeof(self, S): assert not isinstance(S, lltype.Ptr) - return Descr(symbolic.get_size(S)) - - @staticmethod - def numof(S): - return 4 - - ##addresssuffix = '4' + return self.getdescr(symbolic.get_size(S)) def cast_adr_to_int(self, adr): return llimpl.cast_adr_to_int(self.memo_cast, adr) @@ -291,18 +278,14 @@ BaseCPU.__init__(self, *args, **kwds) self.fielddescrof_vtable = self.fielddescrof(rclass.OBJECT, 'typeptr') - @staticmethod - def fielddescrof(S, fieldname): + def fielddescrof(self, S, fieldname): ofs, size = symbolic.get_field_token(S, fieldname) token = history.getkind(getattr(S, fieldname)) - res = Descr(ofs, token[0]) - res.name = fieldname - return res + return self.getdescr(ofs, token[0], name=fieldname) - @staticmethod - def calldescrof(FUNC, ARGS, RESULT, extrainfo=None): + def calldescrof(self, FUNC, ARGS, RESULT, extrainfo=None): token = history.getkind(RESULT) - return Descr(0, token[0], extrainfo=extrainfo) + return self.getdescr(0, token[0], extrainfo=extrainfo) def get_exception(self): return self.cast_adr_to_int(llimpl.get_exception()) @@ -310,13 +293,12 @@ def get_exc_value(self): return llimpl.get_exc_value() - @staticmethod - def arraydescrof(A): + def arraydescrof(self, A): assert isinstance(A, lltype.GcArray) assert A.OF != lltype.Void size = symbolic.get_size(A) token = history.getkind(A.OF) - return Descr(size, token[0]) + return self.getdescr(size, token[0]) # ---------- the backend-dependent operations ---------- From arigo at codespeak.net Mon Nov 30 15:38:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 15:38:18 +0100 (CET) Subject: [pypy-svn] r69770 - pypy/branch/virtual-forcing/pypy Message-ID: <20091130143818.B5815168023@codespeak.net> Author: arigo Date: Mon Nov 30 15:38:18 2009 New Revision: 69770 Modified: pypy/branch/virtual-forcing/pypy/testrunner_cfg.py Log: Found the cause of "py.test failed" on buildbot. Modified: pypy/branch/virtual-forcing/pypy/testrunner_cfg.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/testrunner_cfg.py (original) +++ pypy/branch/virtual-forcing/pypy/testrunner_cfg.py Mon Nov 30 15:38:18 2009 @@ -7,7 +7,8 @@ reldir.startswith('rlib/test') or reldir.startswith('rpython/memory/') or reldir.startswith('jit/backend/x86/') or - reldir.startswith('jit/backend/cli')): + #reldir.startswith('jit/backend/cli') or + 0): testdirs.extend(tests) else: testdirs.append(reldir) From pedronis at codespeak.net Mon Nov 30 16:33:22 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Mon, 30 Nov 2009 16:33:22 +0100 (CET) Subject: [pypy-svn] r69771 - pypy/build/bot2/pypybuildbot Message-ID: <20091130153322.99B8A168027@codespeak.net> Author: pedronis Date: Mon Nov 30 16:33:21 2009 New Revision: 69771 Modified: pypy/build/bot2/pypybuildbot/master.py Log: experimental, add a builder that would test the same optimisations as with the jit but which builds something not using the jit, so to stress the optimisations alone Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Mon Nov 30 16:33:21 2009 @@ -71,6 +71,13 @@ pypyjit=True ) +pypy_OjitTranslatedTestFactory = pypybuilds.Translated( + translationArgs=['-Ojit', '--gc=hybrid', '--no-translation-jit' + '--gcrootfinder=asmgcc'], + lib_python=True, + app_tests=True + ) + pypyJITBenchmarkFactory = pypybuilds.JITBenchmark() pypyTranslatedLibPythonMaemoTestFactory = pypybuilds.TranslatedScratchbox() @@ -85,6 +92,7 @@ APPLVLFREEBSD64 = 'pypy-c-app-level-freebsd-7-x86-64' JITLINUX32 = "pypy-c-jit-linux-x86-32" +OJITLINUX32 = "pypy-c-Ojit-no-jit-linux-x86-32" JITONLYLINUX32 = "jitonly-own-linux-x86-32" JITBENCH = "jit-benchmark-linux-x86-32" @@ -97,7 +105,7 @@ 'change_source': [], 'schedulers': [ Nightly("nightly", [LINUX32, APPLVLLINUX32, APPLVLWIN32, - STACKLESSAPPLVLLINUX32, JITLINUX32, + STACKLESSAPPLVLLINUX32, JITLINUX32, OJITLINUX32, MACOSX32], hour=4, minute=45), Nightly("nightly-benchmark", [JITBENCH], @@ -133,7 +141,13 @@ "builddir": STACKLESSAPPLVLLINUX32, "factory": pypyStacklessTranslatedAppLevelTestFactory, "category": 'stackless' - }, + }, + {"name": OJITLINUX32, + "slavenames": ["wyvern", "cobra"], + "builddir": OJITLINUX32, + "factory": pypy_OjitTranslatedTestFactory, + "category": 'applevel' + }, {"name": APPLVLWIN32, "slavenames": ["bigboard"], "builddir": APPLVLWIN32, From cfbolz at codespeak.net Mon Nov 30 17:13:05 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 17:13:05 +0100 (CET) Subject: [pypy-svn] r69772 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091130161305.B9CF0168029@codespeak.net> Author: cfbolz Date: Mon Nov 30 17:13:04 2009 New Revision: 69772 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/optimizeutil.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: Share the field descr lists between all virtual info objects that have the same layout, not just those coming from the same specific virtual instance. This is done by having a sharing-dict-like (or SELF map-like) approach. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon Nov 30 17:13:04 2009 @@ -9,7 +9,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode -from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs +from pypy.jit.metainterp.optimizeutil import av_newdict2, av_newdict_int, _findall from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile from pypy.jit.metainterp.typesystem import llhelper, oohelper @@ -173,20 +173,60 @@ raise NotImplementedError("abstract base") +class FieldMap(object): + def __init__(self, prev, newfield): + if prev is not None: + assert newfield is not None + self.fieldindexes = prev.fieldindexes.copy() + self.fieldindexes[newfield] = len(self.fieldindexes) + self.fieldlist = prev.fieldlist + [newfield] + else: + assert newfield is None + self.fieldindexes = av_newdict_int() + self.fieldlist = [] + self.nextmaps = av_newdict2() + + def getindex(self, field): + return self.fieldindexes.get(field, -1) + + def nextmap(self, field): + result = self.nextmaps.get(field, None) + if result is None: + result = FieldMap(self, field) + self.nextmaps[field] = result + return result + +def get_no_fields_map(cpu): + if hasattr(cpu, '_optimizeopt_fieldmap'): + return cpu._optimizeopt_fieldmap + res = FieldMap(None, None) + cpu._optimizeopt_fieldmap = res + return res +get_no_fields_map._annspecialcase_ = 'specialize:memo' + class AbstractVirtualStructValue(AbstractVirtualValue): - _attrs_ = ('_fields', '_cached_sorted_fields') + _attrs_ = ('_fieldmap', '_fieldvalues') def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) - self._fields = av_newdict2() - self._cached_sorted_fields = None + self._fieldmap = get_no_fields_map(optimizer.cpu) + self._fieldvalues = [] def getfield(self, ofs, default): - return self._fields.get(ofs, default) + i = self._fieldmap.getindex(ofs) + if i == -1: + return default + return self._fieldvalues[i] def setfield(self, ofs, fieldvalue): assert isinstance(fieldvalue, OptValue) - self._fields[ofs] = fieldvalue + i = self._fieldmap.getindex(ofs) + if i != -1: + self._fieldvalues[i] = fieldvalue + else: + self._fieldmap = self._fieldmap.nextmap(ofs) + self._fieldvalues.append(fieldvalue) + def _really_force(self): assert self.source_op is not None @@ -194,39 +234,30 @@ newoperations.append(self.source_op) self.box = box = self.source_op.result # - iteritems = self._fields.iteritems() - if not we_are_translated(): #random order is fine, except for tests - iteritems = list(iteritems) - iteritems.sort(key = lambda (x,y): x.sort_key()) - for ofs, value in iteritems: + #if not we_are_translated(): #random order is fine, except for tests + # iteritems = self._fields.iteritems() + # iteritems = list(iteritems) + # iteritems.sort(key = lambda (x,y): x.sort_key()) + for i in range(len(self._fieldmap.fieldlist)): + ofs = self._fieldmap.fieldlist[i] + value = self._fieldvalues[i] subbox = value.force_box() op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, descr=ofs) newoperations.append(op) - self._fields = None + self._fieldvalues = None def _get_field_descr_list(self): - # this shares only per instance and not per type, but better than nothing - _cached_sorted_fields = self._cached_sorted_fields - if (_cached_sorted_fields is not None and - len(self._fields) == len(_cached_sorted_fields)): - lst = self._cached_sorted_fields - else: - lst = self._fields.keys() - sort_descrs(lst) - self._cached_sorted_fields = lst - return lst + return self._fieldmap.fieldlist def get_args_for_fail(self, modifier): + # modifier.already_seen_virtual() + # checks for recursion: it is False unless + # we have already seen the very same keybox if self.box is None and not modifier.already_seen_virtual(self.keybox): - # modifier.already_seen_virtual() - # checks for recursion: it is False unless - # we have already seen the very same keybox - lst = self._get_field_descr_list() - fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] + fieldboxes = [value.get_key_box() for value in self._fieldvalues] modifier.register_virtual_fields(self.keybox, fieldboxes) - for ofs in lst: - fieldvalue = self._fields[ofs] + for fieldvalue in self._fieldvalues: fieldvalue.get_args_for_fail(modifier) Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Mon Nov 30 17:13:04 2009 @@ -24,6 +24,9 @@ # the values... return r_dict(av_eq, av_hash) +def av_newdict_int(): + return r_dict(av_eq, av_hash) + def _findall(Class, name_prefix): result = [] for value, name in resoperation.opname.items(): Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 30 17:13:04 2009 @@ -63,16 +63,28 @@ assert fdescr.rd_consts == [] def test_sharing_field_lists_of_virtual(): - virt1 = optimizeopt.AbstractVirtualStructValue(None, None) + class FakeOptimizer(object): + class cpu(object): + pass + opt = FakeOptimizer() + virt1 = optimizeopt.AbstractVirtualStructValue(opt, None) lst1 = virt1._get_field_descr_list() assert lst1 == [] lst2 = virt1._get_field_descr_list() assert lst1 is lst2 virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) - lst1 = virt1._get_field_descr_list() - assert lst1 == [LLtypeMixin.valuedescr] - lst2 = virt1._get_field_descr_list() - assert lst1 is lst2 + lst3 = virt1._get_field_descr_list() + assert lst3 == [LLtypeMixin.valuedescr] + lst4 = virt1._get_field_descr_list() + assert lst3 is lst4 + + virt2 = optimizeopt.AbstractVirtualStructValue(opt, None) + lst5 = virt2._get_field_descr_list() + assert lst5 is lst1 + virt2.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) + lst6 = virt1._get_field_descr_list() + assert lst6 is lst3 + def test_reuse_vinfo(): class FakeVInfo(object): @@ -90,6 +102,15 @@ assert vinfo3 is vinfo4 +def test_fieldmap(): + map = optimizeopt.FieldMap(None, None) + assert map.getindex(LLtypeMixin.valuedescr) == -1 + map2 = map.nextmap(LLtypeMixin.valuedescr) + assert map2.getindex(LLtypeMixin.valuedescr) == 0 + assert map2.fieldlist == [LLtypeMixin.valuedescr] + map3 = map.nextmap(LLtypeMixin.valuedescr) + assert map2 is map3 + # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Mon Nov 30 17:13:04 2009 @@ -358,13 +358,18 @@ assert metainterp.framestack == fs2 +class FakeOptimizer_VirtualValue(object): + class cpu: + pass +fakeoptimizer = FakeOptimizer_VirtualValue() + def virtual_value(keybox, value, next): - vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), keybox) + vv = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), keybox) if not isinstance(next, OptValue): next = OptValue(next) - vv.setfield(LLtypeMixin.nextdescr, next) vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) + vv.setfield(LLtypeMixin.nextdescr, next) return vv def test_rebuild_from_resumedata_two_guards_w_virtuals(): @@ -797,18 +802,16 @@ modifier.liveboxes = {} modifier.vfieldboxes = {} - v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), b2s) - v2._fields = {LLtypeMixin.nextdescr: b4s, - LLtypeMixin.valuedescr: c1s} - v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] - v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu), b4s) - v4._fields = {LLtypeMixin.nextdescr: b2s, - LLtypeMixin.valuedescr: b3s, - LLtypeMixin.otherdescr: b5s} - v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, - LLtypeMixin.otherdescr] + v4.setfield(LLtypeMixin.nextdescr, OptValue(b2s)) + v4.setfield(LLtypeMixin.valuedescr, OptValue(b3s)) + v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s)) + v2 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), b2s) + v2.setfield(LLtypeMixin.nextdescr, v4) + v2.setfield(LLtypeMixin.valuedescr, OptValue(c1s)) + modifier.register_virtual_fields(b2s, [b4s, c1s]) modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) values = {b2s: v2, b4s: v4} @@ -870,6 +873,8 @@ modifier.vfieldboxes = {} class FakeOptimizer(object): + class cpu: + pass def new_const_item(self, descr): return None v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) @@ -918,9 +923,9 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} - v2 = VStructValue(None, LLtypeMixin.ssize, b2s) - v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} - v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] + v2 = VStructValue(fakeoptimizer, LLtypeMixin.ssize, b2s) + v2.setfield(LLtypeMixin.adescr, OptValue(c1s)) + v2.setfield(LLtypeMixin.bdescr, OptValue(b4s)) modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] modifier._number_virtuals(liveboxes, {b2s: v2}, 0) From cfbolz at codespeak.net Mon Nov 30 17:13:21 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 17:13:21 +0100 (CET) Subject: [pypy-svn] r69773 - pypy/extradoc/planning Message-ID: <20091130161321.D109E168029@codespeak.net> Author: cfbolz Date: Mon Nov 30 17:13:21 2009 New Revision: 69773 Modified: pypy/extradoc/planning/jit.txt Log: this is done Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Mon Nov 30 17:13:21 2009 @@ -13,8 +13,6 @@ - forcing virtualizables should only force fields affected, not everything -- share lists of field descrs - - compress resume data (?) - think out looking into functions or not, based on arguments, From arigo at codespeak.net Mon Nov 30 17:54:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 30 Nov 2009 17:54:16 +0100 (CET) Subject: [pypy-svn] r69774 - in pypy/branch/virtual-forcing/pypy: jit/backend/x86 rpython/lltypesystem Message-ID: <20091130165416.CE68A16802D@codespeak.net> Author: arigo Date: Mon Nov 30 17:54:16 2009 New Revision: 69774 Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Log: Fix a translation issue: we cannot write 'rffi.ptradd(self.all_null_registers, 16)' because that's constant-foldable, but the result is an ll2ctypes array of unknown length, which is not translatable. Crash a bit more explicitly in ll2ctypes in that case. Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/assembler.py Mon Nov 30 17:54:16 2009 @@ -878,8 +878,9 @@ arglocs.append(loc) return arglocs[:] - def grab_frame_values(self, bytecode, frame_addr, registers): - self.fail_ebp = registers[ebp.op] + def grab_frame_values(self, bytecode, frame_addr, allregisters): + # no malloc allowed here!! + self.fail_ebp = allregisters[16 + ebp.op] num = 0 value_hi = 0 while 1: @@ -916,11 +917,10 @@ break code >>= 2 if kind == self.DESCR_FLOAT: - xmmregisters = rffi.ptradd(registers, -16) - value = xmmregisters[2*code] - value_hi = xmmregisters[2*code + 1] + value = allregisters[2*code] + value_hi = allregisters[2*code + 1] else: - value = registers[code] + value = allregisters[16 + code] # store the loaded value into fail_boxes_ if kind == self.DESCR_INT: @@ -943,14 +943,14 @@ def setup_failure_recovery(self): def failure_recovery_func(registers): - # no malloc allowed here!! # 'registers' is a pointer to a structure containing the # original value of the registers, optionally the original # value of XMM registers, and finally a reference to the # recovery bytecode. See _build_failure_recovery() for details. stack_at_ebp = registers[ebp.op] bytecode = rffi.cast(rffi.UCHARP, registers[8]) - return self.grab_frame_values(bytecode, stack_at_ebp, registers) + allregisters = rffi.ptradd(registers, -16) + return self.grab_frame_values(bytecode, stack_at_ebp, allregisters) self.failure_recovery_func = failure_recovery_func self.failure_recovery_code = [0, 0, 0, 0] Modified: pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/virtual-forcing/pypy/jit/backend/x86/runner.py Mon Nov 30 17:54:16 2009 @@ -105,7 +105,7 @@ fail_index_2 = self.assembler.grab_frame_values( bytecode, addr_of_force_index - FORCE_INDEX_OFS, - rffi.ptradd(self.all_null_registers, 16)) + self.all_null_registers) self.assembler.leave_jitted_hook() # end of "no gc operation!" block assert fail_index == fail_index_2 Modified: pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/virtual-forcing/pypy/rpython/lltypesystem/ll2ctypes.py Mon Nov 30 17:54:16 2009 @@ -443,6 +443,9 @@ self._storage._setitem(index, value, boundscheck=False) def getitems(self): + if self._TYPE.OF != lltype.Char: + raise Exception("cannot get all items of an unknown-length " + "array of %r" % self._TYPE.OF) _items = [] i = 0 while 1: From cfbolz at codespeak.net Mon Nov 30 17:55:39 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 30 Nov 2009 17:55:39 +0100 (CET) Subject: [pypy-svn] r69775 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20091130165539.426D116802D@codespeak.net> Author: cfbolz Date: Mon Nov 30 17:55:38 2009 New Revision: 69775 Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py pypy/trunk/pypy/jit/metainterp/optimizeutil.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/test/test_resume.py Log: revert r69772, Armin thinks it's too complex as it cannot be understood immediately just by reading the diff. Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt.py Mon Nov 30 17:55:38 2009 @@ -9,7 +9,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import VirtualStructSpecNode -from pypy.jit.metainterp.optimizeutil import av_newdict2, av_newdict_int, _findall +from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs from pypy.jit.metainterp.optimizeutil import InvalidLoop from pypy.jit.metainterp import resume, compile from pypy.jit.metainterp.typesystem import llhelper, oohelper @@ -173,60 +173,20 @@ raise NotImplementedError("abstract base") -class FieldMap(object): - def __init__(self, prev, newfield): - if prev is not None: - assert newfield is not None - self.fieldindexes = prev.fieldindexes.copy() - self.fieldindexes[newfield] = len(self.fieldindexes) - self.fieldlist = prev.fieldlist + [newfield] - else: - assert newfield is None - self.fieldindexes = av_newdict_int() - self.fieldlist = [] - self.nextmaps = av_newdict2() - - def getindex(self, field): - return self.fieldindexes.get(field, -1) - - def nextmap(self, field): - result = self.nextmaps.get(field, None) - if result is None: - result = FieldMap(self, field) - self.nextmaps[field] = result - return result - -def get_no_fields_map(cpu): - if hasattr(cpu, '_optimizeopt_fieldmap'): - return cpu._optimizeopt_fieldmap - res = FieldMap(None, None) - cpu._optimizeopt_fieldmap = res - return res -get_no_fields_map._annspecialcase_ = 'specialize:memo' - class AbstractVirtualStructValue(AbstractVirtualValue): - _attrs_ = ('_fieldmap', '_fieldvalues') + _attrs_ = ('_fields', '_cached_sorted_fields') def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) - self._fieldmap = get_no_fields_map(optimizer.cpu) - self._fieldvalues = [] + self._fields = av_newdict2() + self._cached_sorted_fields = None def getfield(self, ofs, default): - i = self._fieldmap.getindex(ofs) - if i == -1: - return default - return self._fieldvalues[i] + return self._fields.get(ofs, default) def setfield(self, ofs, fieldvalue): assert isinstance(fieldvalue, OptValue) - i = self._fieldmap.getindex(ofs) - if i != -1: - self._fieldvalues[i] = fieldvalue - else: - self._fieldmap = self._fieldmap.nextmap(ofs) - self._fieldvalues.append(fieldvalue) - + self._fields[ofs] = fieldvalue def _really_force(self): assert self.source_op is not None @@ -234,30 +194,39 @@ newoperations.append(self.source_op) self.box = box = self.source_op.result # - #if not we_are_translated(): #random order is fine, except for tests - # iteritems = self._fields.iteritems() - # iteritems = list(iteritems) - # iteritems.sort(key = lambda (x,y): x.sort_key()) - for i in range(len(self._fieldmap.fieldlist)): - ofs = self._fieldmap.fieldlist[i] - value = self._fieldvalues[i] + iteritems = self._fields.iteritems() + if not we_are_translated(): #random order is fine, except for tests + iteritems = list(iteritems) + iteritems.sort(key = lambda (x,y): x.sort_key()) + for ofs, value in iteritems: subbox = value.force_box() op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, descr=ofs) newoperations.append(op) - self._fieldvalues = None + self._fields = None def _get_field_descr_list(self): - return self._fieldmap.fieldlist + # this shares only per instance and not per type, but better than nothing + _cached_sorted_fields = self._cached_sorted_fields + if (_cached_sorted_fields is not None and + len(self._fields) == len(_cached_sorted_fields)): + lst = self._cached_sorted_fields + else: + lst = self._fields.keys() + sort_descrs(lst) + self._cached_sorted_fields = lst + return lst def get_args_for_fail(self, modifier): - # modifier.already_seen_virtual() - # checks for recursion: it is False unless - # we have already seen the very same keybox if self.box is None and not modifier.already_seen_virtual(self.keybox): - fieldboxes = [value.get_key_box() for value in self._fieldvalues] + # modifier.already_seen_virtual() + # checks for recursion: it is False unless + # we have already seen the very same keybox + lst = self._get_field_descr_list() + fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] modifier.register_virtual_fields(self.keybox, fieldboxes) - for fieldvalue in self._fieldvalues: + for ofs in lst: + fieldvalue = self._fields[ofs] fieldvalue.get_args_for_fail(modifier) Modified: pypy/trunk/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeutil.py Mon Nov 30 17:55:38 2009 @@ -24,9 +24,6 @@ # the values... return r_dict(av_eq, av_hash) -def av_newdict_int(): - return r_dict(av_eq, av_hash) - def _findall(Class, name_prefix): result = [] for value, name in resoperation.opname.items(): @@ -55,3 +52,5 @@ def sort_descrs(lst): quicksort(lst, 0, len(lst)-1) + + Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Nov 30 17:55:38 2009 @@ -63,28 +63,16 @@ assert fdescr.rd_consts == [] def test_sharing_field_lists_of_virtual(): - class FakeOptimizer(object): - class cpu(object): - pass - opt = FakeOptimizer() - virt1 = optimizeopt.AbstractVirtualStructValue(opt, None) + virt1 = optimizeopt.AbstractVirtualStructValue(None, None) lst1 = virt1._get_field_descr_list() assert lst1 == [] lst2 = virt1._get_field_descr_list() assert lst1 is lst2 virt1.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) - lst3 = virt1._get_field_descr_list() - assert lst3 == [LLtypeMixin.valuedescr] - lst4 = virt1._get_field_descr_list() - assert lst3 is lst4 - - virt2 = optimizeopt.AbstractVirtualStructValue(opt, None) - lst5 = virt2._get_field_descr_list() - assert lst5 is lst1 - virt2.setfield(LLtypeMixin.valuedescr, optimizeopt.OptValue(None)) - lst6 = virt1._get_field_descr_list() - assert lst6 is lst3 - + lst1 = virt1._get_field_descr_list() + assert lst1 == [LLtypeMixin.valuedescr] + lst2 = virt1._get_field_descr_list() + assert lst1 is lst2 def test_reuse_vinfo(): class FakeVInfo(object): @@ -102,15 +90,6 @@ assert vinfo3 is vinfo4 -def test_fieldmap(): - map = optimizeopt.FieldMap(None, None) - assert map.getindex(LLtypeMixin.valuedescr) == -1 - map2 = map.nextmap(LLtypeMixin.valuedescr) - assert map2.getindex(LLtypeMixin.valuedescr) == 0 - assert map2.fieldlist == [LLtypeMixin.valuedescr] - map3 = map.nextmap(LLtypeMixin.valuedescr) - assert map2 is map3 - # ____________________________________________________________ def equaloplists(oplist1, oplist2, strict_fail_args=True, remap={}): Modified: pypy/trunk/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_resume.py Mon Nov 30 17:55:38 2009 @@ -358,18 +358,13 @@ assert metainterp.framestack == fs2 -class FakeOptimizer_VirtualValue(object): - class cpu: - pass -fakeoptimizer = FakeOptimizer_VirtualValue() - def virtual_value(keybox, value, next): - vv = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, - LLtypeMixin.cpu), keybox) + vv = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), keybox) if not isinstance(next, OptValue): next = OptValue(next) - vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) vv.setfield(LLtypeMixin.nextdescr, next) + vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) return vv def test_rebuild_from_resumedata_two_guards_w_virtuals(): @@ -802,16 +797,18 @@ modifier.liveboxes = {} modifier.vfieldboxes = {} - v4 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr2, - LLtypeMixin.cpu), b4s) - v4.setfield(LLtypeMixin.nextdescr, OptValue(b2s)) - v4.setfield(LLtypeMixin.valuedescr, OptValue(b3s)) - v4.setfield(LLtypeMixin.otherdescr, OptValue(b5s)) - v2 = VirtualValue(fakeoptimizer, ConstAddr(LLtypeMixin.node_vtable_adr, + v2 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), b2s) - v2.setfield(LLtypeMixin.nextdescr, v4) - v2.setfield(LLtypeMixin.valuedescr, OptValue(c1s)) - + v2._fields = {LLtypeMixin.nextdescr: b4s, + LLtypeMixin.valuedescr: c1s} + v2._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr] + v4 = VirtualValue(None, ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), b4s) + v4._fields = {LLtypeMixin.nextdescr: b2s, + LLtypeMixin.valuedescr: b3s, + LLtypeMixin.otherdescr: b5s} + v4._cached_sorted_fields = [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, + LLtypeMixin.otherdescr] modifier.register_virtual_fields(b2s, [b4s, c1s]) modifier.register_virtual_fields(b4s, [b2s, b3s, b5s]) values = {b2s: v2, b4s: v4} @@ -873,8 +870,6 @@ modifier.vfieldboxes = {} class FakeOptimizer(object): - class cpu: - pass def new_const_item(self, descr): return None v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s) @@ -923,9 +918,9 @@ modifier.liveboxes_from_env = {} modifier.liveboxes = {} modifier.vfieldboxes = {} - v2 = VStructValue(fakeoptimizer, LLtypeMixin.ssize, b2s) - v2.setfield(LLtypeMixin.adescr, OptValue(c1s)) - v2.setfield(LLtypeMixin.bdescr, OptValue(b4s)) + v2 = VStructValue(None, LLtypeMixin.ssize, b2s) + v2._fields = {LLtypeMixin.adescr: c1s, LLtypeMixin.bdescr: b4s} + v2._cached_sorted_fields = [LLtypeMixin.adescr, LLtypeMixin.bdescr] modifier.register_virtual_fields(b2s, [c1s, b4s]) liveboxes = [] modifier._number_virtuals(liveboxes, {b2s: v2}, 0) From afa at codespeak.net Mon Nov 30 17:57:52 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 30 Nov 2009 17:57:52 +0100 (CET) Subject: [pypy-svn] r69776 - in pypy/build/testrunner: . test Message-ID: <20091130165752.3307316802D@codespeak.net> Author: afa Date: Mon Nov 30 17:57:51 2009 New Revision: 69776 Modified: pypy/build/testrunner/runner.py pypy/build/testrunner/test/test_runner.py Log: Fix testrunner on Windows: - prefix the executable with the current dir, as the docs for subprocess say """this directory is not considered when searching the executable, so you can?t specify the program?s path relative to cwd.""" - when win32api is not installed, use ctypes to access the kernel API and kill the running process. --Cette ligne, et les suivantes ci-dessous, seront ignor?es-- M test/test_runner.py M runner.py Modified: pypy/build/testrunner/runner.py ============================================================================== --- pypy/build/testrunner/runner.py (original) +++ pypy/build/testrunner/runner.py Mon Nov 30 17:57:51 2009 @@ -8,7 +8,11 @@ import win32api, pywintypes except ImportError: def _kill(pid, sig): - print >>sys.stderr, "no process killing support without pywin32" + import ctypes + winapi = ctypes.windll.kernel32 + proch = winapi.OpenProcess(PROCESS_TERMINATE, 0, pid) + winapi.TerminateProcess(proch, 1) == 1 + winapi.CloseHandle(proch) else: def _kill(pid, sig): try: @@ -18,6 +22,7 @@ except pywintypes.error, e: pass + SIGKILL = SIGTERM = 0 READ_MODE = 'rU' WRITE_MODE = 'wb' else: @@ -26,6 +31,9 @@ os.kill(pid, sig) except OSError: pass + + SIGKILL = signal.SIGKILL + SIGTERM = signal.SIGTERM READ_MODE = 'r' WRITE_MODE = 'w' @@ -56,11 +64,11 @@ tnow = time.time() if (tnow-t0) > timeout: if timedout: - _kill(p.pid, signal.SIGKILL) + _kill(p.pid, SIGKILL) return TIMEDOUT else: timedout = TIMEDOUT - _kill(p.pid, signal.SIGTERM) + _kill(p.pid, SIGTERM) time.sleep(min(timeout, 10)) finally: f.close() @@ -81,10 +89,12 @@ def execute_test(cwd, test, out, logfname, interp, test_driver, do_dry_run=False, timeout=None): - args = interp+test_driver + args = interp + test_driver args += ['--resultlog=%s' % logfname, test] args = map(str, args) + args[0] = os.path.join(str(cwd), args[0]) + if do_dry_run: runfunc = dry_run else: Modified: pypy/build/testrunner/test/test_runner.py ============================================================================== --- pypy/build/testrunner/test/test_runner.py (original) +++ pypy/build/testrunner/test/test_runner.py Mon Nov 30 17:57:51 2009 @@ -72,7 +72,7 @@ test_driver=['driver', 'darg'], timeout='secs') - expected = ['INTERP', 'IARG', + expected = ['/wd' + os.sep + 'INTERP', 'IARG', 'driver', 'darg', '--resultlog=LOGFILE', 'test_one'] From lac at codespeak.net Mon Nov 30 23:14:41 2009 From: lac at codespeak.net (lac at codespeak.net) Date: Mon, 30 Nov 2009 23:14:41 +0100 (CET) Subject: [pypy-svn] r69786 - pypy/dist/pypy/doc Message-ID: <20091130221441.8BDBA16803B@codespeak.net> Author: lac Date: Mon Nov 30 23:14:39 2009 New Revision: 69786 Modified: pypy/dist/pypy/doc/getting-started.txt Log: fix tiny typo I just noticed Modified: pypy/dist/pypy/doc/getting-started.txt ============================================================================== --- pypy/dist/pypy/doc/getting-started.txt (original) +++ pypy/dist/pypy/doc/getting-started.txt Mon Nov 30 23:14:39 2009 @@ -50,7 +50,7 @@ source in ``pypy-dist/pypy`` and documentation files in ``pypy-dist/pypy/doc``. If you would prefer to check out the "cutting edge" version of PyPy - which may not always be stable! - then check out -from ``http://codespeak.net/svn/pypy/trunk`` intead. +from ``http://codespeak.net/svn/pypy/trunk`` instead. Where to go from here ----------------------