From fijal at codespeak.net Thu Jul 2 10:21:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Jul 2009 10:21:05 +0200 (CEST) Subject: [pypy-svn] r66089 - in pypy/trunk/pypy/translator: c goal Message-ID: <20090702082105.67E04168417@codespeak.net> Author: fijal Date: Thu Jul 2 10:21:02 2009 New Revision: 66089 Modified: pypy/trunk/pypy/translator/c/dlltool.py pypy/trunk/pypy/translator/goal/sharedpypy.py Log: Actually pass around config obj Modified: pypy/trunk/pypy/translator/c/dlltool.py ============================================================================== --- pypy/trunk/pypy/translator/c/dlltool.py (original) +++ pypy/trunk/pypy/translator/c/dlltool.py Thu Jul 2 10:21:02 2009 @@ -35,10 +35,10 @@ return self.so_name class DLLDef(object): - def __init__(self, name, functions=[], policy=None): + def __init__(self, name, functions=[], policy=None, config=None): self.name = name self.functions = functions # [(function, annotation), ...] - self.driver = TranslationDriver() + self.driver = TranslationDriver(config=config) self.driver.setup_library(self, policy=policy) def compile(self): Modified: pypy/trunk/pypy/translator/goal/sharedpypy.py ============================================================================== --- pypy/trunk/pypy/translator/goal/sharedpypy.py (original) +++ pypy/trunk/pypy/translator/goal/sharedpypy.py Thu Jul 2 10:21:02 2009 @@ -2,13 +2,14 @@ import sys from pypy.translator.c.dlltool import DLLDef from pypy.config.translationoption import get_combined_translation_config -from pypy.rpython.lltypesystem.rffi import charp2str, CCHARP +from pypy.rpython.lltypesystem.rffi import charp2str, CCHARP, VOIDP from pypy.tool.option import make_objspace from pypy.interpreter.error import OperationError from pypy.config.pypyoption import pypy_optiondescription, set_pypy_opt_level from pypy.interpreter.pyopcode import prepare_exec from pypy.translator.goal.ann_override import PyPyAnnotatorPolicy from pypy.config.translationoption import set_opt_level +from pypy.config.pypyoption import enable_allworkingmodules OVERRIDES = { 'translation.debug': False, @@ -22,12 +23,12 @@ config.translating = True set_opt_level(config, '1') set_pypy_opt_level(config, '1') - print config + enable_allworkingmodules(config) space = make_objspace(config) policy = PyPyAnnotatorPolicy(single_space = space) - def interpret(source): + def interpret(source, context): source = charp2str(source) w_dict = space.newdict() try: @@ -41,7 +42,8 @@ return 1 return 0 - dll = DLLDef('pypylib', [(interpret, [CCHARP])], policy=policy) + dll = DLLDef('pypylib', [(interpret, [CCHARP, VOIDP])], policy=policy, + config=config) exe_name = dll.compile() if __name__ == '__main__': From cfbolz at codespeak.net Thu Jul 2 16:46:11 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Jul 2009 16:46:11 +0200 (CEST) Subject: [pypy-svn] r66090 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090702144611.D358E16840C@codespeak.net> Author: cfbolz Date: Thu Jul 2 16:46:09 2009 New Revision: 66090 Added: pypy/extradoc/talk/icooolps2009/talk/ pypy/extradoc/talk/icooolps2009/talk/Makefile pypy/extradoc/talk/icooolps2009/talk/beamerouterthememy.sty pypy/extradoc/talk/icooolps2009/talk/beamerthemeWarsaw.sty pypy/extradoc/talk/icooolps2009/talk/talk.tex Log: Start with the ICOOOLPS slides for the tracing talk. This is how I have given the talk in Bad Honnef, needs updating. Added: pypy/extradoc/talk/icooolps2009/talk/Makefile ============================================================================== --- (empty file) +++ pypy/extradoc/talk/icooolps2009/talk/Makefile Thu Jul 2 16:46:09 2009 @@ -0,0 +1,12 @@ +%.pdf: %.eps + epstopdf $< + +viewtalk: talk.pdf + evince talk.pdf & + +clean: + rm talk.pdf + +talk.pdf: talk.tex beamerouterthememy.sty beamerthemeWarsaw.sty interp.pdf + pdflatex talk + Added: pypy/extradoc/talk/icooolps2009/talk/beamerouterthememy.sty ============================================================================== --- (empty file) +++ pypy/extradoc/talk/icooolps2009/talk/beamerouterthememy.sty Thu Jul 2 16:46:09 2009 @@ -0,0 +1,39 @@ +\ProvidesPackageRCS $Header: /cvsroot/latex-beamer/latex-beamer/themes/outer/beamerouterthemesplit.sty,v 1.4 2004/10/07 22:21:16 tantau Exp $ + +% Copyright 2003 by Till Tantau +% +% This program can be redistributed and/or modified under the terms +% of the GNU Public License, version 2. + +\mode + +\setbeamercolor{section in head/foot}{parent=palette quaternary} +\setbeamercolor{subsection in head/foot}{parent=palette primary} + +\setbeamercolor{author in head/foot}{parent=section in head/foot} +\setbeamercolor{title in head/foot}{parent=subsection in head/foot} + + + +\usesectionheadtemplate + {\hfill\insertsectionhead} + {\hfill\color{fg!50!bg}\insertsectionhead} + + + + +\defbeamertemplate*{footline}{split theme} +{% + \leavevmode% + \hbox{\begin{beamercolorbox}[wd=.6\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm plus1fill,rightskip=.3cm]{author in head/foot}% + \usebeamerfont{author in head/foot}\insertshortauthor + \end{beamercolorbox}% + \begin{beamercolorbox}[wd=.4\paperwidth,ht=2.5ex,dp=1.125ex,leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}% + \usebeamerfont{title in head/foot}\insertshorttitle + \end{beamercolorbox}}% + \vskip0pt% +} + + +\mode + Added: pypy/extradoc/talk/icooolps2009/talk/beamerthemeWarsaw.sty ============================================================================== --- (empty file) +++ pypy/extradoc/talk/icooolps2009/talk/beamerthemeWarsaw.sty Thu Jul 2 16:46:09 2009 @@ -0,0 +1,18 @@ +\ProvidesPackageRCS $Header: /cvsroot/latex-beamer/latex-beamer/themes/theme/beamerthemeWarsaw.sty,v 1.8 2004/10/07 20:53:10 tantau Exp $ + +% Copyright 2003 by Till Tantau +% +% This program can be redistributed and/or modified under the terms +% of the GNU Public License, version 2. + +\mode + +\useinnertheme[shadow=true]{rounded} +\useoutertheme{my} +\usecolortheme{orchid} +\usecolortheme{whale} + +\setbeamerfont{block title}{size={}} + +\mode + Added: pypy/extradoc/talk/icooolps2009/talk/talk.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/icooolps2009/talk/talk.tex Thu Jul 2 16:46:09 2009 @@ -0,0 +1,539 @@ +\documentclass[utf8x]{beamer} + +\mode +{ + \usetheme{Warsaw} + + %\setbeamercovered{transparent} +} + + +\usepackage[english]{babel} + +\usepackage[utf8x]{inputenc} + +\usepackage{times} +\usepackage[T1]{fontenc} + +\title[PyPy's Tracing JIT Compiler]{ + Tracing the Meta-Level: PyPy's Tracing JIT Compiler +} +\author[Bolz, Cuni, Fijalkowski, Rigo] +{ + \textcolor{green!50!black}{Carl~Friedrich~Bolz}\inst{1} \and + Antonio Cuni\inst{2} \and + Maciej Fijalkowski\inst{3} \and + Armin Rigo +} + +\institute[D?sseldorf] +{ + \inst{1} + Softwaretechnik und Programmiersprachen\\ Heinrich-Heine-Universit\"at D\"usseldorf + \and% + \vskip-2mm + \inst{2} + University of Genova, Italy + \and% + \vskip-2mm + \inst{3} + merlinux GmbH +} + +\date{ICOOOLPS 2009 XXX} + + +% Delete this, if you do not want the table of contents to pop up at +% the beginning of each subsection: +%\AtBeginSubsection[] +%{ +% \begin{frame} +% \frametitle{Outline} +% \tableofcontents[currentsection,currentsubsection] +% \end{frame} +%} + + +% If you wish to uncover everything in a step-wise fashion, uncomment +% the following command: + +%\beamerdefaultoverlayspecification{<+->} + + +\begin{document} + +\begin{frame} + \titlepage +\end{frame} + +%\begin{frame} +% \frametitle{Outline} +% \tableofcontents + % You might wish to add the option [pausesections] +%\end{frame} + +\begin{frame} + \frametitle{Motivation} + \begin{itemize} + \item writing good JIT compilers for dynamic programming languages is hard and error-prone + \item tracing JIT compilers are a new approach to JITs that are supposed to be easier + \item what happens when a tracing JIT is applied ``one level down'', i.e. to an interpreter + \item how to solve the occurring problems + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Context: The PyPy Project} + \begin{itemize} + \item a general environment for implementing dynamic languages + \item contains a compiler for a subset of Python (``RPython'') + \item interpreters for dynamic languages written in that subset + \item various interpreters written with PyPy: Python, Prolog, Smalltalk, Scheme, JavaScript, GameBoy emulator + \item can be translated to a variety of target environment: C, JVM, .NET + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Tracing JIT Compilers} + \begin{itemize} + \item idea from Dynamo project: dynamic rewriting of machine code + \item later used for a lightweight Java JIT + \item seems to also work for dynamic languages + \item incorporated into Mozilla's JavaScript VM ("TraceMonkey") + \end{itemize} + \pause + \begin{block}{Basic Assumption of a Tracing JIT} + \begin{itemize} + \item programs spend most of their time executing loops + \item several iterations of a loop are likely to take similar code paths + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \frametitle{Tracing JIT Compilers} + \begin{itemize} + \item mixed-mode execuction environment + \item at first, everything is interpreted + \item lightweight profiling to discover hot loops + \item code generation only for common paths of hot loops + \item when a hot loop is discovered, start to produce a trace + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Tracing} + \begin{itemize} + \item a \emph{trace} is a sequential list of operations + \item a trace is produced by recording every operation the interpreter executes + \item tracing ends when the tracer sees a position in the program it has seen before + \item to identify these places, the \emph{position key} is used + \item the position key encodes the current point of execution + \item a trace thus corresponds to exactly one loop + \item that means it ends with a jump to its beginning + \end{itemize} + \pause + \begin{block}{Guards} + \begin{itemize} + \item the trace is only one of the possible code paths through the loop + \item at places where the path \emph{could} diverge, a guard is placed + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \frametitle{Code Generation and Execution} + \begin{itemize} + \item being linear, the trace can easily be turned into machine code + \item the machine code can be immediately executed + \item execution stops when a guard fails + \item after a guard failure, go back to interpreting program + \end{itemize} +\end{frame} + + +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Example} + \begin{verbatim} +def strange_sum(n): + result = 0 + while n >= 0: + result = f(result, n) + n -= 1 + return result + +def f(a, b): + if b % 46 == 41: + return a - b + else: + return a + b + + + + + + + + + + +\end{verbatim} +} + +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Example} + \begin{verbatim} +def strange_sum(n): + result = 0 + while n >= 0: + result = f(result, n) + n -= 1 + return result + +def f(a, b): + if b % 46 == 41: + return a - b + else: + return a + b + +# loop_header(result0, n0) +# i0 = int_mod(n0, Const(46)) +# i1 = int_eq(i0, Const(41)) +# guard_false(i1) +# result1 = int_add(result0, n0) +# n1 = int_sub(n0, Const(1)) +# i2 = int_ge(n1, Const(0)) +# guard_true(i2) +# jump(result1, n1) +\end{verbatim} +} + +\begin{frame} + \frametitle{(Dis-)Advantages of Tracing JITs} + \begin{block}{Good Points of the Approach} + \begin{itemize} + \item easy and fast machine code generation: needs so support only one path + \item interpreter does a lot of the work + \item can be added to an existing interpreter unobtrusively + \item automatic inlining + \item produces very little code + \end{itemize} + \end{block} + \pause + \begin{block}{Bad Points of the Approach} + \begin{itemize} + \item unclear whether assumptions are true often enough + \item switching between interpretation and machine code execution takes time + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \frametitle{Applying a Tracing JIT to an Interpreter} + \begin{itemize} + \item Question: What happens if the program is itself a bytecode interpreter? + \item the (usually only) hot loop of a bytecode interpreter is the bytecode dispatch loop + \item Assumption violated: two iterations of the dispatch loop will usually take very different code paths + \end{itemize} + \pause + \begin{block}{Terminology} + \begin{itemize} + \item \emph{tracing interpreter:} the interpreter that originally runs the program and produces traces + \item \emph{language interpreter:} the bytecode interpreter runs on top + \item \emph{user program:} the program run by the language interpreter + \end{itemize} + \end{block} +\end{frame} + +\frame[containsverbatim, plain, shrink=10]{ + \begin{verbatim} +def interpret(bytecode, a): + regs = [0] * 256 + pc = 0 + while True: + opcode = ord(bytecode[pc]) + pc += 1 + if opcode == JUMP_IF_A: + target = ord(bytecode[pc]) + pc += 1 + if a: + pc = target + elif opcode == MOV_A_R: + n = ord(bytecode[pc]) + pc += 1 + regs[n] = a + elif opcode == MOV_R_A: + n = ord(bytecode[pc]) + pc += 1 + a = regs[n] + elif opcode == ADD_R_TO_A: + n = ord(bytecode[pc]) + pc += 1 + a += regs[n] + elif opcode == DECR_A: + a -= 1 + elif opcode == RETURN_A: + return a + \end{verbatim} +} + +\frame[containsverbatim, plain, shrink=10]{ + \begin{verbatim} +def interpret(bytecode, a): | + regs = [0] * 256 | # Example bytecode + pc = 0 | # Square the accumulator: + while True: | + opcode = ord(bytecode[pc]) | MOV_A_R 0 # i = a + pc += 1 | MOV_A_R 1 # copy of 'a' + if opcode == JUMP_IF_A: | + target = ord(bytecode[pc]) | # 4: + pc += 1 | MOV_R_A 0 # i-- + if a: | DECR_A + pc = target | MOV_A_R 0 + elif opcode == MOV_A_R: | + n = ord(bytecode[pc]) | MOV_R_A 2 # res += a + pc += 1 | ADD_R_TO_A 1 + regs[n] = a | MOV_A_R 2 + elif opcode == MOV_R_A: | + n = ord(bytecode[pc]) | MOV_R_A 0 # if i!=0: + pc += 1 | JUMP_IF_A 4 # goto 4 + a = regs[n] | + elif opcode == ADD_R_TO_A: | MOV_R_A 2 # return res + n = ord(bytecode[pc]) | RETURN_A + pc += 1 | + a += regs[n] | + elif opcode == DECR_A: | + a -= 1 | + elif opcode == RETURN_A: | + return a | + \end{verbatim} +} + +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Trace} + ~\\ + Resulting trace when tracing bytecode \texttt{DECR\_A}: + \begin{verbatim} + +loop_start(a0, regs0, bytecode0, pc0) +opcode0 = strgetitem(bytecode0, pc0) +pc1 = int_add(pc0, Const(1)) +guard_value(opcode0, Const(7)) +a1 = int_sub(a0, Const(1)) +jump(a1, regs0, bytecode0, pc1) +\end{verbatim} +} + +\begin{frame} + \frametitle{Idea for a Solution} + \begin{itemize} + \item key idea: try to trace the loops in the user program + \item approach: add things to the position key of the tracer + \item tracing interpreter needs information about the language interpreter + \item provided by adding \emph{hints} to the language interpreter + \end{itemize} + \pause + \begin{block}{Hints Give Information About:} + \begin{itemize} + \item which variables make up the program counter of the language interpreter + \item where the bytecode dispatch loop is + \item which bytecodes can correspond to backward jumps + \end{itemize} + \end{block} +\end{frame} + +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Interpreter with Hints} +\begin{verbatim} +tlrjitdriver = JitDriver(['pc', 'bytecode']) + +def interpret(bytecode, a): + regs = [0] * 256 + pc = 0 + while True: + tlrjitdriver.start_dispatch_loop() + opcode = ord(bytecode[pc]) + pc += 1 + if opcode == JUMP_IF_A: + target = ord(bytecode[pc]) + pc += 1 + if a: + pc = target + if target < pc: + tlrjitdriver.backward_jump() + elif opcode == MOV_A_R: + ... # rest unmodified +\end{verbatim} +} + +\frame[containsverbatim, plain, shrink=20]{ + \frametitle{Result When Tracing \texttt{SQUARE}} +\begin{verbatim} +loop_start(a0, regs0, bytecode0, pc0) +# MOV_R_A 0 +opcode0 = strgetitem(bytecode0, pc0) +pc1 = int_add(pc0, Const(1)) +guard_value(opcode0, Const(2)) +n1 = strgetitem(bytecode0, pc1) +pc2 = int_add(pc1, Const(1)) +a1 = call(Const(<* fn list_getitem>), regs0, n1) +# DECR_A +... +# MOV_A_R 0 +... +# MOV_R_A 2 +... +# ADD_R_TO_A 1 +... +# MOV_A_R 2 +... +# MOV_R_A 0 +... +# JUMP_IF_A 4 +opcode6 = strgetitem(bytecode0, pc13) +pc14 = int_add(pc13, Const(1)) +guard_value(opcode6, Const(3)) +target0 = strgetitem(bytecode0, pc14) +pc15 = int_add(pc14, Const(1)) +i1 = int_is_true(a5) +guard_true(i1) +jump(a5, regs0, bytecode0, target0) +\end{verbatim} +} + +\begin{frame} + \frametitle{What Have We Won?} + \begin{itemize} + \item trace corresponds to one loop of the user program + \item however, most operations concerned with manipulating bytecode and program counter + \item bytecode and program counter are part of the position key + \item thus they are constant at the beginning of the loop + \item therefore they can and should be constant-folded + \end{itemize} +\end{frame} + +\frame[containsverbatim, plain, shrink=20]{ + \frametitle{Result When Tracing \texttt{SQUARE} With Constant-Folding} +\begin{verbatim} +loop_start(a0, regs0) +# MOV_R_A 0 +a1 = call(Const(<* fn list_getitem>), regs0, Const(0)) +# DECR_A +a2 = int_sub(a1, Const(1)) +# MOV_A_R 0 +call(Const(<* fn list_setitem>), regs0, Const(0), a2) +# MOV_R_A 2 +a3 = call(Const(<* fn list_getitem>), regs0, Const(2)) +# ADD_R_TO_A 1 +i0 = call(Const(<* fn list_getitem>), regs0, Const(1)) +a4 = int_add(a3, i0) +# MOV_A_R 2 +call(Const(<* fn list_setitem>), regs0, Const(2), a4) +# MOV_R_A 0 +a5 = call(Const(<* fn list_getitem>), regs0, Const(0)) +# JUMP_IF_A 4 +i1 = int_is_true(a5) +guard_true(i1) +jump(a5, regs0) +\end{verbatim} +} + +\begin{frame} + \frametitle{Results} + \begin{itemize} + \item almost only computations related to the user program remain + \item list of registers is only vestige of language interpreter + \end{itemize} + \pause + \begin{block}{Timing Results Computing Square of 10'000'000} + \begin{tabular}{|l|l|r|r|} +\hline +& &Time (ms) &speedup\\ +\hline +1 &No JIT &442.7 $\pm$ 3.4 &1.00\\ +2 &Normal Trace Compilation &1518.7 $\pm$ 7.2 &0.29\\ +3 &Unrolling of Interp. Loop &737.6 $\pm$ 7.9 &0.60\\ +4 &JIT, Full Optimizations &156.2 $\pm$ 3.8 &2.83\\ +\hline +\end{tabular} +\end{block} +\end{frame} + +\begin{frame} + \frametitle{Scaling to Large Interpreters?} + \begin{itemize} + \item we can apply this approach to PyPy's Python interpreter (70 KLOC) + \item speed-ups promising: factor of 6 faster for simple loops with arithmetic + \item no Python-specific bugs! + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Conclusions} + \begin{itemize} + \item some small changes to a tracing JIT makes it possible to effectively apply it to bytecode interpreters + \item result is similar to a tracing JIT for that language + \item bears resemblance to partial evaluation, arrived at by different means + \item maybe enough to write exactly one tracing JIT? + \end{itemize} + \pause + \begin{block}{Outlook} + \begin{itemize} + \item better optimizations of the traces + \item escape analysis + \item optimize frame objects + \item speed up tracing itself + \item apply to other interpreters and larger programs + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \frametitle{Thank you! Questions?} + \begin{itemize} + \item some small changes to a tracing JIT makes it possible to effectively apply it to bytecode interpreters + \item result is similar to a tracing JIT for that language + \item bears resemblance to partial evaluation, arrived at by different means + \item maybe enough to write exactly one tracing JIT? + \end{itemize} + \begin{block}{Outlook} + \begin{itemize} + \item better optimizations of the traces + \item escape analysis + \item optimize frame objects + \item speed up tracing itself + \item apply to other interpreters and larger programs + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \frametitle{Backup Slides} +\end{frame} + +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Timings for Python Interpreter} +\begin{verbatim} +def f(a): + t = (1, 2, 3) + i = 0 + while i < a: + t = (t[1], t[2], t[0]) + i += t[0] + return i +\end{verbatim} +\begin{block}{Timings} +\begin{tabular}{|l|l|r|r|} +\hline +& &Time (s) &speedup\\ +\hline +1 &PyPy compiled to C, no JIT &23.44 $\pm$ 0.07 &1.00\\ +2 &PyPy comp'd to C, with JIT &3.58 $\pm$ 0.05 &6.54\\ +3 &CPython 2.5.2 &4.96 $\pm$ 0.05 &4.73\\ +4 &CPython 2.5.2 + Psyco 1.6 &1.51 $\pm$ 0.05 &15.57\\\hline +\end{tabular} +\end{block} +} + +\end{document} From cfbolz at codespeak.net Thu Jul 2 17:11:37 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Jul 2009 17:11:37 +0200 (CEST) Subject: [pypy-svn] r66092 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090702151137.2C2521684FA@codespeak.net> Author: cfbolz Date: Thu Jul 2 17:11:36 2009 New Revision: 66092 Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex Log: first round of simplification Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex ============================================================================== --- pypy/extradoc/talk/icooolps2009/talk/talk.tex (original) +++ pypy/extradoc/talk/icooolps2009/talk/talk.tex Thu Jul 2 17:11:36 2009 @@ -118,40 +118,10 @@ \item lightweight profiling to discover hot loops \item code generation only for common paths of hot loops \item when a hot loop is discovered, start to produce a trace + \item when a full loop is traced, the trace is converted to machine code \end{itemize} \end{frame} -\begin{frame} - \frametitle{Tracing} - \begin{itemize} - \item a \emph{trace} is a sequential list of operations - \item a trace is produced by recording every operation the interpreter executes - \item tracing ends when the tracer sees a position in the program it has seen before - \item to identify these places, the \emph{position key} is used - \item the position key encodes the current point of execution - \item a trace thus corresponds to exactly one loop - \item that means it ends with a jump to its beginning - \end{itemize} - \pause - \begin{block}{Guards} - \begin{itemize} - \item the trace is only one of the possible code paths through the loop - \item at places where the path \emph{could} diverge, a guard is placed - \end{itemize} - \end{block} -\end{frame} - -\begin{frame} - \frametitle{Code Generation and Execution} - \begin{itemize} - \item being linear, the trace can easily be turned into machine code - \item the machine code can be immediately executed - \item execution stops when a guard fails - \item after a guard failure, go back to interpreting program - \end{itemize} -\end{frame} - - \frame[containsverbatim, plain, shrink=10]{ \frametitle{Example} \begin{verbatim} @@ -213,10 +183,11 @@ \begin{block}{Good Points of the Approach} \begin{itemize} \item easy and fast machine code generation: needs so support only one path + \item (things are more complex, but let's ignore that for now) \item interpreter does a lot of the work \item can be added to an existing interpreter unobtrusively \item automatic inlining - \item produces very little code + \item produces comparatively little code \end{itemize} \end{block} \pause @@ -232,7 +203,7 @@ \frametitle{Applying a Tracing JIT to an Interpreter} \begin{itemize} \item Question: What happens if the program is itself a bytecode interpreter? - \item the (usually only) hot loop of a bytecode interpreter is the bytecode dispatch loop + \item the (most important) hot loop of a bytecode interpreter is the bytecode dispatch loop \item Assumption violated: two iterations of the dispatch loop will usually take very different code paths \end{itemize} \pause @@ -376,7 +347,7 @@ guard_value(opcode0, Const(2)) n1 = strgetitem(bytecode0, pc1) pc2 = int_add(pc1, Const(1)) -a1 = call(Const(<* fn list_getitem>), regs0, n1) +a1 = list_getitem(regs0, n1) # DECR_A ... # MOV_A_R 0 @@ -417,20 +388,20 @@ \begin{verbatim} loop_start(a0, regs0) # MOV_R_A 0 -a1 = call(Const(<* fn list_getitem>), regs0, Const(0)) +a1 = list_getitem(regs0, Const(0)) # DECR_A a2 = int_sub(a1, Const(1)) # MOV_A_R 0 -call(Const(<* fn list_setitem>), regs0, Const(0), a2) +list_setitem(regs0, Const(0), a2) # MOV_R_A 2 -a3 = call(Const(<* fn list_getitem>), regs0, Const(2)) +list_getitem(regs0, Const(2)) # ADD_R_TO_A 1 -i0 = call(Const(<* fn list_getitem>), regs0, Const(1)) +i0 = list_getitem(regs0, Const(1)) a4 = int_add(a3, i0) # MOV_A_R 2 -call(Const(<* fn list_setitem>), regs0, Const(2), a4) +list_setitem(regs0, Const(2), a4) # MOV_R_A 0 -a5 = call(Const(<* fn list_getitem>), regs0, Const(0)) +a5 = list_getitem(regs0, Const(0)) # JUMP_IF_A 4 i1 = int_is_true(a5) guard_true(i1) From cfbolz at codespeak.net Thu Jul 2 18:09:14 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Jul 2009 18:09:14 +0200 (CEST) Subject: [pypy-svn] r66094 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090702160914.BD7F9168547@codespeak.net> Author: cfbolz Date: Thu Jul 2 18:09:12 2009 New Revision: 66094 Modified: pypy/extradoc/talk/icooolps2009/talk/Makefile Log: that was killed from the talk Modified: pypy/extradoc/talk/icooolps2009/talk/Makefile ============================================================================== --- pypy/extradoc/talk/icooolps2009/talk/Makefile (original) +++ pypy/extradoc/talk/icooolps2009/talk/Makefile Thu Jul 2 18:09:12 2009 @@ -7,6 +7,6 @@ clean: rm talk.pdf -talk.pdf: talk.tex beamerouterthememy.sty beamerthemeWarsaw.sty interp.pdf +talk.pdf: talk.tex beamerouterthememy.sty beamerthemeWarsaw.sty pdflatex talk From fijal at codespeak.net Fri Jul 3 00:27:19 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 3 Jul 2009 00:27:19 +0200 (CEST) Subject: [pypy-svn] r66095 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090702222719.86AB61684C0@codespeak.net> Author: fijal Date: Fri Jul 3 00:27:18 2009 New Revision: 66095 Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex Log: If we're using tex, we can go for a correct spelling Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex ============================================================================== --- pypy/extradoc/talk/icooolps2009/talk/talk.tex (original) +++ pypy/extradoc/talk/icooolps2009/talk/talk.tex Fri Jul 3 00:27:18 2009 @@ -22,7 +22,7 @@ { \textcolor{green!50!black}{Carl~Friedrich~Bolz}\inst{1} \and Antonio Cuni\inst{2} \and - Maciej Fijalkowski\inst{3} \and + Maciej Fija\l{}kowski\inst{3} \and Armin Rigo } From benjamin at codespeak.net Fri Jul 3 01:01:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 3 Jul 2009 01:01:02 +0200 (CEST) Subject: [pypy-svn] r66096 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090702230102.17B521684FA@codespeak.net> Author: benjamin Date: Fri Jul 3 01:01:00 2009 New Revision: 66096 Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (contents, props changed) pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (contents, props changed) Log: implement symbol table builder Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 3 01:01:00 2009 @@ -0,0 +1,408 @@ +""" +Symbol tabling building. +""" + +from pypy.interpreter.astcompiler import ast2 as ast, misc +from pypy.interpreter.pyparser.error import SyntaxError + +# These are for internal use only: +SYM_BLANK = 0 +SYM_GLOBAL = 1 +SYM_ASSIGNED = 2 # Or deleted actually. +SYM_PARAM = 2 << 1 +SYM_USED = 2 << 2 +SYM_BOUND = (SYM_PARAM | SYM_ASSIGNED) + +# codegen.py actually deals with these: +SCOPE_UNKNOWN = 0 +SCOPE_GLOBAL_IMPLICIT = 1 +SCOPE_GLOBAL_EXPLICIT = 2 +SCOPE_LOCAL = 3 +SCOPE_FREE = 4 +SCOPE_CELL = 5 + + +class Scope(object): + + def __init__(self, node, name, optimized): + self.node = node + self.parent = None + self.name = name + self.optimized = optimized + self.symbols = None + self.roles = {} + self.varnames = [] + self.children = [] + self.has_free = False + self.child_has_free = False + self.nested = False + + def lookup(self, name): + return self.symbols.get(name, SCOPE_UNKNOWN) + + def note_symbol(self, identifier, role): + mangled = self.mangle(identifier) + new_role = role + if identifier in self.roles: + old_role = self.roles[identifier] + if old_role & SYM_PARAM and role & SYM_PARAM: + err = "duplicate argument '%s' in function definition" % \ + (identifier,) + raise SyntaxError(err, self.node.lineno, self.node.col_offset) + new_role |= old_role + self.roles[mangled] = new_role + if role & SYM_PARAM: + self.varnames.append(mangled) + return mangled + + def note_yield(self, yield_node): + raise SyntaxError("yield outside function", yield_node.lineno, + yield_node.col_offset) + + def note_return(self, ret): + raise SyntaxError("return outside function", ret.lineno, + ret.col_offset) + + def note_bare_exec(self, exc): + pass + + def note_import_star(self, imp): + pass + + def mangle(self, name): + if self.parent: + return self.parent.mangle(name) + else: + return name + + def add_child(self, child_scope): + child_scope.parent = self + self.children.append(child_scope) + + def _finalize_name(self, name, flags, local, bound, free, globs): + if flags & SYM_GLOBAL: + if flags & SYM_PARAM: + raise SyntaxError("name %r is both local and global" % (name,), + self.node.lineno, self.node.col_offset) + self.symbols[name] = SCOPE_GLOBAL_EXPLICIT + globs[name] = None + if bound: + try: + del bound[name] + except KeyError: + pass + elif flags & SYM_BOUND: + self.symbols[name] = SCOPE_LOCAL + local[name] = None + try: + del globs[name] + except KeyError: + pass + elif bound and name in bound: + self.symbols[name] = SCOPE_FREE + free[name] = None + self.has_free = True + elif name in globs: + self.symbols[name] = SCOPE_GLOBAL_IMPLICIT + else: + if self.nested: + self.has_free = True + self.symbols[name] = SCOPE_GLOBAL_IMPLICIT + + def _pass_on_bindings(self, local, bound, globs, new_bound, new_globs): + new_globs.update(globs) + if bound: + new_bound.update(bound) + + def _finalize_cells(self, free): + pass + + def _check_optimization(self): + pass + + _hide_bound_from_nested_scopes = False + + def finalize(self, bound, free, globs): + self.symbols = {} + local = {} + new_globs = {} + new_bound = {} + new_free = {} + if self._hide_bound_from_nested_scopes: + self._pass_on_bindings(local, bound, globs, new_bound, new_globs) + for name, flags in self.roles.iteritems(): + self._finalize_name(name, flags, local, bound, free, globs) + if not self._hide_bound_from_nested_scopes: + self._pass_on_bindings(local, bound, globs, new_bound, new_globs) + child_frees = {} + for child in self.children: + child_free = new_free.copy() + child.finalize(new_bound.copy(), child_free, new_globs.copy()) + child_frees.update(child_free) + if child.has_free or child.child_has_free: + self.child_has_free = True + new_free.update(child_frees) + self._finalize_cells(new_free) + for name in new_free: + try: + role_here = self.roles[name] + except KeyError: + if name in bound: + self.symbols[name] = SCOPE_FREE + else: + if role_here & (SYM_ASSIGNED | SYM_GLOBAL) and \ + self._hide_bound_from_nested_scopes: + self.symbols[name] = SCOPE_FREE + self._check_optimization() + free.update(new_free) + + +class ModuleScope(Scope): + + def __init__(self, module): + Scope.__init__(self, module, "top", False) + + +class FunctionScope(Scope): + + def __init__(self, func, name): + Scope.__init__(self, func, name, True) + self.has_variable_arg = False + self.has_keywords_arg = False + self.is_generator = False + self.return_with_value = False + self.import_star = None + self.bare_exec = None + + def note_yield(self, yield_node): + if self.return_with_value: + raise SyntaxError("return with value in generator", + yield_node.lineno, yield_node.col_offset) + self.is_generator = True + + def note_return(self, ret): + if self.is_generator and ret.value: + raise SyntaxError("return with value in generator", ret.lineno, + ret.col_offset) + self.return_with_value = True + + def note_exec(self, exc): + self.has_exec = True + if not exc.globals: + self.optimized = False + self.bare_exec = exc + + def note_import_star(self, imp): + self.optimized = False + self.import_star = imp + + def note_variable_arg(self, vararg): + self.has_variable_arg = True + + def note_keywords_arg(self, kwarg): + self.has_keywords_arg = True + + def add_child(self, child_scope): + Scope.add_child(self, child_scope) + child_scope.nested = True + + def _pass_on_bindings(self, local, bound, globs, new_bound, new_globs): + new_bound.update(local) + Scope._pass_on_bindings(self, local, bound, globs, new_bound, new_globs) + + def _finalize_cells(self, free): + for name, role in self.symbols.iteritems(): + if role == SCOPE_LOCAL and name in free: + self.symbols[name] = SCOPE_CELL + del free[name] + + def _check_optimization(self): + if (self.has_free or self.child_has_free) and not self.optimized: + err = None + if self.import_star: + node = self.import_star + if self.bare_exec: + err = "function %r uses import * and bare exec, " \ + "which are illegal because it %s" + else: + err = "import * is not allowed in function %r because it %s" + elif self.bare_exec: + node = self.bare_exec + err = "unqualified exec is not allowed in function %r " \ + "because it %s" + else: + raise AssertionError("unkown reason for unoptimization") + if self.child_has_free: + trailer = "contains a nested function with free variables" + else: + trailer = "is a nested function" + raise SyntaxError(err % (self.name, trailer), node.lineno, + node.col_offset) + + +class ClassScope(Scope): + + _hide_bound_from_nested_scopes = True + + def __init__(self, clsdef): + Scope.__init__(self, clsdef, clsdef.name, False) + + def mangle(self, name): + return misc.mangle(name, self.name) + + +class SymtableBuilder(ast.GenericASTVisitor): + + def __init__(self, space, module): + self.space = space + self.module = module + self.scopes = {} + self.scope = None + self.stack = [] + top = ModuleScope(module) + self.globs = top.roles + self.push_scope(top) + module.walkabout(self) + top.finalize(None, {}, {}) + self.pop_scope() + assert not self.stack + + def push_scope(self, scope): + if self.stack: + self.stack[-1].add_child(scope) + self.stack.append(scope) + self.scopes[scope.node] = scope + # Convenience + self.scope = scope + + def pop_scope(self): + self.stack.pop() + if self.stack: + self.scope = self.stack[-1] + else: + self.scope = None + + def find_scope(self, scope_node): + return self.scopes[scope_node] + + def implicit_arg(self, pos): + name = ".%i" % (pos,) + self.note_symbol(name, SYM_PARAM) + + def note_symbol(self, identifier, role): + mangled = self.scope.note_symbol(identifier, role) + if role & SYM_GLOBAL: + if identifier in self.globs: + role |= self.globs[mangled] + self.globs[mangled] = role + + def visit_FunctionDef(self, func): + self.note_symbol(func.name, SYM_ASSIGNED) + if func.args.defaults: + self.visit_sequence(func.args.defaults) + if func.decorators: + self.visit_sequence(func.decorators) + self.push_scope(FunctionScope(func, func.name)) + func.args.walkabout(self) + self.visit_sequence(func.body) + self.pop_scope() + + def visit_Return(self, ret): + self.scope.note_return(ret) + ast.GenericASTVisitor.visit_Return(self, ret) + + def visit_ClassDef(self, clsdef): + self.note_symbol(clsdef.name, SYM_ASSIGNED) + if clsdef.bases: + self.visit_sequence(clsdef.bases) + self.push_scope(ClassScope(clsdef)) + self.visit_sequence(clsdef.body) + self.pop_scope() + + def visit_ImportFrom(self, imp): + for alias in imp.names: + if self.visit_alias(alias): + self.scope.note_import_star(imp) + + def visit_alias(self, alias): + if alias.asname: + store_name = alias.asname + else: + store_name = alias.name + if store_name == "*": + return True + dot = store_name.find(".") + if dot != -1: + store_name = store_name[:dot] + self.note_symbol(store_name, SYM_ASSIGNED) + return False + + def visit_Exec(self, exc): + self.scope.note_exec(exc) + ast.GenericASTVisitor.visit_Exec(self, exc) + + def visit_Yield(self, yie): + self.scope.note_yield(yie) + ast.GenericASTVisitor.visit_Yield(self, yie) + + def visit_Global(self, glob): + for name in glob.names: + self.note_symbol(name, SYM_GLOBAL) + + def visit_Lambda(self, lamb): + if lamb.args.defaults: + self.visit_sequence(lamb.defaults) + self.push_scope(FunctionScope(lamb, "lambda")) + lamb.args.walkabout(self) + lamb.body.walkabout(self) + self.pop_scope() + + def visit_GeneratorExp(self, genexp): + outer = genexp.generators[0] + outer.iter.walkabout(self) + self.push_scope(FunctionScope(genexp, "genexp")) + self.implicit_arg(0) + outer.target.walkabout(self) + if outer.ifs: + self.visit_sequence(outer.ifs) + self.visit_sequence(genexp.generators[1:]) + genexp.elt.walkabout(self) + self.pop_scope() + + def visit_arguments(self, arguments): + assert isinstance(self.scope, FunctionScope) # Annotator hint. + if arguments.args: + self._handle_params(arguments.args, True) + if arguments.vararg: + self.note_symbol(arguments.vararg, SYM_PARAM) + self.scope.note_variable_arg(arguments.vararg) + if arguments.kwarg: + self.note_symbol(arguments.kwarg, SYM_PARAM) + self.scope.note_keywords_arg(arguments.kwarg) + if arguments.args: + self._handle_nested_params(arguments.args) + + def _handle_params(self, params, is_toplevel): + for i in range(len(params)): + arg = params[i] + if isinstance(arg, ast.Name): + self.note_symbol(arg.id, SYM_PARAM) + elif isinstance(arg, ast.Tuple): + if is_toplevel: + self.implicit_arg(i) + else: + raise AssertionError("unkown parameter type") + if not is_toplevel: + self._handle_nested_params(params) + + def _handle_nested_params(self, params): + for param in params: + if isinstance(param, ast.Tuple): + self._handle_params(param.elts, False) + + def visit_Name(self, name): + if name.ctx is ast.Load: + role = SYM_USED + else: + role = SYM_ASSIGNED + self.note_symbol(name.id, role) Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Fri Jul 3 01:01:00 2009 @@ -0,0 +1,290 @@ +import string +import py +from pypy.interpreter.astcompiler import ast2 as ast, astbuilder, symtable +from pypy.interpreter.pyparser import pyparse +from pypy.interpreter.pyparser.error import SyntaxError + + +class TestSymbolTable: + + def setup_class(cls): + cls.parser = pyparse.PythonParser(cls.space) + + def mod_scope(self, source, mode="exec"): + tree = self.parser.parse_source(source) + module = astbuilder.ast_from_node(self.space, tree) + builder = symtable.SymtableBuilder(self.space, module) + scope = builder.find_scope(module) + assert isinstance(scope, symtable.ModuleScope) + return scope + + def func_scope(self, func_code): + mod_scope = self.mod_scope(func_code) + assert len(mod_scope.children) == 1 + func_name = mod_scope.lookup("f") + assert func_name == symtable.SCOPE_LOCAL + func_scope = mod_scope.children[0] + assert isinstance(func_scope, symtable.FunctionScope) + return func_scope + + def class_scope(self, class_code): + mod_scope = self.mod_scope(class_code) + assert len(mod_scope.children) == 1 + class_name = mod_scope.lookup("x") + assert class_name == symtable.SCOPE_LOCAL + class_scope = mod_scope.children[0] + assert isinstance(class_scope, symtable.ClassScope) + return class_scope + + def gen_scope(self, gen_code): + mod_scope = self.mod_scope(gen_code) + assert len(mod_scope.children) == 1 + gen_scope = mod_scope.children[0] + assert isinstance(gen_scope, symtable.FunctionScope) + assert not gen_scope.children + assert gen_scope.name == "genexp" + return mod_scope, gen_scope + + def check_unknown(self, scp, *names): + for name in names: + assert scp.lookup(name) == symtable.SCOPE_UNKNOWN + + def test_toplevel(self): + scp = self.mod_scope("x = 4") + assert scp.lookup("x") == symtable.SCOPE_LOCAL + assert not scp.optimized + scp = self.mod_scope("x = 4", "single") + assert not scp.optimized + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.mod_scope("x*4*6", "eval") + assert not scp.optimized + assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT + + def test_duplicate_argument(self): + input = "def f(x, x): pass" + exc = py.test.raises(SyntaxError, self.mod_scope, input).value + assert exc.msg == "duplicate argument 'x' in function definition" + + def test_function_defaults(self): + scp = self.mod_scope("y = 4\ndef f(x=y): return x") + self.check_unknown(scp, "x") + assert scp.lookup("y") == symtable.SCOPE_LOCAL + scp = scp.children[0] + assert scp.lookup("x") == symtable.SCOPE_LOCAL + self.check_unknown(scp, "y") + + def test_genexp(self): + scp, gscp = self.gen_scope("(y[1] for y in z)") + assert scp.lookup("z") == symtable.SCOPE_GLOBAL_IMPLICIT + self.check_unknown(scp, "y", "x") + self.check_unknown(gscp, "z") + assert gscp.lookup("y") == symtable.SCOPE_LOCAL + scp, gscp = self.gen_scope("(x for x in z if x)") + self.check_unknown(scp, "x") + assert gscp.lookup("x") == symtable.SCOPE_LOCAL + scp, gscp = self.gen_scope("(x for y in g for f in n if f[h])") + self.check_unknown(scp, "f") + assert gscp.lookup("f") == symtable.SCOPE_LOCAL + + def test_arguments(self): + scp = self.func_scope("def f(): pass") + assert not scp.children + self.check_unknown(scp, "x", "y") + assert not scp.symbols + assert not scp.roles + scp = self.func_scope("def f(x): pass") + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.func_scope("def f(*x): pass") + assert scp.has_variable_arg + assert not scp.has_keywords_arg + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.func_scope("def f(**x): pass") + assert scp.has_keywords_arg + assert not scp.has_variable_arg + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.func_scope("def f((x, y), a): pass") + for name in ("x", "y", "a"): + assert scp.lookup(name) == symtable.SCOPE_LOCAL + scp = self.func_scope("def f(((a, b), c)): pass") + for name in ("a", "b", "c"): + assert scp.lookup(name) == symtable.SCOPE_LOCAL + + def test_function(self): + scp = self.func_scope("def f(): x = 4") + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.func_scope("def f(): x") + assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT + + def test_nested_scopes(self): + def nested_scope(*bodies): + names = enumerate("f" + string.ascii_letters) + lines = [] + for body, (level, name) in zip(bodies, names): + lines.append(" " * level + "def %s():\n" % (name,)) + if body: + if isinstance(body, str): + body = [body] + lines.extend(" " * (level + 1) + line + "\n" + for line in body) + return self.func_scope("".join(lines)) + scp = nested_scope("x = 1", "return x") + assert not scp.has_free + assert scp.child_has_free + assert scp.lookup("x") == symtable.SCOPE_CELL + child = scp.children[0] + assert child.has_free + assert child.lookup("x") == symtable.SCOPE_FREE + scp = nested_scope("x = 1", None, "return x") + assert not scp.has_free + assert scp.child_has_free + assert scp.lookup("x") == symtable.SCOPE_CELL + child = scp.children[0] + assert not child.has_free + assert child.child_has_free + assert child.lookup("x") == symtable.SCOPE_FREE + child = child.children[0] + assert child.has_free + assert not child.child_has_free + assert child.lookup("x") == symtable.SCOPE_FREE + scp = nested_scope("x = 1", "x = 3", "return x") + assert scp.child_has_free + assert not scp.has_free + assert scp.lookup("x") == symtable.SCOPE_LOCAL + child = scp.children[0] + assert child.child_has_free + assert not child.has_free + assert child.lookup("x") == symtable.SCOPE_CELL + child = child.children[0] + assert child.has_free + assert child.lookup("x") == symtable.SCOPE_FREE + + def test_class(self): + scp = self.mod_scope("class x(A, B): pass") + cscp = scp.children[0] + for name in ("A", "B"): + assert scp.lookup(name) == symtable.SCOPE_GLOBAL_IMPLICIT + self.check_unknown(cscp, name) + scp = self.func_scope("""def f(x): + class X: + def n(): + return x + a = x + return X()""") + self.check_unknown(scp, "a") + assert scp.lookup("x") == symtable.SCOPE_CELL + assert scp.lookup("X") == symtable.SCOPE_LOCAL + cscp = scp.children[0] + assert cscp.lookup("a") == symtable.SCOPE_LOCAL + assert cscp.lookup("x") == symtable.SCOPE_FREE + fscp = cscp.children[0] + assert fscp.lookup("x") == symtable.SCOPE_FREE + self.check_unknown(fscp, "a") + + def test_lambda(self): + scp = self.mod_scope("lambda x: y") + self.check_unknown(scp, "x", "y") + assert len(scp.children) == 1 + lscp = scp.children[0] + assert isinstance(lscp, symtable.FunctionScope) + assert lscp.name == "lambda" + assert lscp.lookup("x") == symtable.SCOPE_LOCAL + assert lscp.lookup("y") == symtable.SCOPE_GLOBAL_IMPLICIT + + def test_import(self): + scp = self.mod_scope("import x") + assert scp.lookup("x") == symtable.SCOPE_LOCAL + scp = self.mod_scope("import x as y") + assert scp.lookup("y") == symtable.SCOPE_LOCAL + self.check_unknown(scp, "x") + scp = self.mod_scope("import x.y") + assert scp.lookup("x") == symtable.SCOPE_LOCAL + self.check_unknown(scp, "y") + + def test_from_import(self): + scp = self.mod_scope("from x import y") + self.check_unknown("x") + assert scp.lookup("y") == symtable.SCOPE_LOCAL + scp = self.mod_scope("from a import b as y") + assert scp.lookup("y") == symtable.SCOPE_LOCAL + self.check_unknown(scp, "a", "b") + scp = self.mod_scope("from x import *") + self.check_unknown("x") + scp = self.func_scope("def f(): from x import *") + self.check_unknown(scp, "x") + assert not scp.optimized + assert scp.import_star + + def test_global(self): + scp = self.func_scope("def f():\n global x\n x = 4") + assert scp.lookup("x") == symtable.SCOPE_GLOBAL_EXPLICIT + input = "def f(x):\n global x" + scp = self.func_scope("""def f(): + y = 3 + def x(): + global y + y = 4 + def z(): + return y""") + assert scp.lookup("y") == symtable.SCOPE_CELL + xscp, zscp = scp.children + assert xscp.lookup("y") == symtable.SCOPE_GLOBAL_EXPLICIT + assert zscp.lookup("y") == symtable.SCOPE_FREE + exc = py.test.raises(SyntaxError, self.func_scope, input).value + assert exc.msg == "name 'x' is both local and global" + + def test_optimization(self): + assert not self.mod_scope("").optimized + assert not self.class_scope("class x: pass").optimized + assert self.func_scope("def f(): pass").optimized + + def test_unoptimization_with_nested_scopes(self): + table = ( + ("from x import *; exec 'hi'", "function 'f' uses import * " \ + "and bare exec, which are illegal because it"), + ("from x import *", "import * is not allowed in function 'f' " \ + "because it"), + ("exec 'hi'", "unqualified exec is not allowed in function 'f' " \ + "because it") + ) + for line, error in table: + input = """def n(): + x = 4 + def f(): + %s + return x""" % (line,) + exc = py.test.raises(SyntaxError, self.mod_scope, input).value + assert exc.msg == error + " is a nested function" + input = """def f(): + %s + x = 4 + def n(): + return x""" % (line,) + exc = py.test.raises(SyntaxError, self.mod_scope, input).value + assert exc.msg == error + " contains a nested function with free variables" + + def test_exec(self): + scp = self.func_scope("def f(): exec 'hi'") + assert not scp.optimized + assert isinstance(scp.bare_exec, ast.Exec) + assert scp.has_exec + for line in ("exec 'hi' in g", "exec 'hi' in g, h"): + scp = self.func_scope("def f(): " + line) + assert scp.optimized + assert scp.bare_exec is None + assert scp.has_exec + + def test_yield(self): + scp = self.func_scope("def f(): yield x") + assert scp.is_generator + for input in ("yield x", "class y: yield x"): + exc = py.test.raises(SyntaxError, self.mod_scope, "yield x").value + assert exc.msg == "yield outside function" + for input in ("yield\n return x", "return x\n yield"): + input = "def f():\n " + input + exc = py.test.raises(SyntaxError, self.func_scope, input).value + assert exc.msg == "return with value in generator" + + def test_return(self): + for input in ("class x: return", "return"): + exc = py.test.raises(SyntaxError, self.func_scope, input).value + assert exc.msg == "return outside function" From cfbolz at codespeak.net Fri Jul 3 13:22:28 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 3 Jul 2009 13:22:28 +0200 (CEST) Subject: [pypy-svn] r66097 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20090703112228.1CF0916856C@codespeak.net> Author: cfbolz Date: Fri Jul 3 13:22:27 2009 New Revision: 66097 Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/dictobject.py pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py pypy/trunk/pypy/objspace/std/test/test_dictobject.py pypy/trunk/pypy/objspace/std/test/test_typeobject.py pypy/trunk/pypy/objspace/std/typeobject.py Log: (pedronis, cfbolz): Fix the fact that dict proxies are not updated properly. The fix works only when using multidicts, as otherwise it would be very annoying to fix. Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Fri Jul 3 13:22:27 2009 @@ -1049,6 +1049,7 @@ w_self.implementation = SharedDictImplementation(space) else: w_self.implementation = space.emptydictimpl + w_self.space = space def initialize_content(w_self, list_pairs_w): impl = w_self.implementation @@ -1056,6 +1057,11 @@ impl = impl.setitem(w_k, w_v) w_self.implementation = impl + def initialize_from_strdict_shared(w_self, strdict): + impl = StrDictImplementation(w_self.space) + impl.content = strdict + w_self.implementation = impl + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%s)" % (w_self.__class__.__name__, w_self.implementation) Modified: pypy/trunk/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictobject.py Fri Jul 3 13:22:27 2009 @@ -13,11 +13,18 @@ w_self.content = r_dict(space.eq_w, space.hash_w) else: w_self.content = w_otherdict.content.copy() + w_self.space = space def initialize_content(w_self, list_pairs_w): for w_k, w_v in list_pairs_w: w_self.content[w_k] = w_v + def initialize_from_strdict_shared(w_self, strdict): + # XXX the stuff below is slightly broken, as the dict is not really shared + # this would be very very annoying to fix with non-multidicts + for key, w_value in strdict.items(): + w_self.content[w_self.space.wrap(key)] = w_value + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%s)" % (w_self.__class__.__name__, w_self.content) Modified: pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py Fri Jul 3 13:22:27 2009 @@ -12,6 +12,17 @@ def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + def test_initialize_from_strdict_really_shared(self): + space = self.space + w = space.wrap + d = {"a": w(1), "b": w(2)} + w_d = space.DictObjectCls(space) + w_d.initialize_from_strdict_shared(d) + assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) + assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) + d["c"] = w(41) + assert self.space.eq_w(space.getitem(w_d, w("c")), w(41)) + class AppTest_DictMultiObject(test_dictobject.AppTest_DictObject): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) Modified: pypy/trunk/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_dictobject.py Fri Jul 3 13:22:27 2009 @@ -123,6 +123,16 @@ assert self.space.eq_w(space.call_function(get, w("33")), w(None)) assert self.space.eq_w(space.call_function(get, w("33"), w(44)), w(44)) + def test_initialize_from_strdict_shared(self): + space = self.space + w = space.wrap + d = {"a": w(1), "b": w(2)} + w_d = space.DictObjectCls(space) + w_d.initialize_from_strdict_shared(d) + assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) + assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) + + class AppTest_DictObject: Modified: pypy/trunk/pypy/objspace/std/test/test_typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_typeobject.py Fri Jul 3 13:22:27 2009 @@ -915,14 +915,19 @@ return 0 raises(TypeError, X) +class AppTestWithMultidictTypes: + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + def test_dictproxy_is_updated(self): - skip("fix me") class A(object): x = 1 d = A.__dict__ assert d["x"] == 1 A.y = 2 assert d["y"] == 2 + assert ("x", 1) in d.items() + assert ("y", 2) in d.items() class AppTestMutableBuiltintypes: Modified: pypy/trunk/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/typeobject.py (original) +++ pypy/trunk/pypy/objspace/std/typeobject.py Fri Jul 3 13:22:27 2009 @@ -246,14 +246,8 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - dictspec = [] - for key, w_value in w_self.dict_w.items(): - dictspec.append((space.wrap(key), w_value)) - # speed hack: instantiate a dict object cls directly - # NB: cannot use newdict, because that could return something else - # than an instance of DictObjectCls newdic = space.DictObjectCls(space) - newdic.initialize_content(dictspec) + newdic.initialize_from_strdict_shared(w_self.dict_w) return W_DictProxyObject(newdic) def unwrap(w_self, space): From cfbolz at codespeak.net Fri Jul 3 13:51:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 3 Jul 2009 13:51:25 +0200 (CEST) Subject: [pypy-svn] r66098 - pypy/branch/pyjitpl5/pypy/translator/test Message-ID: <20090703115125.D0AAC16854E@codespeak.net> Author: cfbolz Date: Fri Jul 3 13:51:22 2009 New Revision: 66098 Modified: pypy/branch/pyjitpl5/pypy/translator/test/test_driver.py Log: (pedronis, cfbolz): fix test_driver Modified: pypy/branch/pyjitpl5/pypy/translator/test/test_driver.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/test/test_driver.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/test/test_driver.py Fri Jul 3 13:51:22 2009 @@ -35,7 +35,8 @@ 'compile_cli', 'compile_c', 'run_c', 'run_cli', 'compile_jvm', 'source_jvm', 'run_jvm', - 'prejitbackendopt_lltype', 'pyjitpl_lltype'] + 'prejitbackendopt_lltype', 'pyjitpl_lltype', + 'prejitbackendopt_ootype', 'pyjitpl_ootype'] assert set(td.exposed) == set(expected) td = TranslationDriver({'backend': None, 'type_system': 'lltype'}) From cfbolz at codespeak.net Fri Jul 3 14:15:10 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 3 Jul 2009 14:15:10 +0200 (CEST) Subject: [pypy-svn] r66099 - pypy/branch/pyjitpl5/pypy/translator/jvm Message-ID: <20090703121510.AB3B716846C@codespeak.net> Author: cfbolz Date: Fri Jul 3 14:15:10 2009 New Revision: 66099 Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/database.py pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Log: (pedronis, cfbolz): fix the JVM backend Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/database.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/jvm/database.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/jvm/database.py Fri Jul 3 14:15:10 2009 @@ -464,6 +464,8 @@ return self._translate_record(OOT) if isinstance(OOT, ootype.StaticMethod): return self.record_delegate(OOT) + if OOT is ootype.Object: + return jvm.jObject assert False, "Untranslatable type %s!" % OOT Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Fri Jul 3 14:15:10 2009 @@ -77,6 +77,8 @@ 'oosend': [JvmCallMethod, StoreResult], 'ooupcast': DoNothing, 'oodowncast': [DownCast, StoreResult], + 'cast_to_object': DoNothing, + 'cast_from_object': [DownCast, StoreResult], 'instanceof': [CastTo, StoreResult], 'subclassof': [PushAllArgs, jvm.SWAP, jvm.CLASSISASSIGNABLEFROM, StoreResult], 'classof': [PushAllArgs, jvm.OBJECTGETCLASS, StoreResult], From arigo at codespeak.net Fri Jul 3 14:28:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Jul 2009 14:28:06 +0200 (CEST) Subject: [pypy-svn] r66100 - pypy/branch/pyjitpl5/pypy/rpython Message-ID: <20090703122806.8A89116857B@codespeak.net> Author: arigo Date: Fri Jul 3 14:28:06 2009 New Revision: 66100 Modified: pypy/branch/pyjitpl5/pypy/rpython/llinterp.py Log: Add operations (not implemented at this level) to make test_lloperation pass. Modified: pypy/branch/pyjitpl5/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rpython/llinterp.py (original) +++ pypy/branch/pyjitpl5/pypy/rpython/llinterp.py Fri Jul 3 14:28:06 2009 @@ -878,6 +878,15 @@ def op_gc_set_max_heap_size(self, maxsize): raise NotImplementedError("gc_set_max_heap_size") + def op_do_malloc_fixedsize_clear(self): + raise NotImplementedError("do_malloc_fixedsize_clear") + + def op_do_malloc_varsize_clear(self): + raise NotImplementedError("do_malloc_varsize_clear") + + def op_get_write_barrier_failing_case(self): + raise NotImplementedError("get_write_barrier_failing_case") + def op_yield_current_frame_to_caller(self): raise NotImplementedError("yield_current_frame_to_caller") From antocuni at codespeak.net Fri Jul 3 14:32:48 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Jul 2009 14:32:48 +0200 (CEST) Subject: [pypy-svn] r66101 - in pypy/trunk/pypy/module/__builtin__: . test Message-ID: <20090703123248.C8FFE168469@codespeak.net> Author: antocuni Date: Fri Jul 3 14:32:48 2009 New Revision: 66101 Modified: pypy/trunk/pypy/module/__builtin__/functional.py pypy/trunk/pypy/module/__builtin__/test/test_range.py Log: (brian brazil, antocuni, Harald Massa around, based on a patch by jasonpjason) a test and a fix (the contributors line was already too long to write a longer commit message :-)) Modified: pypy/trunk/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/functional.py Fri Jul 3 14:32:48 2009 @@ -62,17 +62,15 @@ try: # save duplication by redirecting every error to applevel - x = space.int_w(w_x) + x = space.int_w(space.int(w_x)) if space.is_w(w_y, space.w_None): start, stop = 0, x else: - start, stop = x, space.int_w(w_y) - step = space.int_w(w_step) + start, stop = x, space.int_w(space.int(w_y)) + step = space.int_w(space.int(w_step)) howmany = get_len_of_range(start, stop, step) except OperationError, e: if not e.match(space, space.w_TypeError): - pass - else: raise except (ValueError, OverflowError): pass Modified: pypy/trunk/pypy/module/__builtin__/test/test_range.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_range.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_range.py Fri Jul 3 14:32:48 2009 @@ -17,7 +17,6 @@ def test_range_negstartisstop(self): assert range(-1, -1) == [] - def test_range_zero(self): assert range(0) == [] @@ -67,3 +66,13 @@ def test_range_wrong_type(self): raises(TypeError, range, "42") + def test_range_object_with___int__(self): + class A(object): + def __int__(self): + return 5 + + assert range(A()) == [0, 1, 2, 3, 4] + assert range(0, A()) == [0, 1, 2, 3, 4] + assert range(0, 10, A()) == [0, 5] + + From antocuni at codespeak.net Fri Jul 3 16:06:46 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Jul 2009 16:06:46 +0200 (CEST) Subject: [pypy-svn] r66102 - in pypy/trunk/pypy/module/__builtin__: . test Message-ID: <20090703140646.76CC0169E0D@codespeak.net> Author: antocuni Date: Fri Jul 3 16:06:44 2009 New Revision: 66102 Modified: pypy/trunk/pypy/module/__builtin__/functional.py pypy/trunk/pypy/module/__builtin__/test/test_functional.py pypy/trunk/pypy/module/__builtin__/test/test_range.py Log: (brian brazil, based on a patch by jasonpjason) - Fix using class with __int__ and xrange. Add a unittest - Re-add unittest for range with floats, there doesn't appear to be a reason to ignore it - Add float unittest for xrange, this tests _toint Modified: pypy/trunk/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/functional.py Fri Jul 3 16:06:44 2009 @@ -205,7 +205,7 @@ def _toint(space, w_obj): # trying to support float arguments, just because CPython still does try: - return space.int_w(w_obj) + return space.int_w(space.int(w_obj)) except OperationError, e: if space.is_true(space.isinstance(w_obj, space.w_float)): return space.int_w(space.int(w_obj)) Modified: pypy/trunk/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_functional.py Fri Jul 3 16:06:44 2009 @@ -117,6 +117,18 @@ # test again, to make sure that xrange() is not its own iterator assert iter(x).next() == 2 + def test_xrange_object_with___int__(self): + class A(object): + def __int__(self): + return 5 + + assert list(xrange(A())) == [0, 1, 2, 3, 4] + assert list(xrange(0, A())) == [0, 1, 2, 3, 4] + assert list(xrange(0, 10, A())) == [0, 5] + + def test_xrange_float(self): + assert list(xrange(0.1, 2.0, 1.1)) == [0, 1] + class AppTestReversed: def test_reversed(self): r = reversed("hello") Modified: pypy/trunk/pypy/module/__builtin__/test/test_range.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_range.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_range.py Fri Jul 3 16:06:44 2009 @@ -59,8 +59,8 @@ def test_range_zerostep(self): raises(ValueError, range, 1, 5, 0) - def DONT_test_range_float(self): - "How CPython does it - UGLY, ignored for now." + def test_range_float(self): + "How CPython does it - UGLY." assert range(0.1, 2.0, 1.1) == [0, 1] def test_range_wrong_type(self): From pedronis at codespeak.net Fri Jul 3 17:07:34 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 3 Jul 2009 17:07:34 +0200 (CEST) Subject: [pypy-svn] r66103 - pypy/branch/pyjitpl5/pypy/translator/jvm/test Message-ID: <20090703150734.D930F16854E@codespeak.net> Author: pedronis Date: Fri Jul 3 17:07:33 2009 New Revision: 66103 Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/test/runtest.py Log: fixing tests that want to pass backendopt in Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/jvm/test/runtest.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/jvm/test/runtest.py Fri Jul 3 17:07:33 2009 @@ -128,10 +128,10 @@ def _skip_llinterpreter(self, reason, skipLL=True, skipOO=True): pass - def interpret(self, fn, args, annotation=None): + def interpret(self, fn, args, annotation=None, backendopt=False): detect_missing_support_programs() try: - src = self.compile(fn, args, annotation) + src = self.compile(fn, args, annotation, backendopt=backendopt) res = src(*args) return res except JvmError, e: From antocuni at codespeak.net Sat Jul 4 11:00:17 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 11:00:17 +0200 (CEST) Subject: [pypy-svn] r66107 - pypy/extradoc/talk/rst2beamer-template Message-ID: <20090704090017.03A76168467@codespeak.net> Author: antocuni Date: Sat Jul 4 11:00:15 2009 New Revision: 66107 Added: pypy/extradoc/talk/rst2beamer-template/ pypy/extradoc/talk/rst2beamer-template/Makefile pypy/extradoc/talk/rst2beamer-template/author.latex - copied unchanged from r66036, pypy/extradoc/talk/ep2009/jit/author.latex pypy/extradoc/talk/rst2beamer-template/beamerdefs.txt - copied unchanged from r66034, pypy/extradoc/talk/ep2009/jit/beamerdefs.txt pypy/extradoc/talk/rst2beamer-template/stylesheet.latex - copied unchanged from r66034, pypy/extradoc/talk/ep2009/jit/stylesheet.latex pypy/extradoc/talk/rst2beamer-template/talk.pdf.info - copied, changed from r66034, pypy/extradoc/talk/pycon-italy-2009/status/status.pdf.info pypy/extradoc/talk/rst2beamer-template/talk.txt (contents, props changed) pypy/extradoc/talk/rst2beamer-template/title.latex - copied, changed from r66034, pypy/extradoc/talk/ep2009/jit/title.latex Log: a template directory with all the files needed to produce an rst2beamer based talk Added: pypy/extradoc/talk/rst2beamer-template/Makefile ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rst2beamer-template/Makefile Sat Jul 4 11:00:15 2009 @@ -0,0 +1,17 @@ +# you can find rst2beamer.py here: +# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py + +# WARNING: to work, it needs this patch for docutils +# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 + +talk.pdf: talk.txt author.latex title.latex stylesheet.latex + rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.txt talk.latex || exit + sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit + sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit + pdflatex talk.latex || exit + +view: talk.pdf + evince talk.pdf & + +xpdf: talk.pdf + xpdf talk.pdf & Copied: pypy/extradoc/talk/rst2beamer-template/talk.pdf.info (from r66034, pypy/extradoc/talk/pycon-italy-2009/status/status.pdf.info) ============================================================================== --- pypy/extradoc/talk/pycon-italy-2009/status/status.pdf.info (original) +++ pypy/extradoc/talk/rst2beamer-template/talk.pdf.info Sat Jul 4 11:00:15 2009 @@ -1,6 +1,6 @@ AvailableTransitions=[Crossfade] TransitionDuration = 100 -EstimatedDuration = 60*60 +EstimatedDuration = 60*60 # in seconds MinutesOnly = True PageProps = { Added: pypy/extradoc/talk/rst2beamer-template/talk.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/rst2beamer-template/talk.txt Sat Jul 4 11:00:15 2009 @@ -0,0 +1,7 @@ +.. include:: beamerdefs.txt + +================================ +Title +================================ + +XXX Copied: pypy/extradoc/talk/rst2beamer-template/title.latex (from r66034, pypy/extradoc/talk/ep2009/jit/title.latex) ============================================================================== --- pypy/extradoc/talk/ep2009/jit/title.latex (original) +++ pypy/extradoc/talk/rst2beamer-template/title.latex Sat Jul 4 11:00:15 2009 @@ -1,5 +1,5 @@ \begin{titlepage} \begin{figure}[h] -\includegraphics[width=80px]{../../img/py-web.png} +\includegraphics[width=80px]{../img/py-web.png} \end{figure} \end{titlepage} From pedronis at codespeak.net Sat Jul 4 12:54:04 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 4 Jul 2009 12:54:04 +0200 (CEST) Subject: [pypy-svn] r66109 - pypy/branch/pyjitpl5/pypy/translator/jvm Message-ID: <20090704105404.454D4169DFC@codespeak.net> Author: pedronis Date: Sat Jul 4 12:54:00 2009 New Revision: 66109 Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Log: fix test_op/test_overflow hopefully, the llong ops are not tested it seems Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Sat Jul 4 12:54:00 2009 @@ -124,7 +124,7 @@ 'int_add_nonneg_ovf': jvm.IADDOVF, 'int_sub_ovf': jvm.ISUBOVF, 'int_mul_ovf': jvm.IMULOVF, - 'int_floordiv_ovf': jvm.IDIV, # these can't overflow! + 'int_floordiv_ovf': jvm.IFLOORDIVOVF, 'int_mod_zer': _check_zer(jvm.IREM), 'int_mod_ovf': jvm.IREMOVF, 'int_and_ovf': jvm.IAND, @@ -134,7 +134,7 @@ 'int_rshift_ovf': jvm.ISHR, # these can't overflow! 'int_xor_ovf': jvm.IXOR, - 'int_floordiv_ovf_zer': _check_zer(jvm.IDIV), + 'int_floordiv_ovf_zer': jvm.IFLOORDIVZEROVF, 'int_mod_ovf_zer': _check_zer(jvm.IREMOVF), 'uint_invert': 'bitwise_negate', @@ -180,7 +180,8 @@ 'llong_lshift': [PushAllArgs, jvm.L2I, jvm.LSHL, StoreResult], 'llong_rshift': [PushAllArgs, jvm.L2I, jvm.LSHR, StoreResult], 'llong_xor': jvm.LXOR, - 'llong_floordiv_ovf': jvm.LDIV, # these can't overflow! + 'llong_floordiv_ovf': jvm.LFLOORDIVOVF, + 'llong_floordiv_ovf_zer': jvm.LFLOORDIVZEROVF, 'llong_mod_ovf': jvm.LREMOVF, 'llong_lshift_ovf': jvm.LSHLOVF, From antocuni at codespeak.net Sat Jul 4 12:59:43 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 12:59:43 +0200 (CEST) Subject: [pypy-svn] r66110 - pypy/trunk/pypy/module/__builtin__/test Message-ID: <20090704105943.9B97B169E0D@codespeak.net> Author: antocuni Date: Sat Jul 4 12:59:43 2009 New Revision: 66110 Modified: pypy/trunk/pypy/module/__builtin__/test/test_classobj.py Log: (bbrazil, antocuni around) fix this for 64 bit machines Modified: pypy/trunk/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_classobj.py Sat Jul 4 12:59:43 2009 @@ -505,6 +505,7 @@ raises(TypeError, cmp, a, b) def test_hash(self): + import sys class A: pass hash(A()) # does not crash @@ -528,11 +529,12 @@ return 1 a = A() raises(TypeError, hash, a) + bigint = sys.maxint + 1 class A: # can return long def __hash__(self): - return long(2**31) + return long(bigint) a = A() - assert hash(a) == -2147483648 + assert hash(a) == -bigint def test_index(self): import sys From antocuni at codespeak.net Sat Jul 4 13:09:46 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 13:09:46 +0200 (CEST) Subject: [pypy-svn] r66111 - in pypy/trunk/pypy/module/__builtin__: . test Message-ID: <20090704110946.DEA6216846E@codespeak.net> Author: antocuni Date: Sat Jul 4 13:09:46 2009 New Revision: 66111 Modified: pypy/trunk/pypy/module/__builtin__/functional.py pypy/trunk/pypy/module/__builtin__/test/test_functional.py pypy/trunk/pypy/module/__builtin__/test/test_range.py Log: (bbrazil, antocuni around) more tests and fixes for various range corner cases Modified: pypy/trunk/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/functional.py Sat Jul 4 13:09:46 2009 @@ -61,7 +61,6 @@ get a list in decending order.""" try: - # save duplication by redirecting every error to applevel x = space.int_w(space.int(w_x)) if space.is_w(w_y, space.w_None): start, stop = 0, x @@ -69,23 +68,20 @@ start, stop = x, space.int_w(space.int(w_y)) step = space.int_w(space.int(w_step)) howmany = get_len_of_range(start, stop, step) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - except (ValueError, OverflowError): - pass - else: - if (space.config.objspace.std.withmultilist or - space.config.objspace.std.withrangelist): - return range_withspecialized_implementation(space, start, - step, howmany) - res_w = [None] * howmany - v = start - for idx in range(howmany): - res_w[idx] = space.wrap(v) - v += step - return space.newlist(res_w) - return range_fallback(space, w_x, w_y, w_step) + except (ValueError, OverflowError, OperationError): + # save duplication by redirecting every error to applevel + return range_fallback(space, w_x, w_y, w_step) + + if (space.config.objspace.std.withmultilist or + space.config.objspace.std.withrangelist): + return range_withspecialized_implementation(space, start, + step, howmany) + res_w = [None] * howmany + v = start + for idx in range(howmany): + res_w[idx] = space.wrap(v) + v += step + return space.newlist(res_w) range_int = range range_int.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] del range # don't hide the builtin one Modified: pypy/trunk/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_functional.py Sat Jul 4 13:09:46 2009 @@ -129,6 +129,13 @@ def test_xrange_float(self): assert list(xrange(0.1, 2.0, 1.1)) == [0, 1] + def test_xrange_long(self): + import sys + a = long(10 * sys.maxint) + raises(OverflowError, xrange, a) + raises(OverflowError, xrange, 0, a) + raises(OverflowError, xrange, 0, 1, a) + class AppTestReversed: def test_reversed(self): r = reversed("hello") Modified: pypy/trunk/pypy/module/__builtin__/test/test_range.py ============================================================================== --- pypy/trunk/pypy/module/__builtin__/test/test_range.py (original) +++ pypy/trunk/pypy/module/__builtin__/test/test_range.py Sat Jul 4 13:09:46 2009 @@ -75,4 +75,18 @@ assert range(0, A()) == [0, 1, 2, 3, 4] assert range(0, 10, A()) == [0, 5] + def test_range_long(self): + import sys + assert range(-2**100) == [] + assert range(0, -2**100) == [] + assert range(0, 2**100, -1) == [] + assert range(0, 2**100, -1) == [] + a = long(10 * sys.maxint) + b = long(100 * sys.maxint) + c = long(50 * sys.maxint) + + assert range(a, a+2) == [a, a+1] + assert range(a+2, a, -1L) == [a+2, a+1] + assert range(a+4, a, -2) == [a+4, a+2] + From arigo at codespeak.net Sat Jul 4 14:53:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Jul 2009 14:53:08 +0200 (CEST) Subject: [pypy-svn] r66113 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090704125308.C4441168443@codespeak.net> Author: arigo Date: Sat Jul 4 14:53:05 2009 New Revision: 66113 Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex Log: Small reorganizion of the slide "Idea for a Solution" Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex ============================================================================== --- pypy/extradoc/talk/icooolps2009/talk/talk.tex (original) +++ pypy/extradoc/talk/icooolps2009/talk/talk.tex Sat Jul 4 14:53:05 2009 @@ -204,13 +204,13 @@ \begin{itemize} \item Question: What happens if the program is itself a bytecode interpreter? \item the (most important) hot loop of a bytecode interpreter is the bytecode dispatch loop - \item Assumption violated: two iterations of the dispatch loop will usually take very different code paths + \item Assumption violated: consecutive iterations of the dispatch loop will usually take very different code paths \end{itemize} \pause \begin{block}{Terminology} \begin{itemize} \item \emph{tracing interpreter:} the interpreter that originally runs the program and produces traces - \item \emph{language interpreter:} the bytecode interpreter runs on top + \item \emph{language interpreter:} the bytecode interpreter run on top \item \emph{user program:} the program run by the language interpreter \end{itemize} \end{block} @@ -298,10 +298,10 @@ \begin{frame} \frametitle{Idea for a Solution} \begin{itemize} - \item key idea: try to trace the loops in the user program - \item approach: add things to the position key of the tracer + \item goal: try to trace the loops \emph{in the user program,} + and not just one iteration of the bytecode dispatch loop \item tracing interpreter needs information about the language interpreter - \item provided by adding \emph{hints} to the language interpreter + \item provided by adding \emph{three hints} to the language interpreter \end{itemize} \pause \begin{block}{Hints Give Information About:} @@ -376,7 +376,7 @@ \frametitle{What Have We Won?} \begin{itemize} \item trace corresponds to one loop of the user program - \item however, most operations concerned with manipulating bytecode and program counter + \item however, most operations are concerned with manipulating bytecode and program counter \item bytecode and program counter are part of the position key \item thus they are constant at the beginning of the loop \item therefore they can and should be constant-folded @@ -422,8 +422,8 @@ & &Time (ms) &speedup\\ \hline 1 &No JIT &442.7 $\pm$ 3.4 &1.00\\ -2 &Normal Trace Compilation &1518.7 $\pm$ 7.2 &0.29\\ -3 &Unrolling of Interp. Loop &737.6 $\pm$ 7.9 &0.60\\ +2 &JIT, Normal Trace Compilation &1518.7 $\pm$ 7.2 &0.29\\ +3 &JIT, Unrolling of Interp. Loop &737.6 $\pm$ 7.9 &0.60\\ 4 &JIT, Full Optimizations &156.2 $\pm$ 3.8 &2.83\\ \hline \end{tabular} From antocuni at codespeak.net Sat Jul 4 15:09:48 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 15:09:48 +0200 (CEST) Subject: [pypy-svn] r66114 - in pypy/trunk/pypy: module/mmap/test objspace/std/test Message-ID: <20090704130948.2231A16845E@codespeak.net> Author: antocuni Date: Sat Jul 4 15:09:47 2009 New Revision: 66114 Modified: pypy/trunk/pypy/module/mmap/test/test_mmap.py pypy/trunk/pypy/objspace/std/test/test_stringobject.py Log: (bbrazil) - Don't run string overflow replace test on 64 bit platforms, that would be bad - CPython can also raise a SystemError Modified: pypy/trunk/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/trunk/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/trunk/pypy/module/mmap/test/test_mmap.py Sat Jul 4 15:09:47 2009 @@ -410,7 +410,7 @@ def fn(m): m *= 1 # but it raises((SystemError, TypeError), fn, m) # doesn't def fn(): 1 * m # make much sense - raises(TypeError, fn) + raises((SystemError, TypeError), fn) m.close() f.close() # Modified: pypy/trunk/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_stringobject.py Sat Jul 4 15:09:47 2009 @@ -723,6 +723,9 @@ raises(TypeError, len, iter(iterable)) def test_overflow_replace(self): + import sys + if sys.maxint > 2**31-1: + skip("Wrong platform") x = "A" * (2**16) raises(OverflowError, x.replace, '', x) From antocuni at codespeak.net Sat Jul 4 15:18:15 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 15:18:15 +0200 (CEST) Subject: [pypy-svn] r66115 - in pypy/trunk/pypy: module/posix rlib translator/goal Message-ID: <20090704131815.EDD78168473@codespeak.net> Author: antocuni Date: Sat Jul 4 15:18:15 2009 New Revision: 66115 Modified: pypy/trunk/pypy/module/posix/__init__.py pypy/trunk/pypy/rlib/streamio.py pypy/trunk/pypy/translator/goal/targetpypystandalone.py Log: (yoli, antocuni) (sigh) evil hack to make pypy-{cli,jvm} to translate again on windows; the problem is that streamio.py assumes that the backend is c, so it freely uses rffi externals and win32 APIs to implement ftruncate, but of course there is no chance to make it working on ootype. It is really a design issue, we need to be able to provide alternative implementations based on (platform, backend) at some point. Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Sat Jul 4 15:18:15 2009 @@ -120,6 +120,14 @@ if hasattr(os, name): interpleveldefs[name] = 'interp_posix.' + name + def __init__(self, space, w_name): + backend = space.config.translation.backend + # the Win32 urandom implementation isn't going to translate on JVM or CLI + # so we have to remove it + if backend == 'cli' or backend == 'jvm': + del self.interpleveldefs['urandom'] + MixedModule.__init__(self, space, w_name) + def startup(self, space): from pypy.module.posix import interp_posix interp_posix.get(space).startup(space) Modified: pypy/trunk/pypy/rlib/streamio.py ============================================================================== --- pypy/trunk/pypy/rlib/streamio.py (original) +++ pypy/trunk/pypy/rlib/streamio.py Sat Jul 4 15:18:15 2009 @@ -176,6 +176,10 @@ compilation_info=_eci) SetEndOfFile = rffi.llexternal('SetEndOfFile', [rffi.LONG], rwin32.BOOL, compilation_info=_eci) + + # HACK: These implementations are specific to MSVCRT and the C backend. + # When generating on CLI or JVM, these are patched out. + # See PyPyTarget.target() in targetpypystandalone.py def _setfd_binary(fd): _setmode(fd, os.O_BINARY) Modified: pypy/trunk/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/trunk/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/trunk/pypy/translator/goal/targetpypystandalone.py Sat Jul 4 15:18:15 2009 @@ -199,6 +199,16 @@ wrapstr = 'space.wrap(%r)' % (options) pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr + if config.translation.backend in ["cli", "jvm"] and sys.platform == "win32": + # HACK: The ftruncate implementation in streamio.py which is used for the Win32 platform + # is specific for the C backend and can't be generated on CLI or JVM. Because of that, + # we have to patch it out. + from pypy.rlib import streamio + def ftruncate_win32_dummy(fd, size): pass + def _setfd_binary_dummy(fd): pass + streamio.ftruncate_win32 = ftruncate_win32_dummy + streamio._setfd_binary = _setfd_binary_dummy + return self.get_entry_point(config) def jitpolicy(self, driver): From antocuni at codespeak.net Sat Jul 4 15:21:09 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 15:21:09 +0200 (CEST) Subject: [pypy-svn] r66116 - pypy/trunk/pypy/translator/cli/src Message-ID: <20090704132109.187C016847B@codespeak.net> Author: antocuni Date: Sat Jul 4 15:21:08 2009 New Revision: 66116 Modified: pypy/trunk/pypy/translator/cli/src/ll_os.cs Log: (yoli, antocuni around) don't crash when exiting pypy-cli and you try to close() stdin Modified: pypy/trunk/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/trunk/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/trunk/pypy/translator/cli/src/ll_os.cs Sat Jul 4 15:21:08 2009 @@ -261,7 +261,8 @@ public static void ll_os_close(int fd) { FileStream stream = getfd(fd).GetStream(); - stream.Close(); + if (stream != null) // stdin/stdout/stderr files don't have a stream + stream.Close(); FileDescriptors.Remove(fd); } From benjamin at codespeak.net Sat Jul 4 15:55:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 4 Jul 2009 15:55:44 +0200 (CEST) Subject: [pypy-svn] r66118 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test Message-ID: <20090704135544.8EA1816845A@codespeak.net> Author: benjamin Date: Sat Jul 4 15:55:41 2009 New Revision: 66118 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_metaparser.py Log: represent an accepting state with the second field on the state tuple Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py Sat Jul 4 15:55:41 2009 @@ -140,9 +140,7 @@ arcs = [] for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) - if state.is_final: - arcs.append((0, state_index)) - states.append(arcs) + states.append((arcs, state.is_final)) our_id = gram.symbol_ids[name] gram.dfas[our_id] = (states, self.make_first(gram, name)) gram.start = gram.symbol_ids[self.start_symbol] Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Sat Jul 4 15:55:41 2009 @@ -87,18 +87,18 @@ while True: dfa, state_index, node = self.stack[-1] states, first = dfa - arcs = states[state_index] + arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] if label_index == i: self.shift(next_state, token_type, value, lineno, column) - state_index = next_state - while states[state_index] == [(0, state_index)]: + state = states[next_state] + while state[1] and not state[0]: self.pop() if not self.stack: return True dfa, state_index, node = self.stack[-1] - states = dfa[0] + state = dfa[0][state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id] @@ -107,7 +107,7 @@ column) break else: - if (0, state_index) in arcs: + if is_accepting: self.pop() if not self.stack: raise ParseError("too much input", token_type, value, Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_metaparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_metaparser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_metaparser.py Sat Jul 4 15:55:41 2009 @@ -36,7 +36,7 @@ assert eval_sym in g.dfas assert g.start == eval_sym states, first = g.dfas[eval_sym] - assert states == [[(1, 1)], [(0, 1)]] + assert states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -53,10 +53,10 @@ g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 states = g.dfas[g.symbol_ids["foo"]][0] - last = states[0][0][1] + last = states[0][0][0][1] for state in states[1:-1]: - assert last < state[0][1] - last = state[0][1] + assert last < state[0][0][1] + last = state[0][0][1] def test_alternatives(self): g = self.gram_for("foo: STRING | OP") From antocuni at codespeak.net Sat Jul 4 16:03:52 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 16:03:52 +0200 (CEST) Subject: [pypy-svn] r66119 - pypy/trunk/pypy/rpython/test Message-ID: <20090704140352.DC31C16845A@codespeak.net> Author: antocuni Date: Sat Jul 4 16:03:52 2009 New Revision: 66119 Modified: pypy/trunk/pypy/rpython/test/test_rfloat.py Log: (bbrazil) - r_longlong becomes int on a 64bit system, so update test_longlong_conversion to handle this Modified: pypy/trunk/pypy/rpython/test/test_rfloat.py ============================================================================== --- pypy/trunk/pypy/rpython/test/test_rfloat.py (original) +++ pypy/trunk/pypy/rpython/test/test_rfloat.py Sat Jul 4 16:03:52 2009 @@ -2,7 +2,7 @@ from pypy.translator.translator import TranslationContext from pypy.rpython.test import snippet from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_singlefloat,\ +from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_singlefloat,\ isnan, isinf class TestSnippet(object): @@ -76,12 +76,17 @@ assert res == fn(2.34) def test_longlong_conversion(self): + import sys def fn(f): return r_longlong(f) res = self.interpret(fn, [1.0]) assert res == 1 - assert self.is_of_type(res, r_longlong) + # r_longlong is int on a 64 bit system + if sys.maxint == 2**63 - 1: + assert self.is_of_type(res, int) + else: + assert self.is_of_type(res, r_longlong) res = self.interpret(fn, [2.34]) assert res == fn(2.34) big = float(0x7fffffffffffffff) From antocuni at codespeak.net Sat Jul 4 16:16:08 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 16:16:08 +0200 (CEST) Subject: [pypy-svn] r66120 - pypy/trunk/pypy/translator/jvm Message-ID: <20090704141608.A39EB16846E@codespeak.net> Author: antocuni Date: Sat Jul 4 16:16:08 2009 New Revision: 66120 Modified: pypy/trunk/pypy/translator/jvm/genjvm.py Log: (yole, antocuni around) fix jvm tests on windows. It seems we never run them there :-( Modified: pypy/trunk/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/trunk/pypy/translator/jvm/genjvm.py (original) +++ pypy/trunk/pypy/translator/jvm/genjvm.py Sat Jul 4 16:16:08 2009 @@ -3,6 +3,7 @@ """ import sys +import os import py from py.compat import subprocess @@ -197,7 +198,7 @@ cmd = [getoption('java'), '-Xmx256M', # increase the heapsize so the microbenchmarks run '-cp', - str(self.javadir)+":"+str(self.jnajar), + str(self.javadir)+os.pathsep+str(self.jnajar), self.package+".Main"] + strargs print "Invoking java to run the code" stdout, stderr, retval = self._invoke(cmd, True) From antocuni at codespeak.net Sat Jul 4 16:38:19 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sat, 4 Jul 2009 16:38:19 +0200 (CEST) Subject: [pypy-svn] r66121 - pypy/trunk/pypy/module/mmap/test Message-ID: <20090704143819.8F797168424@codespeak.net> Author: antocuni Date: Sat Jul 4 16:38:19 2009 New Revision: 66121 Modified: pypy/trunk/pypy/module/mmap/test/test_mmap.py Log: (bbrazil, antocuni and pedronis around) CPython raises an OverflowError if you pass a negative length to mmap, RPython raises a TypeError. Make the unittest happy about both. Modified: pypy/trunk/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/trunk/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/trunk/pypy/module/mmap/test/test_mmap.py Sat Jul 4 16:38:19 2009 @@ -42,7 +42,7 @@ raises(TypeError, mmap, 0, 1, 2, 3, 4, 5) raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5) raises(TypeError, mmap, 0, 1, foo="foo") - raises(TypeError, mmap, 0, -1) + raises((TypeError, OverflowError), mmap, 0, -1) raises(OverflowError, mmap, 0, sys.maxint ** 3) raises(ValueError, mmap, 0, 1, flags=2, access=3) raises(ValueError, mmap, 0, 1, access=123) From fijal at codespeak.net Sun Jul 5 18:10:56 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 5 Jul 2009 18:10:56 +0200 (CEST) Subject: [pypy-svn] r66122 - pypy/trunk/pypy/translator/goal Message-ID: <20090705161056.3CA8016803E@codespeak.net> Author: fijal Date: Sun Jul 5 18:10:54 2009 New Revision: 66122 Modified: pypy/trunk/pypy/translator/goal/sharedpypy.py Log: minor tweaks Modified: pypy/trunk/pypy/translator/goal/sharedpypy.py ============================================================================== --- pypy/trunk/pypy/translator/goal/sharedpypy.py (original) +++ pypy/trunk/pypy/translator/goal/sharedpypy.py Sun Jul 5 18:10:54 2009 @@ -1,5 +1,5 @@ -import sys +import sys, pdb, traceback from pypy.translator.c.dlltool import DLLDef from pypy.config.translationoption import get_combined_translation_config from pypy.rpython.lltypesystem.rffi import charp2str, CCHARP, VOIDP @@ -27,6 +27,7 @@ space = make_objspace(config) policy = PyPyAnnotatorPolicy(single_space = space) + policy.allow_someobjects = False def interpret(source, context): source = charp2str(source) @@ -47,4 +48,13 @@ exe_name = dll.compile() if __name__ == '__main__': - main(sys.argv) + try: + main(sys.argv) + except KeyboardInterrupt: + raise + except: + e, v, tb = sys.exc_info() + traceback.print_tb(tb) + print e, v + pdb.post_mortem(tb) + From cfbolz at codespeak.net Sun Jul 5 20:27:46 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 5 Jul 2009 20:27:46 +0200 (CEST) Subject: [pypy-svn] r66123 - pypy/extradoc/talk/icooolps2009/talk Message-ID: <20090705182746.F286D16843A@codespeak.net> Author: cfbolz Date: Sun Jul 5 20:27:44 2009 New Revision: 66123 Added: pypy/extradoc/talk/icooolps2009/talk/talk.pdf (contents, props changed) Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex Log: Some small additions, leaving out some things from the interpreter and traces. Added: pypy/extradoc/talk/icooolps2009/talk/talk.pdf ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/icooolps2009/talk/talk.tex ============================================================================== --- pypy/extradoc/talk/icooolps2009/talk/talk.tex (original) +++ pypy/extradoc/talk/icooolps2009/talk/talk.tex Sun Jul 5 20:27:44 2009 @@ -40,7 +40,7 @@ merlinux GmbH } -\date{ICOOOLPS 2009 XXX} +\date{6th of July 2009, ICOOOLPS '09, Genova} % Delete this, if you do not want the table of contents to pop up at @@ -98,8 +98,7 @@ \begin{itemize} \item idea from Dynamo project: dynamic rewriting of machine code \item later used for a lightweight Java JIT - \item seems to also work for dynamic languages - \item incorporated into Mozilla's JavaScript VM ("TraceMonkey") + \item seems to also work for dynamic languages (see TraceMonkey) \end{itemize} \pause \begin{block}{Basic Assumption of a Tracing JIT} @@ -187,7 +186,7 @@ \item interpreter does a lot of the work \item can be added to an existing interpreter unobtrusively \item automatic inlining - \item produces comparatively little code + \item produces comparatively little machine code \end{itemize} \end{block} \pause @@ -205,6 +204,7 @@ \item Question: What happens if the program is itself a bytecode interpreter? \item the (most important) hot loop of a bytecode interpreter is the bytecode dispatch loop \item Assumption violated: consecutive iterations of the dispatch loop will usually take very different code paths + \item what can we do? \end{itemize} \pause \begin{block}{Terminology} @@ -229,54 +229,52 @@ pc += 1 if a: pc = target - elif opcode == MOV_A_R: - n = ord(bytecode[pc]) - pc += 1 - regs[n] = a elif opcode == MOV_R_A: n = ord(bytecode[pc]) pc += 1 a = regs[n] + elif opcode == MOV_A_R: + ... elif opcode == ADD_R_TO_A: - n = ord(bytecode[pc]) - pc += 1 - a += regs[n] + ... elif opcode == DECR_A: a -= 1 elif opcode == RETURN_A: return a + + + \end{verbatim} } \frame[containsverbatim, plain, shrink=10]{ \begin{verbatim} -def interpret(bytecode, a): | - regs = [0] * 256 | # Example bytecode - pc = 0 | # Square the accumulator: - while True: | - opcode = ord(bytecode[pc]) | MOV_A_R 0 # i = a - pc += 1 | MOV_A_R 1 # copy of 'a' - if opcode == JUMP_IF_A: | - target = ord(bytecode[pc]) | # 4: - pc += 1 | MOV_R_A 0 # i-- - if a: | DECR_A - pc = target | MOV_A_R 0 - elif opcode == MOV_A_R: | - n = ord(bytecode[pc]) | MOV_R_A 2 # res += a - pc += 1 | ADD_R_TO_A 1 - regs[n] = a | MOV_A_R 2 - elif opcode == MOV_R_A: | - n = ord(bytecode[pc]) | MOV_R_A 0 # if i!=0: - pc += 1 | JUMP_IF_A 4 # goto 4 - a = regs[n] | - elif opcode == ADD_R_TO_A: | MOV_R_A 2 # return res - n = ord(bytecode[pc]) | RETURN_A - pc += 1 | - a += regs[n] | +def interpret(bytecode, a): + regs = [0] * 256 | + pc = 0 | # Example bytecode + while True: | # Square the accumulator: + opcode = ord(bytecode[pc]) | + pc += 1 | MOV_A_R 0 # i = a + if opcode == JUMP_IF_A: | MOV_A_R 1 # copy of 'a' + target = ord(bytecode[pc]) | + pc += 1 | # 4: + if a: | MOV_R_A 0 # i-- + pc = target | DECR_A + elif opcode == MOV_R_A: | MOV_A_R 0 + n = ord(bytecode[pc]) | + pc += 1 | MOV_R_A 2 # res += a + a = regs[n] | ADD_R_TO_A 1 + elif opcode == MOV_A_R: | MOV_A_R 2 + ... | + elif opcode == ADD_R_TO_A: | MOV_R_A 0 # if i!=0: + ... | JUMP_IF_A 4 # goto 4 elif opcode == DECR_A: | - a -= 1 | - elif opcode == RETURN_A: | + a -= 1 | MOV_R_A 2 # return res + elif opcode == RETURN_A: | RETURN_A return a | + + + \end{verbatim} } @@ -306,7 +304,7 @@ \pause \begin{block}{Hints Give Information About:} \begin{itemize} - \item which variables make up the program counter of the language interpreter + \item which variables make up the program counter of the language interpreter (together those are called \emph{position key}) \item where the bytecode dispatch loop is \item which bytecodes can correspond to backward jumps \end{itemize} @@ -337,6 +335,19 @@ \end{verbatim} } +\begin{frame} + \frametitle{Modifying Tracing} + \begin{itemize} + \item goal: try to trace the loops \emph{in the user program,} + and not just one iteration of the bytecode dispatch loop + \item tracing interpreter stops tracing only when: + \begin{itemize} + \item it sees a backward jump in the language interpreter + \item the position key of the language interpreter matches an ealier value + \end{itemize} + \item in this way, full user loops are traced + \end{itemize} +\end{frame} \frame[containsverbatim, plain, shrink=20]{ \frametitle{Result When Tracing \texttt{SQUARE}} \begin{verbatim} @@ -349,15 +360,10 @@ pc2 = int_add(pc1, Const(1)) a1 = list_getitem(regs0, n1) # DECR_A -... # MOV_A_R 0 -... # MOV_R_A 2 -... # ADD_R_TO_A 1 -... # MOV_A_R 2 -... # MOV_R_A 0 ... # JUMP_IF_A 4 @@ -434,11 +440,35 @@ \frametitle{Scaling to Large Interpreters?} \begin{itemize} \item we can apply this approach to PyPy's Python interpreter (70 KLOC) - \item speed-ups promising: factor of 6 faster for simple loops with arithmetic + \item speed-ups promising (see next slide) \item no Python-specific bugs! \end{itemize} \end{frame} +\frame[containsverbatim, plain, shrink=10]{ + \frametitle{Timings for Python Interpreter} +\begin{verbatim} +def f(a): + t = (1, 2, 3) + i = 0 + while i < a: + t = (t[1], t[2], t[0]) + i += t[0] + return i +\end{verbatim} +\begin{block}{Timings of \texttt{f(10000000)}} +\begin{tabular}{|l|l|r|r|} +\hline +& &Time (ms) &speedup\\ +\hline +1 &PyPy compiled to C, no JIT &1793 $\pm$ 11 &1.00\\ +2 &PyPy comp'd to C, with JIT &483 $\pm$ 6 &3.71\\ +3 &CPython 2.6 &1869 $\pm$ 11 & 0.96\\ +4 &CPython 2.6 + Psyco 1.6 & 511 $\pm$ 7 &3.51\\\hline +\end{tabular} +\end{block} +} + \begin{frame} \frametitle{Conclusions} @@ -479,32 +509,4 @@ \end{block} \end{frame} -\begin{frame} - \frametitle{Backup Slides} -\end{frame} - -\frame[containsverbatim, plain, shrink=10]{ - \frametitle{Timings for Python Interpreter} -\begin{verbatim} -def f(a): - t = (1, 2, 3) - i = 0 - while i < a: - t = (t[1], t[2], t[0]) - i += t[0] - return i -\end{verbatim} -\begin{block}{Timings} -\begin{tabular}{|l|l|r|r|} -\hline -& &Time (s) &speedup\\ -\hline -1 &PyPy compiled to C, no JIT &23.44 $\pm$ 0.07 &1.00\\ -2 &PyPy comp'd to C, with JIT &3.58 $\pm$ 0.05 &6.54\\ -3 &CPython 2.5.2 &4.96 $\pm$ 0.05 &4.73\\ -4 &CPython 2.5.2 + Psyco 1.6 &1.51 $\pm$ 0.05 &15.57\\\hline -\end{tabular} -\end{block} -} - \end{document} From antocuni at codespeak.net Sun Jul 5 21:21:28 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 5 Jul 2009 21:21:28 +0200 (CEST) Subject: [pypy-svn] r66124 - pypy/extradoc/talk/icooolps2009-dotnet/talk Message-ID: <20090705192128.9259C168563@codespeak.net> Author: antocuni Date: Sun Jul 5 21:21:26 2009 New Revision: 66124 Added: pypy/extradoc/talk/icooolps2009-dotnet/talk/ - copied from r66107, pypy/extradoc/talk/rst2beamer-template/ pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf (contents, props changed) Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/Makefile pypy/extradoc/talk/icooolps2009-dotnet/talk/author.latex pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.txt pypy/extradoc/talk/icooolps2009-dotnet/talk/title.latex Log: the slides for my talk Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/Makefile ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/Makefile (original) +++ pypy/extradoc/talk/icooolps2009-dotnet/talk/Makefile Sun Jul 5 21:21:26 2009 @@ -7,11 +7,11 @@ talk.pdf: talk.txt author.latex title.latex stylesheet.latex rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.txt talk.latex || exit sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit - sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit + #sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit pdflatex talk.latex || exit view: talk.pdf evince talk.pdf & xpdf: talk.pdf - xpdf talk.pdf & + xpdf talk.pdf 50 & Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/author.latex ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/author.latex (original) +++ pypy/extradoc/talk/icooolps2009-dotnet/talk/author.latex Sun Jul 5 21:21:26 2009 @@ -1,8 +1,9 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy: becoming fast]{PyPy: becoming fast} -\author[antocuni, cfbolz, pedronis] -{Antonio Cuni \\ Carl Friedrich Bolz\\ Samuele Pedroni} +\title[Efficient dynamic languages on .NET] +{Faster than C\#: efficient implementation of dynamic languages on .NET} +\author[A. Cuni, D. Ancona, A. Rigo] +{Antonio Cuni \\ Davide Ancona \\ Armin Rigo} -\institute{EuroPython 2009} -\date{June 30 2009} +\institute[icooolps 09]{ICOOOLPS at ECOOP 2009 - Genova, Italy} +\date{July 6, 2009} Added: pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.txt ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/talk.txt (original) +++ pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.txt Sun Jul 5 21:21:26 2009 @@ -1,7 +1,446 @@ .. include:: beamerdefs.txt -================================ -Title -================================ +============== +Faster than C# +============== -XXX +Introduction +------------- + +- Dynamic languages are nice + + * e.g., Python + +- so are .NET and the JVM + +- Problem: slow! + +- Solution: make them faster :-) + +- We concentrate our efforts on .NET + + +State of the art +----------------- + +- IronPython + +- Jython + +- JRuby, Groovy, ... + +|pause| + +- **Self** + +- Javascript: TraceMonkey, V8 + +- ... + + +Why so slow? +------------- + +- Hard to compile efficiently + +- Lack of type information at compile-time + +- VMs not optimized to run them + +- .NET is a multi-language VM? |pause| + + * Sure, as long as the language is C# + +|pause| + +- JVM is in a better shape, but still heavily optimized for Java + + +JIT compiler +---------------------- + +- Wait until you know what you need + +- Interweave compile-time and runtime + +- Exploit runtime information + +|pause| + +|alert<| JIT on top of .NET |>| + +- JIT layering + +- How to extend existing code? + +- Fight the VM + +|end_alert| + + +PyPy +---- + +- Python in Python + +- (lots of features and goals) + +- **JIT compiler generator** + +- Python semantics for free + +- JIT frontend + + * Not limited to Python + +- JIT backends + + - x86 backend + + - **CLI/.NET backend** + +|pause| + +- Note: this talk is about JIT v2 + + +Partial evaluation (PE) +----------------------- + +* Assume the Python bytecode to be constant + +* Constant-propagate it into the Python interpreter. + +* Colors + + - :green:`Green`: compile-time value + + - :red:`Red`: runtime value + + +Partial Evaluation with Colors +------------------------------ + +* :green:`Green operations`: unchanged, executed at compile-time + +* :red:`Red operations`: converted into corresponding code emitting code + +|pause| +|column1| +|example<| Example |>| + +.. raw:: latex + + \smallskip + \begin{rtbliteral} + def~f(\green{x},~\red{y}):~\\ + ~~\green{x2}~=~\green{x}~*~\green{x}~\\ + ~~\red{y2}~=~\red{y}~*~\red{y}~\\ + ~~return~\green{x2}~+~\red{y2} + \end{rtbliteral} + \smallskip + +|end_example| + +|pause| +|column2| +|alert<| case x=10 |>| +:: + + def f_10(y): + y2 = y * y + return 100 + y2 + +|end_alert| +|end_columns| + + +Challenges +---------------------- + +* A shortcoming of PE is that in many cases not much can be really + assumed constant at compile-time: poor results + +* Effective dynamic compilation requires feedback of runtime + information into compile-time + +* For a dynamic language: types are a primary example + +Solution: Promotion +-------------------- + +* "Promote" run-time values to compile-time + +* Promotion guided by few hints in the interpreter + +* Stop the compilation at promotions + +* Execute until promotion points + +* Compile more + + + +Promotion (example) +------------------------ + +|example<| Example |>| + +.. raw:: latex + + \smallskip + \begin{rtbliteral} + def~f(\red{x},~\red{y}):\\ + ~~\green{x1}~=~hint(\red{x},~promote=True)\\ + ~~return~\green{x1}*\green{x1}~+~\red{y}*\red{y} + \end{rtbliteral} + \smallskip + +|end_example| + +|small| +|pause| +|column1| +|alert<| original |>| +:: + + def f_(x, y): + switch x: + pass + default: + compile_more(x) + +|end_alert| + +|pause| +|column2| +|alert<| augmented |>| +:: + + def f_(x, y): + switch x: + case 3: + return 9 + y*y + default: + compile_more(x) + +|end_alert| +|end_columns| +|end_small| + + +Promotion on .NET +------------------ + +- Flexswitch + + * Growable switch + + * Can add new cases at runtime + +- Ideally as efficient as a jump + +- No support from the VM + +- Very costly + +- Still effective as long as it's not in the hot path + + +Flexswitch example +------------------ + +|column1| + +.. image:: ../flexswitch1.png + :scale: 45 + +|column2| + +|end_columns| + + +Flexswitch example +------------------ + +|column1| + +.. image:: ../flexswitch1.png + :scale: 45 + +|column2| + +.. image:: ../flexswitch2.png + :scale: 45 + +|end_columns| + + +Flexswitch for CLI +------------------- + +- Unit of compilation: method + +- Flowgraphs split into multiple methods + +- Primary method + + * Contains a trampoline + + * Array of delegates + +- Secondary methods + + * Stored into that array + +- Jumps between secondary methods go through the trampoline + +- Hard (and slow!) to pass arguments around + + +TLC +----------- + +- Python not (yet) supported :-( + +- Dynamic toy language + +- Designed to be "as slow as Python" + +- Stack manipulation + +- Boxed integers + +- Dynamic lookup of methods + + +Benchmarks (1) +-------------- + +.. raw:: latex + + \begin{table}[ht] + \begin{center} + + \begin{tabular}{l|rrrrrr} + \multicolumn{5}{c}{\textbf{Factorial}} \\ [0.5ex] + + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ + \hline + \textbf{Interp} & 0.031 & 30.984 & N/A & N/A \\ + \textbf{JIT} & 0.422 & 0.453 & 0.859 & 4.844 \\ + \textbf{JIT 2} & 0.000 & 0.047 & 0.453 & 4.641 \\ + \textbf{C\#} & 0.000 & 0.031 & 0.359 & 3.438 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{661.000} & N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{1.500} & \textbf{1.261} & \textbf{1.350} \\ [3ex] + + \end{tabular} + + \end{center} + \end{table} + + +Benchmarks (2) +-------------- + +.. raw:: latex + + \begin{table}[ht] + \begin{center} + + \begin{tabular}{l|rrrrrr} + \multicolumn{5}{c}{\textbf{Fibonacci}} \\ [0.5ex] + + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ + \hline + \textbf{Interp} & 0.031 & 29.359 & N/A & N/A \\ + \textbf{JIT} & 0.453 & 0.469 & 0.688 & 2.953 \\ + \textbf{JIT 2} & 0.000 & 0.016 & 0.250 & 2.500 \\ + \textbf{C\#} & 0.000 & 0.016 & 0.234 & 2.453 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{1879.962}& N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{0.999} & \textbf{1.067} & \textbf{1.019} \\ + + \end{tabular} + + \end{center} + \end{table} + + +Benchmars (3) +-------------- + +|small| + +:: + + def main(n): + if n < 0: + n = -n + obj = new(value, accumulate=count) + else: + obj = new(value, accumulate=add) + obj.value = 0 + while n > 0: + n = n - 1 + obj.accumulate(n) + return obj.value + + def count(x): + this.value = this.value + 1 + def add(x): + this.value = this.value + x + +|end_small| + + +Benchmars (4) +-------------- + +.. raw:: latex + + \begin{table}[ht] + \begin{center} + + \begin{tabular}{l|rrrrrr} + \multicolumn{5}{c}{\textbf{Accumulator}} \\ [0.5ex] + + \textbf{$n$} & $10$ & $10^7$ & $10^8$ & $10^9$ \\ + \hline + \textbf{Interp} & 0.031 & 43.063 & N/A & N/A \\ + \textbf{JIT} & 0.453 & 0.516 & 0.875 & 4.188 \\ + \textbf{JIT 2} & 0.000 & 0.047 & 0.453 & 3.672 \\ + \textbf{C\#} & 0.000 & 0.063 & 0.563 & 5.953 \\ + \textbf{Interp/JIT 2} & N/A & \textbf{918.765} & N/A & N/A \\ + \textbf{JIT 2/C\#} & N/A & \textbf{0.750} & \textbf{0.806} & \textbf{0.617} \\ + + \end{tabular} + \end{center} + \end{table} + + +Future work +------------- + +- Non local jumps are terribly slow + +- Good results only if they are not in the inner loop + +- Recompile hot non-local jumps? + +- Tracing JIT? + + * You have just seen it in the previous talk :-) + + +Contributions +-------------- + +- JIT layering works + + * Optimize different levels of overhead + + * .NET's own JIT could be improved + +- Current VMs are limited + + * How to make them more friendly to dynamic languges? Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/title.latex ============================================================================== --- pypy/extradoc/talk/rst2beamer-template/title.latex (original) +++ pypy/extradoc/talk/icooolps2009-dotnet/talk/title.latex Sun Jul 5 21:21:26 2009 @@ -1,5 +1,5 @@ \begin{titlepage} \begin{figure}[h] -\includegraphics[width=80px]{../img/py-web.png} +\includegraphics[width=80px]{../../img/py-web.png} \end{figure} \end{titlepage} From antocuni at codespeak.net Sun Jul 5 21:26:19 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 5 Jul 2009 21:26:19 +0200 (CEST) Subject: [pypy-svn] r66126 - pypy/extradoc/talk/icooolps2009-dotnet/talk Message-ID: <20090705192619.4859C169E1D@codespeak.net> Author: antocuni Date: Sun Jul 5 21:26:18 2009 New Revision: 66126 Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf.info Log: the talk is actually 25 minutes long Modified: pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf.info ============================================================================== --- pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf.info (original) +++ pypy/extradoc/talk/icooolps2009-dotnet/talk/talk.pdf.info Sun Jul 5 21:26:18 2009 @@ -1,6 +1,6 @@ AvailableTransitions=[Crossfade] TransitionDuration = 100 -EstimatedDuration = 60*60 # in seconds +EstimatedDuration = 25*60 # in seconds MinutesOnly = True PageProps = { From fijal at codespeak.net Mon Jul 6 10:22:16 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 6 Jul 2009 10:22:16 +0200 (CEST) Subject: [pypy-svn] r66127 - pypy/trunk/pypy/tool Message-ID: <20090706082216.3F494169E9A@codespeak.net> Author: fijal Date: Mon Jul 6 10:22:14 2009 New Revision: 66127 Modified: pypy/trunk/pypy/tool/compat.py Log: Another difference among 2.x python versions Modified: pypy/trunk/pypy/tool/compat.py ============================================================================== --- pypy/trunk/pypy/tool/compat.py (original) +++ pypy/trunk/pypy/tool/compat.py Mon Jul 6 10:22:14 2009 @@ -8,3 +8,8 @@ except ImportError: # no _md5 module on this platform from pypy.lib.md5 import md5 + +try: + BaseException +except NameError: + BaseException = Exception From fijal at codespeak.net Mon Jul 6 11:23:25 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 6 Jul 2009 11:23:25 +0200 (CEST) Subject: [pypy-svn] r66128 - pypy/trunk/pypy/translator/c Message-ID: <20090706092325.1EF82169EA7@codespeak.net> Author: fijal Date: Mon Jul 6 11:23:24 2009 New Revision: 66128 Modified: pypy/trunk/pypy/translator/c/node.py Log: use baseexception instead Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Mon Jul 6 11:23:24 2009 @@ -13,6 +13,7 @@ from pypy.rlib.rarithmetic import isinf, isnan from pypy.translator.c import extfunc from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.tool.compat import BaseException def needs_gcheader(T): if not isinstance(T, ContainerType): @@ -892,7 +893,7 @@ return 'Py_None' import types, py if isinstance(value, (type, types.ClassType)): - if (issubclass(value, Exception) and + if (issubclass(value, BaseException) and (value.__module__ == 'exceptions' or value is py.magic.AssertionError)): return 'PyExc_' + value.__name__ From cfbolz at codespeak.net Mon Jul 6 18:17:47 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 6 Jul 2009 18:17:47 +0200 (CEST) Subject: [pypy-svn] r66129 - pypy/extradoc/talk/ecoop2009-tutorial Message-ID: <20090706161747.2964A169EC6@codespeak.net> Author: cfbolz Date: Mon Jul 6 18:17:44 2009 New Revision: 66129 Added: pypy/extradoc/talk/ecoop2009-tutorial/ (props changed) - copied from r66128, pypy/extradoc/talk/rst2beamer-template/ Log: copy rst2beamer template for the tutorial part From cfbolz at codespeak.net Mon Jul 6 23:58:11 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 6 Jul 2009 23:58:11 +0200 (CEST) Subject: [pypy-svn] r66133 - pypy/extradoc/talk/ecoop2009-tutorial Message-ID: <20090706215811.DCB2F16853A@codespeak.net> Author: cfbolz Date: Mon Jul 6 23:58:09 2009 New Revision: 66133 Added: pypy/extradoc/talk/ecoop2009-tutorial/dynlang.png (props changed) - copied unchanged from r66128, pypy/extradoc/talk/ep2007/interpreter/dynlang.png Modified: pypy/extradoc/talk/ecoop2009-tutorial/author.latex pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Log: add some intro slides for the tutorial Modified: pypy/extradoc/talk/ecoop2009-tutorial/author.latex ============================================================================== --- pypy/extradoc/talk/ecoop2009-tutorial/author.latex (original) +++ pypy/extradoc/talk/ecoop2009-tutorial/author.latex Mon Jul 6 23:58:09 2009 @@ -1,8 +1,8 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy: becoming fast]{PyPy: becoming fast} -\author[antocuni, cfbolz, pedronis] -{Antonio Cuni \\ Carl Friedrich Bolz\\ Samuele Pedroni} +\title[Writing Interpreters Using PyPy]{Writing Interpreters for Dynamic Languages Using PyPy (and Getting Them Fast)} +\author[] +{Antonio Cuni \\ Carl Friedrich Bolz\\ Armin Rigo} -\institute{EuroPython 2009} -\date{June 30 2009} +\institute{ECOOP '09 Summer School} +\date{July 6 2009} Modified: pypy/extradoc/talk/ecoop2009-tutorial/talk.txt ============================================================================== --- pypy/extradoc/talk/ecoop2009-tutorial/talk.txt (original) +++ pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Mon Jul 6 23:58:09 2009 @@ -1,7 +1,84 @@ .. include:: beamerdefs.txt -================================ -Title -================================ +============================================================================= +Writing Interpreters for Dynamic Languages Using PyPy (and Getting Them Fast) +============================================================================= -XXX +Motivation +================ + +- good VMs for dynamic languages are very hard +- when writing it is very hard to reconcile + + - performance + - simplicity, maintainability + - flexibility, features + +- this is particularly true in an Open Source / research context + +Python Case +============ + +- **CPython** is a simple maintainable but slow VM, using a simple bytecode + interpreter +- **Stackless** is a more featureful fork of CPython that was never merged for maintainability reasons +- **Psyco** is a very complex JIT compiler for Python, which speeds up Python a lot + + +Motivation (2) +================ + +fixing of early design decisions: + +- when writing a VM in C, some things have to be decided up front +- examples are things like memory model, GC, threading model, etc. +- desicions manifest throughout the source code +- extremely hard to change later + +|pause| +|alert<| Python case |>| + +- reference counting +- OS threads + +|end_alert| + +PyPy's approach to VM construction +================================== + +Goal: achieve flexibility, simplicity and performance together + +- Approach: auto-generate VMs from high-level descriptions + of the language +- high-level description: an interpreter written in a high-level + language +- ... which we translate (i.e. compile) to a VM running in + various target environments, like C/Posix, CLR, JVM + +|pause| + +- the high-level language that is used to implement the interpreters is called *RPython* + +PyPy's approach to VM construction +================================== + +.. image:: dynlang.png + :scale: 50 + +Receipe for a PyPy-based VM +============================ + + #. write an interpreter for your language in RPython + #. test it in pure Python until it works and is bug-free + #. try to compile it to C and fix the type-bugs that this finds + #. apply the JIT (optional and experimental step) + +What is RPython +=============== + +A subset of the Python language on which type inference can be performed + +- single inheritance with mixin support +- most of Python's builtin data structures are supported +- no type mixing: ``l = [1, "string"]`` does not work +- don't think about it too hard, it feels a lot like Python still From antocuni at codespeak.net Tue Jul 7 00:03:48 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 00:03:48 +0200 (CEST) Subject: [pypy-svn] r66134 - pypy/extradoc/talk/ecoop2009-tutorial Message-ID: <20090706220348.B9B1116856C@codespeak.net> Author: antocuni Date: Tue Jul 7 00:03:48 2009 New Revision: 66134 Modified: pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Log: typo Modified: pypy/extradoc/talk/ecoop2009-tutorial/talk.txt ============================================================================== --- pypy/extradoc/talk/ecoop2009-tutorial/talk.txt (original) +++ pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Tue Jul 7 00:03:48 2009 @@ -32,7 +32,7 @@ - when writing a VM in C, some things have to be decided up front - examples are things like memory model, GC, threading model, etc. -- desicions manifest throughout the source code +- decisions manifest throughout the source code - extremely hard to change later |pause| From arigo at codespeak.net Tue Jul 7 10:27:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Jul 2009 10:27:36 +0200 (CEST) Subject: [pypy-svn] r66135 - pypy/extradoc/talk/ecoop2009-tutorial Message-ID: <20090707082736.51DEC1683DF@codespeak.net> Author: arigo Date: Tue Jul 7 10:27:34 2009 New Revision: 66135 Modified: pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Log: Details. Modified: pypy/extradoc/talk/ecoop2009-tutorial/talk.txt ============================================================================== --- pypy/extradoc/talk/ecoop2009-tutorial/talk.txt (original) +++ pypy/extradoc/talk/ecoop2009-tutorial/talk.txt Tue Jul 7 10:27:34 2009 @@ -52,7 +52,7 @@ of the language - high-level description: an interpreter written in a high-level language -- ... which we translate (i.e. compile) to a VM running in +- ... which we translate (i.e. compile) to VMs running in various target environments, like C/Posix, CLR, JVM |pause| @@ -71,7 +71,7 @@ #. write an interpreter for your language in RPython #. test it in pure Python until it works and is bug-free #. try to compile it to C and fix the type-bugs that this finds - #. apply the JIT (optional and experimental step) + #. apply the JIT generator (optional and experimental step) What is RPython =============== From arigo at codespeak.net Tue Jul 7 10:47:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Jul 2009 10:47:33 +0200 (CEST) Subject: [pypy-svn] r66136 - pypy/branch/pyjitpl5/pypy/jit/tl Message-ID: <20090707084733.5CF08169E83@codespeak.net> Author: arigo Date: Tue Jul 7 10:47:31 2009 New Revision: 66136 Added: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (contents, props changed) Log: Start tla.py, "toy language for academics" (thanks antonio for the name :-) Added: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 10:47:31 2009 @@ -0,0 +1,94 @@ + +from pypy.rlib.jit import JitDriver + + +class W_Object: + pass + + +class W_IntObject(W_Object): + + def __init__(self, intvalue): + self.intvalue = intvalue + + def is_true(self): + return self.intvalue != 0 + + def add(self, w_other): + if isinstance(w_other, W_IntObject): + sum = self.intvalue + other.intvalue + return W_IntObject(sum) + else: + raise OperationError + + +class OperationError: + pass + +# ____________________________________________________________ + +CONST_INT = 1 +POP = 2 +ADD = 3 +RETURN = 4 + +# ____________________________________________________________ + + +class Frame(object): + _virtualizable2_ = ['stackpos', 'stack[*]'] + + def __init__(self, bytecode): + self.bytecode = bytecode + self.stack = [None] * 8 + self.stackpos = 0 + + def push(self, w_x): + self.stack[self.stackpos] = w_x + self.stackpos += 1 + + def pop(self): + self.stackpos -= 1 + assert self.stackpos >= 0 + return self.stack[self.stackpos] + + def interp(self): + bytecode = self.bytecode + pc = 0 + + while pc < len(bytecode): + opcode = ord(bytecode[pc]) + pc += 1 + + if opcode == CONST_INT: + value = ord(bytecode[pc]) + pc += 1 + w_z = W_IntObject(value) + self.push(w_z) + + elif opcode == POP: + self.pop() + + elif opcode == ADD: + w_y = self.pop() + w_x = self.pop() + w_z = w_x.add(w_y) + self.push(w_z) + + elif opcode == JUMP_IF: + target = ord(bytecode[pc]) + pc += 1 + w_x = self.pop() + if w_x.is_true(): + pc = target + + elif opcode == RETURN: + w_x = self.pop() + return w_x + + +def run(bytecode, w_arg): + frame = Frame(bytecode) + frame.push(w_arg) + w_result = frame.interp() + return w_result From arigo at codespeak.net Tue Jul 7 10:57:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Jul 2009 10:57:40 +0200 (CEST) Subject: [pypy-svn] r66137 - pypy/branch/pyjitpl5/pypy/jit/tl Message-ID: <20090707085740.5622D169E8F@codespeak.net> Author: arigo Date: Tue Jul 7 10:57:38 2009 New Revision: 66137 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: Typo. Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 10:57:38 2009 @@ -16,7 +16,7 @@ def add(self, w_other): if isinstance(w_other, W_IntObject): - sum = self.intvalue + other.intvalue + sum = self.intvalue + w_other.intvalue return W_IntObject(sum) else: raise OperationError From antocuni at codespeak.net Tue Jul 7 10:59:34 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 10:59:34 +0200 (CEST) Subject: [pypy-svn] r66138 - pypy/branch/pyjitpl5/pypy/jit/tl/test Message-ID: <20090707085934.45DC6169E8E@codespeak.net> Author: antocuni Date: Tue Jul 7 10:59:33 2009 New Revision: 66138 Added: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (contents, props changed) Log: first tla tests Added: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 10:59:33 2009 @@ -0,0 +1,24 @@ +import py +from pypy.jit.tl import tla + +def test_stack(): + f = tla.Frame('') + f.push(1) + f.push(2) + f.push(3) + assert f.pop() == 3 + assert f.pop() == 2 + assert f.pop() == 1 + py.test.raises(AssertionError, f.pop) + + +def test_W_IntObject(): + w_a = tla.W_IntObject(0) + w_b = tla.W_IntObject(10) + w_c = tla.W_IntObject(32) + assert not w_a.is_true() + assert w_b.is_true() + assert w_c.is_true() + assert w_b.add(w_c).intvalue == 42 + + From antocuni at codespeak.net Tue Jul 7 11:02:26 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 11:02:26 +0200 (CEST) Subject: [pypy-svn] r66139 - in pypy/branch/pyjitpl5/pypy/jit/tl: . test Message-ID: <20090707090226.7023F169DFC@codespeak.net> Author: antocuni Date: Tue Jul 7 11:02:25 2009 New Revision: 66139 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: more tests Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:02:25 2009 @@ -22,3 +22,18 @@ assert w_b.add(w_c).intvalue == 42 +def assemble(mylist): + return ''.join([chr(x) for x in mylist]) + +def interp(mylist): + bytecode = assemble(mylist) + f = tla.Frame(bytecode) + return f.interp() + +def test_interp(): + bytecode = [ + tla.CONST_INT, 42, + tla.RETURN + ] + assert interp(bytecode).intvalue == 42 + Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 11:02:25 2009 @@ -31,6 +31,7 @@ POP = 2 ADD = 3 RETURN = 4 +JUMP_IF = 5 # ____________________________________________________________ From antocuni at codespeak.net Tue Jul 7 11:05:05 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 11:05:05 +0200 (CEST) Subject: [pypy-svn] r66140 - pypy/branch/pyjitpl5/pypy/jit/tl/test Message-ID: <20090707090505.40FD8169DFC@codespeak.net> Author: antocuni Date: Tue Jul 7 11:05:04 2009 New Revision: 66140 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Log: use the conveninent tla.run function Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:05:04 2009 @@ -25,15 +25,13 @@ def assemble(mylist): return ''.join([chr(x) for x in mylist]) -def interp(mylist): +def interp(mylist, w_arg): bytecode = assemble(mylist) - f = tla.Frame(bytecode) - return f.interp() + return tla.run(bytecode, w_arg) def test_interp(): bytecode = [ - tla.CONST_INT, 42, tla.RETURN ] - assert interp(bytecode).intvalue == 42 + assert interp(bytecode, tla.W_IntObject(42)).intvalue == 42 From antocuni at codespeak.net Tue Jul 7 11:14:32 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 11:14:32 +0200 (CEST) Subject: [pypy-svn] r66141 - pypy/branch/pyjitpl5/pypy/jit/tl/test Message-ID: <20090707091432.960E7169EB1@codespeak.net> Author: antocuni Date: Tue Jul 7 11:14:31 2009 New Revision: 66141 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Log: more tests Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:14:31 2009 @@ -1,5 +1,6 @@ import py from pypy.jit.tl import tla +from pypy.jit.tl.tla import CONST_INT, POP, ADD, RETURN, JUMP_IF def test_stack(): f = tla.Frame('') @@ -30,8 +31,40 @@ return tla.run(bytecode, w_arg) def test_interp(): - bytecode = [ + code = [ tla.RETURN ] - assert interp(bytecode, tla.W_IntObject(42)).intvalue == 42 + res = interp(code, tla.W_IntObject(42)) + assert res.intvalue == 42 +def test_pop(): + code = [ + tla.CONST_INT, 99, + tla.POP, + tla.RETURN + ] + res = interp(code, tla.W_IntObject(42)) + assert res.intvalue == 42 + +def test_add(): + code = [ + CONST_INT, 20, + ADD, + RETURN + ] + res = interp(code, tla.W_IntObject(22)) + assert res.intvalue == 42 + +def test_jump_if(): + code = [ + JUMP_IF, 5, # jump to target + CONST_INT, 123, + RETURN, + CONST_INT, 234, # target + RETURN + ] + res = interp(code, tla.W_IntObject(0)) + assert res.intvalue == 123 + + res = interp(code, tla.W_IntObject(1)) + assert res.intvalue == 234 From cfbolz at codespeak.net Tue Jul 7 11:17:25 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 7 Jul 2009 11:17:25 +0200 (CEST) Subject: [pypy-svn] r66142 - pypy/branch/pyjitpl5/pypy/jit/tl/test Message-ID: <20090707091725.64113169EB1@codespeak.net> Author: cfbolz Date: Tue Jul 7 11:17:25 2009 New Revision: 66142 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Log: check in a line. anto, please work above it Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:17:25 2009 @@ -68,3 +68,7 @@ res = interp(code, tla.W_IntObject(1)) assert res.intvalue == 234 + + +# ____________________________________________________________ + From antocuni at codespeak.net Tue Jul 7 11:31:43 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 11:31:43 +0200 (CEST) Subject: [pypy-svn] r66143 - in pypy/branch/pyjitpl5/pypy/jit/tl: . test Message-ID: <20090707093143.83E81169ECC@codespeak.net> Author: antocuni Date: Tue Jul 7 11:31:42 2009 New Revision: 66143 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: add a sanity check before RETURN, and implement strings Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:31:42 2009 @@ -1,6 +1,6 @@ import py from pypy.jit.tl import tla -from pypy.jit.tl.tla import CONST_INT, POP, ADD, RETURN, JUMP_IF +from pypy.jit.tl.tla import CONST_INT, POP, ADD, RETURN, JUMP_IF, NEWSTR def test_stack(): f = tla.Frame('') @@ -46,6 +46,13 @@ res = interp(code, tla.W_IntObject(42)) assert res.intvalue == 42 +def test_bogus_return(): + code = [ + CONST_INT, 123, + RETURN # stack depth == 2 here, error! + ] + py.test.raises(AssertionError, "interp(code, tla.W_IntObject(234))") + def test_add(): code = [ CONST_INT, 20, @@ -70,5 +77,26 @@ assert res.intvalue == 234 +def test_newstr(): + code = [ + POP, + NEWSTR, ord('x'), + RETURN + ] + res = interp(code, tla.W_IntObject(0)) + assert isinstance(res, tla.W_StringObject) + assert res.strvalue == 'x' + +def test_add_strings(): + code = [ + NEWSTR, ord('d'), + ADD, + NEWSTR, ord('!'), + ADD, + RETURN + ] + res = interp(code, tla.W_StringObject('Hello worl')) + assert res.strvalue == 'Hello world!' + # ____________________________________________________________ Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 11:31:42 2009 @@ -22,6 +22,21 @@ raise OperationError +class W_StringObject(W_Object): + + def __init__(self, strvalue): + self.strvalue = strvalue + + def is_true(self): + return len(self.strvalue) != 0 + + def add(self, w_other): + if isinstance(w_other, W_StringObject): + concat = self.strvalue + w_other.strvalue + return W_StringObject(concat) + else: + raise OperationError + class OperationError: pass @@ -32,6 +47,7 @@ ADD = 3 RETURN = 4 JUMP_IF = 5 +NEWSTR = 6 # ____________________________________________________________ @@ -83,10 +99,20 @@ if w_x.is_true(): pc = target + elif opcode == NEWSTR: + char = bytecode[pc] + pc += 1 + w_z = W_StringObject(char) + self.push(w_z) + elif opcode == RETURN: w_x = self.pop() + assert self.stackpos == 0 return w_x + else: + assert False, 'Unknown opcode: %d' % opcode + def run(bytecode, w_arg): frame = Frame(bytecode) From cfbolz at codespeak.net Tue Jul 7 11:35:21 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 7 Jul 2009 11:35:21 +0200 (CEST) Subject: [pypy-svn] r66144 - in pypy/branch/pyjitpl5/pypy/jit/tl: . test Message-ID: <20090707093521.CB5FA169EB6@codespeak.net> Author: cfbolz Date: Tue Jul 7 11:35:21 2009 New Revision: 66144 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: add sub and dup bytecodes Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:35:21 2009 @@ -1,6 +1,5 @@ import py from pypy.jit.tl import tla -from pypy.jit.tl.tla import CONST_INT, POP, ADD, RETURN, JUMP_IF, NEWSTR def test_stack(): f = tla.Frame('') @@ -46,29 +45,47 @@ res = interp(code, tla.W_IntObject(42)) assert res.intvalue == 42 +def test_dup(): + code = [ + tla.DUP, + tla.ADD, + tla.RETURN + ] + res = interp(code, tla.W_IntObject(41)) + assert res.intvalue == 2 * 41 + def test_bogus_return(): code = [ - CONST_INT, 123, - RETURN # stack depth == 2 here, error! + tla.CONST_INT, 123, + tla.RETURN # stack depth == 2 here, error! ] py.test.raises(AssertionError, "interp(code, tla.W_IntObject(234))") def test_add(): code = [ - CONST_INT, 20, - ADD, - RETURN + tla.CONST_INT, 20, + tla.ADD, + tla.RETURN ] res = interp(code, tla.W_IntObject(22)) assert res.intvalue == 42 +def test_sub(): + code = [ + tla.CONST_INT, 20, + tla.SUB, + tla.RETURN + ] + res = interp(code, tla.W_IntObject(22)) + assert res.intvalue == 2 + def test_jump_if(): code = [ - JUMP_IF, 5, # jump to target - CONST_INT, 123, - RETURN, - CONST_INT, 234, # target - RETURN + tla.JUMP_IF, 5, # jump to target + tla.CONST_INT, 123, + tla.RETURN, + tla.CONST_INT, 234, # target + tla.RETURN ] res = interp(code, tla.W_IntObject(0)) assert res.intvalue == 123 @@ -79,9 +96,9 @@ def test_newstr(): code = [ - POP, - NEWSTR, ord('x'), - RETURN + tla.POP, + tla.NEWSTR, ord('x'), + tla.RETURN ] res = interp(code, tla.W_IntObject(0)) assert isinstance(res, tla.W_StringObject) @@ -89,11 +106,11 @@ def test_add_strings(): code = [ - NEWSTR, ord('d'), - ADD, - NEWSTR, ord('!'), - ADD, - RETURN + tla.NEWSTR, ord('d'), + tla.ADD, + tla.NEWSTR, ord('!'), + tla.ADD, + tla.RETURN ] res = interp(code, tla.W_StringObject('Hello worl')) assert res.strvalue == 'Hello world!' Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 11:35:21 2009 @@ -21,6 +21,12 @@ else: raise OperationError + def sub(self, w_other): + if isinstance(w_other, W_IntObject): + sum = self.intvalue - w_other.intvalue + return W_IntObject(sum) + else: + raise OperationError class W_StringObject(W_Object): @@ -47,7 +53,9 @@ ADD = 3 RETURN = 4 JUMP_IF = 5 -NEWSTR = 6 +DUP = 6 +SUB = 7 +NEWSTR = 8 # ____________________________________________________________ @@ -86,12 +94,22 @@ elif opcode == POP: self.pop() + elif opcode == DUP: + w_x = self.pop() + self.push(w_x) + self.push(w_x) + elif opcode == ADD: w_y = self.pop() w_x = self.pop() w_z = w_x.add(w_y) self.push(w_z) + elif opcode == SUB: + w_y = self.pop() + w_x = self.pop() + w_z = w_x.sub(w_y) + self.push(w_z) elif opcode == JUMP_IF: target = ord(bytecode[pc]) pc += 1 From antocuni at codespeak.net Tue Jul 7 11:47:39 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 11:47:39 +0200 (CEST) Subject: [pypy-svn] r66145 - in pypy/branch/pyjitpl5/pypy/jit/tl: . test Message-ID: <20090707094739.E357E169ED1@codespeak.net> Author: antocuni Date: Tue Jul 7 11:47:37 2009 New Revision: 66145 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: kill W_StringObject.add, and write some exercises Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 11:47:37 2009 @@ -104,7 +104,13 @@ assert isinstance(res, tla.W_StringObject) assert res.strvalue == 'x' +# ____________________________________________________________ +# EXERCISES +# ____________________________________________________________ + + def test_add_strings(): + py.test.skip('exercise!') code = [ tla.NEWSTR, ord('d'), tla.ADD, @@ -115,5 +121,36 @@ res = interp(code, tla.W_StringObject('Hello worl')) assert res.strvalue == 'Hello world!' +def test_mul(): + py.test.skip('exercise!') + code = [ + tla.CONST_INT, 2, + tla.MUL, + tla.RETURN + ] + res = interp(code, tla.W_IntObject(21)) + assert res.intvalue == 42 + +def test_mul_strings(): + py.test.skip('exercise!') + code = [ + tla.CONST_INT, 3 + tla.MUL, + tla.RETURN + ] + res = interp(code, tla.W_StringObject('foo ')) + assert res.strvalue == 'foo foo foo ' + +def test_div_float(): + py.test.skip('exercise!') + code = [ + tla.CONST_INT, 2 + tla.DIV, + tla.RETURN + ] + res = interp(code, tla.W_IntObject(5)) + assert isinstance(res, tla.W_FloatObject) + assert res.floatval == 2.5 + # ____________________________________________________________ Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 11:47:37 2009 @@ -3,7 +3,13 @@ class W_Object: - pass + + def is_true(self): + raise NotImplementedError + + def add(self): + raise NotImplementedError + class W_IntObject(W_Object): @@ -36,12 +42,6 @@ def is_true(self): return len(self.strvalue) != 0 - def add(self, w_other): - if isinstance(w_other, W_StringObject): - concat = self.strvalue + w_other.strvalue - return W_StringObject(concat) - else: - raise OperationError class OperationError: pass From antocuni at codespeak.net Tue Jul 7 12:03:26 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 12:03:26 +0200 (CEST) Subject: [pypy-svn] r66146 - pypy/branch/pyjitpl5/pypy/jit/tl/test Message-ID: <20090707100326.0EFC4169EE5@codespeak.net> Author: antocuni Date: Tue Jul 7 12:03:25 2009 New Revision: 66146 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Log: typos Modified: pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py Tue Jul 7 12:03:25 2009 @@ -20,6 +20,7 @@ assert w_b.is_true() assert w_c.is_true() assert w_b.add(w_c).intvalue == 42 + assert w_b.getrepr() == '10' def assemble(mylist): @@ -134,7 +135,7 @@ def test_mul_strings(): py.test.skip('exercise!') code = [ - tla.CONST_INT, 3 + tla.CONST_INT, 3, tla.MUL, tla.RETURN ] @@ -144,7 +145,7 @@ def test_div_float(): py.test.skip('exercise!') code = [ - tla.CONST_INT, 2 + tla.CONST_INT, 2, tla.DIV, tla.RETURN ] From antocuni at codespeak.net Tue Jul 7 12:07:53 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 12:07:53 +0200 (CEST) Subject: [pypy-svn] r66147 - pypy/branch/pyjitpl5/pypy/jit/tl Message-ID: <20090707100753.45A15168068@codespeak.net> Author: antocuni Date: Tue Jul 7 12:07:52 2009 New Revision: 66147 Added: pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py (contents, props changed) pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py (contents, props changed) pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py (contents, props changed) Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Log: add a target, an assembler and an example program Added: pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py Tue Jul 7 12:07:52 2009 @@ -0,0 +1,7 @@ +from pypy.jit.tl import tla + +code = [ + tla.CONST_INT, 10, + tla.ADD, + tla.RETURN + ] Added: pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py Tue Jul 7 12:07:52 2009 @@ -0,0 +1,36 @@ +import py +py.magic.autopath() +from pypy.jit.tl import tla + + +def entry_point(args): + """Main entry point of the stand-alone executable: + takes a list of strings and returns the exit code. + """ + if len(args) < 3: + print "Usage: %s filename x" % (args[0],) + return 2 + filename = args[1] + x = int(args[2]) + w_x = tla.W_IntObject(x) + bytecode = load_bytecode(filename) + w_res = tla.run(bytecode, w_x) + print w_res.getrepr() + return 0 + +def load_bytecode(filename): + from pypy.rlib.streamio import open_file_as_stream + f = open_file_as_stream(filename) + bytecode = f.readall() + f.close() + return bytecode + +def target(driver, args): + return entry_point, None + +# ____________________________________________________________ + + +if __name__ == '__main__': + import sys + sys.exit(entry_point(sys.argv)) Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla.py Tue Jul 7 12:07:52 2009 @@ -4,10 +4,16 @@ class W_Object: + def getrepr(self): + """ + Return an RPython string which represent the object + """ + raise NotImplementedError + def is_true(self): raise NotImplementedError - def add(self): + def add(self, w_other): raise NotImplementedError @@ -17,6 +23,9 @@ def __init__(self, intvalue): self.intvalue = intvalue + def getrepr(self): + return str(self.intvalue) + def is_true(self): return self.intvalue != 0 @@ -39,6 +48,9 @@ def __init__(self, strvalue): self.strvalue = strvalue + def getrepr(self): + return self.strvalue + def is_true(self): return len(self.strvalue) != 0 Added: pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py Tue Jul 7 12:07:52 2009 @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import sys +import py +py.magic.autopath() +from pypy.jit.tl.test.test_tla import assemble + +def usage(): + print >> sys.stderr, 'Usage: tla_assembler.py filename.tla.py' + sys.exit(1) + +def main(): + if len(sys.argv) != 2: + usage() + + filename = sys.argv[1] + if not filename.endswith('.tla.py'): + usage() + + outname = filename[:-len('.py')] + mydict = {} + execfile(filename, mydict) + bytecode = assemble(mydict['code']) + f = open(outname, 'w') + f.write(bytecode) + f.close() + print '%s successfully assembled' % outname + +if __name__ == '__main__': + main() From antocuni at codespeak.net Tue Jul 7 12:27:41 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 12:27:41 +0200 (CEST) Subject: [pypy-svn] r66148 - in pypy/branch/pyjitpl5/pypy/jit/tl: . test tla Message-ID: <20090707102741.CD02A169EB5@codespeak.net> Author: antocuni Date: Tue Jul 7 12:27:41 2009 New Revision: 66148 Added: pypy/branch/pyjitpl5/pypy/jit/tl/tla/ pypy/branch/pyjitpl5/pypy/jit/tl/tla/__init__.py (contents, props changed) pypy/branch/pyjitpl5/pypy/jit/tl/tla/add_10.tla.py - copied, changed from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py - copied, changed from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py - copied, changed from r66146, pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py - copied unchanged from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla_assembler.py - copied, changed from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py Removed: pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py Log: move tla files into tl/tla Added: pypy/branch/pyjitpl5/pypy/jit/tl/tla/__init__.py ============================================================================== Copied: pypy/branch/pyjitpl5/pypy/jit/tl/tla/add_10.tla.py (from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/add_10.tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/add_10.tla.py Tue Jul 7 12:27:41 2009 @@ -1,4 +1,4 @@ -from pypy.jit.tl import tla +from pypy.jit.tl.tla import tla code = [ tla.CONST_INT, 10, Copied: pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py (from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/targettla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py Tue Jul 7 12:27:41 2009 @@ -1,6 +1,6 @@ import py py.magic.autopath() -from pypy.jit.tl import tla +from pypy.jit.tl.tla import tla def entry_point(args): Copied: pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py (from r66146, pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/test/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py Tue Jul 7 12:27:41 2009 @@ -1,5 +1,5 @@ import py -from pypy.jit.tl import tla +from pypy.jit.tl.tla import tla def test_stack(): f = tla.Frame('') Copied: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla_assembler.py (from r66147, pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla_assembler.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla_assembler.py Tue Jul 7 12:27:41 2009 @@ -3,7 +3,7 @@ import sys import py py.magic.autopath() -from pypy.jit.tl.test.test_tla import assemble +from pypy.jit.tl.tla.test_tla import assemble def usage(): print >> sys.stderr, 'Usage: tla_assembler.py filename.tla.py' From antocuni at codespeak.net Tue Jul 7 12:30:42 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Tue, 7 Jul 2009 12:30:42 +0200 (CEST) Subject: [pypy-svn] r66149 - pypy/branch/pyjitpl5/pypy/translator/jvm Message-ID: <20090707103042.6D59F169EEF@codespeak.net> Author: antocuni Date: Tue Jul 7 12:30:41 2009 New Revision: 66149 Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Log: ignore this opcode, so that genjvm doesn't crash Modified: pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/jvm/opcodes.py Tue Jul 7 12:30:41 2009 @@ -96,6 +96,7 @@ 'gc__collect': jvm.SYSTEMGC, 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, + 'promote_virtualizable': Ignore, 'debug_assert': [], # TODO: implement? From fijal at codespeak.net Wed Jul 8 17:22:15 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 8 Jul 2009 17:22:15 +0200 (CEST) Subject: [pypy-svn] r66150 - pypy/trunk/pypy/tool Message-ID: <20090708152215.764C9168562@codespeak.net> Author: fijal Date: Wed Jul 8 17:22:13 2009 New Revision: 66150 Modified: pypy/trunk/pypy/tool/compat.py Log: Fix BaseException assignment Modified: pypy/trunk/pypy/tool/compat.py ============================================================================== --- pypy/trunk/pypy/tool/compat.py (original) +++ pypy/trunk/pypy/tool/compat.py Wed Jul 8 17:22:13 2009 @@ -10,6 +10,6 @@ from pypy.lib.md5 import md5 try: - BaseException + BaseException = BaseException except NameError: BaseException = Exception From fijal at codespeak.net Wed Jul 8 18:11:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 8 Jul 2009 18:11:32 +0200 (CEST) Subject: [pypy-svn] r66151 - pypy/trunk/pypy/tool/test Message-ID: <20090708161132.D2E4A168476@codespeak.net> Author: fijal Date: Wed Jul 8 18:11:30 2009 New Revision: 66151 Added: pypy/trunk/pypy/tool/test/test_compat.py (contents, props changed) Log: Test for pypy.tool.compat.BaseException. Unsure about the value of this one... Added: pypy/trunk/pypy/tool/test/test_compat.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/test/test_compat.py Wed Jul 8 18:11:30 2009 @@ -0,0 +1,5 @@ + +def test_base_exc(): + from pypy.tool.compat import BaseException + assert KeyboardInterrupt.__mro__[-2] is BaseException + assert ValueError.__mro__[-2] is BaseException From fijal at codespeak.net Wed Jul 8 18:55:46 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 8 Jul 2009 18:55:46 +0200 (CEST) Subject: [pypy-svn] r66152 - in pypy/trunk/pypy/translator/platform: . test Message-ID: <20090708165546.9191B169F25@codespeak.net> Author: fijal Date: Wed Jul 8 18:55:44 2009 New Revision: 66152 Modified: pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/test/test_distutils.py pypy/trunk/pypy/translator/platform/test/test_maemo.py pypy/trunk/pypy/translator/platform/test/test_platform.py pypy/trunk/pypy/translator/platform/windows.py Log: Don't use -l when creating a shared lib (since you don't need it and messes up the build system sometimes) Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Wed Jul 8 18:55:44 2009 @@ -70,7 +70,12 @@ ofiles.append(self._compile_c_file(self.cc, cfile, compile_args)) return ofiles - def execute(self, executable, args=None, env=None): + def execute(self, executable, args=None, env=None, compilation_info=None): + if env is None: + env = {} + if compilation_info is not None: + env['LD_LIBRARY_PATH'] = ':'.join( + [str(i) for i in compilation_info.library_dirs]) returncode, stdout, stderr = _run_subprocess(str(executable), args, env) return ExecutionResult(returncode, stdout, stderr) @@ -120,9 +125,13 @@ cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) - def _link_args_from_eci(self, eci): - library_dirs = self._libdirs(eci.library_dirs) - libraries = self._libs(eci.libraries) + def _link_args_from_eci(self, eci, standalone): + if standalone: + library_dirs = self._libdirs(eci.library_dirs) + libraries = self._libs(eci.libraries) + else: + library_dirs = [] + libraries = [] link_files = self._linkfiles(eci.link_files) return (library_dirs + libraries + self.link_flags + link_files + list(eci.link_extra)) @@ -137,8 +146,8 @@ exe_name += '.' + self.exe_ext else: exe_name += '.' + self.so_ext - return self._link(self.cc, ofiles, self._link_args_from_eci(eci), - standalone, exe_name) + largs = self._link_args_from_eci(eci, standalone) + return self._link(self.cc, ofiles, largs, standalone, exe_name) # below are some detailed informations for platforms Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Wed Jul 8 18:55:44 2009 @@ -38,8 +38,8 @@ args.append(f) return args - def _link_args_from_eci(self, eci): - args = super(Darwin, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + args = super(Darwin, self)._link_args_from_eci(eci, standalone) frameworks = self._frameworks(eci.frameworks) include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) Modified: pypy/trunk/pypy/translator/platform/test/test_distutils.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_distutils.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_distutils.py Wed Jul 8 18:55:44 2009 @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_shared_no_links(self): + py.test.skip("Unsupported") Modified: pypy/trunk/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_maemo.py Wed Jul 8 18:55:44 2009 @@ -31,3 +31,6 @@ executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) self.check_res(res) + + def test_shared_no_links(self): + py.test.skip("Unsupported") Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Wed Jul 8 18:55:44 2009 @@ -1,5 +1,5 @@ -import py, sys +import py, sys, ctypes from pypy.tool.udir import udir from pypy.translator.platform import CompilationError, Platform from pypy.translator.platform import host @@ -102,6 +102,18 @@ res = self.platform.execute(executable) assert res.out.startswith('4.0') + def test_shared_no_links(self): + eci = ExternalCompilationInfo(libraries = ['xxxxxxxxxxxxxxxxxxx']) + tmpdir = udir.join('shared_no_links').ensure(dir=1) + c_file = tmpdir.join('shared1.c') + c_file.write(''' + int f(int a, int b) + { + return (a + b); + } + ''') + library = self.platform.compile([c_file], eci, standalone=False) + assert ctypes.CDLL(str(library)).f(3, 4) == 7 def test_equality(): class X(Platform): Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Wed Jul 8 18:55:44 2009 @@ -133,8 +133,8 @@ def _args_for_shared(self, args): return ['/dll'] + args - def _link_args_from_eci(self, eci): - args = super(MsvcPlatform, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone) return args + ['/EXPORT:%s' % symbol for symbol in eci.export_symbols] def _compile_c_file(self, cc, cfile, compile_args): From fijal at codespeak.net Fri Jul 10 08:58:10 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 10 Jul 2009 08:58:10 +0200 (CEST) Subject: [pypy-svn] r66154 - pypy/trunk/pypy/translator/platform Message-ID: <20090710065810.2555A169E50@codespeak.net> Author: fijal Date: Fri Jul 10 08:58:08 2009 New Revision: 66154 Modified: pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/linux.py Log: provide possible prefixes for libraries Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Fri Jul 10 08:58:08 2009 @@ -52,6 +52,8 @@ name = "abstract platform" c_environ = None + so_prefixes = [''] + def __init__(self, cc): if self.__class__ is Platform: raise TypeError("You should not instantiate Platform class directly") Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Fri Jul 10 08:58:08 2009 @@ -13,6 +13,7 @@ standalone_only = [] shared_only = [] so_ext = 'so' + so_prefixes = ['lib', ''] def _args_for_shared(self, args): return ['-shared'] + args From fijal at codespeak.net Fri Jul 10 09:02:31 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 10 Jul 2009 09:02:31 +0200 (CEST) Subject: [pypy-svn] r66155 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20090710070231.C5F2916842F@codespeak.net> Author: fijal Date: Fri Jul 10 09:02:31 2009 New Revision: 66155 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: A test and a fix for library in library_paths, but not in libraries. Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Fri Jul 10 09:02:31 2009 @@ -19,6 +19,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress +from pypy.translator.platform import platform def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -813,10 +814,21 @@ cfunc = get_on_lib(ctypes.windll.kernel32, funcname) else: cfunc = None + not_found = [] for libname in libraries: - libpath = ctypes.util.find_library(libname) - if not libpath and os.path.isabs(libname): - libpath = libname + libpath = None + ext = platform.so_ext + prefixes = platform.so_prefixes + for dir in eci.library_dirs: + for prefix in prefixes: + tryfile = os.path.join(dir, prefix + libname + '.' + ext) + if os.path.isfile(tryfile): + libpath = tryfile + break + if not libpath: + libpath = ctypes.util.find_library(libname) + if not libpath and os.path.isabs(libname): + libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') # urgh, cannot pass the flag to dllclass.LoadLibrary @@ -824,11 +836,20 @@ cfunc = get_on_lib(clib, funcname) if cfunc is not None: break + else: + not_found.append(libname) if cfunc is None: # function name not found in any of the libraries if not libraries: place = 'the standard C library (missing libraries=...?)' + elif len(not_found) == len(libraries): + if len(not_found) == 1: + raise NotImplementedError( + 'cannot find the library %r' % (not_found[0],)) + else: + raise NotImplementedError( + 'cannot find any of the libraries %r' % (not_found,)) elif len(libraries) == 1: place = 'library %r' % (libraries[0],) else: Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri Jul 10 09:02:31 2009 @@ -15,6 +15,7 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.rtyper import RPythonTyper +from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -994,3 +995,21 @@ v2 = ctypes2lltype(llmemory.GCREF, ctypes.c_void_p(1235)) assert v2 != v +class TestPlatform(object): + def test_lib_on_libpaths(self): + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo() + so = platform.compile([c_file], eci, standalone=False) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 From fijal at codespeak.net Fri Jul 10 09:11:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 10 Jul 2009 09:11:42 +0200 (CEST) Subject: [pypy-svn] r66156 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20090710071142.50037168470@codespeak.net> Author: fijal Date: Fri Jul 10 09:11:41 2009 New Revision: 66156 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: a test and a fix for prefixes Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Fri Jul 10 09:11:41 2009 @@ -820,11 +820,13 @@ ext = platform.so_ext prefixes = platform.so_prefixes for dir in eci.library_dirs: + if libpath: + break for prefix in prefixes: tryfile = os.path.join(dir, prefix + libname + '.' + ext) - if os.path.isfile(tryfile): - libpath = tryfile - break + if os.path.isfile(tryfile): + libpath = tryfile + break if not libpath: libpath = ctypes.util.find_library(libname) if not libpath and os.path.isabs(libname): Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri Jul 10 09:11:41 2009 @@ -1013,3 +1013,28 @@ f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, compilation_info=eci) assert f(3, 4) == 7 + + def test_prefix(self): + + if sys.platform != 'linux2': + py.test.skip("Not supported") + + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths_prefix') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo() + so = platform.compile([c_file], eci, standalone=False) + sopath = py.path.local(so) + sopath.move(sopath.dirpath().join('libc_file.so')) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 + From afa at codespeak.net Fri Jul 10 10:36:43 2009 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 10 Jul 2009 10:36:43 +0200 (CEST) Subject: [pypy-svn] r66157 - in pypy/trunk/pypy: rpython/lltypesystem/test translator/platform Message-ID: <20090710083643.A3EC4169E29@codespeak.net> Author: afa Date: Fri Jul 10 10:36:40 2009 New Revision: 66157 Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/trunk/pypy/translator/platform/windows.py Log: Fix recent developments when running on Windows Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri Jul 10 10:36:40 2009 @@ -1004,7 +1004,7 @@ tmpdir.ensure(dir=1) c_file = tmpdir.join('c_file.c') c_file.write('int f(int a, int b) { return (a + b); }') - eci = ExternalCompilationInfo() + eci = ExternalCompilationInfo(export_symbols=['f']) so = platform.compile([c_file], eci, standalone=False) eci = ExternalCompilationInfo( libraries = ['c_file'], Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Fri Jul 10 10:36:40 2009 @@ -134,7 +134,8 @@ return ['/dll'] + args def _link_args_from_eci(self, eci, standalone): - args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone) + # Windows needs to resolve all symbols even for DLLs + args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) return args + ['/EXPORT:%s' % symbol for symbol in eci.export_symbols] def _compile_c_file(self, cc, cfile, compile_args): From arigo at codespeak.net Fri Jul 10 11:47:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 11:47:48 +0200 (CEST) Subject: [pypy-svn] r66158 - pypy/branch/pyjitpl5/pypy/jit/backend/llvm Message-ID: <20090710094748.61067168057@codespeak.net> Author: arigo Date: Fri Jul 10 11:47:46 2009 New Revision: 66158 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llvm/llvm_rffi.py Log: Check that llvm is installed before letting the tests run. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llvm/llvm_rffi.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llvm/llvm_rffi.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llvm/llvm_rffi.py Fri Jul 10 11:47:46 2009 @@ -20,6 +20,12 @@ if (not os.path.isfile(libname) or os.path.getmtime(cname) > os.path.getmtime(libname) or os.path.getmtime(cppname) > os.path.getmtime(libname)): + g = os.popen('%s --version' % llvm_config, 'r') + data = g.read() + g.close() + if not data.startswith('2.'): + py.test.skip("llvm (version 2) is required") + if not os.path.isdir(dirname): if not os.path.isdir(cachename): os.mkdir(cachename) From arigo at codespeak.net Fri Jul 10 11:56:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 11:56:35 +0200 (CEST) Subject: [pypy-svn] r66159 - pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test Message-ID: <20090710095635.A15E11684A1@codespeak.net> Author: arigo Date: Fri Jul 10 11:56:34 2009 New Revision: 66159 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_exception.py pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_tl.py Log: Skip most of the tests in the minimal backend. Clean it up if we want to resurrect this backend at some point. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_exception.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_exception.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_exception.py Fri Jul 10 11:56:34 2009 @@ -7,6 +7,7 @@ CPUClass = LLtypeCPU def meta_interp(self, *args, **kwds): + py.test.skip("skipped for now") from pypy.jit.metainterp import simple_optimize kwds['optimizer'] = simple_optimize return CCompiledMixin.meta_interp(self, *args, **kwds) @@ -16,6 +17,7 @@ CPUClass = OOtypeCPU def meta_interp(self, *args, **kwds): + py.test.skip("skipped for now") from pypy.jit.metainterp import simple_optimize kwds['optimizer'] = simple_optimize return CliCompiledMixin.meta_interp(self, *args, **kwds) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_tl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_tl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/minimal/test/test_zrpy_tl.py Fri Jul 10 11:56:34 2009 @@ -4,7 +4,7 @@ class TestOOtype(OOTranslatedJitMixin, ToyLanguageTests): - def test_tl_base(self): + def XXXtest_tl_base(self): # XXX: remove this hack as soon as WarmEnterState is no longer a pbc from pypy.rlib import jit try: @@ -13,7 +13,7 @@ finally: jit.PARAMETERS['hash_bits'] = 14 - def test_tl_2(self): + def XXXtest_tl_2(self): # XXX: remove this hack as soon as WarmEnterState is no longer a pbc from pypy.rlib import jit try: @@ -22,7 +22,7 @@ finally: jit.PARAMETERS['hash_bits'] = 14 - def test_tl_call(self): + def XXXtest_tl_call(self): # XXX: remove this hack as soon as WarmEnterState is no longer a pbc from pypy.rlib import jit try: From arigo at codespeak.net Fri Jul 10 12:11:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 12:11:55 +0200 (CEST) Subject: [pypy-svn] r66160 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090710101155.ECD61169E0D@codespeak.net> Author: arigo Date: Fri Jul 10 12:11:55 2009 New Revision: 66160 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/virtualizable.py Log: Fix for ootype. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/virtualizable.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/virtualizable.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/virtualizable.py Fri Jul 10 12:11:55 2009 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import cast_instance_to_base_ptr +from pypy.rpython.annlowlevel import cast_instance_to_base_obj from pypy.rpython.annlowlevel import cast_base_ptr_to_instance from pypy.rpython import rvirtualizable2 from pypy.rlib.objectmodel import we_are_translated @@ -198,7 +199,10 @@ def cast_instance_to_base_ptr(self, vable_rti): if we_are_translated(): - return cast_instance_to_base_ptr(vable_rti) + if not self.is_oo: + return cast_instance_to_base_ptr(vable_rti) + else: + return cast_instance_to_base_obj(vable_rti) else: vable_rti._TYPE = self.VABLERTI # hack for non-translated mode return vable_rti From arigo at codespeak.net Fri Jul 10 12:28:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 12:28:56 +0200 (CEST) Subject: [pypy-svn] r66161 - pypy/branch/pyjitpl5/pypy/rlib Message-ID: <20090710102856.40F4E168487@codespeak.net> Author: arigo Date: Fri Jul 10 12:28:55 2009 New Revision: 66161 Modified: pypy/branch/pyjitpl5/pypy/rlib/libffi.py pypy/branch/pyjitpl5/pypy/rlib/rmmap.py Log: Fix for r63773 the proper way. Makes test_libffi pass again. Modified: pypy/branch/pyjitpl5/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rlib/libffi.py (original) +++ pypy/branch/pyjitpl5/pypy/rlib/libffi.py Fri Jul 10 12:28:55 2009 @@ -10,7 +10,6 @@ from pypy.tool.autopath import pypydir from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rlib.rmmap import alloc -from pypy.rlib.nonconst import NonConstant import py import os import sys @@ -410,7 +409,7 @@ self.free_list = lltype.nullptr(rffi.VOIDP.TO) def _more(self): - chunk = rffi.cast(CLOSURES, alloc(NonConstant(CHUNK))) + chunk = rffi.cast(CLOSURES, alloc(CHUNK)) count = CHUNK//rffi.sizeof(FFI_CLOSUREP.TO) for i in range(count): rffi.cast(rffi.VOIDPP, chunk)[0] = self.free_list Modified: pypy/branch/pyjitpl5/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rlib/rmmap.py (original) +++ pypy/branch/pyjitpl5/pypy/rlib/rmmap.py Fri Jul 10 12:28:55 2009 @@ -634,6 +634,7 @@ raise MemoryError hint.pos += map_size return res + alloc._annenforceargs_ = (int,) free = c_munmap From arigo at codespeak.net Fri Jul 10 12:29:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 12:29:37 +0200 (CEST) Subject: [pypy-svn] r66162 - pypy/branch/pyjitpl5/pypy/rlib Message-ID: <20090710102937.8224E16849E@codespeak.net> Author: arigo Date: Fri Jul 10 12:29:36 2009 New Revision: 66162 Modified: pypy/branch/pyjitpl5/pypy/rlib/rmmap.py Log: Oups, this needs to be fixed for Windows too. Modified: pypy/branch/pyjitpl5/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rlib/rmmap.py (original) +++ pypy/branch/pyjitpl5/pypy/rlib/rmmap.py Fri Jul 10 12:29:36 2009 @@ -748,6 +748,7 @@ lltype.free(arg, flavor='raw') # ignore errors, just try return res + alloc._annenforceargs_ = (int,) def free(ptr, map_size): VirtualFree(ptr, 0, MEM_RELEASE) From arigo at codespeak.net Fri Jul 10 13:27:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 13:27:16 +0200 (CEST) Subject: [pypy-svn] r66163 - pypy/branch/pyjitpl5/pypy/translator/tool Message-ID: <20090710112716.6555E168428@codespeak.net> Author: arigo Date: Fri Jul 10 13:27:15 2009 New Revision: 66163 Modified: pypy/branch/pyjitpl5/pypy/translator/tool/graphpage.py Log: Fix for translator.viewcg(). Modified: pypy/branch/pyjitpl5/pypy/translator/tool/graphpage.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/tool/graphpage.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/tool/graphpage.py Fri Jul 10 13:27:15 2009 @@ -237,14 +237,14 @@ def graph_name(self, *args): raise NotImplementedError - def compute(self, translator, *args): + def compute(self, translator, *args, **kwds): self.translator = translator self.object_by_name = {} self.name_by_object = {} dotgen = DotGen(self.graph_name(*args)) dotgen.emit('mclimit=15.0') - self.do_compute(dotgen, *args) + self.do_compute(dotgen, *args, **kwds) self.source = dotgen.generate(target=None) From arigo at codespeak.net Fri Jul 10 13:38:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 13:38:21 +0200 (CEST) Subject: [pypy-svn] r66164 - pypy/branch/pyjitpl5/pypy/jit/backend/x86 Message-ID: <20090710113821.070B9168473@codespeak.net> Author: arigo Date: Fri Jul 10 13:38:18 2009 New Revision: 66164 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Log: Fix: in some situations (e.g. test_zrpy_virtualizable:test_external_read) the list is not compatible with the list stored in 'shape' (known-empty list versus list of booleans). Could be really fixed, I guess, but it's a bit of a mess. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Fri Jul 10 13:38:18 2009 @@ -233,7 +233,9 @@ def _get_loop_for_call(self, args, calldescr, ptr): if calldescr.call_loop is not None: - assert calldescr.shape == ([arg.type == history.PTR for arg in args[1:]], ptr) + if not we_are_translated(): + assert (calldescr.shape == + ([arg.type == history.PTR for arg in args[1:]], ptr)) return calldescr.call_loop args = [arg.clonebox() for arg in args] result = self._new_box(ptr) From arigo at codespeak.net Fri Jul 10 13:49:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 13:49:34 +0200 (CEST) Subject: [pypy-svn] r66165 - pypy/branch/pyjitpl5/pypy/jit/backend/x86/test Message-ID: <20090710114934.51D18168417@codespeak.net> Author: arigo Date: Fri Jul 10 13:49:33 2009 New Revision: 66165 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: These tests are meant to be run with simple_optimize. Some fail when run with optimize4 for unknown reasons but that's another matter... Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py Fri Jul 10 13:49:33 2009 @@ -50,6 +50,7 @@ from pypy.annotation.listdef import s_list_of_strings from pypy.translator.translator import TranslationContext from pypy.jit.metainterp.warmspot import apply_jit + from pypy.jit.metainterp import simple_optimize from pypy.translator.c import genc # t = TranslationContext() @@ -60,7 +61,7 @@ t.buildannotator().build_types(f, [s_list_of_strings]) t.buildrtyper().specialize() if kwds['jit']: - apply_jit(t, CPUClass=CPU386) + apply_jit(t, CPUClass=CPU386, optimizer=simple_optimize) cbuilder = genc.CStandaloneBuilder(t, f, t.config) cbuilder.generate_source() cbuilder.compile() From arigo at codespeak.net Fri Jul 10 19:15:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 19:15:38 +0200 (CEST) Subject: [pypy-svn] r66166 - in pypy/branch/pyjitpl5/pypy/jit: backend/x86/test tl Message-ID: <20090710171538.72377168478@codespeak.net> Author: arigo Date: Fri Jul 10 19:15:36 2009 New Revision: 66166 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_tl.py pypy/branch/pyjitpl5/pypy/jit/tl/tlc.py Log: Fix test_zrpy_tl, tediously. I blame fijal for leaving around a test that has always been failing -- I could not find a single revision in which it passed. Indeed, the amount of effort required to fix it seems to show that such a revision never existed. Additional hint: the second "assert res == 13" is bogus, as fib(5) == 5. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_tl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_tl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_tl.py Fri Jul 10 19:15:36 2009 @@ -10,8 +10,9 @@ pass class TestTLC(Jit386Mixin): - def _get_interp(self, bytecode, pool): + def _get_interp(self, bytecode_, pool_): def interp(inputarg): + bytecode, pool = tlc.non_constant(bytecode_, pool_) args = [tlc.IntObj(inputarg)] obj = tlc.interp_eval(bytecode, 0, args, pool) return obj.int_o() @@ -28,6 +29,3 @@ code = path.read() res = self.exec_code(code, 20) assert res == 6765 - res = self.exec_code(code, 5) - assert res == 13 - Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tlc.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tlc.py Fri Jul 10 19:15:36 2009 @@ -454,6 +454,17 @@ interp_without_call, interp_eval_without_call = make_interp(supports_call = False) interp_nonjit , interp_eval_nonjit = make_interp(supports_call = True, jitted=False) +def non_constant(bytecode, pool): + from pypy.rlib.nonconst import NonConstant + if NonConstant(False): + pool = ConstantPool() + pool.add_string("foo") + pool.add_string("bazz") + pool.add_classdescr(["a", "bc"], [("foo", 3), ("y", 5)]) + return "123", pool + else: + return bytecode, pool + if __name__ == '__main__': import sys from pypy.jit.tl.test.test_tl import FACTORIAL_SOURCE From arigo at codespeak.net Fri Jul 10 19:19:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 19:19:12 +0200 (CEST) Subject: [pypy-svn] r66167 - pypy/branch/pyjitpl5/pypy/jit/backend/x86/test Message-ID: <20090710171912.465131684A9@codespeak.net> Author: arigo Date: Fri Jul 10 19:19:11 2009 New Revision: 66167 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_recursive.py Log: Skip a test that is not supposed to be translated. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_recursive.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_recursive.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_recursive.py Fri Jul 10 19:19:11 2009 @@ -1,7 +1,9 @@ - +import py from pypy.jit.metainterp.test.test_recursive import RecursiveTests from pypy.jit.backend.x86.test.test_zrpy_slist import Jit386Mixin class TestRecursive(Jit386Mixin, RecursiveTests): - pass + + def test_inline_faulty_can_inline(self): + py.test.skip("this test is not supposed to be translated") From arigo at codespeak.net Fri Jul 10 19:40:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 19:40:23 +0200 (CEST) Subject: [pypy-svn] r66169 - pypy/branch/pyjitpl5/lib-python Message-ID: <20090710174023.4E0CC168450@codespeak.net> Author: arigo Date: Fri Jul 10 19:40:22 2009 New Revision: 66169 Modified: pypy/branch/pyjitpl5/lib-python/conftest.py Log: Record that some extra tests require threads. Modified: pypy/branch/pyjitpl5/lib-python/conftest.py ============================================================================== --- pypy/branch/pyjitpl5/lib-python/conftest.py (original) +++ pypy/branch/pyjitpl5/lib-python/conftest.py Fri Jul 10 19:40:22 2009 @@ -231,7 +231,7 @@ RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fnmatch.py', core=True), - RegrTest('test_fork1.py'), + RegrTest('test_fork1.py', usemodules="thread"), RegrTest('test_format.py', core=True), RegrTest('test_fpformat.py', core=True), RegrTest('test_frozen.py', skip="unsupported extension module"), @@ -346,7 +346,7 @@ RegrTest('test_pyclbr.py'), RegrTest('test_pyexpat.py'), - RegrTest('test_queue.py'), + RegrTest('test_queue.py', usemodules='thread'), RegrTest('test_quopri.py'), RegrTest('test_random.py'), RegrTest('test_re.py', core=True), @@ -465,7 +465,7 @@ RegrTest('test_code.py'), RegrTest('test_coding.py'), RegrTest('test_complex_args.py'), - RegrTest('test_contextlib.py'), + RegrTest('test_contextlib.py', usemodules="thread"), # we skip test ctypes, since we adapted it massively in order # to test what we want to support. There are real failures, # but it's about missing features that we don't want to support @@ -481,13 +481,13 @@ RegrTest('test_pep352.py'), RegrTest('test_platform.py'), RegrTest('test_runpy.py'), - RegrTest('test_sqlite.py'), + RegrTest('test_sqlite.py', usemodules="thread"), RegrTest('test_startfile.py', skip="bogus test"), RegrTest('test_structmembers.py', skip="depends on _testcapi"), - RegrTest('test_urllib2_localnet.py'), + RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_uuid.py'), - RegrTest('test_wait3.py'), - RegrTest('test_wait4.py'), + RegrTest('test_wait3.py', usemodules="thread"), + RegrTest('test_wait4.py', usemodules="thread"), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), From arigo at codespeak.net Fri Jul 10 20:19:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Jul 2009 20:19:13 +0200 (CEST) Subject: [pypy-svn] r66173 - pypy/branch/pyjitpl5/pypy/translator/c/src Message-ID: <20090710181913.7E938168456@codespeak.net> Author: arigo Date: Fri Jul 10 20:19:13 2009 New Revision: 66173 Modified: pypy/branch/pyjitpl5/pypy/translator/c/src/support.h Log: Use %ld for the 'long' argument. Modified: pypy/branch/pyjitpl5/pypy/translator/c/src/support.h ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/c/src/support.h (original) +++ pypy/branch/pyjitpl5/pypy/translator/c/src/support.h Fri Jul 10 20:19:13 2009 @@ -55,7 +55,7 @@ void RPyAssertFailed(const char* filename, long lineno, const char* function, const char *msg) { fprintf(stderr, - "PyPy assertion failed at %s:%d:\n" + "PyPy assertion failed at %s:%ld:\n" "in %s: %s\n", filename, lineno, function, msg); abort(); From arigo at codespeak.net Sun Jul 12 13:04:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Jul 2009 13:04:01 +0200 (CEST) Subject: [pypy-svn] r66182 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090712110401.D0777169F3A@codespeak.net> Author: arigo Date: Sun Jul 12 13:03:59 2009 New Revision: 66182 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Log: Kill some code that used to be related to virtualizables. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Sun Jul 12 13:03:59 2009 @@ -121,9 +121,8 @@ if node.startbox: node.cls = None assert not node.virtual - ofs = perfect_specializer.adapt_for_match(old_loop) + perfect_specializer.adapt_for_match(old_loop) perfect_specializer.optimize_loop() - perfect_specializer.update_loop(ofs, old_loop) return old_loop return None # no loop matches @@ -551,64 +550,10 @@ jump_op = self.loop.operations[-1] assert jump_op.opnum == rop.JUMP self.specnodes = old_loop.specnodes - all_offsets = [] for i in range(len(old_loop.specnodes)): old_specnode = old_loop.specnodes[i] new_instnode = self.getnode(jump_op.args[i]) - offsets = [] - old_specnode.adapt_to(new_instnode, offsets) - all_offsets.append(offsets) - return all_offsets - - def _patch(self, origargs, newargs): - i = 0 - res = [] - for arg in newargs: - if arg is None: - res.append(origargs[i]) - i += 1 - else: - res.append(arg) - return res - - def _patch_loop(self, operations, inpargs, rebuild_ops, loop): - for op in operations: - if op.is_guard(): - if op.suboperations[-1].opnum == rop.FAIL: - op.suboperations = (op.suboperations[:-1] + rebuild_ops + - [op.suboperations[-1]]) - else: - self._patch_loop(op.suboperations, inpargs, rebuild_ops, - loop) - jump = operations[-1] - if jump.opnum == rop.JUMP and jump.jump_target is loop: - jump.args = self._patch(jump.args, inpargs) - - def update_loop(self, offsets, loop): - if loop.operations is None: # special loops 'done_with_this_frame' - return # and 'exit_frame_with_exception' - j = 0 - new_inputargs = [] - prev_ofs = 0 - rebuild_ops = [] - memo = {} - for i in range(len(offsets)): - for specnode, descr, parentnode, rel_ofs, node in offsets[i]: - while parentnode.source != loop.inputargs[j]: - j += 1 - ofs = j + rel_ofs + 1 - new_inputargs.extend([None] * (ofs - prev_ofs)) - prev_ofs = ofs - boxlist = [] - specnode.expand_boxlist(node, boxlist) - new_inputargs.extend(boxlist) - box = self.prepare_rebuild_ops(node, rebuild_ops, memo) - assert isinstance(descr, AbstractDescr) - rebuild_ops.append(ResOperation(rop.SETFIELD_GC, - [parentnode.source, box], None, descr)) - new_inputargs.extend([None] * (len(loop.inputargs) - prev_ofs)) - loop.inputargs = self._patch(loop.inputargs, new_inputargs) - self._patch_loop(loop.operations, new_inputargs, rebuild_ops, loop) + old_specnode.adapt_to(new_instnode, None) # --------------------------------------------------------------- From arigo at codespeak.net Sun Jul 12 13:28:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Jul 2009 13:28:32 +0200 (CEST) Subject: [pypy-svn] r66183 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090712112832.E2B02169E19@codespeak.net> Author: arigo Date: Sun Jul 12 13:28:32 2009 New Revision: 66183 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Log: Test and fix. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Sun Jul 12 13:28:32 2009 @@ -117,10 +117,14 @@ for old_loop in old_loops: if perfect_specializer.match(old_loop): # xxx slow, maybe + # XXX the next loop is a big hack. Ideally it should set cls=None + # to prevent assuming something about the cls -- but only if there + # is no code in the previous loop that checks the cls. for node in perfect_specializer.nodes.values(): if node.startbox: node.cls = None assert not node.virtual + perfect_specializer.propagate_escapes() perfect_specializer.adapt_for_match(old_loop) perfect_specializer.optimize_loop() return old_loop @@ -144,7 +148,7 @@ node = InstanceNode(box, escaped=True, const=True) else: assert self._allow_automatic_node_creation - node = InstanceNode(box, escaped=False, startbox=True) + node = InstanceNode(box, escaped=True, startbox=True) self.nodes[box] = node return node @@ -266,6 +270,9 @@ if isinstance(other_box, Box): self.nodes[box].add_to_dependency_graph(self.nodes[other_box], self.dependency_graph) + self.propagate_escapes() + + def propagate_escapes(self): # XXX find efficient algorithm, we're too fried for that by now done = False while not done: Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Sun Jul 12 13:28:32 2009 @@ -881,3 +881,22 @@ # It is ok to reorder just the 'getfield_gc[n1], n2' operation, # but the three remaining getfields/setfields *must* be in that order. equaloplists(spec.loop.operations, P.ops) + +# ____________________________________________________________ + +class Q: + locals().update(A.__dict__) # :-) + ops = [ + ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, + size_of_node), + ResOperation('setfield_gc', [n2, n1], None, ofs_next), + ResOperation('jump', [], None), + ] + +def test_Q_find_nodes(): + spec = PerfectSpecializer(Loop(None, Q.ops)) + spec.find_nodes() + spec.propagate_escapes() + # 'n2' should be marked as 'escaped', so that 'n1' is too + assert spec.nodes[Q.n2].escaped + assert spec.nodes[Q.n1].escaped From arigo at codespeak.net Sun Jul 12 13:53:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Jul 2009 13:53:37 +0200 (CEST) Subject: [pypy-svn] r66184 - in pypy/trunk/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/platform translator/platform/test Message-ID: <20090712115337.2A24D169F48@codespeak.net> Author: arigo Date: Sun Jul 12 13:53:35 2009 New Revision: 66184 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/linux.py pypy/trunk/pypy/translator/platform/test/test_distutils.py pypy/trunk/pypy/translator/platform/test/test_maemo.py pypy/trunk/pypy/translator/platform/test/test_platform.py pypy/trunk/pypy/translator/platform/windows.py Log: Bulk-revert all changes from r66152 onwards. Fijal did not do anything since we discussed the issue two days ago on irc, so better have a working base from which to re-check-in correct changes than leaving the current broken files. Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Sun Jul 12 13:53:35 2009 @@ -19,7 +19,6 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress -from pypy.translator.platform import platform def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -814,23 +813,10 @@ cfunc = get_on_lib(ctypes.windll.kernel32, funcname) else: cfunc = None - not_found = [] for libname in libraries: - libpath = None - ext = platform.so_ext - prefixes = platform.so_prefixes - for dir in eci.library_dirs: - if libpath: - break - for prefix in prefixes: - tryfile = os.path.join(dir, prefix + libname + '.' + ext) - if os.path.isfile(tryfile): - libpath = tryfile - break - if not libpath: - libpath = ctypes.util.find_library(libname) - if not libpath and os.path.isabs(libname): - libpath = libname + libpath = ctypes.util.find_library(libname) + if not libpath and os.path.isabs(libname): + libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') # urgh, cannot pass the flag to dllclass.LoadLibrary @@ -838,20 +824,11 @@ cfunc = get_on_lib(clib, funcname) if cfunc is not None: break - else: - not_found.append(libname) if cfunc is None: # function name not found in any of the libraries if not libraries: place = 'the standard C library (missing libraries=...?)' - elif len(not_found) == len(libraries): - if len(not_found) == 1: - raise NotImplementedError( - 'cannot find the library %r' % (not_found[0],)) - else: - raise NotImplementedError( - 'cannot find any of the libraries %r' % (not_found,)) elif len(libraries) == 1: place = 'library %r' % (libraries[0],) else: Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Sun Jul 12 13:53:35 2009 @@ -15,7 +15,6 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.rtyper import RPythonTyper -from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -995,46 +994,3 @@ v2 = ctypes2lltype(llmemory.GCREF, ctypes.c_void_p(1235)) assert v2 != v -class TestPlatform(object): - def test_lib_on_libpaths(self): - from pypy.translator.platform import platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - tmpdir = udir.join('lib_on_libppaths') - tmpdir.ensure(dir=1) - c_file = tmpdir.join('c_file.c') - c_file.write('int f(int a, int b) { return (a + b); }') - eci = ExternalCompilationInfo(export_symbols=['f']) - so = platform.compile([c_file], eci, standalone=False) - eci = ExternalCompilationInfo( - libraries = ['c_file'], - library_dirs = [str(so.dirpath())] - ) - f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, - compilation_info=eci) - assert f(3, 4) == 7 - - def test_prefix(self): - - if sys.platform != 'linux2': - py.test.skip("Not supported") - - from pypy.translator.platform import platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - tmpdir = udir.join('lib_on_libppaths_prefix') - tmpdir.ensure(dir=1) - c_file = tmpdir.join('c_file.c') - c_file.write('int f(int a, int b) { return (a + b); }') - eci = ExternalCompilationInfo() - so = platform.compile([c_file], eci, standalone=False) - sopath = py.path.local(so) - sopath.move(sopath.dirpath().join('libc_file.so')) - eci = ExternalCompilationInfo( - libraries = ['c_file'], - library_dirs = [str(so.dirpath())] - ) - f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, - compilation_info=eci) - assert f(3, 4) == 7 - Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Sun Jul 12 13:53:35 2009 @@ -52,8 +52,6 @@ name = "abstract platform" c_environ = None - so_prefixes = [''] - def __init__(self, cc): if self.__class__ is Platform: raise TypeError("You should not instantiate Platform class directly") @@ -72,12 +70,7 @@ ofiles.append(self._compile_c_file(self.cc, cfile, compile_args)) return ofiles - def execute(self, executable, args=None, env=None, compilation_info=None): - if env is None: - env = {} - if compilation_info is not None: - env['LD_LIBRARY_PATH'] = ':'.join( - [str(i) for i in compilation_info.library_dirs]) + def execute(self, executable, args=None, env=None): returncode, stdout, stderr = _run_subprocess(str(executable), args, env) return ExecutionResult(returncode, stdout, stderr) @@ -127,13 +120,9 @@ cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) - def _link_args_from_eci(self, eci, standalone): - if standalone: - library_dirs = self._libdirs(eci.library_dirs) - libraries = self._libs(eci.libraries) - else: - library_dirs = [] - libraries = [] + def _link_args_from_eci(self, eci): + library_dirs = self._libdirs(eci.library_dirs) + libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) return (library_dirs + libraries + self.link_flags + link_files + list(eci.link_extra)) @@ -148,8 +137,8 @@ exe_name += '.' + self.exe_ext else: exe_name += '.' + self.so_ext - largs = self._link_args_from_eci(eci, standalone) - return self._link(self.cc, ofiles, largs, standalone, exe_name) + return self._link(self.cc, ofiles, self._link_args_from_eci(eci), + standalone, exe_name) # below are some detailed informations for platforms Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Sun Jul 12 13:53:35 2009 @@ -38,8 +38,8 @@ args.append(f) return args - def _link_args_from_eci(self, eci, standalone): - args = super(Darwin, self)._link_args_from_eci(eci, standalone) + def _link_args_from_eci(self, eci): + args = super(Darwin, self)._link_args_from_eci(eci) frameworks = self._frameworks(eci.frameworks) include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Sun Jul 12 13:53:35 2009 @@ -13,7 +13,6 @@ standalone_only = [] shared_only = [] so_ext = 'so' - so_prefixes = ['lib', ''] def _args_for_shared(self, args): return ['-shared'] + args Modified: pypy/trunk/pypy/translator/platform/test/test_distutils.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_distutils.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_distutils.py Sun Jul 12 13:53:35 2009 @@ -8,6 +8,3 @@ def test_nice_errors(self): py.test.skip("Unsupported") - - def test_shared_no_links(self): - py.test.skip("Unsupported") Modified: pypy/trunk/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_maemo.py Sun Jul 12 13:53:35 2009 @@ -31,6 +31,3 @@ executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) self.check_res(res) - - def test_shared_no_links(self): - py.test.skip("Unsupported") Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Sun Jul 12 13:53:35 2009 @@ -1,5 +1,5 @@ -import py, sys, ctypes +import py, sys from pypy.tool.udir import udir from pypy.translator.platform import CompilationError, Platform from pypy.translator.platform import host @@ -102,18 +102,6 @@ res = self.platform.execute(executable) assert res.out.startswith('4.0') - def test_shared_no_links(self): - eci = ExternalCompilationInfo(libraries = ['xxxxxxxxxxxxxxxxxxx']) - tmpdir = udir.join('shared_no_links').ensure(dir=1) - c_file = tmpdir.join('shared1.c') - c_file.write(''' - int f(int a, int b) - { - return (a + b); - } - ''') - library = self.platform.compile([c_file], eci, standalone=False) - assert ctypes.CDLL(str(library)).f(3, 4) == 7 def test_equality(): class X(Platform): Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Sun Jul 12 13:53:35 2009 @@ -133,9 +133,8 @@ def _args_for_shared(self, args): return ['/dll'] + args - def _link_args_from_eci(self, eci, standalone): - # Windows needs to resolve all symbols even for DLLs - args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) + def _link_args_from_eci(self, eci): + args = super(MsvcPlatform, self)._link_args_from_eci(eci) return args + ['/EXPORT:%s' % symbol for symbol in eci.export_symbols] def _compile_c_file(self, cc, cfile, compile_args): From benjamin at codespeak.net Sun Jul 12 19:28:12 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 12 Jul 2009 19:28:12 +0200 (CEST) Subject: [pypy-svn] r66185 - pypy/branch/parser-compiler/pypy/tool Message-ID: <20090712172812.93750169ECA@codespeak.net> Author: benjamin Date: Sun Jul 12 19:28:10 2009 New Revision: 66185 Modified: pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py Log: add opcodes directly to the namespace Modified: pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py Sun Jul 12 19:28:10 2009 @@ -18,6 +18,7 @@ for name in __all__: if name in opcode_dict: globals()[name] = opcode_dict[name] +globals().update(opmap) opcode_method_names = ['MISSING_OPCODE'] * 256 for name, index in opmap.items(): From arigo at codespeak.net Mon Jul 13 15:14:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Jul 2009 15:14:05 +0200 (CEST) Subject: [pypy-svn] r66186 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090713131405.5D8DB169E14@codespeak.net> Author: arigo Date: Mon Jul 13 15:14:04 2009 New Revision: 66186 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Log: Revert r63474, which looks obscure and wrong. It contains no bug description nor any test that would show the bug, so it's cancelled until it can be clearly explained. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Mon Jul 13 15:14:04 2009 @@ -349,7 +349,7 @@ op_fail = op.suboperations[0] assert op_fail.opnum == rop.FAIL for box in op_fail.args: - if isinstance(box, Const) or box not in self.nodes: + if isinstance(box, Const): continue self.prepare_rebuild_ops(self.nodes[box], rebuild_ops, memo, box) From arigo at codespeak.net Mon Jul 13 15:43:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Jul 2009 15:43:32 +0200 (CEST) Subject: [pypy-svn] r66187 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090713134332.4D2D5169EAC@codespeak.net> Author: arigo Date: Mon Jul 13 15:43:31 2009 New Revision: 66187 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Log: Test and (one-line) fix. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Mon Jul 13 15:43:31 2009 @@ -469,6 +469,7 @@ elif opnum == rop.NEW_WITH_VTABLE: # self.nodes[op.result] keeps the value from Steps (1,2) instnode = self.nodes[op.result] + instnode.curfields = r_dict(av_eq, av_hash) if not instnode.escaped: instnode.virtual = True assert instnode.cls is not None Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Mon Jul 13 15:43:31 2009 @@ -5,10 +5,10 @@ from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.jit.backend.llgraph import runner -from pypy.jit.metainterp import resoperation +from pypy.jit.metainterp import resoperation, history from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - ConstAddr, TreeLoop) + Const, ConstAddr, TreeLoop) from pypy.jit.metainterp.optimize4 import PerfectSpecializer from pypy.jit.metainterp.specnode4 import (FixedClassSpecNode, NotSpecNode, @@ -85,6 +85,25 @@ assert op.is_guard(), op op.suboperations = [ResOperation('fail', args, None)] + +class CheckPerfectSpecializer(PerfectSpecializer): + def optimize_loop(self): + PerfectSpecializer.optimize_loop(self) + check_operations(self.loop.inputargs, self.loop.operations) + +def check_operations(inputargs, operations, indent=' |'): + seen = dict.fromkeys(inputargs) + for op in operations: + print indent, op + for x in op.args: + assert x in seen or isinstance(x, Const) + assert op.descr is None or isinstance(op.descr, history.AbstractDescr) + if op.is_guard(): + check_operations(seen.keys(), op.suboperations, indent+' ') + if op.result is not None: + seen[op.result] = True + assert operations[-1].opnum in (rop.FAIL, rop.JUMP) + # ____________________________________________________________ class A: @@ -116,7 +135,7 @@ ] def test_A_find_nodes(): - spec = PerfectSpecializer(Loop(A.inputargs, A.ops)) + spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) spec.find_nodes() assert spec.nodes[A.sum] is not spec.nodes[A.sum2] assert spec.nodes[A.n1] is not spec.nodes[A.n2] @@ -131,7 +150,7 @@ assert spec.nodes[A.n2].curfields[A.ofs_value] is spec.nodes[A.v2] def test_A_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(A.inputargs, A.ops)) + spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) spec.find_nodes() spec.intersect_input_and_output() assert len(spec.specnodes) == 2 @@ -143,7 +162,7 @@ assert isinstance(spec_n.fields[0][1], NotSpecNode) def test_A_optimize_loop(): - spec = PerfectSpecializer(Loop(A.inputargs, A.ops)) + spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -173,7 +192,7 @@ ] def test_B_find_nodes(): - spec = PerfectSpecializer(Loop(B.inputargs, B.ops)) + spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) spec.find_nodes() assert spec.nodes[B.n1].cls.source.value == node_vtable_adr assert spec.nodes[B.n1].escaped @@ -181,7 +200,7 @@ assert spec.nodes[B.n2].escaped def test_B_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(B.inputargs, B.ops)) + spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) spec.find_nodes() spec.intersect_input_and_output() assert len(spec.specnodes) == 2 @@ -191,7 +210,7 @@ assert spec_n.known_class.value == node_vtable_adr def test_B_optimize_loop(): - spec = PerfectSpecializer(Loop(B.inputargs, B.ops)) + spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -228,14 +247,14 @@ ] def test_C_find_nodes(): - spec = PerfectSpecializer(Loop(C.inputargs, C.ops)) + spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) spec.find_nodes() assert spec.nodes[C.n1].cls.source.value == node_vtable_adr assert spec.nodes[C.n1].escaped assert spec.nodes[C.n2].cls.source.value == node_vtable_adr def test_C_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(C.inputargs, C.ops)) + spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) spec.find_nodes() spec.intersect_input_and_output() assert spec.nodes[C.n2].escaped @@ -246,7 +265,7 @@ assert spec_n.known_class.value == node_vtable_adr def test_C_optimize_loop(): - spec = PerfectSpecializer(Loop(C.inputargs, C.ops)) + spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -283,7 +302,7 @@ def test_D_intersect_input_and_output(): py.test.skip("nowadays, this compiles, just without making a virtual") - spec = PerfectSpecializer(Loop(D.inputargs, D.ops)) + spec = CheckPerfectSpecializer(Loop(D.inputargs, D.ops)) spec.find_nodes() py.test.raises(CancelInefficientLoop, spec.intersect_input_and_output) @@ -306,7 +325,7 @@ set_guard(ops[-2], [sum2, n2]) def test_E_optimize_loop(): - spec = PerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) + spec = CheckPerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -329,7 +348,7 @@ ]) ##def test_E_rebuild_after_failure(): -## spec = PerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) +## spec = CheckPerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) ## spec.find_nodes() ## spec.intersect_input_and_output() ## spec.optimize_loop() @@ -379,13 +398,13 @@ set_guard(ops[-6], [sum2, n2, n3]) def test_F_find_nodes(): - spec = PerfectSpecializer(Loop(F.inputargs, F.ops)) + spec = CheckPerfectSpecializer(Loop(F.inputargs, F.ops)) spec.find_nodes() assert not spec.nodes[F.n1].escaped assert not spec.nodes[F.n2].escaped def test_F_optimize_loop(): - spec = PerfectSpecializer(Loop(F.inputargs, F.ops), cpu=cpu) + spec = CheckPerfectSpecializer(Loop(F.inputargs, F.ops), cpu=cpu) spec.find_nodes() spec.intersect_input_and_output() assert spec.nodes[F.n3].escaped @@ -418,7 +437,7 @@ set_guard(ops[-3], [n2]) def test_F2_optimize_loop(): - spec = PerfectSpecializer(Loop(F2.inputargs, F2.ops), cpu=cpu) + spec = CheckPerfectSpecializer(Loop(F2.inputargs, F2.ops), cpu=cpu) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -448,7 +467,7 @@ set_guard(ops[-2], [sum2, n2]) def test_G_optimize_loop(): - spec = PerfectSpecializer(Loop(G.inputargs, G.ops), cpu=cpu) + spec = CheckPerfectSpecializer(Loop(G.inputargs, G.ops), cpu=cpu) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -498,7 +517,7 @@ ] def test_H_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(H.inputargs, H.ops)) + spec = CheckPerfectSpecializer(Loop(H.inputargs, H.ops)) spec.find_nodes() spec.intersect_input_and_output() assert spec.nodes[H.n0].escaped @@ -526,7 +545,7 @@ ] def test_I_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(I.inputargs, I.ops)) + spec = CheckPerfectSpecializer(Loop(I.inputargs, I.ops)) spec.find_nodes() spec.intersect_input_and_output() assert spec.nodes[I.n0].escaped @@ -555,7 +574,7 @@ ] def test_J_intersect_input_and_output(): - spec = PerfectSpecializer(Loop(J.inputargs, J.ops)) + spec = CheckPerfectSpecializer(Loop(J.inputargs, J.ops)) spec.find_nodes() spec.intersect_input_and_output() assert not spec.nodes[J.n0].escaped @@ -582,7 +601,7 @@ def test_K0_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(K0.inputargs, K0.ops)) + spec = CheckPerfectSpecializer(Loop(K0.inputargs, K0.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -617,7 +636,7 @@ def test_K1_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(K1.inputargs, K1.ops)) + spec = CheckPerfectSpecializer(Loop(K1.inputargs, K1.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -651,7 +670,7 @@ def test_K_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(K.inputargs, K.ops)) + spec = CheckPerfectSpecializer(Loop(K.inputargs, K.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -683,7 +702,7 @@ def test_L_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(L.inputargs, L.ops)) + spec = CheckPerfectSpecializer(Loop(L.inputargs, L.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -715,7 +734,7 @@ def test_M_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(M.inputargs, M.ops)) + spec = CheckPerfectSpecializer(Loop(M.inputargs, M.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -747,7 +766,7 @@ def test_N_optimize_loop(): py.test.skip("Disabled") - spec = PerfectSpecializer(Loop(N.inputargs, N.ops)) + spec = CheckPerfectSpecializer(Loop(N.inputargs, N.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -776,7 +795,7 @@ set_guard(ops[-2], []) def test_O1_optimize_loop(): - spec = PerfectSpecializer(Loop(O1.inputargs, O1.ops)) + spec = CheckPerfectSpecializer(Loop(O1.inputargs, O1.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -806,7 +825,7 @@ set_guard(ops[-2], []) def test_O2_optimize_loop(): - spec = PerfectSpecializer(Loop(O2.inputargs, O2.ops)) + spec = CheckPerfectSpecializer(Loop(O2.inputargs, O2.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -838,7 +857,7 @@ set_guard(ops[-2], []) def test_O3_optimize_loop(): - spec = PerfectSpecializer(Loop(O3.inputargs, O3.ops)) + spec = CheckPerfectSpecializer(Loop(O3.inputargs, O3.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -872,7 +891,7 @@ set_guard(ops[-3], []) def test_P_optimize_loop(): - spec = PerfectSpecializer(Loop(P.inputargs, P.ops)) + spec = CheckPerfectSpecializer(Loop(P.inputargs, P.ops)) spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() @@ -894,9 +913,33 @@ ] def test_Q_find_nodes(): - spec = PerfectSpecializer(Loop(None, Q.ops)) + spec = CheckPerfectSpecializer(Loop(None, Q.ops)) spec.find_nodes() spec.propagate_escapes() # 'n2' should be marked as 'escaped', so that 'n1' is too assert spec.nodes[Q.n2].escaped assert spec.nodes[Q.n1].escaped + +# ____________________________________________________________ + +class R: + locals().update(A.__dict__) # :-) + inputargs = [sum] + ops = [ + ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, + size_of_node), + ResOperation('int_is_true', [sum], n1nz), + ResOperation('guard_true', [n1nz], None), + ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, + size_of_node), + ResOperation('setfield_gc', [n1, n2], None, ofs_next), + ResOperation('int_sub', [sum, ConstInt(1)], sum2), + ResOperation('jump', [sum2], None), + ] + set_guard(ops[2], [n1]) + +def test_R_find_nodes(): + spec = CheckPerfectSpecializer(Loop(R.inputargs, R.ops), cpu=cpu) + spec.find_nodes() + spec.intersect_input_and_output() + spec.optimize_loop() From arigo at codespeak.net Mon Jul 13 15:49:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Jul 2009 15:49:06 +0200 (CEST) Subject: [pypy-svn] r66188 - pypy/trunk/pypy/tool/test Message-ID: <20090713134906.22F41169EBF@codespeak.net> Author: arigo Date: Mon Jul 13 15:49:05 2009 New Revision: 66188 Modified: pypy/trunk/pypy/tool/test/test_compat.py Log: Fix the test, mostly randomly (what is the point of this test?). Modified: pypy/trunk/pypy/tool/test/test_compat.py ============================================================================== --- pypy/trunk/pypy/tool/test/test_compat.py (original) +++ pypy/trunk/pypy/tool/test/test_compat.py Mon Jul 13 15:49:05 2009 @@ -1,5 +1,5 @@ def test_base_exc(): from pypy.tool.compat import BaseException - assert KeyboardInterrupt.__mro__[-2] is BaseException - assert ValueError.__mro__[-2] is BaseException + assert issubclass(KeyboardInterrupt, BaseException) + assert issubclass(ValueError, BaseException) From arigo at codespeak.net Mon Jul 13 17:44:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Jul 2009 17:44:49 +0200 (CEST) Subject: [pypy-svn] r66189 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090713154449.AB66B169EC4@codespeak.net> Author: arigo Date: Mon Jul 13 17:44:47 2009 New Revision: 66189 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py Log: Forgot to attach 'specnodes' to the new loop. Test missing :-( It seems we really need tests for what occurs when compiling multiple loops and bridges... Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py Mon Jul 13 17:44:47 2009 @@ -200,21 +200,23 @@ class TerminatingLoop(TreeLoop): pass +prebuiltNotSpecNode = NotSpecNode() + # pseudo-loops to make the life of optimize.py easier _loop = TerminatingLoop('done_with_this_frame_int') -_loop.specnodes = [NotSpecNode()] +_loop.specnodes = [prebuiltNotSpecNode] _loop.inputargs = [BoxInt()] _loop.finishdescr = done_with_this_frame_descr_int loops_done_with_this_frame_int = [_loop] _loop = TerminatingLoop('done_with_this_frame_ptr') -_loop.specnodes = [NotSpecNode()] +_loop.specnodes = [prebuiltNotSpecNode] _loop.inputargs = [BoxPtr()] _loop.finishdescr = done_with_this_frame_descr_ptr loops_done_with_this_frame_ptr = [_loop] _loop = TerminatingLoop('done_with_this_frame_obj') -_loop.specnodes = [NotSpecNode()] +_loop.specnodes = [prebuiltNotSpecNode] _loop.inputargs = [BoxObj()] _loop.finishdescr = done_with_this_frame_descr_obj loops_done_with_this_frame_obj = [_loop] @@ -226,13 +228,13 @@ loops_done_with_this_frame_void = [_loop] _loop = TerminatingLoop('exit_frame_with_exception_ptr') -_loop.specnodes = [NotSpecNode()] +_loop.specnodes = [prebuiltNotSpecNode] _loop.inputargs = [BoxPtr()] _loop.finishdescr = exit_frame_with_exception_descr_ptr loops_exit_frame_with_exception_ptr = [_loop] _loop = TerminatingLoop('exit_frame_with_exception_obj') -_loop.specnodes = [NotSpecNode()] +_loop.specnodes = [prebuiltNotSpecNode] _loop.inputargs = [BoxObj()] _loop.finishdescr = exit_frame_with_exception_descr_obj loops_exit_frame_with_exception_obj = [_loop] @@ -355,6 +357,7 @@ metainterp.history.inputargs = redkey new_loop.greenkey = greenkey new_loop.inputargs = redkey + new_loop.specnodes = [prebuiltNotSpecNode] * len(redkey) send_loop_to_backend(metainterp, new_loop, None, "entry bridge") metainterp_sd.stats.loops.append(new_loop) # send the new_loop to warmspot.py, to be called directly the next time From benjamin at codespeak.net Mon Jul 13 22:32:26 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 13 Jul 2009 22:32:26 +0200 (CEST) Subject: [pypy-svn] r66191 - pypy/branch/parser-compiler/pypy/tool Message-ID: <20090713203226.E34A1169F18@codespeak.net> Author: benjamin Date: Mon Jul 13 22:32:26 2009 New Revision: 66191 Modified: pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py Log: add useful constants Modified: pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py ============================================================================== --- pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py (original) +++ pypy/branch/parser-compiler/pypy/tool/stdlib_opcode.py Mon Jul 13 22:32:26 2009 @@ -19,6 +19,9 @@ if name in opcode_dict: globals()[name] = opcode_dict[name] globals().update(opmap) +SLICE = opmap["SLICE+0"] +STORE_SLICE = opmap["STORE_SLICE+0"] +DELETE_SLICE = opmap["DELETE_SLICE+0"] opcode_method_names = ['MISSING_OPCODE'] * 256 for name, index in opmap.items(): From benjamin at codespeak.net Mon Jul 13 23:05:37 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 13 Jul 2009 23:05:37 +0200 (CEST) Subject: [pypy-svn] r66192 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090713210537.3C0A8169F42@codespeak.net> Author: benjamin Date: Mon Jul 13 23:05:35 2009 New Revision: 66192 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: support parsing of octal and hex literals Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Mon Jul 13 23:05:35 2009 @@ -1029,15 +1029,25 @@ def parse_number(self, raw): base = 10 + if raw.startswith("0"): + if len(raw) > 2 and raw[1] in "Xx": + base = 16 + elif len(raw) > 1: + base = 8 + raw = raw.rstrip("0xX") + if not raw: + raw = "0" w_num_str = self.space.wrap(raw) + w_index = None + w_base = self.space.wrap(base) if raw[-1] in "lL": tp = self.space.w_long - return self.space.call_function(tp, w_num_str) + return self.space.call_function(tp, w_num_str, w_base) elif raw[-1] in "jJ": tp = self.space.w_complex return self.space.call_function(tp, w_num_str) try: - return self.space.call_function(self.space.w_int, w_num_str) + return self.space.call_function(self.space.w_int, w_num_str, w_base) except error.OperationError, e: if not e.match(self.space, self.space.w_ValueError): raise Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Mon Jul 13 23:05:35 2009 @@ -1032,6 +1032,13 @@ assert space.eq_w(get_num("32l"), space.newlong(32)) assert space.eq_w(get_num("13j"), space.wrap(13j)) assert space.eq_w(get_num("13J"), space.wrap(13J)) + assert space.eq_w(get_num("053"), space.wrap(053)) + assert space.eq_w(get_num("00053"), space.wrap(053)) + for num in ("0x53", "0X53", "0x0000053", "0X00053"): + assert space.eq_w(get_num(num), space.wrap(0x53)) + assert space.eq_w(get_num("0X53"), space.wrap(0x53)) + assert space.eq_w(get_num("0"), space.wrap(0)) + assert space.eq_w(get_num("00000"), space.wrap(0)) def check_comprehension(self, brackets, ast_type): def brack(s): From benjamin at codespeak.net Tue Jul 14 00:07:09 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 00:07:09 +0200 (CEST) Subject: [pypy-svn] r66193 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090713220709.26518168416@codespeak.net> Author: benjamin Date: Tue Jul 14 00:07:09 2009 New Revision: 66193 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: this should be lstrip() Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 14 00:07:09 2009 @@ -1034,7 +1034,7 @@ base = 16 elif len(raw) > 1: base = 8 - raw = raw.rstrip("0xX") + raw = raw.lstrip("0xX") if not raw: raw = "0" w_num_str = self.space.wrap(raw) From benjamin at codespeak.net Tue Jul 14 00:19:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 00:19:35 +0200 (CEST) Subject: [pypy-svn] r66194 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090713221935.540CA168416@codespeak.net> Author: benjamin Date: Tue Jul 14 00:19:34 2009 New Revision: 66194 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: fold '-' on constant numbers Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 14 00:19:34 2009 @@ -868,6 +868,19 @@ return result def handle_factor(self, factor_node): + # Fold '-' on constants. + if factor_node.children[0].type == tokens.MINUS and \ + len(factor_node.children) == 2: + factor = factor_node.children[1] + if factor.type == syms.factor and len(factor.children) == 1: + power = factor.children[0] + if power.type == syms.power and len(power.children) == 1: + atom = power.children[0] + if atom.type == syms.atom and \ + atom.children[0].type == tokens.NUMBER: + num = atom.children[0] + num.value = "-" + num.value + return self.handle_atom(atom) expr = self.handle_expr(factor_node.children[1]) op_type = factor_node.children[0].type if op_type == tokens.PLUS: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 14 00:19:34 2009 @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import random import string +import sys import py from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.pyparser import pyparse @@ -1039,6 +1040,10 @@ assert space.eq_w(get_num("0X53"), space.wrap(0x53)) assert space.eq_w(get_num("0"), space.wrap(0)) assert space.eq_w(get_num("00000"), space.wrap(0)) + assert space.eq_w(get_num("-3"), space.wrap(-3)) + assert space.eq_w(get_num("-0"), space.wrap(0)) + n = get_num(str(-sys.maxint - 1)) + assert space.is_true(space.isinstance(n, space.w_int)) def check_comprehension(self, brackets, ast_type): def brack(s): From benjamin at codespeak.net Tue Jul 14 05:33:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 05:33:59 +0200 (CEST) Subject: [pypy-svn] r66195 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: test tools Message-ID: <20090714033359.E9FA3168440@codespeak.net> Author: benjamin Date: Tue Jul 14 05:33:58 2009 New Revision: 66195 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: implement simples sums with integers like cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 14 05:33:58 2009 @@ -78,21 +78,21 @@ assert isinstance(d, ast.Delete) assert len(d.targets) == 1 assert isinstance(d.targets[0], ast.Name) - assert d.targets[0].ctx is ast.Del + assert d.targets[0].ctx == ast.Del d = self.get_first_stmt("del x, y") assert len(d.targets) == 2 - assert d.targets[0].ctx is ast.Del - assert d.targets[1].ctx is ast.Del + assert d.targets[0].ctx == ast.Del + assert d.targets[1].ctx == ast.Del d = self.get_first_stmt("del x.y") assert len(d.targets) == 1 attr = d.targets[0] assert isinstance(attr, ast.Attribute) - assert attr.ctx is ast.Del + assert attr.ctx == ast.Del d = self.get_first_stmt("del x[:]") assert len(d.targets) == 1 sub = d.targets[0] assert isinstance(sub, ast.Subscript) - assert sub.ctx is ast.Del + assert sub.ctx == ast.Del def test_break(self): br = self.get_first_stmt("while True: break").body[0] @@ -249,7 +249,7 @@ if_ = self.get_first_stmt("if x: 4") assert isinstance(if_, ast.If) assert isinstance(if_.test, ast.Name) - assert if_.test.ctx is ast.Load + assert if_.test.ctx == ast.Load assert len(if_.body) == 1 assert isinstance(if_.body[0].value, ast.Num) assert if_.orelse is None @@ -283,7 +283,7 @@ wh = self.get_first_stmt("while x: pass") assert isinstance(wh, ast.While) assert isinstance(wh.test, ast.Name) - assert wh.test.ctx is ast.Load + assert wh.test.ctx == ast.Load assert len(wh.body) == 1 assert isinstance(wh.body[0], ast.Pass) assert wh.orelse is None @@ -298,27 +298,27 @@ fr = self.get_first_stmt("for x in y: pass") assert isinstance(fr, ast.For) assert isinstance(fr.target, ast.Name) - assert fr.target.ctx is ast.Store + assert fr.target.ctx == ast.Store assert isinstance(fr.iter, ast.Name) - assert fr.iter.ctx is ast.Load + assert fr.iter.ctx == ast.Load assert len(fr.body) == 1 assert isinstance(fr.body[0], ast.Pass) assert fr.orelse is None fr = self.get_first_stmt("for x, in y: pass") tup = fr.target assert isinstance(tup, ast.Tuple) - assert tup.ctx is ast.Store + assert tup.ctx == ast.Store assert len(tup.elts) == 1 assert isinstance(tup.elts[0], ast.Name) - assert tup.elts[0].ctx is ast.Store + assert tup.elts[0].ctx == ast.Store fr = self.get_first_stmt("for x, y in g: pass") tup = fr.target assert isinstance(tup, ast.Tuple) - assert tup.ctx is ast.Store + assert tup.ctx == ast.Store assert len(tup.elts) == 2 for elt in tup.elts: assert isinstance(elt, ast.Name) - assert elt.ctx is ast.Store + assert elt.ctx == ast.Store fr = self.get_first_stmt("for x in g: pass\nelse: 4") assert len(fr.body) == 1 assert isinstance(fr.body[0], ast.Pass) @@ -348,7 +348,7 @@ assert len(tr.handlers) == 1 handler = tr.handlers[0] assert isinstance(handler.type, ast.Name) - assert handler.type.ctx is ast.Load + assert handler.type.ctx == ast.Load assert handler.name is None assert len(handler.body) == 1 assert tr.orelse is None @@ -357,7 +357,7 @@ handler = tr.handlers[0] assert isinstance(handler.type, ast.Name) assert isinstance(handler.name, ast.Name) - assert handler.name.ctx is ast.Store + assert handler.name.ctx == ast.Store assert handler.name.id == "e" assert len(handler.body) == 1 tr = self.get_first_stmt("try: x\nexcept: pass\nelse: 4") @@ -411,12 +411,12 @@ assert isinstance(wi.context_expr, ast.Name) assert len(wi.body) == 1 assert isinstance(wi.optional_vars, ast.Name) - assert wi.optional_vars.ctx is ast.Store + assert wi.optional_vars.ctx == ast.Store wi = self.get_first_stmt("with x as (y,): pass") assert isinstance(wi.optional_vars, ast.Tuple) assert len(wi.optional_vars.elts) == 1 - assert wi.optional_vars.ctx is ast.Store - assert wi.optional_vars.elts[0].ctx is ast.Store + assert wi.optional_vars.ctx == ast.Store + assert wi.optional_vars.elts[0].ctx == ast.Store input = "with x hi y: pass" exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == "expected \"with [expr] as [var]\"" @@ -434,13 +434,13 @@ assert len(cls.bases) == 1 base = cls.bases[0] assert isinstance(base, ast.Name) - assert base.ctx is ast.Load + assert base.ctx == ast.Load assert base.id == "Y" cls = self.get_first_stmt("class X(Y, Z): pass") assert len(cls.bases) == 2 for b in cls.bases: assert isinstance(b, ast.Name) - assert b.ctx is ast.Load + assert b.ctx == ast.Load def test_function(self): func = self.get_first_stmt("def f(): pass") @@ -460,10 +460,10 @@ a1, a2 = args.args assert isinstance(a1, ast.Name) assert a1.id == "a" - assert a1.ctx is ast.Param + assert a1.ctx == ast.Param assert isinstance(a2, ast.Name) assert a2.id == "b" - assert a2.ctx is ast.Param + assert a2.ctx == ast.Param assert args.vararg is None assert args.kwarg is None args = self.get_first_stmt("def f(a=b): pass").args @@ -471,12 +471,12 @@ arg = args.args[0] assert isinstance(arg, ast.Name) assert arg.id == "a" - assert arg.ctx is ast.Param + assert arg.ctx == ast.Param assert len(args.defaults) == 1 default = args.defaults[0] assert isinstance(default, ast.Name) assert default.id == "b" - assert default.ctx is ast.Load + assert default.ctx == ast.Load args = self.get_first_stmt("def f(*a): pass").args assert args.args is None assert args.defaults is None @@ -494,14 +494,14 @@ assert len(args.args) == 1 tup = args.args[0] assert isinstance(tup, ast.Tuple) - assert tup.ctx is ast.Store + assert tup.ctx == ast.Store assert len(tup.elts) == 2 e1, e2 = tup.elts assert isinstance(e1, ast.Name) - assert e1.ctx is ast.Store + assert e1.ctx == ast.Store assert e1.id == "a" assert isinstance(e2, ast.Name) - assert e2.ctx is ast.Store + assert e2.ctx == ast.Store assert e2.id == "b" args = self.get_first_stmt("def f((a, (b, c))): pass").args assert len(args.args) == 1 @@ -510,20 +510,20 @@ assert len(tup.elts) == 2 tup2 = tup.elts[1] assert isinstance(tup2, ast.Tuple) - assert tup2.ctx is ast.Store + assert tup2.ctx == ast.Store for elt in tup2.elts: assert isinstance(elt, ast.Name) - assert elt.ctx is ast.Store + assert elt.ctx == ast.Store assert tup2.elts[0].id == "b" assert tup2.elts[1].id == "c" args = self.get_first_stmt("def f(a, b, c=d, *e, **f): pass").args assert len(args.args) == 3 for arg in args.args: assert isinstance(arg, ast.Name) - assert arg.ctx is ast.Param + assert arg.ctx == ast.Param assert len(args.defaults) == 1 assert isinstance(args.defaults[0], ast.Name) - assert args.defaults[0].ctx is ast.Load + assert args.defaults[0].ctx == ast.Load assert args.vararg == "e" assert args.kwarg == "f" input = "def f(a=b, c): pass" @@ -537,12 +537,12 @@ dec = func.decorators[0] assert isinstance(dec, ast.Name) assert dec.id == "dec" - assert dec.ctx is ast.Load + assert dec.ctx == ast.Load func = self.get_first_stmt("@mod.hi.dec\ndef f(): pass") assert len(func.decorators) == 1 dec = func.decorators[0] assert isinstance(dec, ast.Attribute) - assert dec.ctx is ast.Load + assert dec.ctx == ast.Load assert dec.attr == "dec" assert isinstance(dec.value, ast.Attribute) assert dec.value.attr == "hi" @@ -552,7 +552,7 @@ assert len(func.decorators) == 2 for dec in func.decorators: assert isinstance(dec, ast.Name) - assert dec.ctx is ast.Load + assert dec.ctx == ast.Load assert func.decorators[0].id == "dec" assert func.decorators[1].id == "dec2" func = self.get_first_stmt("@dec()\ndef f(): pass") @@ -596,7 +596,7 @@ assert isinstance(assign, ast.AugAssign) assert assign.op is ast_type assert isinstance(assign.target, ast.Name) - assert assign.target.ctx is ast.Store + assert assign.target.ctx == ast.Store assert isinstance(assign.value, ast.Num) def test_assign(self): @@ -605,28 +605,28 @@ assert len(assign.targets) == 1 name = assign.targets[0] assert isinstance(name, ast.Name) - assert name.ctx is ast.Store + assert name.ctx == ast.Store value = assign.value assert self.space.eq_w(value.n, self.space.wrap(32)) assign = self.get_first_stmt("hi, = something") assert len(assign.targets) == 1 tup = assign.targets[0] assert isinstance(tup, ast.Tuple) - assert tup.ctx is ast.Store + assert tup.ctx == ast.Store assert len(tup.elts) == 1 assert isinstance(tup.elts[0], ast.Name) - assert tup.elts[0].ctx is ast.Store + assert tup.elts[0].ctx == ast.Store def test_name(self): name = self.get_first_expr("hi") assert isinstance(name, ast.Name) - assert name.ctx is ast.Load + assert name.ctx == ast.Load def test_tuple(self): tup = self.get_first_expr("()") assert isinstance(tup, ast.Tuple) assert tup.elts is None - assert tup.ctx is ast.Load + assert tup.ctx == ast.Load tup = self.get_first_expr("(3,)") assert len(tup.elts) == 1 assert self.space.eq_w(tup.elts[0].n, self.space.wrap(3)) @@ -637,7 +637,7 @@ seq = self.get_first_expr("[]") assert isinstance(seq, ast.List) assert seq.elts is None - assert seq.ctx is ast.Load + assert seq.ctx == ast.Load seq = self.get_first_expr("[3,]") assert len(seq.elts) == 1 assert self.space.eq_w(seq.elts[0].n, self.space.wrap(3)) @@ -658,17 +658,17 @@ key1, key2 = d.keys assert isinstance(key1, ast.Num) assert isinstance(key2, ast.Name) - assert key2.ctx is ast.Load + assert key2.ctx == ast.Load v1, v2 = d.values assert isinstance(v1, ast.Name) - assert v1.ctx is ast.Load + assert v1.ctx == ast.Load assert isinstance(v2, ast.Num) def test_set_context(self): tup = self.get_ast("(a, b) = c").body[0].targets[0] - assert all(elt.ctx is ast.Store for elt in tup.elts) + assert all(elt.ctx == ast.Store for elt in tup.elts) seq = self.get_ast("[a, b] = c").body[0].targets[0] - assert all(elt.ctx is ast.Store for elt in seq.elts) + assert all(elt.ctx == ast.Store for elt in seq.elts) invalid_stores = ( ("(lambda x: x)", "lambda"), ("f()", "call"), @@ -749,30 +749,30 @@ ifexp = self.get_first_expr("x if y else g") assert isinstance(ifexp, ast.IfExp) assert isinstance(ifexp.test, ast.Name) - assert ifexp.test.ctx is ast.Load + assert ifexp.test.ctx == ast.Load assert isinstance(ifexp.body, ast.Name) - assert ifexp.body.ctx is ast.Load + assert ifexp.body.ctx == ast.Load assert isinstance(ifexp.orelse, ast.Name) - assert ifexp.orelse.ctx is ast.Load + assert ifexp.orelse.ctx == ast.Load def test_boolop(self): for ast_type, op in ((ast.And, "and"), (ast.Or, "or")): bo = self.get_first_expr("x %s a" % (op,)) assert isinstance(bo, ast.BoolOp) - assert bo.op is ast_type + assert bo.op == ast_type assert len(bo.values) == 2 assert isinstance(bo.values[0], ast.Name) assert isinstance(bo.values[1], ast.Name) bo = self.get_first_expr("x %s a %s b" % (op, op)) - assert bo.op is ast_type + assert bo.op == ast_type assert len(bo.values) == 3 def test_not(self): n = self.get_first_expr("not x") assert isinstance(n, ast.UnaryOp) - assert n.op is ast.Not + assert n.op == ast.Not assert isinstance(n.operand, ast.Name) - assert n.operand.ctx is ast.Load + assert n.operand.ctx == ast.Load def test_comparison(self): compares = ( @@ -792,12 +792,12 @@ comp = self.get_first_expr("x %s y" % (op,)) assert isinstance(comp, ast.Compare) assert isinstance(comp.left, ast.Name) - assert comp.left.ctx is ast.Load + assert comp.left.ctx == ast.Load assert len(comp.ops) == 1 - assert comp.ops[0] is ast_type + assert comp.ops[0] == ast_type assert len(comp.comparators) == 1 assert isinstance(comp.comparators[0], ast.Name) - assert comp.comparators[0].ctx is ast.Load + assert comp.comparators[0].ctx == ast.Load # Just for fun let's randomly combine operators. :) for j in range(10): vars = string.ascii_letters[:random.randint(3, 7)] @@ -827,14 +827,14 @@ for op, ast_type in binops: bin = self.get_first_expr("a %s b" % (op,)) assert isinstance(bin, ast.BinOp) - assert bin.op is ast_type + assert bin.op == ast_type assert isinstance(bin.left, ast.Name) assert isinstance(bin.right, ast.Name) - assert bin.left.ctx is ast.Load - assert bin.right.ctx is ast.Load + assert bin.left.ctx == ast.Load + assert bin.right.ctx == ast.Load bin = self.get_first_expr("a %s b %s c" % (op, op)) assert isinstance(bin.left, ast.BinOp) - assert bin.left.op is ast_type + assert bin.left.op == ast_type assert isinstance(bin.right, ast.Name) def test_yield(self): @@ -856,16 +856,16 @@ for op, ast_type in unary_ops: unary = self.get_first_expr("%sx" % (op,)) assert isinstance(unary, ast.UnaryOp) - assert unary.op is ast_type + assert unary.op == ast_type assert isinstance(unary.operand, ast.Name) - assert unary.operand.ctx is ast.Load + assert unary.operand.ctx == ast.Load def test_power(self): power = self.get_first_expr("x**5") assert isinstance(power, ast.BinOp) - assert power.op is ast.Pow + assert power.op == ast.Pow assert isinstance(power.left , ast.Name) - assert power.left.ctx is ast.Load + assert power.left.ctx == ast.Load assert isinstance(power.right, ast.Num) def test_call(self): @@ -876,7 +876,7 @@ assert call.starargs is None assert call.kwargs is None assert isinstance(call.func, ast.Name) - assert call.func.ctx is ast.Load + assert call.func.ctx == ast.Load call = self.get_first_expr("f(2, 3)") assert len(call.args) == 2 assert isinstance(call.args[0], ast.Num) @@ -895,10 +895,10 @@ assert call.args is None assert isinstance(call.starargs, ast.Name) assert call.starargs.id == "a" - assert call.starargs.ctx is ast.Load + assert call.starargs.ctx == ast.Load assert isinstance(call.kwargs, ast.Name) assert call.kwargs.id == "b" - assert call.kwargs.ctx is ast.Load + assert call.kwargs.ctx == ast.Load call = self.get_first_expr("f(a, b, x=4, *m, **f)") assert len(call.args) == 2 assert isinstance(call.args[0], ast.Name) @@ -927,23 +927,23 @@ attr = self.get_first_expr("x.y") assert isinstance(attr, ast.Attribute) assert isinstance(attr.value, ast.Name) - assert attr.value.ctx is ast.Load + assert attr.value.ctx == ast.Load assert attr.attr == "y" - assert attr.ctx is ast.Load + assert attr.ctx == ast.Load assign = self.get_first_stmt("x.y = 54") assert isinstance(assign, ast.Assign) assert len(assign.targets) == 1 attr = assign.targets[0] assert isinstance(attr, ast.Attribute) - assert attr.value.ctx is ast.Load - assert attr.ctx is ast.Store + assert attr.value.ctx == ast.Load + assert attr.ctx == ast.Store def test_subscript_and_slices(self): sub = self.get_first_expr("x[y]") assert isinstance(sub, ast.Subscript) assert isinstance(sub.value, ast.Name) - assert sub.value.ctx is ast.Load - assert sub.ctx is ast.Load + assert sub.value.ctx == ast.Load + assert sub.ctx == ast.Load assert isinstance(sub.slice, ast.Index) assert isinstance(sub.slice.value, ast.Name) for input in (":", "::"): @@ -988,7 +988,7 @@ assert isinstance(slc, ast.Index) assert isinstance(slc.value, ast.Tuple) assert len(slc.value.elts) == 3 - assert slc.value.ctx is ast.Load + assert slc.value.ctx == ast.Load slc = self.get_first_expr("x[1,3:4]").slice assert isinstance(slc, ast.ExtSlice) assert len(slc.dims) == 2 @@ -1051,14 +1051,14 @@ gen = self.get_first_expr(brack("x for x in y")) assert isinstance(gen, ast_type) assert isinstance(gen.elt, ast.Name) - assert gen.elt.ctx is ast.Load + assert gen.elt.ctx == ast.Load assert len(gen.generators) == 1 comp = gen.generators[0] assert isinstance(comp, ast.comprehension) assert comp.ifs is None assert isinstance(comp.target, ast.Name) assert isinstance(comp.iter, ast.Name) - assert comp.target.ctx is ast.Store + assert comp.target.ctx == ast.Store gen = self.get_first_expr(brack("x for x in y if w")) comp = gen.generators[0] assert len(comp.ifs) == 1 @@ -1068,7 +1068,7 @@ tup = gen.generators[0].target assert isinstance(tup, ast.Tuple) assert len(tup.elts) == 1 - assert tup.ctx is ast.Store + assert tup.ctx == ast.Store gen = self.get_first_expr(brack("a for w in x for m in p if g")) gens = gen.generators assert len(gens) == 2 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Tue Jul 14 05:33:58 2009 @@ -50,19 +50,23 @@ class ASTNodeVisitor(ASDLVisitor): def visitType(self, tp): - self.emit("class %s(AST):" % (tp.name,)) self.visit(tp.value, tp.name) - self.emit("") def visitSum(self, sum, base): - self.emit("pass", 1) - self.emit("") - is_simple = self.is_simple_sum(sum) - for cons in sum.types: - self.visit(cons, base, is_simple, sum.attributes) + if self.is_simple_sum(sum): + for i, cons in enumerate(sum.types): + self.emit("%s = %i" % (cons.name, i + 1)) + self.emit("") + else: + self.emit("class %s(AST):" % (base,)) + self.emit("pass", 1) self.emit("") + for cons in sum.types: + self.visit(cons, base, sum.attributes) + self.emit("") def visitProduct(self, product, name): + self.emit("class %s(AST):" % (name,)) self.emit("") self.make_constructor(product.fields) self.emit("") @@ -79,18 +83,13 @@ self.emit("def __init__(self):", 1) self.emit("pass", 2) - def visitConstructor(self, cons, base, is_enum, extra_attributes): - if is_enum: - self.emit("class _%s(%s):" % (cons.name, base)) - self.emit("pass", 1) - self.emit("%s = _%s()" % (cons.name, cons.name)) - else: - self.emit("class %s(%s):" % (cons.name, base)) - self.emit("") - self.make_constructor(cons.fields + extra_attributes) - self.emit("") - self.emit("def walkabout(self, visitor):", 1) - self.emit("visitor.visit_%s(self)" % (cons.name,), 2) + def visitConstructor(self, cons, base, extra_attributes): + self.emit("class %s(%s):" % (cons.name, base)) + self.emit("") + self.make_constructor(cons.fields + extra_attributes) + self.emit("") + self.emit("def walkabout(self, visitor):", 1) + self.emit("visitor.visit_%s(self)" % (cons.name,), 2) def visitField(self, field): self.emit("self.%s = %s" % (field.name, field.name), 2) From fijal at codespeak.net Tue Jul 14 12:54:37 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 12:54:37 +0200 (CEST) Subject: [pypy-svn] r66198 - in pypy/trunk/pypy: tool tool/test translator/c Message-ID: <20090714105437.08286168440@codespeak.net> Author: fijal Date: Tue Jul 14 12:54:34 2009 New Revision: 66198 Removed: pypy/trunk/pypy/tool/test/test_compat.py Modified: pypy/trunk/pypy/tool/compat.py pypy/trunk/pypy/translator/c/node.py Log: Kill the BaseException hack. Reuse py.builtin instead Modified: pypy/trunk/pypy/tool/compat.py ============================================================================== --- pypy/trunk/pypy/tool/compat.py (original) +++ pypy/trunk/pypy/tool/compat.py Tue Jul 14 12:54:34 2009 @@ -8,8 +8,3 @@ except ImportError: # no _md5 module on this platform from pypy.lib.md5 import md5 - -try: - BaseException = BaseException -except NameError: - BaseException = Exception Modified: pypy/trunk/pypy/translator/c/node.py ============================================================================== --- pypy/trunk/pypy/translator/c/node.py (original) +++ pypy/trunk/pypy/translator/c/node.py Tue Jul 14 12:54:34 2009 @@ -13,7 +13,7 @@ from pypy.rlib.rarithmetic import isinf, isnan from pypy.translator.c import extfunc from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.compat import BaseException +from py.builtin import BaseException def needs_gcheader(T): if not isinstance(T, ContainerType): From fijall at gmail.com Tue Jul 14 12:47:41 2009 From: fijall at gmail.com (Maciej Fijalkowski) Date: Tue, 14 Jul 2009 04:47:41 -0600 Subject: [pypy-svn] r66184 - in pypy/trunk/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/platform translator/platform/test In-Reply-To: <20090712115337.2A24D169F48@codespeak.net> References: <20090712115337.2A24D169F48@codespeak.net> Message-ID: <693bc9ab0907140347j64849668ua7b9e81dc4cb8137@mail.gmail.com> I know I should have changed stuff, but I'm unhappy anyway. Each time you break stuff, I try to fix it or create a branch with bugs, so you can work there. Instead you just bulk revert stuff, instead of even checking which revision broke it... On Sun, Jul 12, 2009 at 5:53 AM, wrote: > Author: arigo > Date: Sun Jul 12 13:53:35 2009 > New Revision: 66184 > > Modified: > ? pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py > ? pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py > ? pypy/trunk/pypy/translator/platform/__init__.py > ? pypy/trunk/pypy/translator/platform/darwin.py > ? pypy/trunk/pypy/translator/platform/linux.py > ? pypy/trunk/pypy/translator/platform/test/test_distutils.py > ? pypy/trunk/pypy/translator/platform/test/test_maemo.py > ? pypy/trunk/pypy/translator/platform/test/test_platform.py > ? pypy/trunk/pypy/translator/platform/windows.py > Log: > Bulk-revert all changes from r66152 onwards. > Fijal did not do anything since we discussed the > issue two days ago on irc, so better have a > working base from which to re-check-in correct > changes than leaving the current broken files. > From fijall at gmail.com Tue Jul 14 13:09:24 2009 From: fijall at gmail.com (Maciej Fijalkowski) Date: Tue, 14 Jul 2009 05:09:24 -0600 Subject: [pypy-svn] r66186 - pypy/branch/pyjitpl5/pypy/jit/metainterp In-Reply-To: <20090713131405.5D8DB169E14@codespeak.net> References: <20090713131405.5D8DB169E14@codespeak.net> Message-ID: <693bc9ab0907140409x7121aff3wf96cb4ecf89c4093@mail.gmail.com> This one is related to a problem of bridges adding new fields. Since we don't have a solution to this problem, we likely need to think in more general way first (and relevant tests were happily deleted when virtualizables changed style). On Mon, Jul 13, 2009 at 7:14 AM, wrote: > Author: arigo > Date: Mon Jul 13 15:14:04 2009 > New Revision: 66186 > > Modified: > ? pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py > Log: > Revert r63474, which looks obscure and wrong. ?It contains no bug > description nor any test that would show the bug, so it's cancelled > until it can be clearly explained. > > > Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py > ============================================================================== > --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ? ? ? (original) > +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ? ? ? Mon Jul 13 15:14:04 2009 > @@ -349,7 +349,7 @@ > ? ? ? ? op_fail = op.suboperations[0] > ? ? ? ? assert op_fail.opnum == rop.FAIL > ? ? ? ? for box in op_fail.args: > - ? ? ? ? ? ?if isinstance(box, Const) or box not in self.nodes: > + ? ? ? ? ? ?if isinstance(box, Const): > ? ? ? ? ? ? ? ? continue > ? ? ? ? ? ? self.prepare_rebuild_ops(self.nodes[box], rebuild_ops, memo, box) > > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From fijal at codespeak.net Tue Jul 14 13:18:47 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 13:18:47 +0200 (CEST) Subject: [pypy-svn] r66199 - in pypy/trunk/pypy/translator/platform: . test Message-ID: <20090714111847.E0C11168411@codespeak.net> Author: fijal Date: Tue Jul 14 13:18:47 2009 New Revision: 66199 Modified: pypy/trunk/pypy/translator/platform/__init__.py pypy/trunk/pypy/translator/platform/darwin.py pypy/trunk/pypy/translator/platform/linux.py pypy/trunk/pypy/translator/platform/test/test_platform.py pypy/trunk/pypy/translator/platform/windows.py Log: Re-revert irrelevant changes that were reverted :-( svn merge -r 66184:66183 . and fix tests, by reverting correct changes Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Tue Jul 14 13:18:47 2009 @@ -52,6 +52,8 @@ name = "abstract platform" c_environ = None + so_prefixes = [''] + def __init__(self, cc): if self.__class__ is Platform: raise TypeError("You should not instantiate Platform class directly") @@ -70,7 +72,12 @@ ofiles.append(self._compile_c_file(self.cc, cfile, compile_args)) return ofiles - def execute(self, executable, args=None, env=None): + def execute(self, executable, args=None, env=None, compilation_info=None): + if env is None: + env = {} + if compilation_info is not None: + env['LD_LIBRARY_PATH'] = ':'.join( + [str(i) for i in compilation_info.library_dirs]) returncode, stdout, stderr = _run_subprocess(str(executable), args, env) return ExecutionResult(returncode, stdout, stderr) @@ -120,7 +127,7 @@ cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) - def _link_args_from_eci(self, eci): + def _link_args_from_eci(self, eci, standalone): library_dirs = self._libdirs(eci.library_dirs) libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) @@ -137,8 +144,8 @@ exe_name += '.' + self.exe_ext else: exe_name += '.' + self.so_ext - return self._link(self.cc, ofiles, self._link_args_from_eci(eci), - standalone, exe_name) + largs = self._link_args_from_eci(eci, standalone) + return self._link(self.cc, ofiles, largs, standalone, exe_name) # below are some detailed informations for platforms Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Tue Jul 14 13:18:47 2009 @@ -38,8 +38,8 @@ args.append(f) return args - def _link_args_from_eci(self, eci): - args = super(Darwin, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + args = super(Darwin, self)._link_args_from_eci(eci, standalone) frameworks = self._frameworks(eci.frameworks) include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) Modified: pypy/trunk/pypy/translator/platform/linux.py ============================================================================== --- pypy/trunk/pypy/translator/platform/linux.py (original) +++ pypy/trunk/pypy/translator/platform/linux.py Tue Jul 14 13:18:47 2009 @@ -13,6 +13,7 @@ standalone_only = [] shared_only = [] so_ext = 'so' + so_prefixes = ['lib', ''] def _args_for_shared(self, args): return ['-shared'] + args Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Tue Jul 14 13:18:47 2009 @@ -1,5 +1,5 @@ -import py, sys +import py, sys, ctypes from pypy.tool.udir import udir from pypy.translator.platform import CompilationError, Platform from pypy.translator.platform import host @@ -102,7 +102,6 @@ res = self.platform.execute(executable) assert res.out.startswith('4.0') - def test_equality(): class X(Platform): def __init__(self): Modified: pypy/trunk/pypy/translator/platform/windows.py ============================================================================== --- pypy/trunk/pypy/translator/platform/windows.py (original) +++ pypy/trunk/pypy/translator/platform/windows.py Tue Jul 14 13:18:47 2009 @@ -133,8 +133,9 @@ def _args_for_shared(self, args): return ['/dll'] + args - def _link_args_from_eci(self, eci): - args = super(MsvcPlatform, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + # Windows needs to resolve all symbols even for DLLs + args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) return args + ['/EXPORT:%s' % symbol for symbol in eci.export_symbols] def _compile_c_file(self, cc, cfile, compile_args): From fijal at codespeak.net Tue Jul 14 13:20:08 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 13:20:08 +0200 (CEST) Subject: [pypy-svn] r66200 - in pypy/trunk/pypy/rpython/lltypesystem: . test Message-ID: <20090714112008.59EA6168440@codespeak.net> Author: fijal Date: Tue Jul 14 13:20:07 2009 New Revision: 66200 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Log: Another part of re-revert of bulk revert. Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Tue Jul 14 13:20:07 2009 @@ -19,6 +19,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress +from pypy.translator.platform import platform def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -813,10 +814,23 @@ cfunc = get_on_lib(ctypes.windll.kernel32, funcname) else: cfunc = None + not_found = [] for libname in libraries: - libpath = ctypes.util.find_library(libname) - if not libpath and os.path.isabs(libname): - libpath = libname + libpath = None + ext = platform.so_ext + prefixes = platform.so_prefixes + for dir in eci.library_dirs: + if libpath: + break + for prefix in prefixes: + tryfile = os.path.join(dir, prefix + libname + '.' + ext) + if os.path.isfile(tryfile): + libpath = tryfile + break + if not libpath: + libpath = ctypes.util.find_library(libname) + if not libpath and os.path.isabs(libname): + libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') # urgh, cannot pass the flag to dllclass.LoadLibrary @@ -824,11 +838,20 @@ cfunc = get_on_lib(clib, funcname) if cfunc is not None: break + else: + not_found.append(libname) if cfunc is None: # function name not found in any of the libraries if not libraries: place = 'the standard C library (missing libraries=...?)' + elif len(not_found) == len(libraries): + if len(not_found) == 1: + raise NotImplementedError( + 'cannot find the library %r' % (not_found[0],)) + else: + raise NotImplementedError( + 'cannot find any of the libraries %r' % (not_found,)) elif len(libraries) == 1: place = 'library %r' % (libraries[0],) else: Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Tue Jul 14 13:20:07 2009 @@ -15,6 +15,7 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.rtyper import RPythonTyper +from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -994,3 +995,46 @@ v2 = ctypes2lltype(llmemory.GCREF, ctypes.c_void_p(1235)) assert v2 != v +class TestPlatform(object): + def test_lib_on_libpaths(self): + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo(export_symbols=['f']) + so = platform.compile([c_file], eci, standalone=False) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 + + def test_prefix(self): + + if sys.platform != 'linux2': + py.test.skip("Not supported") + + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths_prefix') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo() + so = platform.compile([c_file], eci, standalone=False) + sopath = py.path.local(so) + sopath.move(sopath.dirpath().join('libc_file.so')) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 + From fijal at codespeak.net Tue Jul 14 13:44:37 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 13:44:37 +0200 (CEST) Subject: [pypy-svn] r66201 - pypy/trunk/pypy/rpython/tool/test Message-ID: <20090714114437.8212A168449@codespeak.net> Author: fijal Date: Tue Jul 14 13:44:37 2009 New Revision: 66201 Modified: pypy/trunk/pypy/rpython/tool/test/test_rffi_platform.py Log: Additional test for 66200 Modified: pypy/trunk/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/trunk/pypy/rpython/tool/test/test_rffi_platform.py Tue Jul 14 13:44:37 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.platform import platform def import_ctypes(): try: @@ -234,3 +235,24 @@ a = rffi_platform.memory_alignment() print a assert a % struct.calcsize("P") == 0 + +def test_external_lib(): + # XXX this one seems to be a bit too platform-specific. Check + # how to test it on windows correctly (using so_prefix?) + # and what are alternatives to LD_LIBRARY_PATH + eci = ExternalCompilationInfo() + c_source = """ + int f(int a, int b) + { + return (a + b); + } + """ + tmpdir = udir.join('external_lib').ensure(dir=1) + c_file = tmpdir.join('libc_lib.c') + c_file.write(c_source) + l = platform.compile([c_file], eci, standalone=False) + eci = ExternalCompilationInfo( + libraries = ['c_lib'], + library_dirs = [str(tmpdir)] + ) + rffi_platform.verify_eci(eci) From arigo at codespeak.net Tue Jul 14 14:10:14 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 14:10:14 +0200 (CEST) Subject: [pypy-svn] r66202 - pypy/extradoc/talk/ep2009 Message-ID: <20090714121014.EE61116842E@codespeak.net> Author: arigo Date: Tue Jul 14 14:10:13 2009 New Revision: 66202 Modified: pypy/extradoc/talk/ep2009/abstract.txt Log: Link to the talk. Modified: pypy/extradoc/talk/ep2009/abstract.txt ============================================================================== --- pypy/extradoc/talk/ep2009/abstract.txt (original) +++ pypy/extradoc/talk/ep2009/abstract.txt Tue Jul 14 14:10:13 2009 @@ -4,7 +4,7 @@ Slides: 1. http://codespeak.net/svn/pypy/extradoc/talk/ep2009/status/pypy-status.pdf -2. not ready +2. http://codespeak.net/svn/pypy/extradoc/talk/ep2009/jit/pypy-jit.pdf Part 1 - becoming complete - 30 minutes of the projected 1 hour From arigo at codespeak.net Tue Jul 14 14:42:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 14:42:52 +0200 (CEST) Subject: [pypy-svn] r66203 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090714124252.CEAC7168440@codespeak.net> Author: arigo Date: Tue Jul 14 14:42:52 2009 New Revision: 66203 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/warmspot.py Log: Comment. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/warmspot.py Tue Jul 14 14:42:52 2009 @@ -185,6 +185,8 @@ graph.startblock = support.split_before_jit_merge_point( *find_jit_merge_point([graph])) graph.startblock.isstartblock = True + # a crash in the following checkgraph() means that you forgot + # to list some variable in greens=[] or reds=[] in JitDriver. checkgraph(graph) for v in graph.getargs(): assert isinstance(v, Variable) From arigo at codespeak.net Tue Jul 14 14:44:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 14:44:43 +0200 (CEST) Subject: [pypy-svn] r66204 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090714124443.7BC4716844B@codespeak.net> Author: arigo Date: Tue Jul 14 14:44:42 2009 New Revision: 66204 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py Log: After a guard_value, replace the Box with the corresponding Const everywhere it is stored, in addition to returning it. Allows better code to be generated for a case like the one shown in test_virtualizable, without needing hint(promote=True) all over the place. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py Tue Jul 14 14:44:42 2009 @@ -896,6 +896,7 @@ if isinstance(box, Box): promoted_box = box.constbox() self.generate_guard(pc, rop.GUARD_VALUE, box, [promoted_box]) + self.metainterp.replace_box(box, promoted_box) return promoted_box else: return box # no promotion needed, already a Const @@ -1644,6 +1645,18 @@ descr=vinfo.array_descrs[k]) assert i + 1 == len(self.virtualizable_boxes) + def replace_box(self, oldbox, newbox): + for frame in self.framestack: + boxes = frame.env + for i in range(len(boxes)): + if boxes[i] is oldbox: + boxes[i] = newbox + if self.staticdata.virtualizable_info is not None: + boxes = self.virtualizable_boxes + for i in range(len(boxes)): + if boxes[i] is oldbox: + boxes[i] = newbox + class GenerateMergePoint(Exception): def __init__(self, args, target_loop): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py Tue Jul 14 14:44:42 2009 @@ -3,6 +3,7 @@ from pypy.rpython.annlowlevel import llhelper from pypy.jit.metainterp.policy import StopAtXPolicy from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.rarithmetic import intmask from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin from pypy.rpython.lltypesystem.rvirtualizable2 import VABLERTIPTR from pypy.rpython.lltypesystem.rvirtualizable2 import VirtualizableAccessor @@ -641,6 +642,33 @@ " only sometimes, so that it's not seen during tracing") + def test_promote_index_in_virtualizable_list(self): + jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], + virtualizables = ['frame']) + class Frame(object): + _virtualizable2_ = ['stackpos', 'stack[*]'] + + def f(n): + frame = Frame() + frame.stack = [42, 0, 0] + frame.stackpos = 1 + while n > 0: + jitdriver.can_enter_jit(frame=frame, n=n) + jitdriver.jit_merge_point(frame=frame, n=n) + popped = frame.stack[frame.stackpos] + frame.stackpos -= 1 + to_push = intmask(popped * 3) + frame.stack[frame.stackpos] = to_push + frame.stackpos += 1 + n -= 1 + return frame.stack[0] + + res = self.meta_interp(f, [70], listops=True) + assert res == intmask(42 ** 70) + self.check_loops(int_add=0, + int_sub=1) # for 'n -= 1' only + + class TestOOtype(#ExplicitVirtualizableTests, ImplicitVirtualizableTests, OOJitMixin): From cfbolz at codespeak.net Tue Jul 14 14:47:49 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 14 Jul 2009 14:47:49 +0200 (CEST) Subject: [pypy-svn] r66205 - pypy/branch/pyjitpl5/pypy/jit/tl/tla Message-ID: <20090714124749.E2A8616845E@codespeak.net> Author: cfbolz Date: Tue Jul 14 14:47:48 2009 New Revision: 66205 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Log: (arigo, cfbolz): a version of tla with JIT hints Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/targettla.py Tue Jul 14 14:47:48 2009 @@ -28,6 +28,10 @@ def target(driver, args): return entry_point, None +from pypy.jit.metainterp.policy import JitPolicy + +def jitpolicy(driver): + return JitPolicy() # ____________________________________________________________ Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py Tue Jul 14 14:47:48 2009 @@ -155,3 +155,22 @@ # ____________________________________________________________ +from pypy.jit.metainterp.test.test_basic import LLJitMixin +from pypy.jit.metainterp import optimize4 + +class TestLLtype(LLJitMixin): + def test_loop(self): + code = [ + tla.CONST_INT, 1, + tla.SUB, + tla.DUP, + tla.JUMP_IF, 0, + tla.RETURN + ] + def interp_w(intvalue): + w_result = interp(code, tla.W_IntObject(intvalue)) + assert isinstance(w_result, tla.W_IntObject) + return w_result.intvalue + res = self.meta_interp(interp_w, [42], listops=True, + optimizer=optimize4) + assert res == 0 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Tue Jul 14 14:47:48 2009 @@ -1,5 +1,6 @@ -from pypy.rlib.jit import JitDriver +from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.objectmodel import UnboxedValue class W_Object: @@ -18,7 +19,7 @@ -class W_IntObject(W_Object): +class W_IntObject(W_Object, UnboxedValue): def __init__(self, intvalue): self.intvalue = intvalue @@ -55,7 +56,7 @@ return len(self.strvalue) != 0 -class OperationError: +class OperationError(Exception): pass # ____________________________________________________________ @@ -71,6 +72,9 @@ # ____________________________________________________________ +jitdriver = JitDriver(greens=['bytecode', 'pc'], + reds=['self'], + virtualizables=['self']) class Frame(object): _virtualizable2_ = ['stackpos', 'stack[*]'] @@ -81,19 +85,22 @@ self.stackpos = 0 def push(self, w_x): - self.stack[self.stackpos] = w_x - self.stackpos += 1 + stackpos = hint(self.stackpos, promote=True) + self.stack[stackpos] = w_x + self.stackpos = stackpos + 1 def pop(self): - self.stackpos -= 1 - assert self.stackpos >= 0 - return self.stack[self.stackpos] + stackpos = hint(self.stackpos, promote=True) - 1 + assert stackpos >= 0 + self.stackpos = stackpos + return self.stack[stackpos] def interp(self): bytecode = self.bytecode pc = 0 while pc < len(bytecode): + jitdriver.jit_merge_point(bytecode=bytecode, pc=pc, self=self) opcode = ord(bytecode[pc]) pc += 1 @@ -128,6 +135,7 @@ w_x = self.pop() if w_x.is_true(): pc = target + jitdriver.can_enter_jit(bytecode=bytecode, pc=pc, self=self) elif opcode == NEWSTR: char = bytecode[pc] From cfbolz at codespeak.net Tue Jul 14 14:51:15 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 14 Jul 2009 14:51:15 +0200 (CEST) Subject: [pypy-svn] r66206 - pypy/branch/pyjitpl5/pypy/jit/tl/tla Message-ID: <20090714125115.6447616845E@codespeak.net> Author: cfbolz Date: Tue Jul 14 14:51:11 2009 New Revision: 66206 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Log: kill this, I just added it without testing in the demo to David Ungar Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Tue Jul 14 14:51:11 2009 @@ -1,6 +1,5 @@ from pypy.rlib.jit import JitDriver, hint -from pypy.rlib.objectmodel import UnboxedValue class W_Object: @@ -19,7 +18,7 @@ -class W_IntObject(W_Object, UnboxedValue): +class W_IntObject(W_Object): def __init__(self, intvalue): self.intvalue = intvalue From arigo at codespeak.net Tue Jul 14 14:54:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 14:54:06 +0200 (CEST) Subject: [pypy-svn] r66207 - pypy/branch/pyjitpl5/pypy/rlib Message-ID: <20090714125406.F3DAF168479@codespeak.net> Author: arigo Date: Tue Jul 14 14:54:05 2009 New Revision: 66207 Modified: pypy/branch/pyjitpl5/pypy/rlib/jit.py Log: Don't clash on the name 'self' if used in the kwds arguments too. Modified: pypy/branch/pyjitpl5/pypy/rlib/jit.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rlib/jit.py (original) +++ pypy/branch/pyjitpl5/pypy/rlib/jit.py Tue Jul 14 14:54:05 2009 @@ -112,13 +112,13 @@ def _freeze_(self): return True - def jit_merge_point(self, **livevars): + def jit_merge_point(_self, **livevars): # special-cased by ExtRegistryEntry - assert dict.fromkeys(livevars) == self._alllivevars + assert dict.fromkeys(livevars) == _self._alllivevars - def can_enter_jit(self, **livevars): + def can_enter_jit(_self, **livevars): # special-cased by ExtRegistryEntry - assert dict.fromkeys(livevars) == self._alllivevars + assert dict.fromkeys(livevars) == _self._alllivevars def _set_param(self, name, value): # special-cased by ExtRegistryEntry From arigo at codespeak.net Tue Jul 14 14:55:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 14:55:02 +0200 (CEST) Subject: [pypy-svn] r66208 - pypy/branch/pyjitpl5/pypy/jit/tl/tla Message-ID: <20090714125502.32511168488@codespeak.net> Author: arigo Date: Tue Jul 14 14:55:01 2009 New Revision: 66208 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Log: The need for hint(promote=True) is gone now, since r66204. Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/tla.py Tue Jul 14 14:55:01 2009 @@ -1,5 +1,5 @@ -from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.jit import JitDriver class W_Object: @@ -84,12 +84,11 @@ self.stackpos = 0 def push(self, w_x): - stackpos = hint(self.stackpos, promote=True) - self.stack[stackpos] = w_x - self.stackpos = stackpos + 1 + self.stack[self.stackpos] = w_x + self.stackpos += 1 def pop(self): - stackpos = hint(self.stackpos, promote=True) - 1 + stackpos = self.stackpos - 1 assert stackpos >= 0 self.stackpos = stackpos return self.stack[stackpos] From benjamin at codespeak.net Tue Jul 14 17:03:33 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 17:03:33 +0200 (CEST) Subject: [pypy-svn] r66209 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090714150333.D15DE16855A@codespeak.net> Author: benjamin Date: Tue Jul 14 17:03:32 2009 New Revision: 66209 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: mangle names when looking them up Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Tue Jul 14 17:03:32 2009 @@ -38,7 +38,7 @@ self.nested = False def lookup(self, name): - return self.symbols.get(name, SCOPE_UNKNOWN) + return self.symbols.get(self.mangle(name), SCOPE_UNKNOWN) def note_symbol(self, identifier, role): mangled = self.mangle(identifier) From benjamin at codespeak.net Tue Jul 14 17:04:12 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 17:04:12 +0200 (CEST) Subject: [pypy-svn] r66210 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090714150412.9583C168560@codespeak.net> Author: benjamin Date: Tue Jul 14 17:04:12 2009 New Revision: 66210 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: use == Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Tue Jul 14 17:04:12 2009 @@ -401,7 +401,7 @@ self._handle_params(param.elts, False) def visit_Name(self, name): - if name.ctx is ast.Load: + if name.ctx == ast.Load: role = SYM_USED else: role = SYM_ASSIGNED From benjamin at codespeak.net Tue Jul 14 17:32:41 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 17:32:41 +0200 (CEST) Subject: [pypy-svn] r66211 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090714153241.0433E168483@codespeak.net> Author: benjamin Date: Tue Jul 14 17:32:40 2009 New Revision: 66211 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: fix handling of statements with semicolons Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 14 17:32:40 2009 @@ -62,8 +62,9 @@ if sub_stmts_count == 1: stmts.append(self.handle_stmt(stmt)) else: + stmt = stmt.children[0] for j in range(sub_stmts_count): - small_stmt = stmt.children[j] + small_stmt = stmt.children[j * 2] stmts.append(self.handle_stmt(small_stmt)) return ast.Module(stmts) elif n.type == syms.eval_input: @@ -116,13 +117,13 @@ sequence = None expr_type = expr.__class__ if expr_type is ast.Attribute: - if ctx is ast.Store: + if ctx == ast.Store: self.check_forbidden_name(expr.attr, node) expr.ctx = ctx elif expr_type is ast.Subscript: expr.ctx = ctx elif expr_type is ast.Name: - if ctx is ast.Store: + if ctx == ast.Store: self.check_forbidden_name(expr.id, node) expr.ctx = ctx elif expr_type is ast.List: @@ -161,7 +162,7 @@ else: raise AssertionError("unkown expression in set_context()") if error is not None: - if ctx is ast.Store: + if ctx == ast.Store: action = "assign to" else: action = "delete" Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 14 17:32:40 2009 @@ -46,6 +46,12 @@ assert isinstance(mod, ast.Interactive) assert len(mod.body) == 1 + mod = self.get_ast("x = 23; y = 23; b = 23") + assert isinstance(mod, ast.Module) + assert len(mod.body) == 3 + for stmt in mod.body: + assert isinstance(stmt, ast.Assign) + def test_print(self): pri = self.get_first_stmt("print x") assert isinstance(pri, ast.Print) From benjamin at codespeak.net Tue Jul 14 18:39:56 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 18:39:56 +0200 (CEST) Subject: [pypy-svn] r66212 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090714163956.ECC17169E06@codespeak.net> Author: benjamin Date: Tue Jul 14 18:39:55 2009 New Revision: 66212 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: add a temporary variable for list comps Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Tue Jul 14 18:39:55 2009 @@ -259,6 +259,7 @@ self.scopes = {} self.scope = None self.stack = [] + self.tmp_name_counter = 0 top = ModuleScope(module) self.globs = top.roles self.push_scope(top) @@ -289,6 +290,10 @@ name = ".%i" % (pos,) self.note_symbol(name, SYM_PARAM) + def new_temporary_name(self): + self.note_symbol("_[%i]" % (self.tmp_name_counter,), SYM_ASSIGNED) + self.tmp_name_counter += 1 + def note_symbol(self, identifier, role): mangled = self.scope.note_symbol(identifier, role) if role & SYM_GLOBAL: @@ -369,6 +374,10 @@ genexp.elt.walkabout(self) self.pop_scope() + def visit_ListComp(self, lc): + self.new_temporary_name() + ast.GenericASTVisitor.visit_ListComp(self, lc) + def visit_arguments(self, arguments): assert isinstance(self.scope, FunctionScope) # Annotator hint. if arguments.args: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Tue Jul 14 18:39:55 2009 @@ -288,3 +288,7 @@ for input in ("class x: return", "return"): exc = py.test.raises(SyntaxError, self.func_scope, input).value assert exc.msg == "return outside function" + + def test_listcomp_tmpname(self): + scp = self.mod_scope("[x for x in y]") + assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL From arigo at codespeak.net Tue Jul 14 18:47:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 14 Jul 2009 18:47:57 +0200 (CEST) Subject: [pypy-svn] r66213 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090714164757.231DD169E0C@codespeak.net> Author: arigo Date: Tue Jul 14 18:47:56 2009 New Revision: 66213 Added: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizebridge4.py (contents, props changed) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Log: Start writing tests for the case of optimize_bridge(). Test and fix for missing InstanceNodes on a FAIL (this is probably the correct fix for r66186). Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Tue Jul 14 18:47:56 2009 @@ -187,6 +187,8 @@ # for op in self.loop.operations: #print '| ' + op.repr() + if op.is_guard(): + self.find_nodes_guard(op) opnum = op.opnum if opnum == rop.JUMP: break @@ -245,8 +247,6 @@ for box in op.args: if isinstance(box, Box): self.getnode(box).escaped = True - if op.is_guard(): - self.find_nodes_guard(op) box = op.result if box is not None: self.nodes[box] = InstanceNode(box, escaped=True) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Tue Jul 14 18:47:56 2009 @@ -1,5 +1,4 @@ import py -import copy from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE @@ -81,10 +80,6 @@ opnum = getattr(rop, opname.upper()) return resoperation.ResOperation(opnum, args, result, descr) -def set_guard(op, args): - assert op.is_guard(), op - op.suboperations = [ResOperation('fail', args, None)] - class CheckPerfectSpecializer(PerfectSpecializer): def optimize_loop(self): @@ -134,6 +129,12 @@ ResOperation('jump', [sum2, n2], None), ] + def set_guard(op, args): + assert op.is_guard(), op + op.suboperations = [ResOperation('fail', args, None)] + + set_guard(ops[0], []) + def test_A_find_nodes(): spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) spec.find_nodes() @@ -190,6 +191,7 @@ ResOperation('escape', [n2], None), # <== escaping ResOperation('jump', [sum2, n2], None), ] + set_guard(ops[0], []) def test_B_find_nodes(): spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) @@ -245,6 +247,7 @@ ResOperation('setfield_gc', [n2, v2], None, ofs_value), ResOperation('jump', [sum2, n2], None), ] + set_guard(ops[0], []) def test_C_find_nodes(): spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) @@ -322,6 +325,7 @@ ResOperation('guard_true', [v2], None), ResOperation('jump', [sum2, n2], None), ] + set_guard(ops[0], []) set_guard(ops[-2], [sum2, n2]) def test_E_optimize_loop(): @@ -393,6 +397,7 @@ ResOperation('guard_true', [vbool3], None), ResOperation('jump', [sum2, n2, n3], None), ] + set_guard(ops[0], []) set_guard(ops[-2], [sum2, n2, n3]) set_guard(ops[-4], [sum2, n2, n3]) set_guard(ops[-6], [sum2, n2, n3]) @@ -464,6 +469,7 @@ ResOperation('guard_true', [v2], None), ResOperation('jump', [sum2, n2], None), ] + set_guard(ops[0], []) set_guard(ops[-2], [sum2, n2]) def test_G_optimize_loop(): @@ -903,25 +909,6 @@ # ____________________________________________________________ -class Q: - locals().update(A.__dict__) # :-) - ops = [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, - size_of_node), - ResOperation('setfield_gc', [n2, n1], None, ofs_next), - ResOperation('jump', [], None), - ] - -def test_Q_find_nodes(): - spec = CheckPerfectSpecializer(Loop(None, Q.ops)) - spec.find_nodes() - spec.propagate_escapes() - # 'n2' should be marked as 'escaped', so that 'n1' is too - assert spec.nodes[Q.n2].escaped - assert spec.nodes[Q.n1].escaped - -# ____________________________________________________________ - class R: locals().update(A.__dict__) # :-) inputargs = [sum] Added: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizebridge4.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizebridge4.py Tue Jul 14 18:47:56 2009 @@ -0,0 +1,48 @@ + +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, + Const, ConstAddr, TreeLoop) +from pypy.jit.metainterp.optimize4 import PerfectSpecializer +from pypy.jit.metainterp.test.test_optimize4 import A, ResOperation +from pypy.jit.metainterp.test.test_optimize4 import node_vtable, cpu + +def Bridge(operations): + loop = TreeLoop("test") + loop.inputargs = None + loop.operations = operations + return loop + +# ____________________________________________________________ + +class B: + locals().update(A.__dict__) # :-) + ops = [ + ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, + size_of_node), + ResOperation('setfield_gc', [n2, n1], None, ofs_next), + ResOperation('jump', [], None), + ] + +def test_B_find_nodes(): + spec = PerfectSpecializer(Bridge(B.ops)) + spec.find_nodes() + spec.propagate_escapes() + # 'n2' should be marked as 'escaped', so that 'n1' is too + assert spec.nodes[B.n2].escaped + assert spec.nodes[B.n1].escaped + +# ____________________________________________________________ + +class C: + locals().update(A.__dict__) # :-) + ops = [ + ResOperation('guard_value', [n1, ConstInt(1)], None), + ResOperation('jump', [], None), + ] + set_guard(ops[0], [v]) + +def test_C_optimize_loop(): + spec = PerfectSpecializer(Bridge(C.ops)) + spec.find_nodes() + spec.propagate_escapes() + spec.specnodes = [] + spec.optimize_loop() From fijal at codespeak.net Tue Jul 14 19:21:58 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 19:21:58 +0200 (CEST) Subject: [pypy-svn] r66214 - pypy/trunk/pypy/translator/c/test Message-ID: <20090714172158.4C463169E0F@codespeak.net> Author: fijal Date: Tue Jul 14 19:21:57 2009 New Revision: 66214 Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py Log: use os.environ instead of putenv/getenv Modified: pypy/trunk/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_standalone.py (original) +++ pypy/trunk/pypy/translator/c/test/test_standalone.py Tue Jul 14 19:21:57 2009 @@ -77,11 +77,11 @@ cbuilder.compile() counters_fname = udir.join("_counters_") - os.putenv('_INSTRUMENT_COUNTERS', str(counters_fname)) + os.environ['_INSTRUMENT_COUNTERS'] = str(counters_fname) try: data = cbuilder.cmdexec() finally: - os.unsetenv('_INSTRUMENT_COUNTERS') + del os.environ['_INSTRUMENT_COUNTERS'] f = counters_fname.open('rb') counters_data = f.read() From fijal at codespeak.net Tue Jul 14 19:22:19 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 19:22:19 +0200 (CEST) Subject: [pypy-svn] r66215 - pypy/trunk/pypy/translator/platform Message-ID: <20090714172219.67BB9169E16@codespeak.net> Author: fijal Date: Tue Jul 14 19:22:18 2009 New Revision: 66215 Modified: pypy/trunk/pypy/translator/platform/__init__.py Log: nicely inherit environment Modified: pypy/trunk/pypy/translator/platform/__init__.py ============================================================================== --- pypy/trunk/pypy/translator/platform/__init__.py (original) +++ pypy/trunk/pypy/translator/platform/__init__.py Tue Jul 14 19:22:18 2009 @@ -74,7 +74,7 @@ def execute(self, executable, args=None, env=None, compilation_info=None): if env is None: - env = {} + env = os.environ.copy() if compilation_info is not None: env['LD_LIBRARY_PATH'] = ':'.join( [str(i) for i in compilation_info.library_dirs]) From fijal at codespeak.net Tue Jul 14 19:43:35 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Jul 2009 19:43:35 +0200 (CEST) Subject: [pypy-svn] r66216 - pypy/trunk/pypy/translator/platform/test Message-ID: <20090714174335.193E2168488@codespeak.net> Author: fijal Date: Tue Jul 14 19:43:34 2009 New Revision: 66216 Modified: pypy/trunk/pypy/translator/platform/test/test_maemo.py pypy/trunk/pypy/translator/platform/test/test_platform.py Log: Add a precise test for env behavior. Maemo fails, I don't know how to pass env correctly for scratchbox Modified: pypy/trunk/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_maemo.py Tue Jul 14 19:43:34 2009 @@ -31,3 +31,6 @@ executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) self.check_res(res) + + def test_environment_inheritance(self): + py.test.skip("FIXME") Modified: pypy/trunk/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/trunk/pypy/translator/platform/test/test_platform.py (original) +++ pypy/trunk/pypy/translator/platform/test/test_platform.py Tue Jul 14 19:43:34 2009 @@ -1,5 +1,5 @@ -import py, sys, ctypes +import py, sys, ctypes, os from pypy.tool.udir import udir from pypy.translator.platform import CompilationError, Platform from pypy.translator.platform import host @@ -102,6 +102,19 @@ res = self.platform.execute(executable) assert res.out.startswith('4.0') + def test_environment_inheritance(self): + # make sure that environment is inherited + cmd = 'import os; print os.environ["_SOME_VARIABLE_%d"]' + res = self.platform.execute('python', ['-c', cmd % 1], + env={'_SOME_VARIABLE_1':'xyz'}) + assert 'xyz' in res.out + os.environ['_SOME_VARIABLE_2'] = 'zyz' + try: + res = self.platform.execute('python', ['-c', cmd % 2]) + assert 'zyz' in res.out + finally: + del os.environ['_SOME_VARIABLE_2'] + def test_equality(): class X(Platform): def __init__(self): From benjamin at codespeak.net Tue Jul 14 23:11:23 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 14 Jul 2009 23:11:23 +0200 (CEST) Subject: [pypy-svn] r66221 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090714211123.076D6168576@codespeak.net> Author: benjamin Date: Tue Jul 14 23:11:22 2009 New Revision: 66221 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: fix error for an exec not in a function Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Tue Jul 14 23:11:22 2009 @@ -63,7 +63,7 @@ raise SyntaxError("return outside function", ret.lineno, ret.col_offset) - def note_bare_exec(self, exc): + def note_exec(self, exc): pass def note_import_star(self, imp): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Tue Jul 14 23:11:22 2009 @@ -263,6 +263,7 @@ assert exc.msg == error + " contains a nested function with free variables" def test_exec(self): + self.mod_scope("exec 'hi'") scp = self.func_scope("def f(): exec 'hi'") assert not scp.optimized assert isinstance(scp.bare_exec, ast.Exec) From benjamin at codespeak.net Wed Jul 15 02:05:18 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 02:05:18 +0200 (CEST) Subject: [pypy-svn] r66222 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715000518.2E8C016847F@codespeak.net> Author: benjamin Date: Wed Jul 15 02:05:15 2009 New Revision: 66222 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: fix 0 long literal parsing Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 02:05:15 2009 @@ -1051,6 +1051,8 @@ raw = raw.lstrip("0xX") if not raw: raw = "0" + elif not raw[0].isdigit(): + raw = "0" + raw w_num_str = self.space.wrap(raw) w_index = None w_base = self.space.wrap(base) @@ -1072,7 +1074,7 @@ first_child_type = first_child.type if first_child_type == tokens.NAME: return ast.Name(first_child.value, ast.Load, - atom_node.lineno, atom_node.column) + first_child.lineno, first_child.column) elif first_child_type == tokens.STRING: space = self.space sub_strings_w = [parsestring.parsestr(space, self.encoding, s.value) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 02:05:15 2009 @@ -1037,6 +1037,8 @@ assert space.eq_w(get_num("32.5"), space.wrap(32.5)) assert space.eq_w(get_num("32L"), space.newlong(32)) assert space.eq_w(get_num("32l"), space.newlong(32)) + assert space.eq_w(get_num("0L"), space.newlong(0)) + assert space.eq_w(get_num("2"), space.wrap(2)) assert space.eq_w(get_num("13j"), space.wrap(13j)) assert space.eq_w(get_num("13J"), space.wrap(13J)) assert space.eq_w(get_num("053"), space.wrap(053)) From benjamin at codespeak.net Wed Jul 15 06:04:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 06:04:16 +0200 (CEST) Subject: [pypy-svn] r66223 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler astcompiler/test pyparser pyparser/test Message-ID: <20090715040416.697BA168489@codespeak.net> Author: benjamin Date: Wed Jul 15 06:04:13 2009 New Revision: 66223 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: hook the new compiler up to the interpreter also provide a nice way, CompileInfo, to pass information like filename around Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 06:04:13 2009 @@ -5,8 +5,8 @@ from pypy.interpreter.pyparser import parsestring -def ast_from_node(space, n): - return ASTBuilder(space, n).build_ast() +def ast_from_node(space, n, compile_info): + return ASTBuilder(space, n, compile_info).build_ast() augassign_operator_map = { @@ -41,13 +41,9 @@ class ASTBuilder(object): - def __init__(self, space, n): + def __init__(self, space, n, compile_info): self.space = space - if n.type == syms.encoding_decl: - self.encoding = n.value - n = n.children[0] - else: - self.encoding = None + self.compile_info = compile_info self.root_node = n def build_ast(self): @@ -103,7 +99,7 @@ raise AssertionError("non-statement node") def error(self, msg, n): - raise SyntaxError(msg, n.lineno, n.column) + raise SyntaxError(msg, n.lineno, n.column, self.compile_info.filename) def check_forbidden_name(self, name, node): if name == "None": @@ -1077,7 +1073,8 @@ first_child.lineno, first_child.column) elif first_child_type == tokens.STRING: space = self.space - sub_strings_w = [parsestring.parsestr(space, self.encoding, s.value) + encoding = self.compile_info.encoding + sub_strings_w = [parsestring.parsestr(space, encoding, s.value) for s in atom_node.children] if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 06:04:13 2009 @@ -253,7 +253,7 @@ class SymtableBuilder(ast.GenericASTVisitor): - def __init__(self, space, module): + def __init__(self, space, module, compile_info): self.space = space self.module = module self.scopes = {} @@ -263,8 +263,12 @@ top = ModuleScope(module) self.globs = top.roles self.push_scope(top) - module.walkabout(self) - top.finalize(None, {}, {}) + try: + module.walkabout(self) + top.finalize(None, {}, {}) + except SyntaxError, e: + e.filename = compile_info.filename + raise self.pop_scope() assert not self.stack Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 06:04:13 2009 @@ -16,8 +16,9 @@ cls.parser = pyparse.PythonParser(cls.space) def get_ast(self, source, p_mode="exec"): - tree = self.parser.parse_source(source, p_mode) - ast_node = ast_from_node(self.space, tree) + info = pyparse.CompileInfo("", p_mode) + tree = self.parser.parse_source(source, info) + ast_node = ast_from_node(self.space, tree, info) return ast_node def get_first_expr(self, source): @@ -1019,9 +1020,10 @@ assert space.eq_w(s.s, space.wrap("hi implicitly extra")) sentence = u"Die M?nner ?rgen sich!" source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,) - tree = self.parser.parse_source(source.encode("utf-7")) - assert tree.value == "utf-7" - s = ast_from_node(space, tree).body[0].value + info = pyparse.CompileInfo("", "exec") + tree = self.parser.parse_source(source.encode("utf-7"), info) + assert info.encoding == "utf-7" + s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap(sentence)) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Wed Jul 15 06:04:13 2009 @@ -1,4 +1,7 @@ import py +from pypy.interpreter.astcompiler import codegen, astbuilder +from pypy.interpreter.pyparser import pyparse + from pypy.interpreter.astcompiler import misc, pycodegen, opt from pypy.interpreter.pyparser.test.support import source2ast from pypy.interpreter.pyparser.test import expressions @@ -6,20 +9,11 @@ from pypy.interpreter.pyparser.error import SyntaxError, IndentationError def compile_with_astcompiler(expr, mode, space): - ast = source2ast(expr, mode, space) - misc.set_filename('', ast) - ast = opt.optimize_ast_tree(space, ast) - if mode == 'exec': - Generator = pycodegen.ModuleCodeGenerator - elif mode == 'single': - Generator = pycodegen.InteractiveCodeGenerator - elif mode == 'eval': - Generator = pycodegen.ExpressionCodeGenerator - codegen = Generator(space, ast) - rcode = codegen.getCode() - assert isinstance(rcode, PyCode) - assert rcode.co_filename == '' - return rcode + p = pyparse.PythonParser(space) + info = pyparse.CompileInfo("", mode) + cst = p.parse_source(expr, info) + ast = astbuilder.ast_from_node(space, cst, info) + return codegen.compile_ast(space, ast, info) class TestCompiler: @@ -654,7 +648,7 @@ try: self.simple_test(source, None, None) except IndentationError, e: - assert e.msg == 'expected an indented block' + assert e.msg == 'expected indented block' else: raise Exception("DID NOT RAISE") Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 06:04:13 2009 @@ -11,9 +11,10 @@ cls.parser = pyparse.PythonParser(cls.space) def mod_scope(self, source, mode="exec"): - tree = self.parser.parse_source(source) - module = astbuilder.ast_from_node(self.space, tree) - builder = symtable.SymtableBuilder(self.space, module) + info = pyparse.CompileInfo("", mode) + tree = self.parser.parse_source(source, info) + module = astbuilder.ast_from_node(self.space, tree, info) + builder = symtable.SymtableBuilder(self.space, module, info) scope = builder.find_scope(module) assert isinstance(scope, symtable.ModuleScope) return scope Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Wed Jul 15 06:04:13 2009 @@ -205,6 +205,7 @@ ######## + class PythonAstCompiler(PyCodeCompiler): """Uses the stdlib's python implementation of compiler @@ -215,44 +216,30 @@ """ def __init__(self, space, override_version=None): - from pyparser.pythonparse import make_pyparser + from pypy.interpreter.pyparser.pyparse import PythonParser PyCodeCompiler.__init__(self, space) - self.grammar_version = override_version or "2.5" - self.parser = make_pyparser(self.grammar_version) + self.parser = PythonParser(space) self.additional_rules = {} - if self.grammar_version >= '2.5': - self.futureFlags = future.futureFlags_2_5 - else: - self.futureFlags = future.futureFlags_2_4 + self.futureFlags = future.futureFlags_2_5 self.compiler_flags = self.futureFlags.allowed_flags def compile(self, source, filename, mode, flags): from pypy.interpreter.pyparser.error import SyntaxError, IndentationError - from pypy.interpreter import astcompiler - from pypy.interpreter.astcompiler.pycodegen import ModuleCodeGenerator - from pypy.interpreter.astcompiler.pycodegen import InteractiveCodeGenerator - from pypy.interpreter.astcompiler.pycodegen import ExpressionCodeGenerator - from pypy.interpreter.astcompiler.ast import Node - from pypy.interpreter.astcompiler import opt - from pyparser.astbuilder import AstBuilder from pypy.interpreter.pycode import PyCode - from pypy.interpreter.function import Function - + from pypy.interpreter.pyparser.pyparse import CompileInfo from pypy.interpreter.pyparser.future import getFutures from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError + from pypy.interpreter.astcompiler.astbuilder import ast_from_node + from pypy.interpreter.astcompiler.codegen import compile_ast -## flags |= stdlib___future__.generators.compiler_flag # always on (2.2 compat) space = self.space space.timer.start("PythonAST compile") try: - builder = AstBuilder(self.parser, self.grammar_version, space=space) - for rulename, buildfunc in self.additional_rules.iteritems(): - assert isinstance(buildfunc, Function) - builder.user_build_rules[rulename] = buildfunc flags |= getFutures(self.futureFlags, source) - self.parser.parse_source(source, mode, builder, flags) - ast_tree = builder.rule_stack[-1] - encoding = builder.source_encoding + info = CompileInfo(filename, mode, flags) + parse_tree = self.parser.parse_source(source, info) + module = ast_from_node(space, parse_tree, info) + code = compile_ast(space, module, info) except IndentationError, e: raise OperationError(space.w_IndentationError, e.wrap_info(space, filename)) @@ -262,36 +249,9 @@ except SyntaxError, e: raise OperationError(space.w_SyntaxError, e.wrap_info(space, filename)) - ast_tree = opt.optimize_ast_tree(space, ast_tree) - - if not space.is_w(self.w_compile_hook, space.w_None): - try: - w_ast_tree = space.call_function(self.w_compile_hook, - space.wrap(ast_tree), - space.wrap(encoding), - space.wrap(filename)) - ast_tree = space.interp_w(Node, w_ast_tree) - except OperationError: - self.w_compile_hook = space.w_None - raise - try: - astcompiler.misc.set_filename(filename, ast_tree) - flag_names = self.futureFlags.get_flag_names(space, flags) - if mode == 'exec': - codegenerator = ModuleCodeGenerator(space, ast_tree, flag_names) - elif mode == 'single': - codegenerator = InteractiveCodeGenerator(space, ast_tree, flag_names) - else: # mode == 'eval': - codegenerator = ExpressionCodeGenerator(space, ast_tree, flag_names) - c = codegenerator.getCode() - except SyntaxError, e: - raise OperationError(space.w_SyntaxError, - e.wrap_info(space, filename)) - except (ValueError, TypeError), e: - raise OperationError(space.w_SystemError, space.wrap(str(e))) - assert isinstance(c, PyCode) + assert isinstance(code, PyCode) space.timer.stop("PythonAST compile") - return c + return code # interface for pypy.module.recparser def get_parser(self): Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 06:04:13 2009 @@ -56,6 +56,15 @@ return pytokenizer.match_encoding_declaration(line[i:]) +class CompileInfo(object): + + def __init__(self, filename, mode="exec", flags=0): + self.filename = filename + self.mode = mode + self.encoding = None + self.flags = flags + + _targets = { 'eval' : pygram.syms.eval_input, 'single' : pygram.syms.single_input, @@ -68,7 +77,7 @@ parser.Parser.__init__(self, grammar) self.space = space - def parse_source(self, textsrc, mode="exec", flags=0): + def parse_source(self, textsrc, compile_info=None): """Parse a python source according to goal""" # Detect source encoding. enc = None @@ -93,9 +102,9 @@ raise error.SyntaxError("Unknown encoding: %s" % enc) raise - self.prepare(_targets[mode]) + self.prepare(_targets[compile_info.mode]) try: - tokens = pytokenizer.generate_tokens(textsrc, flags) + tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags) for tp, value, lineno, column, line in tokens: if self.add_token(tp, value, lineno, column, line): break @@ -108,12 +117,12 @@ else: new_err = error.SyntaxError msg = "invalid syntax" - raise new_err(msg, e.lineno, e.column, e.line) + raise new_err(msg, e.lineno, e.column, e.line, + compile_info.filename) else: tree = self.root finally: self.root = None if enc is not None: - # Wrap the tree in an encoding_decl node for the AST builder. - tree = parser.Node(pygram.syms.encoding_decl, enc, [tree], 0, 0) + compile_info.encoding = enc return tree Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 06:04:13 2009 @@ -10,27 +10,33 @@ def setup_class(self): self.parser = pyparse.PythonParser(self.space) + def parse(self, source, mode="exec", info=None): + if info is None: + info = pyparse.CompileInfo("", mode) + return self.parser.parse_source(source, info) + def test_clear_state(self): assert self.parser.root is None - tree = self.parser.parse_source("name = 32") + tree = self.parse("name = 32") assert self.parser.root is None def test_encoding(self): - tree = self.parser.parse_source("""# coding: latin-1 + info = pyparse.CompileInfo("", "exec") + tree = self.parse("""# coding: latin-1 stuff = "nothing" -""") - assert tree.type == syms.encoding_decl - assert tree.value == "iso-8859-1" +""", info=info) + assert tree.type == syms.file_input + assert info.encoding == "iso-8859-1" sentence = u"u'Die M?nner ?rgen sich!'" input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") - tree = self.parser.parse_source(input) - assert tree.value == "utf-7" + tree = self.parse(input, info=info) + assert info.encoding == "utf-7" input = "# coding: not-here" - exc = py.test.raises(SyntaxError, self.parser.parse_source, input).value + exc = py.test.raises(SyntaxError, self.parse, input).value assert exc.msg == "Unknown encoding: not-here" def test_syntax_error(self): - parse = self.parser.parse_source + parse = self.parse exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 @@ -44,11 +50,11 @@ py.test.raises(SyntaxError, parse, input) def test_is(self): - self.parser.parse_source("x is y") - self.parser.parse_source("x is not y") + self.parse("x is y") + self.parse("x is not y") def test_indentation_error(self): - parse = self.parser.parse_source + parse = self.parse input = """ def f(): pass""" @@ -65,9 +71,9 @@ assert exc.msg == "unindent does not match any outer indentation level" def test_mode(self): - assert self.parser.parse_source("x = 43*54").type == syms.file_input - tree = self.parser.parse_source("43**54", "eval") + assert self.parse("x = 43*54").type == syms.file_input + tree = self.parse("43**54", "eval") assert tree.type == syms.eval_input - py.test.raises(SyntaxError, self.parser.parse_source, "x = 54", "eval") - tree = self.parser.parse_source("x = 43", "single") + py.test.raises(SyntaxError, self.parse, "x = 54", "eval") + tree = self.parse("x = 43", "single") assert tree.type == syms.single_input From antocuni at codespeak.net Wed Jul 15 11:00:14 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 15 Jul 2009 11:00:14 +0200 (CEST) Subject: [pypy-svn] r66224 - pypy/branch/pyjitpl5/pypy/jit/backend/cli Message-ID: <20090715090014.4B825168400@codespeak.net> Author: antocuni Date: Wed Jul 15 11:00:12 2009 New Revision: 66224 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Log: special-case array of voids. This fixes test_zrpy_slist.test_array_of_voids Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Wed Jul 15 11:00:12 2009 @@ -19,6 +19,7 @@ LoopDelegate = CLR.pypy.runtime.LoopDelegate DelegateHolder = CLR.pypy.runtime.DelegateHolder InputArgs = CLR.pypy.runtime.InputArgs +ListOfVoid = CLR.pypy.runtime.ListOfVoid cVoid = ootype.nullruntimeclass @@ -597,9 +598,21 @@ descr = op.descr assert isinstance(descr, runner.TypeDescr) item_clitype = descr.get_clitype() + if item_clitype is None: + return self.emit_new_arrayofvoids(op) op.args[0].load(self) self.il.Emit(OpCodes.Newarr, item_clitype) - self.store_result(op) + self.store_result(op) + + def emit_new_arrayofvoids(self, op): + clitype = dotnet.typeof(ListOfVoid) + ctor = clitype.GetConstructor(dotnet.new_array(System.Type, 0)) + _ll_resize = clitype.GetMethod('_ll_resize') + self.il.Emit(OpCodes.Newobj, ctor) + self.il.Emit(OpCodes.Dup) + op.args[0].load(self) + self.il.Emit(OpCodes.Callvirt, _ll_resize) + self.store_result(op) def lltype_only(self, op): print 'Operation %s is lltype specific, should not get here!' % op.getopname() From arigo at codespeak.net Wed Jul 15 11:24:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 11:24:41 +0200 (CEST) Subject: [pypy-svn] r66225 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090715092441.C3BE516846E@codespeak.net> Author: arigo Date: Wed Jul 15 11:24:41 2009 New Revision: 66225 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Log: Test and fix, of the kind "how could it ever work". Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Wed Jul 15 11:24:41 2009 @@ -366,6 +366,7 @@ # cpu.execute_operations()). rebuild_ops.append(op_fail) op1 = op.clone() + op1.args = self.new_arguments(op1) op1.suboperations = rebuild_ops op.optimized = op1 return op1 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Wed Jul 15 11:24:41 2009 @@ -909,6 +909,33 @@ # ____________________________________________________________ +class Q: + locals().update(A.__dict__) # :-) + inputargs = [sum] + ops = [ + ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, + size_of_node), + ResOperation('setfield_gc', [n1, sum], None, ofs_value), + ResOperation('getfield_gc', [n1], sum2, ofs_value), + ResOperation('guard_true', [sum2], None), + ResOperation('int_sub', [sum, ConstInt(1)], v), + ResOperation('jump', [v], None), + ] + set_guard(ops[-3], [sum]) + +def test_Q_optimize_loop(): + spec = CheckPerfectSpecializer(Loop(Q.inputargs, Q.ops), cpu=cpu) + spec.find_nodes() + spec.intersect_input_and_output() + spec.optimize_loop() + equaloplists(spec.loop.operations, [ + ResOperation('guard_true', [Q.sum], None), + ResOperation('int_sub', [Q.sum, ConstInt(1)], Q.v), + ResOperation('jump', [Q.v], None), + ]) + +# ____________________________________________________________ + class R: locals().update(A.__dict__) # :-) inputargs = [sum] From arigo at codespeak.net Wed Jul 15 11:27:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 11:27:11 +0200 (CEST) Subject: [pypy-svn] r66226 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090715092711.D44C316846E@codespeak.net> Author: arigo Date: Wed Jul 15 11:27:11 2009 New Revision: 66226 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Log: Reformatting. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Wed Jul 15 11:27:11 2009 @@ -361,9 +361,8 @@ op_fail.args = newboxes # NB. we mutate op_fail in-place above. That's bad. Hopefully # it does not really matter because no-one is going to look again - # at its unoptimized version. We cannot really clone it because - # of how the rest works (e.g. it is returned by - # cpu.execute_operations()). + # at its unoptimized version. We cannot really clone it because of + # how the rest works (e.g. it is returned by cpu.execute_operations()). rebuild_ops.append(op_fail) op1 = op.clone() op1.args = self.new_arguments(op1) From arigo at codespeak.net Wed Jul 15 11:39:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 11:39:35 +0200 (CEST) Subject: [pypy-svn] r66227 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090715093935.2D14A1684B0@codespeak.net> Author: arigo Date: Wed Jul 15 11:39:34 2009 New Revision: 66227 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py Log: This test passes nowadays, when written to check the number of TreeLoops. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py Wed Jul 15 11:39:34 2009 @@ -315,7 +315,6 @@ self.check_loop_count(3) def test_indirect_call_unknown_object_2(self): - py.test.skip("XXX fix me!!!!!!! problem in optimize.py") myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'state']) def getvalue2(): return 2 @@ -347,7 +346,9 @@ res = self.meta_interp(f, [198], policy=StopAtXPolicy(State.externfn.im_func)) assert res == f(198) - self.check_loop_count(3) + # we get two TreeLoops: an initial one, and one entering from + # the interpreter + self.check_tree_loop_count(2) def test_two_behaviors(self): py.test.skip("XXX fix me!!!!!!! problem in optimize.py") From arigo at codespeak.net Wed Jul 15 11:51:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 11:51:49 +0200 (CEST) Subject: [pypy-svn] r66228 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090715095149.DE02716856A@codespeak.net> Author: arigo Date: Wed Jul 15 11:51:49 2009 New Revision: 66228 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py Log: Found a real bug when trying to finish writing this test. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_send.py Wed Jul 15 11:51:49 2009 @@ -425,8 +425,19 @@ self.check_tree_loop_count(2) def test_bug1(self): - py.test.skip("BOOM") + py.test.skip("BUG") myjitdriver = JitDriver(greens = [], reds = ['node', 'n']) + class Base: + pass + class A(Base): + def decr(self, n): + return n - 2 + class B(Base): + def __init__(self, n): + self.n = n + def decr(self, n): + assert n == self.n + return self.n - 1 def extern(n): if n <= 21: return B(n) @@ -440,8 +451,8 @@ n = node.decr(n) node = extern(n) return n - res = self.meta_interp(f, [40], policy=StopAtXPolicy(extern)) - assert res == f(40) + res = self.meta_interp(f, [60], policy=StopAtXPolicy(extern)) + assert res == f(60) def test_recursive_call_to_portal_from_blackhole(self): from pypy.rpython.annlowlevel import hlstr From fijal at codespeak.net Wed Jul 15 11:55:26 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 11:55:26 +0200 (CEST) Subject: [pypy-svn] r66229 - pypy/branch/pyjitpl5/pypy/jit/tl/spli/test Message-ID: <20090715095526.3B08516857E@codespeak.net> Author: fijal Date: Wed Jul 15 11:55:25 2009 New Revision: 66229 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Log: use default optimize, tests still pass. Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Wed Jul 15 11:55:25 2009 @@ -5,7 +5,6 @@ from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.backend.llgraph import runner from pypy.rpython.annlowlevel import llstr, hlstr -from pypy.jit.metainterp import optimize2 class TestSPLIJit(JitMixin): type_system = 'lltype' @@ -29,8 +28,7 @@ d['coderepr'] = coderepr d['space'] = space exec source.compile() in d - return self.meta_interp(d['bootstrap'], args, listops=True, - optimizer=optimize2) + return self.meta_interp(d['bootstrap'], args, listops=True) def test_basic(self): def f(): From arigo at codespeak.net Wed Jul 15 12:10:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 12:10:56 +0200 (CEST) Subject: [pypy-svn] r66230 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090715101056.3E5E6168483@codespeak.net> Author: arigo Date: Wed Jul 15 12:10:55 2009 New Revision: 66230 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode4.py Log: Get rid of the attribute 'const' which is passed inside the constructor. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py Wed Jul 15 12:10:55 2009 @@ -20,14 +20,12 @@ return self.sort_key() class InstanceNode(object): - def __init__(self, source, escaped=True, startbox=False, const=False): - if isinstance(source, Const): - assert const + def __init__(self, source, escaped=True, startbox=False): self.source = source # a Box self.escaped = escaped self.startbox = startbox self.virtual = False - self.const = const + self.const = isinstance(source, Const) # forced to True by guard_value self.cls = None self.origfields = r_dict(av_eq, av_hash) self.curfields = r_dict(av_eq, av_hash) @@ -145,7 +143,7 @@ return self.nodes[box] except KeyError: if isinstance(box, Const): - node = InstanceNode(box, escaped=True, const=True) + node = InstanceNode(box, escaped=True) else: assert self._allow_automatic_node_creation node = InstanceNode(box, escaped=True, startbox=True) @@ -195,7 +193,7 @@ elif opnum == rop.NEW_WITH_VTABLE: box = op.result instnode = InstanceNode(box, escaped=False) - instnode.cls = InstanceNode(op.args[0], const=True) + instnode.cls = InstanceNode(op.args[0]) self.nodes[box] = instnode continue elif opnum == rop.SETFIELD_GC: @@ -220,15 +218,7 @@ elif opnum == rop.GUARD_CLASS: instnode = self.getnode(op.args[0]) if instnode.cls is None: - instnode.cls = InstanceNode(op.args[1], const=True) - continue - elif opnum == rop.GUARD_VALUE: - instnode = self.getnode(op.args[0]) - assert isinstance(op.args[1], Const) - # XXX need to think more about the 'const' attribute - # (see test_send.test_indirect_call_unknown_object_1) - #self.nodes[instnode.source] = InstanceNode(op.args[1], - # const=True) + instnode.cls = InstanceNode(op.args[1]) continue elif op.is_always_pure(): is_pure = True @@ -239,8 +229,7 @@ box = op.result assert box is not None self.nodes[box] = InstanceNode(box.constbox(), - escaped=True, - const=True) + escaped=True) continue elif not op.has_no_side_effect(): # default case @@ -444,7 +433,7 @@ if instnode.cls is not None: assert op.args[1].equals(instnode.cls.source) continue - instnode.cls = InstanceNode(op.args[1], const=True) + instnode.cls = InstanceNode(op.args[1]) newoperations.append(self.optimize_guard(op)) continue elif opnum == rop.GUARD_VALUE: @@ -487,7 +476,7 @@ # or known to be non-zero. if instnode.virtual or instnode.is_nonzero(): box = op.result - instnode = InstanceNode(box.constbox(), const=True) + instnode = InstanceNode(box.constbox()) self.nodes[box] = instnode continue elif (opnum == rop.OOIS or @@ -505,7 +494,7 @@ (instnode_x.is_zero() and instnode_y.is_zero())): # box = op.result - instnode = InstanceNode(box.constbox(), const=True) + instnode = InstanceNode(box.constbox()) self.nodes[box] = instnode continue # default handling of arguments and return value @@ -519,7 +508,7 @@ # all constant arguments: constant-fold away box = op.result assert box is not None - instnode = InstanceNode(box.constbox(), const=True) + instnode = InstanceNode(box.constbox()) self.nodes[box] = instnode continue if op.can_raise(): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode4.py Wed Jul 15 12:10:55 2009 @@ -16,7 +16,7 @@ from pypy.jit.metainterp.optimize4 import InstanceNode if instnode.cls is None: - instnode.cls = InstanceNode(self.known_class, const=True) + instnode.cls = InstanceNode(self.known_class) else: assert instnode.cls.source.equals(self.known_class) From arigo at codespeak.net Wed Jul 15 12:13:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 12:13:24 +0200 (CEST) Subject: [pypy-svn] r66231 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090715101324.4D6CA168483@codespeak.net> Author: arigo Date: Wed Jul 15 12:13:23 2009 New Revision: 66231 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Log: A failing test (skipped). Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py Wed Jul 15 12:13:23 2009 @@ -957,3 +957,28 @@ spec.find_nodes() spec.intersect_input_and_output() spec.optimize_loop() + +# ____________________________________________________________ + +class S: + locals().update(A.__dict__) # :-) + n1subnode = lltype.malloc(NODE2) + n2subnode = lltype.malloc(NODE2) + n1sub = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, n1subnode)) + n2sub = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, n2subnode)) + inputargs = [n1sub] + ops = [ + ResOperation('guard_class', [n1sub, ConstAddr(node2_vtable, cpu)], + None), + ResOperation('escape', [], n2sub), + ResOperation('jump', [n2sub], None), + ] + set_guard(ops[0], [n1sub]) + +def test_S_find_nodes(): + py.test.skip("in-progress") + spec = CheckPerfectSpecializer(Loop(S.inputargs, S.ops), cpu=cpu) + spec.find_nodes() + spec.intersect_input_and_output() + spec.optimize_loop() + equaloplists(spec.loop.operations, S.ops) From fijal at codespeak.net Wed Jul 15 14:33:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 14:33:54 +0200 (CEST) Subject: [pypy-svn] r66232 - pypy/branch/pyjitpl5/pypy/jit/tl/spli/test Message-ID: <20090715123354.1218B168482@codespeak.net> Author: fijal Date: Wed Jul 15 14:33:53 2009 New Revision: 66232 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Log: showcase the bad case. Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Wed Jul 15 14:33:53 2009 @@ -51,3 +51,17 @@ return total self.interpret(f, [1, 10]) + + def test_bridge_bad_case(self): + def f(a, b): + i = 0 + while i < 100: + if i & 1: + a = a + 1 + else: + b = b + 1 + i = i + 1 + return a + b + + self.interpret(f, [1, 10]) + From fijal at codespeak.net Wed Jul 15 14:49:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 14:49:05 +0200 (CEST) Subject: [pypy-svn] r66233 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20090715124905.BDC95169DB5@codespeak.net> Author: fijal Date: Wed Jul 15 14:49:05 2009 New Revision: 66233 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: Kill RTLD_GLOBAL. It breaks tests in very unobvious way, if the symbol is looked up more than once. Some other hackery is needed, but I need a machine to reproduce the failure first :-( this is essentially a partial revert of 50475 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Jul 15 14:49:05 2009 @@ -834,7 +834,7 @@ if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') # urgh, cannot pass the flag to dllclass.LoadLibrary - clib = dllclass._dlltype(libpath, ctypes.RTLD_GLOBAL) + clib = dllclass._dlltype(libpath) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break From fijal at codespeak.net Wed Jul 15 14:58:27 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 14:58:27 +0200 (CEST) Subject: [pypy-svn] r66234 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20090715125827.B2FBC169E04@codespeak.net> Author: fijal Date: Wed Jul 15 14:58:24 2009 New Revision: 66234 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: remove wrong comment Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Jul 15 14:58:24 2009 @@ -833,7 +833,6 @@ libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') - # urgh, cannot pass the flag to dllclass.LoadLibrary clib = dllclass._dlltype(libpath) cfunc = get_on_lib(clib, funcname) if cfunc is not None: From fijal at codespeak.net Wed Jul 15 15:02:36 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 15:02:36 +0200 (CEST) Subject: [pypy-svn] r66235 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20090715130236.9A80D169E0D@codespeak.net> Author: fijal Date: Wed Jul 15 15:02:36 2009 New Revision: 66235 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: write down explanation in a comment why I removed RTLD_GLOBAL Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Jul 15 15:02:36 2009 @@ -833,6 +833,9 @@ libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') + # on ie slackware there was need for RTLD_GLOBAL here. + # this breaks a lot of things, since passing RTLD_GLOBAL + # creates symbol conflicts on C level. clib = dllclass._dlltype(libpath) cfunc = get_on_lib(clib, funcname) if cfunc is not None: From fijal at codespeak.net Wed Jul 15 15:12:49 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 15:12:49 +0200 (CEST) Subject: [pypy-svn] r66236 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20090715131249.89C08169E1C@codespeak.net> Author: fijal Date: Wed Jul 15 15:12:49 2009 New Revision: 66236 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: Use official interface Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Wed Jul 15 15:12:49 2009 @@ -836,7 +836,7 @@ # on ie slackware there was need for RTLD_GLOBAL here. # this breaks a lot of things, since passing RTLD_GLOBAL # creates symbol conflicts on C level. - clib = dllclass._dlltype(libpath) + clib = dllclass.LoadLibrary(libpath) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break From antocuni at codespeak.net Wed Jul 15 16:17:20 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 15 Jul 2009 16:17:20 +0200 (CEST) Subject: [pypy-svn] r66237 - in pypy/branch/pyjitpl5/pypy/jit: backend backend/llgraph metainterp metainterp/test Message-ID: <20090715141720.4DE7316845C@codespeak.net> Author: antocuni Date: Wed Jul 15 16:17:18 2009 New Revision: 66237 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/model.py pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: implement subclassof for ootype jit Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Wed Jul 15 16:17:18 2009 @@ -91,6 +91,7 @@ 'oois' : (('ptr', 'ptr'), 'bool'), 'ooisnot' : (('ptr', 'ptr'), 'bool'), 'instanceof' : (('ptr',), 'bool'), + 'subclassof' : (('ptr', 'ptr'), 'bool'), 'setfield_gc' : (('ptr', 'intorptr'), None), 'getfield_gc' : (('ptr',), 'intorptr'), 'getfield_gc_pure': (('ptr',), 'intorptr'), @@ -767,6 +768,9 @@ inst = ootype.cast_from_object(ootype.ROOT, obj) return ootype.instanceof(inst, typedescr.TYPE) + def op_subclassof(self, *args): + raise NotImplementedError + def _cast_exception(self, exception): return ootype.cast_from_object(ootype.Class, exception) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Wed Jul 15 16:17:18 2009 @@ -469,6 +469,14 @@ assert len(args) == 1 return typedescr.instanceof(args[0]) + def do_subclassof(self, args, descr): + assert len(args) == 2 + box1, box2 = args + cls1 = ootype.cast_from_object(ootype.Class, box1.getobj()) + cls2 = ootype.cast_from_object(ootype.Class, box2.getobj()) + res = ootype.subclassof(cls1, cls2) + return history.BoxInt(res) + def do_getfield_gc(self, args, fielddescr): assert isinstance(fielddescr, FieldDescr) return fielddescr.getfield(args[0]) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/model.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/model.py Wed Jul 15 16:17:18 2009 @@ -171,3 +171,6 @@ def do_instanceof(self, args, descr=None): raise NotImplementedError + + def do_subclassof(self, args, descr=None): + raise NotImplementedError Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py Wed Jul 15 16:17:18 2009 @@ -322,6 +322,10 @@ def opimpl_instanceof(self, box, typedescr): self.execute(rop.INSTANCEOF, [box], descr=typedescr) + @arguments("box", "box") + def opimpl_subclassof(self, box1, box2): + self.execute(rop.SUBCLASSOF, [box1, box2], descr=None) + @arguments("box") def opimpl_ooidentityhash(self, box): self.execute(rop.OOIDENTITYHASH, [box], descr=None) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py Wed Jul 15 16:17:18 2009 @@ -168,9 +168,10 @@ # ootype operations OOIDENTITYHASH = 85 INSTANCEOF = 86 - OOSEND_PURE = 87 + SUBCLASSOF = 87 + OOSEND_PURE = 88 # - _ALWAYS_PURE_LAST = 87 # ----- end of always_pure operations ----- + _ALWAYS_PURE_LAST = 88 # ----- end of always_pure operations ----- GETARRAYITEM_GC = 120 GETFIELD_GC = 121 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Wed Jul 15 16:17:18 2009 @@ -729,6 +729,7 @@ self.meta_interp(f, [40, 0]) + class TestOOtype(BasicTests, OOJitMixin): def test_oohash(self): @@ -779,6 +780,27 @@ def test_r_dict(self): py.test.skip('in-progress') + + def test_subclassof(self): + A = ootype.Instance("A", ootype.ROOT) + B = ootype.Instance("B", A) + clsA = ootype.runtimeClass(A) + clsB = ootype.runtimeClass(B) + def f(n): + if n: + obj = ootype.ooupcast(A, ootype.new(B)) + else: + obj = ootype.new(A) + cls = ootype.classof(obj) + return ootype.subclassof(cls, clsB) + + res = self.interp_operations(f, [True]) + assert res + res = self.interp_operations(f, [False]) + assert not res + + + class TestLLtype(BasicTests, LLJitMixin): def test_oops_on_nongc(self): From benjamin at codespeak.net Wed Jul 15 16:34:17 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 16:34:17 +0200 (CEST) Subject: [pypy-svn] r66238 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test Message-ID: <20090715143417.437161683EB@codespeak.net> Author: benjamin Date: Wed Jul 15 16:34:16 2009 New Revision: 66238 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: some fiddling to get identation handled properly in the tokenizer Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 16:34:16 2009 @@ -102,9 +102,14 @@ raise error.SyntaxError("Unknown encoding: %s" % enc) raise + flags = compile_info.flags + source_lines = textsrc.splitlines(True) + if textsrc and textsrc[-1] == "\n": + flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT + self.prepare(_targets[compile_info.mode]) try: - tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags) + tokens = pytokenizer.generate_tokens(source_lines, flags) for tp, value, lineno, column, line in tokens: if self.add_token(tp, value, lineno, column, line): break Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Wed Jul 15 16:34:16 2009 @@ -44,7 +44,7 @@ return None -def generate_tokens(textsrc, flags): +def generate_tokens(lines, flags): """ This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since the original function is not RPYTHON (uses yield) @@ -83,8 +83,7 @@ endDFA = automata.DFA([], []) # make the annotator happy line = '' - lines = textsrc.splitlines(True) - lines.append('') + lines.append("") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -157,9 +156,12 @@ end = pseudomatch if start == end: - # Nothing matched!!! - raise TokenError("Unknown character", line, - lnum, start, token_list) + if line[pos] == ' ': + break + else: + # Nothing matched!!! + raise TokenError("Unknown character", line, + lnum, start, token_list) pos = end token, initial = line[start:end], line[start] Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 16:34:16 2009 @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import codeop import py from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.pygram import syms, tokens @@ -15,6 +16,12 @@ info = pyparse.CompileInfo("", mode) return self.parser.parse_source(source, info) + def test_dont_imply_dedent(self): + info = pyparse.CompileInfo("", "single", + codeop.PyCF_DONT_IMPLY_DEDENT) + self.parse('if 1:\n x\n', info=info) + self.parse('x = 5 ', info=info) + def test_clear_state(self): assert self.parser.root is None tree = self.parse("name = 32") From benjamin at codespeak.net Wed Jul 15 16:44:30 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 16:44:30 +0200 (CEST) Subject: [pypy-svn] r66239 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715144430.F1FE5168469@codespeak.net> Author: benjamin Date: Wed Jul 15 16:44:30 2009 New Revision: 66239 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: actually generate multiple statements for single input with semicolons Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 16:44:30 2009 @@ -81,7 +81,7 @@ stmt = first_child.children[i] if stmt.type == tokens.NEWLINE: break - stmts.append(self.handle_stmt(first_child)) + stmts.append(self.handle_stmt(stmt)) return ast.Interactive(stmts) else: raise AssertionError("unkown root node") Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 16:44:30 2009 @@ -46,6 +46,12 @@ mod = self.get_ast("x = 23", p_mode="single") assert isinstance(mod, ast.Interactive) assert len(mod.body) == 1 + mod = self.get_ast("x = 23; y = 23; b = 23", p_mode="single") + assert isinstance(mod, ast.Interactive) + assert len(mod.body) == 3 + for stmt in mod.body: + assert isinstance(stmt, ast.Assign) + assert mod.body[-1].targets[0].id == "b" mod = self.get_ast("x = 23; y = 23; b = 23") assert isinstance(mod, ast.Module) From benjamin at codespeak.net Wed Jul 15 16:49:22 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 16:49:22 +0200 (CEST) Subject: [pypy-svn] r66240 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715144922.AE43A16846E@codespeak.net> Author: benjamin Date: Wed Jul 15 16:49:22 2009 New Revision: 66240 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: correctly handle lambdas with default arguments Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 16:49:22 2009 @@ -360,7 +360,7 @@ def visit_Lambda(self, lamb): if lamb.args.defaults: - self.visit_sequence(lamb.defaults) + self.visit_sequence(lamb.args.defaults) self.push_scope(FunctionScope(lamb, "lambda")) lamb.args.walkabout(self) lamb.body.walkabout(self) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 16:49:22 2009 @@ -190,6 +190,11 @@ assert lscp.name == "lambda" assert lscp.lookup("x") == symtable.SCOPE_LOCAL assert lscp.lookup("y") == symtable.SCOPE_GLOBAL_IMPLICIT + scp = self.mod_scope("lambda x=a: b") + self.check_unknown(scp, "x", "b") + assert scp.lookup("a") == symtable.SCOPE_GLOBAL_IMPLICIT + lscp = scp.children[0] + self.check_unknown(lscp, "a") def test_import(self): scp = self.mod_scope("import x") From arigo at codespeak.net Wed Jul 15 18:36:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 18:36:03 +0200 (CEST) Subject: [pypy-svn] r66241 - pypy/branch/pyjitpl5-optimize4 Message-ID: <20090715163603.53ACC169E03@codespeak.net> Author: arigo Date: Wed Jul 15 18:36:01 2009 New Revision: 66241 Added: pypy/branch/pyjitpl5-optimize4/ - copied from r66240, pypy/branch/pyjitpl5/ Log: A branch in which to attempt refactoring of optimize4 to take optimize3's idea of separating InstanceNodes and InstanceValues. From benjamin at codespeak.net Wed Jul 15 19:35:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 19:35:44 +0200 (CEST) Subject: [pypy-svn] r66242 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090715173544.D5C5C169E1F@codespeak.net> Author: benjamin Date: Wed Jul 15 19:35:43 2009 New Revision: 66242 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: implment a utility for syntax warnings Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Wed Jul 15 19:35:43 2009 @@ -1,5 +1,26 @@ +from pypy.interpreter import gateway from pypy.interpreter.astcompiler import ast + +app = gateway.applevel(""" +def syntax_warning(msg, fn, lineno, offset): + import warnings + try: + warnings.warn_explicit(msg, SyntaxWarning, fn, lineno) + except SyntaxWarning: + raise SyntaxError(msg, fn, lineno, offset) +""", filename=__file__) +_emit_syntax_warning = app.interphook("syntax_warning") +del app + +def syntax_warning(space, msg, fn, lineno, offset): + w_msg = space.wrap(msg) + w_filename = space.wrap(fn) + w_lineno = space.wrap(lineno) + w_offset = space.wrap(offset) + _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset) + + def flatten(tup): elts = [] for elt in tup: From benjamin at codespeak.net Wed Jul 15 19:37:22 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 19:37:22 +0200 (CEST) Subject: [pypy-svn] r66243 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090715173722.76F501684BE@codespeak.net> Author: benjamin Date: Wed Jul 15 19:37:21 2009 New Revision: 66243 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: add warnings for misplaced global statements Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 19:37:21 2009 @@ -40,11 +40,14 @@ def lookup(self, name): return self.symbols.get(self.mangle(name), SCOPE_UNKNOWN) + def lookup_role(self, name): + return self.roles.get(self.mangle(name), SYM_BLANK) + def note_symbol(self, identifier, role): mangled = self.mangle(identifier) new_role = role - if identifier in self.roles: - old_role = self.roles[identifier] + if mangled in self.roles: + old_role = self.roles[mangled] if old_role & SYM_PARAM and role & SYM_PARAM: err = "duplicate argument '%s' in function definition" % \ (identifier,) @@ -256,6 +259,7 @@ def __init__(self, space, module, compile_info): self.space = space self.module = module + self.compile_info = compile_info self.scopes = {} self.scope = None self.stack = [] @@ -301,7 +305,7 @@ def note_symbol(self, identifier, role): mangled = self.scope.note_symbol(identifier, role) if role & SYM_GLOBAL: - if identifier in self.globs: + if mangled in self.globs: role |= self.globs[mangled] self.globs[mangled] = role @@ -356,6 +360,14 @@ def visit_Global(self, glob): for name in glob.names: + old_role = self.scope.lookup_role(name) + if old_role & (SYM_USED | SYM_ASSIGNED): + if old_role & SYM_ASSIGNED: + msg = "name '%s' is assigned to before global declaration" + else: + msg = "name '%s' is used prior to global declaration" + misc.syntax_warning(self.space, msg, self.compile_info.filename, + glob.lineno, glob.col_offset) self.note_symbol(name, SYM_GLOBAL) def visit_Lambda(self, lamb): From benjamin at codespeak.net Wed Jul 15 20:25:14 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 20:25:14 +0200 (CEST) Subject: [pypy-svn] r66244 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715182514.1BFF6168518@codespeak.net> Author: benjamin Date: Wed Jul 15 20:25:14 2009 New Revision: 66244 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: fix a corner case of passing free vars through class scopes Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 20:25:14 2009 @@ -33,6 +33,7 @@ self.roles = {} self.varnames = [] self.children = [] + self.free_vars = [] self.has_free = False self.child_has_free = False self.nested = False @@ -103,6 +104,7 @@ pass elif bound and name in bound: self.symbols[name] = SCOPE_FREE + self.free_vars.append(name) free[name] = None self.has_free = True elif name in globs: @@ -152,10 +154,16 @@ except KeyError: if name in bound: self.symbols[name] = SCOPE_FREE + self.free_vars.append(name) else: - if role_here & (SYM_ASSIGNED | SYM_GLOBAL) and \ + if role_here & (SYM_BOUND | SYM_GLOBAL) and \ self._hide_bound_from_nested_scopes: - self.symbols[name] = SCOPE_FREE + # This happens when a class level attribute or method has + # the same name as a free variable passing through the class + # scope. We add the name to the class scope's list of free + # vars, so it will be passed through by the interpreter, but + # we leave the scope alone, so it can be local on its own. + self.free_vars.append(name) self._check_optimization() free.update(new_free) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 20:25:14 2009 @@ -180,6 +180,18 @@ fscp = cscp.children[0] assert fscp.lookup("x") == symtable.SCOPE_FREE self.check_unknown(fscp, "a") + scp = self.func_scope("""def f(n): + class X: + def n(): + return y + def x(): + return n""") + assert scp.lookup("n") == symtable.SCOPE_CELL + cscp = scp.children[0] + assert cscp.lookup("n") == symtable.SCOPE_LOCAL + assert "n" in cscp.free_vars + xscp = cscp.children[1] + assert xscp.lookup("n") == symtable.SCOPE_FREE def test_lambda(self): scp = self.mod_scope("lambda x: y") From benjamin at codespeak.net Wed Jul 15 20:31:49 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 20:31:49 +0200 (CEST) Subject: [pypy-svn] r66245 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715183149.44492168548@codespeak.net> Author: benjamin Date: Wed Jul 15 20:31:48 2009 New Revision: 66245 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: add a temporary name for the with statement Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 20:31:48 2009 @@ -402,6 +402,10 @@ self.new_temporary_name() ast.GenericASTVisitor.visit_ListComp(self, lc) + def visit_With(self, wih): + self.new_temporary_name() + ast.GenericASTVisitor.visit_With(self, wih) + def visit_arguments(self, arguments): assert isinstance(self.scope, FunctionScope) # Annotator hint. if arguments.args: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 20:31:48 2009 @@ -308,6 +308,12 @@ exc = py.test.raises(SyntaxError, self.func_scope, input).value assert exc.msg == "return outside function" - def test_listcomp_tmpname(self): + def test_tmpnames(self): scp = self.mod_scope("[x for x in y]") assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + scp = self.mod_scope("with x: pass") + assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + # Just in case. + scp = self.mod_scope("with [x for y in z]: pass") + assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL From benjamin at codespeak.net Wed Jul 15 20:36:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 20:36:44 +0200 (CEST) Subject: [pypy-svn] r66246 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715183644.55690169E18@codespeak.net> Author: benjamin Date: Wed Jul 15 20:36:43 2009 New Revision: 66246 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: we need a temporary variable for the ctx result, too Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 15 20:36:43 2009 @@ -404,6 +404,8 @@ def visit_With(self, wih): self.new_temporary_name() + if wih.optional_vars: + self.new_temporary_name() ast.GenericASTVisitor.visit_With(self, wih) def visit_arguments(self, arguments): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 20:36:43 2009 @@ -313,6 +313,9 @@ assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL scp = self.mod_scope("with x: pass") assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + scp = self.mod_scope("with x as y: pass") + assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL # Just in case. scp = self.mod_scope("with [x for y in z]: pass") assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL From benjamin at codespeak.net Wed Jul 15 21:15:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:15:25 +0200 (CEST) Subject: [pypy-svn] r66247 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test Message-ID: <20090715191525.DD1AC168478@codespeak.net> Author: benjamin Date: Wed Jul 15 21:15:17 2009 New Revision: 66247 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: implement future statements requiring a change in the parser (like with) Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Wed Jul 15 21:15:17 2009 @@ -20,6 +20,16 @@ self.labels = [0] self.token_ids = {} + def shared_copy(self): + new = self.__class__() + new.symbol_ids = self.symbol_ids + new.symbols_names = self.symbol_names + new.keyword_ids = self.keyword_ids + new.dfas = self.dfas + new.labels = self.labels + new.token_ids = self.token_ids + return new + def _freeze_(self): # Remove some attributes not used in parsing. del self.symbol_to_label Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py Wed Jul 15 21:15:17 2009 @@ -19,6 +19,11 @@ python_grammar = _get_python_grammar() +python_grammar_no_with_statement = python_grammar.shared_copy() +python_grammar_no_with_statement.keyword_ids = \ + python_grammar_no_with_statement.keyword_ids.copy() +del python_grammar_no_with_statement.keyword_ids["with"] +del python_grammar_no_with_statement.keyword_ids["as"] class _Tokens(object): pass Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 21:15:17 2009 @@ -2,6 +2,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.interpreter.pyparser import parser, pytokenizer, pygram, error +from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT _recode_to_utf8 = gateway.applevel(r''' @@ -103,6 +104,10 @@ raise flags = compile_info.flags + if flags & CO_FUTURE_WITH_STATEMENT: + self.grammar = pygram.python_grammar + else: + self.grammar = pygram.python_grammar_no_with_statement source_lines = textsrc.splitlines(True) if textsrc and textsrc[-1] == "\n": flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 21:15:17 2009 @@ -16,6 +16,10 @@ info = pyparse.CompileInfo("", mode) return self.parser.parse_source(source, info) + def test_with_and_as_no_future(self): + self.parse("with = 23") + self.parse("as = 2") + def test_dont_imply_dedent(self): info = pyparse.CompileInfo("", "single", codeop.PyCF_DONT_IMPLY_DEDENT) From arigo at codespeak.net Wed Jul 15 21:17:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 15 Jul 2009 21:17:51 +0200 (CEST) Subject: [pypy-svn] r66248 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090715191751.5E2B8169E30@codespeak.net> Author: arigo Date: Wed Jul 15 21:17:49 2009 New Revision: 66248 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py - copied, changed from r66241, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py - copied, changed from r66241, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py Log: Start writing the new version of optimize.py. So far, only find_nodes is implemented. Work in progress. Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (from r66241, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize4.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize4.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Wed Jul 15 21:17:49 2009 @@ -1,16 +1,7 @@ -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.history import (Box, Const, ConstInt, BoxInt, BoxPtr, - ResOperation, AbstractDescr, - Options, AbstractValue, ConstPtr, - ConstObj) -from pypy.jit.metainterp.specnode4 import (FixedClassSpecNode, - prebuiltNotSpecNode, - VirtualInstanceSpecNode) -from pypy.jit.metainterp import executor -from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import r_dict +from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.metainterp import resoperation +from pypy.jit.metainterp.history import AbstractValue def av_eq(self, other): @@ -19,559 +10,130 @@ def av_hash(self): return self.sort_key() +def av_newdict(): + return r_dict(av_eq, av_hash) + +def _findall(Class, name_prefix): + result = [] + for value, name in resoperation.opname.items(): + if hasattr(Class, name_prefix + name): + result.append((value, getattr(Class, name_prefix + name))) + return unrolling_iterable(result) + +# ____________________________________________________________ + class InstanceNode(object): - def __init__(self, source, escaped=True, startbox=False): - self.source = source # a Box + """For the first phase: InstanceNode is used to match the start and + the end of the loop, so it contains both 'origfields' that represents + the field's status at the start and 'curfields' that represents it + at the current point (== the end when the first phase is complete). + """ + origfields = None # optimization; equivalent to an empty dict + curfields = None # optimization; equivalent to an empty dict + dependencies = None + + def __init__(self, escaped, startbox=False): self.escaped = escaped self.startbox = startbox - self.virtual = False - self.const = isinstance(source, Const) # forced to True by guard_value - self.cls = None - self.origfields = r_dict(av_eq, av_hash) - self.curfields = r_dict(av_eq, av_hash) - - def is_nonzero(self): - return self.cls is not None or (self.const and self.source.get_() != 0) - - def is_zero(self): - return self.const and self.source.get_() == 0 - - def escape_if_startbox(self, memo, cpu): - if self in memo: - return - memo[self] = None - if self.startbox: - self.escaped = True - for node in self.curfields.values(): - node.escape_if_startbox(memo, cpu) - def add_to_dependency_graph(self, other, dep_graph): - dep_graph.append((self, other)) - for ofs, node in self.origfields.items(): - if ofs in other.curfields: - node.add_to_dependency_graph(other.curfields[ofs], dep_graph) - - def intersect(self, other, nodes): - if not other.cls: - return prebuiltNotSpecNode - if self.cls: - if not self.cls.source.equals(other.cls.source): - return prebuiltNotSpecNode - known_class = self.cls.source - else: - known_class = other.cls.source - if other.escaped: - if self.cls is None: - return prebuiltNotSpecNode - return FixedClassSpecNode(known_class) - else: - assert self is not other - fields = [] - d = other.curfields - lst = d.keys() - sort_descrs(lst) - for ofs in lst: - node = d[ofs] - if ofs not in self.origfields: - box = node.source.clonebox() - self.origfields[ofs] = InstanceNode(box, escaped=False) - self.origfields[ofs].cls = node.cls - nodes[box] = self.origfields[ofs] - specnode = self.origfields[ofs].intersect(node, nodes) - fields.append((ofs, specnode)) - return VirtualInstanceSpecNode(known_class, fields) + def add_escape_dependency(self, other): + assert not self.escaped + if self.dependencies is None: + self.dependencies = [] + self.dependencies.append(other) + + def mark_escaped(self): + # invariant: if escaped=True, then dependencies is None + if not self.escaped: + self.escaped = True + if self.dependencies is not None: + deps = self.dependencies + self.dependencies = None + for box in deps: + box.mark_escaped() def __repr__(self): flags = '' - if self.escaped: flags += 'e' - if self.startbox: flags += 's' - if self.const: flags += 'c' - if self.virtual: flags += 'v' - return "" % (self.source, flags) - -def optimize_loop(options, old_loops, loop, cpu=None): - if not options.specialize: # for tests only - if old_loops: - return old_loops[0] - else: - return None + if self.escaped: flags += 'e' + if self.startbox: flags += 's' + return "" % (flags,) + + +class PerfectSpecializationFinder(object): - # This does "Perfect specialization" as per doc/jitpl5.txt. - perfect_specializer = PerfectSpecializer(loop, options, cpu) - perfect_specializer.find_nodes() - perfect_specializer.intersect_input_and_output() - for old_loop in old_loops: - if perfect_specializer.match_exactly(old_loop): - return old_loop - perfect_specializer.optimize_loop() - return None - -def optimize_bridge(options, old_loops, loop, cpu=None): - if not options.specialize: # for tests only - return old_loops[0] - - perfect_specializer = PerfectSpecializer(loop, options, cpu) - perfect_specializer.find_nodes() - for old_loop in old_loops: - if perfect_specializer.match(old_loop): - # xxx slow, maybe - # XXX the next loop is a big hack. Ideally it should set cls=None - # to prevent assuming something about the cls -- but only if there - # is no code in the previous loop that checks the cls. - for node in perfect_specializer.nodes.values(): - if node.startbox: - node.cls = None - assert not node.virtual - perfect_specializer.propagate_escapes() - perfect_specializer.adapt_for_match(old_loop) - perfect_specializer.optimize_loop() - return old_loop - return None # no loop matches - -class PerfectSpecializer(object): - _allow_automatic_node_creation = False - - def __init__(self, loop, options=Options(), cpu=None): - self.loop = loop - self.options = options - self.cpu = cpu - self.nodes = {} - self.dependency_graph = [] + def __init__(self): + self.nodes = {} # Box -> InstanceNode + self.node_escaped = InstanceNode(escaped=True) + + def clear(self): + self.nodes.clear() def getnode(self, box): - try: - return self.nodes[box] - except KeyError: - if isinstance(box, Const): - node = InstanceNode(box, escaped=True) + return self.nodes.get(box, self.node_escaped) + + def find_nodes(self, loop): + self.clear() + for box in loop.inputargs: + self.nodes[box] = InstanceNode(escaped=False, startbox=True) + # + for op in loop.operations: + opnum = op.opnum + for value, func in find_nodes_ops: + if opnum == value: + func(self, op) + break else: - assert self._allow_automatic_node_creation - node = InstanceNode(box, escaped=True, startbox=True) - self.nodes[box] = node - return node - - def getsource(self, box): - if isinstance(box, Const): - return box - return self.nodes[box].source - - def find_nodes_setfield(self, instnode, ofs, fieldnode): - assert isinstance(ofs, AbstractValue) - instnode.curfields[ofs] = fieldnode - self.dependency_graph.append((instnode, fieldnode)) + self.find_nodes_default(op) - def find_nodes_getfield(self, instnode, field, box): + def find_nodes_default(self, op): + if not op.has_no_side_effect(): + # default case: mark the arguments as escaping + for box in op.args: + self.getnode(box).mark_escaped() + + def find_nodes_NEW_WITH_VTABLE(self, op): + self.nodes[op.result] = InstanceNode(escaped=False) + + def find_nodes_SETFIELD_GC(self, op): + instnode = self.getnode(op.args[0]) + if instnode.escaped: + return # nothing to be gained from tracking the field + fieldnode = self.getnode(op.args[1]) + field = op.descr + assert isinstance(field, AbstractValue) + if instnode.curfields is None: + instnode.curfields = av_newdict() + instnode.curfields[field] = fieldnode + instnode.add_escape_dependency(fieldnode) + + def find_nodes_GETFIELD_GC(self, op): + instnode = self.getnode(op.args[0]) + if instnode.escaped: + return # nothing to be gained from tracking the field + field = op.descr assert isinstance(field, AbstractValue) - if field in instnode.curfields: + if instnode.curfields is not None and field in instnode.curfields: fieldnode = instnode.curfields[field] - elif field in instnode.origfields: + elif instnode.origfields is not None and field in instnode.origfields: fieldnode = instnode.origfields[field] else: - fieldnode = InstanceNode(box, escaped=False) - if instnode.startbox: - fieldnode.startbox = True - self.dependency_graph.append((instnode, fieldnode)) + fieldnode = InstanceNode(escaped=False, startbox=instnode.startbox) + instnode.add_escape_dependency(fieldnode) + if instnode.origfields is None: + instnode.origfields = av_newdict() instnode.origfields[field] = fieldnode - self.nodes[box] = fieldnode + self.nodes[op.result] = fieldnode - def find_nodes(self): - # Steps (1) and (2) - if self.loop.inputargs is not None: - for box in self.loop.inputargs: - self.nodes[box] = InstanceNode(box, escaped=False, - startbox=True) - else: - self._allow_automatic_node_creation = True - # - for op in self.loop.operations: - #print '| ' + op.repr() - if op.is_guard(): - self.find_nodes_guard(op) - opnum = op.opnum - if opnum == rop.JUMP: - break - elif opnum == rop.NEW_WITH_VTABLE: - box = op.result - instnode = InstanceNode(box, escaped=False) - instnode.cls = InstanceNode(op.args[0]) - self.nodes[box] = instnode - continue - elif opnum == rop.SETFIELD_GC: - instnode = self.getnode(op.args[0]) - field = op.descr - self.find_nodes_setfield(instnode, field, - self.getnode(op.args[1])) - continue - elif opnum == rop.GETFIELD_GC: - instnode = self.getnode(op.args[0]) - field = op.descr - box = op.result - self.find_nodes_getfield(instnode, field, box) - continue - elif opnum == rop.GETFIELD_GC_PURE: - instnode = self.getnode(op.args[0]) - field = op.descr - if not instnode.const: - box = op.result - self.find_nodes_getfield(instnode, field, box) - continue - elif opnum == rop.GUARD_CLASS: - instnode = self.getnode(op.args[0]) - if instnode.cls is None: - instnode.cls = InstanceNode(op.args[1]) - continue - elif op.is_always_pure(): - is_pure = True - for arg in op.args: - if not self.getnode(arg).const: - is_pure = False - if is_pure: - box = op.result - assert box is not None - self.nodes[box] = InstanceNode(box.constbox(), - escaped=True) - continue - elif not op.has_no_side_effect(): - # default case - for box in op.args: - if isinstance(box, Box): - self.getnode(box).escaped = True - box = op.result - if box is not None: - self.nodes[box] = InstanceNode(box, escaped=True) - - def find_nodes_guard(self, op): - assert len(op.suboperations) == 1 - for arg in op.suboperations[0].args: - self.getnode(arg) - - def recursively_find_escaping_values(self): - end_args = self.loop.operations[-1].args - assert len(self.loop.inputargs) == len(end_args) - memo = {} - for i in range(len(end_args)): - end_box = end_args[i] - if isinstance(end_box, Box): - self.nodes[end_box].escape_if_startbox(memo, self.cpu) - for i in range(len(end_args)): - box = self.loop.inputargs[i] - other_box = end_args[i] - if isinstance(other_box, Box): - self.nodes[box].add_to_dependency_graph(self.nodes[other_box], - self.dependency_graph) - self.propagate_escapes() - - def propagate_escapes(self): - # XXX find efficient algorithm, we're too fried for that by now - done = False - while not done: - done = True - for instnode, fieldnode in self.dependency_graph: - if instnode.escaped: ## and not instnode.virtualized: - if not fieldnode.escaped: - fieldnode.escaped = True - done = False - - def intersect_input_and_output(self): - # Step (3) - self.recursively_find_escaping_values() - jump = self.loop.operations[-1] - assert jump.opnum == rop.JUMP - specnodes = [] - for i in range(len(self.loop.inputargs)): - enternode = self.nodes[self.loop.inputargs[i]] - leavenode = self.getnode(jump.args[i]) - specnodes.append(enternode.intersect(leavenode, self.nodes)) - self.specnodes = specnodes - - def expanded_version_of(self, boxlist): - newboxlist = [] - assert len(boxlist) == len(self.specnodes) - for i in range(len(boxlist)): - box = boxlist[i] - specnode = self.specnodes[i] - specnode.expand_boxlist(self.nodes[box], newboxlist) - return newboxlist - - def prepare_rebuild_ops(self, instnode, rebuild_ops, memo, box=None): - if box is None: - box = instnode.source - if not isinstance(box, Box): - return box - if box in memo: - return box - if instnode.virtual: - ld = instnode.cls.source - if self.cpu.is_oo and isinstance(ld, ConstObj): - # it's probably a ootype new - cls = ld.getobj() - typedescr = self.cpu.class_sizes[cls] # XXX this is probably not rpython - op = ResOperation(rop.NEW_WITH_VTABLE, [ld], box, - descr=typedescr) - else: - assert not self.cpu.is_oo - vtable = ld.getint() - if self.cpu.translate_support_code: - vtable_addr = self.cpu.cast_int_to_adr(vtable) - size = self.cpu.class_sizes[vtable_addr] - else: - size = self.cpu.class_sizes[vtable] - op = ResOperation(rop.NEW_WITH_VTABLE, [ld], box, - descr=size) - rebuild_ops.append(op) - memo[box] = None - for ofs, node in instnode.curfields.items(): - fieldbox = self.prepare_rebuild_ops(node, rebuild_ops, memo) - assert isinstance(ofs, AbstractDescr) - op = ResOperation(rop.SETFIELD_GC, [box, fieldbox], - None, descr=ofs) - rebuild_ops.append(op) - return box - memo[box] = None - return box - - def optimize_guard(self, op): - # Make a list of operations to run to rebuild the unoptimized objects. - rebuild_ops = [] - memo = {} - assert len(op.suboperations) == 1 - op_fail = op.suboperations[0] - assert op_fail.opnum == rop.FAIL - for box in op_fail.args: - if isinstance(box, Const): - continue - self.prepare_rebuild_ops(self.nodes[box], rebuild_ops, memo, box) - - newboxes = [] - for box in op_fail.args: - if box in self.nodes: - box = self.nodes[box].source - newboxes.append(box) - op_fail.args = newboxes - # NB. we mutate op_fail in-place above. That's bad. Hopefully - # it does not really matter because no-one is going to look again - # at its unoptimized version. We cannot really clone it because of - # how the rest works (e.g. it is returned by cpu.execute_operations()). - rebuild_ops.append(op_fail) - op1 = op.clone() - op1.args = self.new_arguments(op1) - op1.suboperations = rebuild_ops - op.optimized = op1 - return op1 - - def new_arguments(self, op): - newboxes = [] - for box in op.args: - if isinstance(box, Box): - instnode = self.nodes[box] - assert not instnode.virtual - box = instnode.source - newboxes.append(box) - return newboxes - - def optimize_getfield(self, instnode, ofs, box): - assert isinstance(ofs, AbstractValue) - if instnode.virtual: - assert ofs in instnode.curfields - return True # this means field is never actually - return False - - def optimize_setfield(self, instnode, ofs, valuenode, valuebox): - assert isinstance(ofs, AbstractValue) - if instnode.virtual: ## or instnode.virtualized: - instnode.curfields[ofs] = valuenode - return True - else: - assert not valuenode.virtual - return False - # we never perform this operation here, note - - def optimize_loop(self): - self._allow_automatic_node_creation = False - newoperations = [] - exception_might_have_happened = False - if self.loop.inputargs is not None: - # closing a loop - assert len(self.loop.inputargs) == len(self.specnodes) - for i in range(len(self.specnodes)): - box = self.loop.inputargs[i] - self.specnodes[i].mutate_nodes(self.nodes[box]) - newinputargs = self.expanded_version_of(self.loop.inputargs) - else: - # making a bridge - newinputargs = None - # - for op in self.loop.operations: - opnum = op.opnum - if opnum == rop.JUMP: - args = self.expanded_version_of(op.args) - for arg in args: - if arg in self.nodes: - assert not self.nodes[arg].virtual - op = op.clone() - op.args = args - newoperations.append(op) - break - elif opnum == rop.GUARD_NO_EXCEPTION: - if not exception_might_have_happened: - continue - exception_might_have_happened = False - newoperations.append(self.optimize_guard(op)) - continue - elif opnum == rop.GUARD_EXCEPTION: - newoperations.append(self.optimize_guard(op)) - continue - elif (opnum == rop.GUARD_TRUE or - opnum == rop.GUARD_FALSE): - instnode = self.nodes[op.args[0]] - if instnode.const: - continue - newoperations.append(self.optimize_guard(op)) - continue - elif opnum == rop.GUARD_CLASS: - instnode = self.nodes[op.args[0]] - if instnode.cls is not None: - assert op.args[1].equals(instnode.cls.source) - continue - instnode.cls = InstanceNode(op.args[1]) - newoperations.append(self.optimize_guard(op)) - continue - elif opnum == rop.GUARD_VALUE: - instnode = self.nodes[op.args[0]] - assert isinstance(op.args[1], Const) - if instnode.const: - continue - instnode.const = True - newoperations.append(self.optimize_guard(op)) - continue - elif opnum == rop.GETFIELD_GC: - instnode = self.nodes[op.args[0]] - if self.optimize_getfield(instnode, op.descr, op.result): - continue - # otherwise we need this getfield, but it does not - # invalidate caches - elif opnum == rop.GETFIELD_GC_PURE: - instnode = self.nodes[op.args[0]] - if not instnode.const: - if self.optimize_getfield(instnode, op.descr, op.result): - continue - elif opnum == rop.NEW_WITH_VTABLE: - # self.nodes[op.result] keeps the value from Steps (1,2) - instnode = self.nodes[op.result] - instnode.curfields = r_dict(av_eq, av_hash) - if not instnode.escaped: - instnode.virtual = True - assert instnode.cls is not None - continue - elif opnum == rop.SETFIELD_GC: - instnode = self.nodes[op.args[0]] - valuenode = self.nodes[op.args[1]] - ofs = op.descr - if self.optimize_setfield(instnode, ofs, valuenode, op.args[1]): - continue - elif (opnum == rop.OOISNULL or - opnum == rop.OONONNULL): - instnode = self.getnode(op.args[0]) - # we know the result is constant if instnode is a virtual, - # or known to be non-zero. - if instnode.virtual or instnode.is_nonzero(): - box = op.result - instnode = InstanceNode(box.constbox()) - self.nodes[box] = instnode - continue - elif (opnum == rop.OOIS or - opnum == rop.OOISNOT): - instnode_x = self.getnode(op.args[0]) - instnode_y = self.getnode(op.args[1]) - # we know the result is constant in one of these 5 cases: - if (instnode_x.virtual or # x is a virtual (even if y isn't) - instnode_y.virtual or # y is a virtual (even if x isn't) - # x != NULL and y == NULL - (instnode_x.is_nonzero() and instnode_y.is_zero()) or - # x == NULL and y != NULL - (instnode_x.is_zero() and instnode_y.is_nonzero()) or - # x == NULL and y == NULL - (instnode_x.is_zero() and instnode_y.is_zero())): - # - box = op.result - instnode = InstanceNode(box.constbox()) - self.nodes[box] = instnode - continue - # default handling of arguments and return value - op = op.clone() - op.args = self.new_arguments(op) - if op.is_always_pure(): - for box in op.args: - if isinstance(box, Box): - break - else: - # all constant arguments: constant-fold away - box = op.result - assert box is not None - instnode = InstanceNode(box.constbox()) - self.nodes[box] = instnode - continue - if op.can_raise(): - exception_might_have_happened = True - box = op.result - if box is not None: - instnode = InstanceNode(box) - self.nodes[box] = instnode - newoperations.append(op) - # - self.loop.specnodes = self.specnodes - self.loop.inputargs = newinputargs - self.loop.operations = newoperations - - def match_exactly(self, old_loop): - assert len(old_loop.specnodes) == len(self.specnodes) - for i in range(len(self.specnodes)): - old_specnode = old_loop.specnodes[i] - new_specnode = self.specnodes[i] - if not old_specnode.equals(new_specnode): - return False - return True - - def match(self, old_loop): - jump_op = self.loop.operations[-1] - assert jump_op.opnum == rop.JUMP - assert len(old_loop.specnodes) == len(jump_op.args) - for i in range(len(old_loop.specnodes)): - old_specnode = old_loop.specnodes[i] - new_instnode = self.getnode(jump_op.args[i]) - if not old_specnode.matches(new_instnode): - return False - return True - - def adapt_for_match(self, old_loop): - jump_op = self.loop.operations[-1] - assert jump_op.opnum == rop.JUMP - self.specnodes = old_loop.specnodes - for i in range(len(old_loop.specnodes)): - old_specnode = old_loop.specnodes[i] - new_instnode = self.getnode(jump_op.args[i]) - old_specnode.adapt_to(new_instnode, None) - -# --------------------------------------------------------------- - -def partition(array, left, right): - last_item = array[right] - pivot = last_item.sort_key() - storeindex = left - for i in range(left, right): - if array[i].sort_key() <= pivot: - array[i], array[storeindex] = array[storeindex], array[i] - storeindex += 1 - # Move pivot to its final place - array[storeindex], array[right] = last_item, array[storeindex] - return storeindex - -def quicksort(array, left, right): - # sort array[left:right+1] (i.e. bounds included) - if right > left: - pivotnewindex = partition(array, left, right) - quicksort(array, left, pivotnewindex - 1) - quicksort(array, pivotnewindex + 1, right) + def find_nodes_GETFIELD_GC_PURE(self, op): + self.find_nodes_GETFIELD_GC(op) + + def find_nodes_GUARD_CLASS(self, op): + pass # prevent the default handling + + def find_nodes_JUMP(self, op): + pass # prevent the default handling + +find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') +perfect_specialization_finder = PerfectSpecializationFinder() -def sort_descrs(lst): - quicksort(lst, 0, len(lst)-1) +# ____________________________________________________________ Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (from r66241, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Wed Jul 15 21:17:49 2009 @@ -8,10 +8,10 @@ from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, ConstAddr, TreeLoop) -from pypy.jit.metainterp.optimize4 import PerfectSpecializer -from pypy.jit.metainterp.specnode4 import (FixedClassSpecNode, - NotSpecNode, - VirtualInstanceSpecNode) +from pypy.jit.metainterp.optimize import perfect_specialization_finder +from pypy.jit.metainterp.specnode import (FixedClassSpecNode, + NotSpecNode, + VirtualInstanceSpecNode) cpu = runner.LLtypeCPU(None) @@ -80,25 +80,6 @@ opnum = getattr(rop, opname.upper()) return resoperation.ResOperation(opnum, args, result, descr) - -class CheckPerfectSpecializer(PerfectSpecializer): - def optimize_loop(self): - PerfectSpecializer.optimize_loop(self) - check_operations(self.loop.inputargs, self.loop.operations) - -def check_operations(inputargs, operations, indent=' |'): - seen = dict.fromkeys(inputargs) - for op in operations: - print indent, op - for x in op.args: - assert x in seen or isinstance(x, Const) - assert op.descr is None or isinstance(op.descr, history.AbstractDescr) - if op.is_guard(): - check_operations(seen.keys(), op.suboperations, indent+' ') - if op.result is not None: - seen[op.result] = True - assert operations[-1].opnum in (rop.FAIL, rop.JUMP) - # ____________________________________________________________ class A: @@ -126,6 +107,7 @@ ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, size_of_node), ResOperation('setfield_gc', [n2, v2], None, ofs_value), + ResOperation('setfield_gc', [n2, n2], None, ofs_next), ResOperation('jump', [sum2, n2], None), ] @@ -136,849 +118,15 @@ set_guard(ops[0], []) def test_A_find_nodes(): - spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) - spec.find_nodes() - assert spec.nodes[A.sum] is not spec.nodes[A.sum2] - assert spec.nodes[A.n1] is not spec.nodes[A.n2] - assert spec.nodes[A.n1].cls.source.value == node_vtable_adr - assert not spec.nodes[A.n1].escaped - assert spec.nodes[A.n2].cls.source.value == node_vtable_adr - assert not spec.nodes[A.n2].escaped - - assert len(spec.nodes[A.n1].curfields) == 0 - assert spec.nodes[A.n1].origfields[A.ofs_value] is spec.nodes[A.v] - assert len(spec.nodes[A.n2].origfields) == 0 - assert spec.nodes[A.n2].curfields[A.ofs_value] is spec.nodes[A.v2] - -def test_A_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert isinstance(spec_n, VirtualInstanceSpecNode) - assert spec_n.known_class.value == node_vtable_adr - assert spec_n.fields[0][0] == A.ofs_value - assert isinstance(spec_n.fields[0][1], NotSpecNode) - -def test_A_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(A.inputargs, A.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [A.sum, A.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [A.v, ConstInt(1)], A.v2), - ResOperation('int_add', [A.sum, A.v], A.sum2), - ResOperation('jump', [A.sum2, A.v2], None), - ]) - -# ____________________________________________________________ - -class B: - locals().update(A.__dict__) # :-) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('escape', [n1], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('escape', [n2], None), # <== escaping - ResOperation('jump', [sum2, n2], None), - ] - set_guard(ops[0], []) - -def test_B_find_nodes(): - spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) - spec.find_nodes() - assert spec.nodes[B.n1].cls.source.value == node_vtable_adr - assert spec.nodes[B.n1].escaped - assert spec.nodes[B.n2].cls.source.value == node_vtable_adr - assert spec.nodes[B.n2].escaped - -def test_B_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert type(spec_n) is FixedClassSpecNode - assert spec_n.known_class.value == node_vtable_adr - -def test_B_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(B.inputargs, B.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [B.sum, B.n1] - equaloplists(spec.loop.operations, [ - # guard_class is gone - ResOperation('escape', [B.n1], None), - ResOperation('getfield_gc', [B.n1], B.v, B.ofs_value), - ResOperation('int_sub', [B.v, ConstInt(1)], B.v2), - ResOperation('int_add', [B.sum, B.v], B.sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], B.n2, - B.size_of_node), - ResOperation('setfield_gc', [B.n2, B.v2], None, B.ofs_value), - ResOperation('escape', [B.n2], None), # <== escaping - ResOperation('jump', [B.sum2, B.n2], None), - ]) - -# ____________________________________________________________ - -class C: - locals().update(A.__dict__) # :-) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('escape', [n1], None), # <== escaping - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('escape', [n1], None), # <== escaping - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('jump', [sum2, n2], None), - ] - set_guard(ops[0], []) - -def test_C_find_nodes(): - spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) - spec.find_nodes() - assert spec.nodes[C.n1].cls.source.value == node_vtable_adr - assert spec.nodes[C.n1].escaped - assert spec.nodes[C.n2].cls.source.value == node_vtable_adr - -def test_C_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert spec.nodes[C.n2].escaped - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert type(spec_n) is FixedClassSpecNode - assert spec_n.known_class.value == node_vtable_adr - -def test_C_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(C.inputargs, C.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [C.sum, C.n1] - equaloplists(spec.loop.operations, [ - # guard_class is gone - ResOperation('escape', [C.n1], None), # <== escaping - ResOperation('getfield_gc', [C.n1], C.v, C.ofs_value), - ResOperation('int_sub', [C.v, ConstInt(1)], C.v2), - ResOperation('int_add', [C.sum, C.v], C.sum2), - ResOperation('escape', [C.n1], None), # <== escaping - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], C.n2, - C.size_of_node), - ResOperation('setfield_gc', [C.n2, C.v2], None, C.ofs_value), - ResOperation('jump', [C.sum2, C.n2], None), - ]) - -# ____________________________________________________________ - -class D: - locals().update(A.__dict__) # :-) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node2_vtable, cpu)], None), - # the only difference is different vtable ^^^^^^^^^^^^ - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('jump', [sum2, n2], None), - ] - -def test_D_intersect_input_and_output(): - py.test.skip("nowadays, this compiles, just without making a virtual") - spec = CheckPerfectSpecializer(Loop(D.inputargs, D.ops)) - spec.find_nodes() - py.test.raises(CancelInefficientLoop, spec.intersect_input_and_output) - -# ____________________________________________________________ - -class E: - locals().update(A.__dict__) # :-) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('guard_true', [v2], None), - ResOperation('jump', [sum2, n2], None), - ] - set_guard(ops[0], []) - set_guard(ops[-2], [sum2, n2]) - -def test_E_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [E.sum, E.v] - equaloplists(spec.loop.operations, [ - # guard_class is gone - ResOperation('int_sub', [E.v, ConstInt(1)], E.v2), - ResOperation('int_add', [E.sum, E.v], E.sum2), - ResOperation('guard_true', [E.v2], None), - ResOperation('jump', [E.sum2, E.v2], None), - ]) - guard_op = spec.loop.operations[-2] - assert guard_op.getopname() == 'guard_true' - _, n2 = guard_op.suboperations[-1].args - equaloplists(guard_op.suboperations, [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable_adr, cpu)], n2, - E.size_of_node), - ResOperation('setfield_gc', [n2, E.v2], None, E.ofs_value), - ResOperation('fail', [E.sum2, n2], None), - ]) - -##def test_E_rebuild_after_failure(): -## spec = CheckPerfectSpecializer(Loop(E.inputargs, E.ops), cpu=cpu) -## spec.find_nodes() -## spec.intersect_input_and_output() -## spec.optimize_loop() -## guard_op = spec.loop.operations[-2] -## v_sum_b = BoxInt(13) -## v_v_b = BoxInt(14) -## history = History(cpu) -## newboxes = rebuild_boxes_from_guard_failure(guard_op, cpu, history, -## [v_sum_b, v_v_b]) -## assert len(newboxes) == 2 -## assert newboxes[0] == v_sum_b -## p = newboxes[1].getptr(lltype.Ptr(NODE)) -## assert p.value == 14 -## assert len(history.operations) == 2 -## assert ([op.getopname() for op in history.operations] == -## ['new_with_vtable', 'setfield_gc']) - -# ____________________________________________________________ - -class F: - locals().update(A.__dict__) # :-) - nextnode = lltype.malloc(NODE) - nextnode.value = 32 - n3 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, nextnode)) - vbool1 = BoxInt(1) - vbool2 = BoxInt(0) - vbool3 = BoxInt(1) - inputargs = [sum, n1, n3] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('ooisnot', [n2, n3], vbool1), - ResOperation('guard_true', [vbool1], None), - ResOperation('ooisnull', [n2], vbool2), - ResOperation('guard_false', [vbool2], None), - ResOperation('oononnull', [n3], vbool3), - ResOperation('guard_true', [vbool3], None), - ResOperation('jump', [sum2, n2, n3], None), - ] - set_guard(ops[0], []) - set_guard(ops[-2], [sum2, n2, n3]) - set_guard(ops[-4], [sum2, n2, n3]) - set_guard(ops[-6], [sum2, n2, n3]) - -def test_F_find_nodes(): - spec = CheckPerfectSpecializer(Loop(F.inputargs, F.ops)) - spec.find_nodes() - assert not spec.nodes[F.n1].escaped - assert not spec.nodes[F.n2].escaped - -def test_F_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(F.inputargs, F.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - assert spec.nodes[F.n3].escaped - spec.optimize_loop() - assert spec.loop.inputargs == [F.sum, F.v, F.n3] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [F.v, ConstInt(1)], F.v2), - ResOperation('int_add', [F.sum, F.v], F.sum2), - ResOperation('oononnull', [F.n3], F.vbool3), - ResOperation('guard_true', [F.vbool3], None), - ResOperation('jump', [F.sum2, F.v2, F.n3], None), - ]) - -class F2: - locals().update(A.__dict__) # :-) - node2 = lltype.malloc(NODE) - node3 = lltype.malloc(NODE) - node4 = lltype.malloc(NODE) - n2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node2)) - n3 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node3)) - n4 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node4)) - vbool1 = BoxInt(0) - inputargs = [n2, n3] - ops = [ - ResOperation('oois', [n2, n3], vbool1), - ResOperation('guard_true', [vbool1], None), - ResOperation('escape', [], n4), - ResOperation('jump', [n2, n4], None), - ] - set_guard(ops[-3], [n2]) - -def test_F2_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(F2.inputargs, F2.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - equaloplists(spec.loop.operations, F2.ops) - -# ____________________________________________________________ - -class G: - locals().update(A.__dict__) # :-) - v3 = BoxInt(123) - v4 = BoxInt(124) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, ConstInt(123)], None, ofs_value), - ResOperation('getfield_gc', [n2], v3, ofs_value), - ResOperation('int_add', [v3, ConstInt(1)], v4), - ResOperation('setfield_gc', [n2, v4], None, ofs_value), - ResOperation('guard_true', [v2], None), - ResOperation('jump', [sum2, n2], None), - ] - set_guard(ops[0], []) - set_guard(ops[-2], [sum2, n2]) - -def test_G_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(G.inputargs, G.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [G.sum, G.v] - equaloplists(spec.loop.operations, [ - # guard_class is gone - ResOperation('int_sub', [G.v, ConstInt(1)], G.v2), - ResOperation('int_add', [G.sum, G.v], G.sum2), - ResOperation('guard_true', [G.v2], None), - ResOperation('jump', [G.sum2, ConstInt(124)], None), - ]) - guard_op = spec.loop.operations[-2] - assert guard_op.getopname() == 'guard_true' - _, n2 = guard_op.suboperations[-1].args - equaloplists(guard_op.suboperations, [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - G.size_of_node), - ResOperation('setfield_gc', [n2, ConstInt(124)], None, G.ofs_value), - ResOperation('fail', [G.sum2, n2], None), - ]) - -# ____________________________________________________________ - -class H: - locals().update(A.__dict__) # :-) - # - containernode = lltype.malloc(NODE) - containernode.next = lltype.malloc(NODE) - containernode.next.value = 20 - n0 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, containernode)) - n1 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, containernode.next)) - nextnode = lltype.malloc(NODE) - nextnode.value = 19 - n2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, nextnode)) - v = BoxInt(containernode.next.value) - v2 = BoxInt(nextnode.value) - inputargs = [n0] - ops = [ - ResOperation('getfield_gc', [n0], n1, ofs_next), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, v2], None, ofs_value), - ResOperation('setfield_gc', [n0, n2], None, ofs_next), - ResOperation('jump', [n0], None), - ] - -def test_H_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(H.inputargs, H.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert spec.nodes[H.n0].escaped - assert spec.nodes[H.n1].escaped - assert spec.nodes[H.n2].escaped - -# ____________________________________________________________ - -class I: - locals().update(A.__dict__) # :-) - # - containernode = lltype.malloc(NODE) - containernode.next = lltype.malloc(NODE) - containernode.next.value = 20 - n0 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, containernode)) - nextnode = lltype.malloc(NODE) - nextnode.value = 19 - n2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, nextnode)) - inputargs = [n0] - ops = [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, n0], None, ofs_next), - ResOperation('jump', [n2], None), - ] - -def test_I_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(I.inputargs, I.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert spec.nodes[I.n0].escaped - assert spec.nodes[I.n2].escaped - -# ____________________________________________________________ - -class J: - locals().update(A.__dict__) # :-) - # - containernode = lltype.malloc(NODE) - containernode.next = lltype.malloc(NODE) - containernode.next.value = 20 - n0 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, containernode)) - nextnode = lltype.malloc(NODE) - nextnode.value = 19 - n2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, nextnode)) - n1 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, nextnode)) - inputargs = [n0] - ops = [ - ResOperation('getfield_gc', [n0], n1, ofs_next), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n2, n1], None, ofs_next), - ResOperation('jump', [n2], None), - ] - -def test_J_intersect_input_and_output(): - spec = CheckPerfectSpecializer(Loop(J.inputargs, J.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - assert not spec.nodes[J.n0].escaped - assert spec.nodes[J.n1].escaped - assert not spec.nodes[J.n2].escaped - -# ____________________________________________________________ - -class K0: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('getfield_gc', [n1], v3, ofs_value), - ResOperation('int_add', [sum2, v3], sum3), - ResOperation('escape', [n1], None), - ResOperation('jump', [sum3, n1], None), - ] - -def test_K0_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(K0.inputargs, K0.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - v4 = spec.loop.operations[-1].args[-1] - assert spec.loop.inputargs == [K0.sum, K0.n1, K0.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [K0.v, ConstInt(1)], K0.v2), - ResOperation('int_add', [K0.sum, K0.v], K0.sum2), - ResOperation('int_add', [K0.sum2, K0.v], K0.sum3), - ResOperation('escape', [K0.n1], None), - ResOperation('getfield_gc', [K0.n1], v4, K0.ofs_value), - ResOperation('jump', [K0.sum3, K0.n1, v4], None), - ]) - - -class K1: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('setfield_gc', [n1, sum], None, ofs_value), - ResOperation('getfield_gc', [n1], v3, ofs_value), - ResOperation('int_add', [sum2, v3], sum3), - ResOperation('escape', [n1], None), - ResOperation('jump', [sum3, n1], None), - ] - -def test_K1_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(K1.inputargs, K1.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - v4 = spec.loop.operations[-1].args[-1] - assert spec.loop.inputargs == [K1.sum, K1.n1, K1.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [K1.v, ConstInt(1)], K1.v2), - ResOperation('int_add', [K1.sum, K1.v], K1.sum2), - ResOperation('int_add', [K1.sum2, K1.sum], K1.sum3), - ResOperation('setfield_gc', [K1.n1, K1.sum], None, K1.ofs_value), - ResOperation('escape', [K1.n1], None), - ResOperation('getfield_gc', [K1.n1], v4, K1.ofs_value), - ResOperation('jump', [K1.sum3, K1.n1, v4], None), - ]) - - -class K: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('getfield_gc', [n1], v3, ofs_value), - ResOperation('int_add', [sum2, v3], sum3), - ResOperation('jump', [sum3, n1], None), - ] - -def test_K_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(K.inputargs, K.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [K.sum, K.n1, K.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [K.v, ConstInt(1)], K.v2), - ResOperation('int_add', [K.sum, K.v], K.sum2), - ResOperation('int_add', [K.sum2, K.v], K.sum3), - ResOperation('jump', [K.sum3, K.n1, K.v], None), - ]) - -# ____________________________________________________________ - -class L: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('escape', [n1], None), - ResOperation('getfield_gc', [n1], v3, ofs_value), - ResOperation('int_add', [sum2, v3], sum3), - ResOperation('jump', [sum3, n1], None), - ] - -def test_L_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(L.inputargs, L.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [L.sum, L.n1, L.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [L.v, ConstInt(1)], L.v2), - ResOperation('int_add', [L.sum, L.v], L.sum2), - ResOperation('escape', [L.n1], None), - ResOperation('getfield_gc', [L.n1], L.v3, L.ofs_value), - ResOperation('int_add', [L.sum2, L.v3], L.sum3), - ResOperation('jump', [L.sum3, L.n1, L.v3], None), - ]) - -# ____________________________________________________________ - -class M: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('escape', [n1], None), - ResOperation('jump', [sum2, n1], None), - ] - -def test_M_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(M.inputargs, M.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - v4 = spec.loop.operations[-1].args[-1] - assert spec.loop.inputargs == [M.sum, M.n1, M.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [M.v, ConstInt(1)], M.v2), - ResOperation('int_add', [M.sum, M.v], M.sum2), - ResOperation('escape', [M.n1], None), - ResOperation('getfield_gc', [M.n1], v4, M.ofs_value), - ResOperation('jump', [M.sum2, M.n1, v4], None), - ]) - -# ____________________________________________________________ - -class N: - locals().update(A.__dict__) # :-) - sum3 = BoxInt(3) - v3 = BoxInt(4) - inputargs = [sum, n1] - ops = [ - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('getfield_gc', [n1], v, ofs_value), - ResOperation('int_sub', [v, ConstInt(1)], v2), - ResOperation('int_add', [sum, v], sum2), - ResOperation('escape', [n1], None), - ResOperation('jump', [sum2, n1], None), - ] - -def test_N_optimize_loop(): - py.test.skip("Disabled") - spec = CheckPerfectSpecializer(Loop(N.inputargs, N.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - v4 = spec.loop.operations[-1].args[-1] - assert spec.loop.inputargs == [N.sum, N.n1, N.v] - equaloplists(spec.loop.operations, [ - ResOperation('int_sub', [N.v, ConstInt(1)], N.v2), - ResOperation('int_add', [N.sum, N.v], N.sum2), - ResOperation('escape', [N.n1], None), - ResOperation('getfield_gc', [N.n1], v4, N.ofs_value), - ResOperation('jump', [N.sum2, N.n1, v4], None), - ]) - -# ____________________________________________________________ - -class O1: - locals().update(A.__dict__) # :-) - inputargs = [] - ops = [ - ResOperation('escape', [], n1), - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('jump', [], None), - ] - set_guard(ops[-3], []) - set_guard(ops[-2], []) - -def test_O1_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(O1.inputargs, O1.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [] - equaloplists(spec.loop.operations, [ - ResOperation('escape', [], O1.n1), - # only the first guard_class is left - ResOperation('guard_class', [O1.n1, ConstAddr(node_vtable, cpu)], - None), - ResOperation('jump', [], None), - ]) - -# ____________________________________________________________ - -class O2: - locals().update(A.__dict__) # :-) - v1 = BoxInt(1) - inputargs = [] - ops = [ - ResOperation('escape', [], n1), - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('oononnull', [n1], v1), - ResOperation('guard_true', [v1], None), - ResOperation('jump', [], None), - ] - set_guard(ops[-4], []) - set_guard(ops[-2], []) - -def test_O2_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(O2.inputargs, O2.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [] - equaloplists(spec.loop.operations, [ - ResOperation('escape', [], O2.n1), - ResOperation('guard_class', [O2.n1, ConstAddr(node_vtable, cpu)], - None), - # the oononnull and guard_true are gone, because we know they - # return True -- as there was already a guard_class done on n1 - ResOperation('jump', [], None), - ]) - -# ____________________________________________________________ - -class O3: - locals().update(A.__dict__) # :-) - v1 = BoxInt(1) - inputargs = [] - ops = [ - ResOperation('escape', [], n1), - ResOperation('guard_class', [n1, ConstAddr(node_vtable, cpu)], None), - ResOperation('oois', [n1, ConstPtr(lltype.nullptr(llmemory.GCREF.TO))], - v1), - ResOperation('guard_false', [v1], None), - ResOperation('jump', [], None), - ] - set_guard(ops[-4], []) - set_guard(ops[-2], []) - -def test_O3_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(O3.inputargs, O3.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - assert spec.loop.inputargs == [] - equaloplists(spec.loop.operations, [ - ResOperation('escape', [], O3.n1), - ResOperation('guard_class', [O3.n1, ConstAddr(node_vtable, cpu)], - None), - # the oois and guard_false are gone, because we know they - # return False -- as there was already a guard_class done on n1 - ResOperation('jump', [], None), - ]) - -# ____________________________________________________________ - -class P: - locals().update(A.__dict__) # :-) - thirdnode = lltype.malloc(NODE) - n3 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, thirdnode)) - f = BoxInt(0) # False - inputargs = [n1, n3] - ops = [ - ResOperation('getfield_gc', [n3], v, ofs_value), - ResOperation('setfield_gc', [n1, ConstInt(1)], None, ofs_value), - ResOperation('getfield_gc', [n3], v2, ofs_value), - ResOperation('int_eq', [v, v2], f), - ResOperation('guard_false', [f], None), - ResOperation('getfield_gc', [n1], n2, ofs_next), - ResOperation('jump', [n2, n3], None), - ] - set_guard(ops[-3], []) - -def test_P_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(P.inputargs, P.ops)) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - # Optimization should not remove any operation. - # If it does, then aliasing is not correctly detected. - # It is ok to reorder just the 'getfield_gc[n1], n2' operation, - # but the three remaining getfields/setfields *must* be in that order. - equaloplists(spec.loop.operations, P.ops) - -# ____________________________________________________________ - -class Q: - locals().update(A.__dict__) # :-) - inputargs = [sum] - ops = [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, - size_of_node), - ResOperation('setfield_gc', [n1, sum], None, ofs_value), - ResOperation('getfield_gc', [n1], sum2, ofs_value), - ResOperation('guard_true', [sum2], None), - ResOperation('int_sub', [sum, ConstInt(1)], v), - ResOperation('jump', [v], None), - ] - set_guard(ops[-3], [sum]) - -def test_Q_optimize_loop(): - spec = CheckPerfectSpecializer(Loop(Q.inputargs, Q.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - equaloplists(spec.loop.operations, [ - ResOperation('guard_true', [Q.sum], None), - ResOperation('int_sub', [Q.sum, ConstInt(1)], Q.v), - ResOperation('jump', [Q.v], None), - ]) - -# ____________________________________________________________ - -class R: - locals().update(A.__dict__) # :-) - inputargs = [sum] - ops = [ - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n1, - size_of_node), - ResOperation('int_is_true', [sum], n1nz), - ResOperation('guard_true', [n1nz], None), - ResOperation('new_with_vtable', [ConstAddr(node_vtable, cpu)], n2, - size_of_node), - ResOperation('setfield_gc', [n1, n2], None, ofs_next), - ResOperation('int_sub', [sum, ConstInt(1)], sum2), - ResOperation('jump', [sum2], None), - ] - set_guard(ops[2], [n1]) - -def test_R_find_nodes(): - spec = CheckPerfectSpecializer(Loop(R.inputargs, R.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - -# ____________________________________________________________ - -class S: - locals().update(A.__dict__) # :-) - n1subnode = lltype.malloc(NODE2) - n2subnode = lltype.malloc(NODE2) - n1sub = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, n1subnode)) - n2sub = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, n2subnode)) - inputargs = [n1sub] - ops = [ - ResOperation('guard_class', [n1sub, ConstAddr(node2_vtable, cpu)], - None), - ResOperation('escape', [], n2sub), - ResOperation('jump', [n2sub], None), - ] - set_guard(ops[0], [n1sub]) - -def test_S_find_nodes(): - py.test.skip("in-progress") - spec = CheckPerfectSpecializer(Loop(S.inputargs, S.ops), cpu=cpu) - spec.find_nodes() - spec.intersect_input_and_output() - spec.optimize_loop() - equaloplists(spec.loop.operations, S.ops) + perfect_specialization_finder.find_nodes(Loop(A.inputargs, A.ops)) + nodes = perfect_specialization_finder.nodes + assert A.sum in nodes + assert A.sum2 not in nodes + assert nodes[A.n1] is not nodes[A.n2] + assert not nodes[A.n1].escaped + assert not nodes[A.n2].escaped + + assert not nodes[A.n1].curfields + assert nodes[A.n1].origfields[A.ofs_value] is nodes[A.v] + assert not nodes[A.n2].origfields + assert nodes[A.n2].curfields[A.ofs_next] is nodes[A.n2] From benjamin at codespeak.net Wed Jul 15 21:19:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:19:44 +0200 (CEST) Subject: [pypy-svn] r66249 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090715191944.396AF169E19@codespeak.net> Author: benjamin Date: Wed Jul 15 21:19:43 2009 New Revision: 66249 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py Log: 'as' must stay as a keyword Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py Wed Jul 15 21:19:43 2009 @@ -23,7 +23,6 @@ python_grammar_no_with_statement.keyword_ids = \ python_grammar_no_with_statement.keyword_ids.copy() del python_grammar_no_with_statement.keyword_ids["with"] -del python_grammar_no_with_statement.keyword_ids["as"] class _Tokens(object): pass From benjamin at codespeak.net Wed Jul 15 21:24:06 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:24:06 +0200 (CEST) Subject: [pypy-svn] r66250 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090715192406.CA238169E4C@codespeak.net> Author: benjamin Date: Wed Jul 15 21:24:05 2009 New Revision: 66250 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py Log: revert 66249; still in 2.5.... Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pygram.py Wed Jul 15 21:24:05 2009 @@ -23,6 +23,7 @@ python_grammar_no_with_statement.keyword_ids = \ python_grammar_no_with_statement.keyword_ids.copy() del python_grammar_no_with_statement.keyword_ids["with"] +del python_grammar_no_with_statement.keyword_ids["as"] class _Tokens(object): pass From benjamin at codespeak.net Wed Jul 15 21:31:23 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:31:23 +0200 (CEST) Subject: [pypy-svn] r66251 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715193123.A48671683EA@codespeak.net> Author: benjamin Date: Wed Jul 15 21:31:23 2009 New Revision: 66251 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: since 'as' is not a keyword we have to check for it Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 21:31:23 2009 @@ -226,6 +226,8 @@ if import_name_type == syms.import_as_name: name = import_name.children[0].value if len(import_name.children) == 3: + if import_name.children[1].value != "as": + self.error("must use 'as' in import", import_name) as_name = import_name.children[2].value self.check_forbidden_name(as_name, import_name.children[2]) else: @@ -236,6 +238,8 @@ if len(import_name.children) == 1: import_name = import_name.children[0] continue + if import_name.children[1].value != "as": + self.error("must use 'as' in import", import_name) alias = self.alias_for_import_name(import_name.children[0]) asname_node = import_name.children[2] alias.asname = asname_node.value Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 21:31:23 2009 @@ -165,6 +165,8 @@ assert a1.asname is None assert a2.name == "y" assert a2.asname == "w" + exc = py.test.raises(SyntaxError, self.get_ast, "import x a b").value + assert exc.msg == "must use 'as' in import" def test_from_import(self): im = self.get_first_stmt("from x import y") @@ -209,6 +211,9 @@ assert a1.asname == "b" assert a2.name == "w" assert a2.asname is None + input + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == "must use 'as' in import" input = "from x import a, b," exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == "trailing comma is only allowed with surronding " \ From benjamin at codespeak.net Wed Jul 15 21:41:05 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:41:05 +0200 (CEST) Subject: [pypy-svn] r66252 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090715194105.72123169E9F@codespeak.net> Author: benjamin Date: Wed Jul 15 21:41:04 2009 New Revision: 66252 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: add forgotten line Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 21:41:04 2009 @@ -211,7 +211,7 @@ assert a1.asname == "b" assert a2.name == "w" assert a2.asname is None - input + input = "from x import y a b" exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == "must use 'as' in import" input = "from x import a, b," From benjamin at codespeak.net Wed Jul 15 21:43:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:43:42 +0200 (CEST) Subject: [pypy-svn] r66253 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090715194342.3974B169E9F@codespeak.net> Author: benjamin Date: Wed Jul 15 21:43:41 2009 New Revision: 66253 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: run these tests with the 'with' statement enabled Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Wed Jul 15 21:43:41 2009 @@ -7,7 +7,7 @@ from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.astcompiler.astbuilder import ast_from_node -from pypy.interpreter.astcompiler import ast2 as ast +from pypy.interpreter.astcompiler import ast2 as ast, consts class TestAstBuilder: @@ -16,7 +16,8 @@ cls.parser = pyparse.PythonParser(cls.space) def get_ast(self, source, p_mode="exec"): - info = pyparse.CompileInfo("", p_mode) + info = pyparse.CompileInfo("", p_mode, + consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) ast_node = ast_from_node(self.space, tree, info) return ast_node Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 21:43:41 2009 @@ -1,6 +1,7 @@ import string import py -from pypy.interpreter.astcompiler import ast2 as ast, astbuilder, symtable +from pypy.interpreter.astcompiler import (ast2 as ast, astbuilder, symtable, + consts) from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.error import SyntaxError @@ -11,7 +12,8 @@ cls.parser = pyparse.PythonParser(cls.space) def mod_scope(self, source, mode="exec"): - info = pyparse.CompileInfo("", mode) + info = pyparse.CompileInfo("", mode, + consts.CO_FUTURE_WITH_STATEMENT) tree = self.parser.parse_source(source, info) module = astbuilder.ast_from_node(self.space, tree, info) builder = symtable.SymtableBuilder(self.space, module, info) From benjamin at codespeak.net Wed Jul 15 21:47:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 21:47:21 +0200 (CEST) Subject: [pypy-svn] r66254 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090715194721.A819E169EB9@codespeak.net> Author: benjamin Date: Wed Jul 15 21:47:21 2009 New Revision: 66254 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: fix argument order Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Wed Jul 15 21:47:21 2009 @@ -228,7 +228,7 @@ start = pos if start Author: benjamin Date: Wed Jul 15 22:26:39 2009 New Revision: 66255 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: undo this change; it was not well thought out Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Wed Jul 15 22:26:39 2009 @@ -156,12 +156,8 @@ end = pseudomatch if start == end: - if line[pos] == ' ': - break - else: - # Nothing matched!!! - raise TokenError("Unknown character", line, - lnum, start, token_list) + raise TokenError("Unknown character", line, + lnum, start + 1, token_list) pos = end token, initial = line[start:end], line[start] From fijal at codespeak.net Wed Jul 15 22:27:03 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 15 Jul 2009 22:27:03 +0200 (CEST) Subject: [pypy-svn] r66256 - in pypy/branch/pyjitpl5/pypy/module/micronumpy: . test Message-ID: <20090715202703.BFC49169E70@codespeak.net> Author: fijal Date: Wed Jul 15 22:27:03 2009 New Revision: 66256 Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/ (props changed) pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py (contents, props changed) pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py (contents, props changed) pypy/branch/pyjitpl5/pypy/module/micronumpy/test/ (props changed) pypy/branch/pyjitpl5/pypy/module/micronumpy/test/__init__.py (contents, props changed) pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py (contents, props changed) Log: A start of 'micronumpy' module. This module is supposed to provide enough interface of numpy to show interesting properties of the jit. Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py Wed Jul 15 22:27:03 2009 @@ -0,0 +1,12 @@ + +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + + applevel_name = 'numpy' + + interpleveldefs = { + 'zeros' : 'numarray.zeros', + } + + appleveldefs = {} Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Wed Jul 15 22:27:03 2009 @@ -0,0 +1,39 @@ + +from pypy.interpreter.baseobjspace import ObjSpace, W_Root, Wrappable +from pypy.interpreter.error import OperationError +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, NoneNotWrapped + +class NumArray(Wrappable): + def __init__(self, space, dim, dtype): + self.dim = dim + # ignore dtype for now + assert len(dim) == 1 + self.storage = [0] * dim[0] + + def descr_getitem(self, space, index): + return space.wrap(self.storage[index]) + descr_getitem.unwrap_spec = ['self', ObjSpace, int] + +NumArray.typedef = TypeDef( + 'NumArray', + __getitem__ = interp2app(NumArray.descr_getitem), +) + +def unpack_dim(space, w_dim): + if space.is_true(space.isinstance(w_dim, space.w_int)): + return [space.int_w(w_dim)] + else: + raise NotImplementedError + +def unpack_dtype(space, w_dtype): + if space.is_w(w_dtype, space.w_int): + return 'i' + else: + raise NotImplementedError + +def zeros(space, w_dim, w_dtype): + dim = unpack_dim(space, w_dim) + dtype = unpack_dtype(space, w_dtype) + return space.wrap(NumArray(space, dim, dtype)) +zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/test/__init__.py ============================================================================== Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Wed Jul 15 22:27:03 2009 @@ -0,0 +1,12 @@ + +from pypy.conftest import gettestobjspace + +class AppTestNumpy(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('micronumpy',)) + + def test_zeroes(self): + from numpy import zeros + ar = zeros(3, dtype=int) + assert ar[0] == 0 + From benjamin at codespeak.net Wed Jul 15 22:33:57 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 22:33:57 +0200 (CEST) Subject: [pypy-svn] r66257 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090715203357.0E5D6169E9A@codespeak.net> Author: benjamin Date: Wed Jul 15 22:33:56 2009 New Revision: 66257 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: pass correct argument order Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 15 22:33:56 2009 @@ -99,7 +99,8 @@ raise AssertionError("non-statement node") def error(self, msg, n): - raise SyntaxError(msg, n.lineno, n.column, self.compile_info.filename) + raise SyntaxError(msg, n.lineno, n.column, + filename=self.compile_info.filename) def check_forbidden_name(self, name, node): if name == "None": From benjamin at codespeak.net Wed Jul 15 22:52:01 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 22:52:01 +0200 (CEST) Subject: [pypy-svn] r66258 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090715205201.9CFBD168530@codespeak.net> Author: benjamin Date: Wed Jul 15 22:52:01 2009 New Revision: 66258 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: add another test Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 15 22:52:01 2009 @@ -281,6 +281,14 @@ return x""" % (line,) exc = py.test.raises(SyntaxError, self.mod_scope, input).value assert exc.msg == error + " contains a nested function with free variables" + input = """def f(): + %s + x = 4 + class Y: + def n(): + return x""" % (line,) + exc = py.test.raises(SyntaxError, self.mod_scope, input).value + assert exc.msg == error + " contains a nested function with free variables" def test_exec(self): self.mod_scope("exec 'hi'") From benjamin at codespeak.net Wed Jul 15 23:16:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 23:16:31 +0200 (CEST) Subject: [pypy-svn] r66259 - pypy/branch/parser-compiler/pypy/interpreter/test Message-ID: <20090715211631.F269B169DB5@codespeak.net> Author: benjamin Date: Wed Jul 15 23:16:30 2009 New Revision: 66259 Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Log: the new (c)python compiler is less strict about this Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Wed Jul 15 23:16:30 2009 @@ -155,6 +155,7 @@ def f(): from a import * + x def g(): x @@ -169,23 +170,27 @@ def f(): from a import * + x class g: x def f(): exec "hi" + x = 5 class g: def h(): x def f(): from a import * + x = 4 class g: def h(): x def f(x): exec "hi" + x = 4 class g: x @@ -196,12 +201,14 @@ def f(x): exec "hi" + x = 5 class g: def h(): x def f(x): from a import * + x = 5 class g: def h(): x From benjamin at codespeak.net Wed Jul 15 23:36:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 15 Jul 2009 23:36:44 +0200 (CEST) Subject: [pypy-svn] r66260 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test Message-ID: <20090715213644.E7C80169E9A@codespeak.net> Author: benjamin Date: Wed Jul 15 23:36:44 2009 New Revision: 66260 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: if the source doesn't end with a newline, append one Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 23:36:44 2009 @@ -109,6 +109,8 @@ else: self.grammar = pygram.python_grammar_no_with_statement source_lines = textsrc.splitlines(True) + if source_lines and not source_lines[-1].endswith("\n"): + source_lines[-1] += '\n' if textsrc and textsrc[-1] == "\n": flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 23:36:44 2009 @@ -52,7 +52,7 @@ assert exc.msg == "invalid syntax" assert exc.lineno == 1 assert exc.offset == 12 - assert exc.text == "name another for" + assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "\"blah").value assert exc.msg == "EOL while scanning single-quoted string" exc = py.test.raises(SyntaxError, parse, "'''\n").value @@ -72,7 +72,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "expected indented block" assert exc.lineno == 3 - assert exc.text == "pass" + assert exc.text.startswith("pass") assert exc.offset == 4 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value From benjamin at codespeak.net Thu Jul 16 00:05:34 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 00:05:34 +0200 (CEST) Subject: [pypy-svn] r66261 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler/tools test Message-ID: <20090715220534.B286F169E70@codespeak.net> Author: benjamin Date: Thu Jul 16 00:05:33 2009 New Revision: 66261 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Log: add a default method for unimplemented visitor methods Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 16 00:05:33 2009 @@ -105,6 +105,9 @@ self.emit("for node in seq:", 2) self.emit("node.walkabout(self)", 3) self.emit("") + self.emit("def default_visitor(self, node):", 1) + self.emit("raise NodeVisitorNotImplemented", 2) + self.emit("") super(ASTVisitorVisitor, self).visitModule(mod) self.emit("") @@ -115,11 +118,11 @@ def visitProduct(self, prod, name): self.emit("def visit_%s(self, node):" % (name,), 1) - self.emit("raise NodeVisitorNotImplemented", 2) + self.emit("self.default_visitor(node)", 2) def visitConstructor(self, cons, _): self.emit("def visit_%s(self, node):" % (cons.name,), 1) - self.emit("raise NodeVisitorNotImplemented", 2) + self.emit("self.default_visitor(node)", 2) class GenericASTVisitorVisitor(ASDLVisitor): Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Thu Jul 16 00:05:33 2009 @@ -197,21 +197,25 @@ assert not space.eq_w(w_const, space.wrap("b")) assert not space.eq_w(w_const, space.wrap("c")) + _unicode_error_kind = "w_UnicodeError" + def test_unicodeliterals(self): + w_error = getattr(self.space, self._unicode_error_kind) + e = py.test.raises(OperationError, self.eval_string, "u'\\Ufffffffe'") ex = e.value ex.normalize_exception(self.space) - assert ex.match(self.space, self.space.w_UnicodeError) + assert ex.match(self.space, w_error) e = py.test.raises(OperationError, self.eval_string, "u'\\Uffffffff'") ex = e.value ex.normalize_exception(self.space) - assert ex.match(self.space, self.space.w_UnicodeError) + assert ex.match(self.space, w_error) e = py.test.raises(OperationError, self.eval_string, "u'\\U%08x'" % 0x110000) ex = e.value ex.normalize_exception(self.space) - assert ex.match(self.space, self.space.w_UnicodeError) + assert ex.match(self.space, w_error) def test_unicode_docstring(self): space = self.space @@ -637,6 +641,8 @@ def setup_method(self, method): self.compiler = CPythonCompiler(self.space) + _unicode_error_kind = "w_SyntaxError" + if sys.version_info < (2, 4): def skip_on_2_3(self): py.test.skip("syntax not supported by the CPython 2.3 compiler") From benjamin at codespeak.net Thu Jul 16 00:05:53 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 00:05:53 +0200 (CEST) Subject: [pypy-svn] r66262 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090715220553.4434B169E93@codespeak.net> Author: benjamin Date: Thu Jul 16 00:05:52 2009 New Revision: 66262 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: add space Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 16 00:05:52 2009 @@ -72,6 +72,7 @@ self.emit("") self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (name,), 2) + self.emit("") def make_constructor(self, fields): if fields: From benjamin at codespeak.net Thu Jul 16 00:21:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 00:21:15 +0200 (CEST) Subject: [pypy-svn] r66263 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090715222115.7D33A169E4C@codespeak.net> Author: benjamin Date: Thu Jul 16 00:21:14 2009 New Revision: 66263 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: having an exec at all unoptimizes the scope Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Thu Jul 16 00:21:14 2009 @@ -34,6 +34,7 @@ self.varnames = [] self.children = [] self.free_vars = [] + self.has_exec = False self.has_free = False self.child_has_free = False self.nested = False @@ -68,7 +69,7 @@ ret.col_offset) def note_exec(self, exc): - pass + self.has_exec = True def note_import_star(self, imp): pass @@ -198,7 +199,7 @@ self.return_with_value = True def note_exec(self, exc): - self.has_exec = True + Scope.note_exec(self, exc) if not exc.globals: self.optimized = False self.bare_exec = exc @@ -249,6 +250,7 @@ trailer = "is a nested function" raise SyntaxError(err % (self.name, trailer), node.lineno, node.col_offset) + self.optimized = self.optimized and not self.has_exec class ClassScope(Scope): From benjamin at codespeak.net Thu Jul 16 00:40:17 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 00:40:17 +0200 (CEST) Subject: [pypy-svn] r66264 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090715224017.01514168568@codespeak.net> Author: benjamin Date: Thu Jul 16 00:40:16 2009 New Revision: 66264 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: ugh. implementation details of temporary names Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Thu Jul 16 00:40:16 2009 @@ -273,7 +273,7 @@ self.scopes = {} self.scope = None self.stack = [] - self.tmp_name_counter = 0 + self.tmp_name_counter = 1 top = ModuleScope(module) self.globs = top.roles self.push_scope(top) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Thu Jul 16 00:40:16 2009 @@ -320,13 +320,13 @@ def test_tmpnames(self): scp = self.mod_scope("[x for x in y]") - assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL scp = self.mod_scope("with x: pass") - assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL scp = self.mod_scope("with x as y: pass") - assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[2]") == symtable.SCOPE_LOCAL # Just in case. scp = self.mod_scope("with [x for y in z]: pass") - assert scp.lookup("_[0]") == symtable.SCOPE_LOCAL assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL + assert scp.lookup("_[2]") == symtable.SCOPE_LOCAL From benjamin at codespeak.net Thu Jul 16 01:04:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 01:04:35 +0200 (CEST) Subject: [pypy-svn] r66265 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090715230435.57F19169EAD@codespeak.net> Author: benjamin Date: Thu Jul 16 01:04:33 2009 New Revision: 66265 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: add test for jumps with EXTENDED_ARG Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Thu Jul 16 01:04:33 2009 @@ -50,6 +50,16 @@ st = simple_test + def test_long_jump(self): + func = """def f(x): + y = 0 + if x: +%s return 1 + else: + return 0""" % (" y += 1\n" * 9000,) + yield self.st, func, "f(1)", 1 + yield self.st, func, "f(0)", 0 + def test_argtuple(self): yield (self.simple_test, "def f( x, (y,z) ): return x,y,z", "f((1,2),(3,4))", ((1,2),3,4)) From benjamin at codespeak.net Thu Jul 16 01:05:48 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 01:05:48 +0200 (CEST) Subject: [pypy-svn] r66266 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090715230548.82AF2169EAD@codespeak.net> Author: benjamin Date: Thu Jul 16 01:05:48 2009 New Revision: 66266 Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (contents, props changed) pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (contents, props changed) Log: add new AST compiler Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Thu Jul 16 01:05:48 2009 @@ -0,0 +1,547 @@ +""" +Python control flow graph generation and bytecode assembly. +""" + +from pypy.interpreter.astcompiler import ast2 as ast, symtable +from pypy.interpreter import pycode +from pypy.tool import stdlib_opcode as ops + +from pypy.interpreter.error import OperationError +from pypy.rlib.objectmodel import we_are_translated + + +class Instruction(object): + + def __init__(self, opcode, lineno, arg=0): + assert lineno != -1 + self.opcode = opcode + self.arg = arg + self.lineno = lineno + self.jump = None + + def size(self): + if self.opcode >= ops.HAVE_ARGUMENT: + if self.arg > 0xFFFF: + return 6 + else: + return 3 + else: + return 1 + + def jump_to(self, target, absolute=False): + self.jump = (target, absolute) + + + def __repr__(self): + data = [ops.opname[self.opcode]] + template = "<%s" + if self.opcode >= ops.HAVE_ARGUMENT: + data.append(self.arg) + template += " %i" + if self.jump: + data.append(self.jump[0]) + template += " %s" + template += ">" + return template % tuple(data) + + +class Block(object): + + def __init__(self): + self.instructions = [] + self.next_block = None + self.marked = False + self.have_return = False + + def _post_order(self, blocks): + if self.marked: + return + self.marked = True + if self.next_block is not None: + self.next_block._post_order(blocks) + for instr in self.instructions: + if instr.jump: + instr.jump[0]._post_order(blocks) + blocks.append(self) + self.marked = True + + def post_order(self): + blocks = [] + self._post_order(blocks) + blocks.reverse() + return blocks + + def code_size(self): + i = 0 + for instr in self.instructions: + i += instr.size() + return i + + def get_code(self): + code = [] + for instr in self.instructions: + opcode = instr.opcode + if opcode >= ops.HAVE_ARGUMENT: + arg = instr.arg + if instr.arg > 0xFFFF: + ext = arg >> 16 + code.append(chr(ops.EXTENDED_ARG)) + code.append(chr(ext & 0xFF)) + code.append(chr(ext >> 8)) + arg &= 0xFFFF + code.append(chr(opcode)) + code.append(chr(arg & 0xFF)) + code.append(chr(arg >> 8)) + else: + code.append(chr(opcode)) + return ''.join(code) + + +def _make_index_dict_filter(syms, flag): + i = 0 + result = {} + for name, scope in syms.iteritems(): + if scope == flag: + result[name] = i + i += 1 + return result + +def _list_to_dict(l, offset=0): + result = {} + index = offset + for i in range(len(l)): + result[l[i]] = index + index += 1 + return result + + +class PythonCodeMaker(ast.ASTVisitor): + + def __init__(self, space, name, first_lineno, scope, compile_info): + self.space = space + self.name = name + self.first_lineno = first_lineno + self.compile_info = compile_info + self.lineno = -1 + self.first_block = self.new_block() + self.use_block(self.first_block) + self.names = {} + self.var_names = _list_to_dict(scope.varnames) + self.cell_vars = _make_index_dict_filter(scope.symbols, + symtable.SCOPE_CELL) + self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars)) + self.w_consts = space.newdict() + self.argcount = 0 + self.add_none_to_final_return = True + + def new_block(self): + return Block() + + def use_block(self, block): + self.current_block = block + self.instrs = block.instructions + + def use_next_block(self, block=None): + if block is None: + block = self.new_block() + self.current_block.next_block = block + self.use_block(block) + return block + + def emit_op(self, op): + instr = Instruction(op, self.lineno) + self.instrs.append(instr) + if op == ops.RETURN_VALUE: + self.current_block.have_return = True + return instr + + def emit_op_arg(self, op, arg): + self.instrs.append(Instruction(op, self.lineno, arg)) + + def emit_op_name(self, op, container, name): + self.emit_op_arg(op, self.add_name(container, name)) + + def emit_jump(self, op, block_to, absolute=False): + self.emit_op(op).jump_to(block_to, absolute) + + def add_name(self, container, name): + name = self.scope.mangle(name) + try: + index = container[name] + except KeyError: + index = len(container) + container[name] = index + return index + + def add_const(self, obj): + space = self.space + w_key = space.newtuple([obj, space.type(obj)]) + w_len = space.finditem(self.w_consts, w_key) + if w_len is None: + w_len = space.len(self.w_consts) + space.setitem(self.w_consts, w_key, w_len) + return space.int_w(w_len) + + def load_const(self, obj): + index = self.add_const(obj) + self.emit_op_arg(ops.LOAD_CONST, index) + + def update_position(self, node): + self.lineno = node.lineno + if self.first_lineno == -1: + self.first_lineno = node.lineno + + def _resolve_block_targets(self, blocks): + last_extended_arg_count = 0 + while True: + extended_arg_count = 0 + offset = 0 + for block in blocks: + block.offset = offset + offset += block.code_size() + for block in blocks: + offset = block.offset + for instr in block.instructions: + offset += instr.size() + if instr.jump: + target, absolute = instr.jump + if absolute: + jump_arg = target.offset + else: + jump_arg = target.offset - offset + instr.arg = jump_arg + if jump_arg > 0xFFFF: + extended_arg_count += 1 + if extended_arg_count == last_extended_arg_count: + break + else: + last_extended_arg_count = extended_arg_count + + def _build_consts_array(self): + w_consts = self.w_consts + space = self.space + consts_w = [space.w_None] * space.int_w(space.len(w_consts)) + w_iter = space.iter(w_consts) + first = space.wrap(0) + while True: + try: + w_key = space.next(w_iter) + except OperationError, e: + if not e.match(space, space.w_StopIteration): + raise + break + w_index = space.getitem(w_consts, w_key) + consts_w[space.int_w(w_index)] = space.getitem(w_key, first) + return consts_w + + def _get_code_flags(self): + raise NotImplementedError + + def _stacksize(self, blocks): + for block in blocks: + block.marked = False + block.initial_depth = -1000 + return self._recursive_stack_depth_walk(blocks[0], 0, 0) + + def _recursive_stack_depth_walk(self, block, depth, max_depth): + if block.marked or block.initial_depth >= depth: + return max_depth + block.marked = True + block.initial_depth = depth + for instr in block.instructions: + depth += _opcode_stack_effect(instr.opcode, instr.arg) + if depth >= max_depth: + max_depth = depth + if instr.jump: + max_depth = self._recursive_stack_depth_walk(instr.jump[0], + depth, max_depth) + if instr.opcode == ops.JUMP_ABSOLUTE or \ + instr.opcode == ops.JUMP_FORWARD: + break + if block.next_block: + max_depth = self._recursive_stack_depth_walk(block.next_block, + depth, max_depth) + block.marked = False + return max_depth + + def _build_lnotab(self, blocks): + lineno_table_builder = LinenoTableBuilder(self.first_lineno) + for block in blocks: + offset = block.offset + for instr in block.instructions: + lineno_table_builder.note_lineno_here(offset, instr.lineno) + offset += instr.size() + return lineno_table_builder.get_table() + + def assemble(self): + if self.lineno == -1: + self.lineno = self.first_lineno + if not self.current_block.have_return: + self.use_next_block() + if self.add_none_to_final_return: + self.load_const(self.space.w_None) + self.emit_op(ops.RETURN_VALUE) + blocks = self.first_block.post_order() + self._resolve_block_targets(blocks) + lnotab = self._build_lnotab(blocks) + stack_depth = self._stacksize(blocks) + consts_w = self._build_consts_array() + names = _list_from_dict(self.names) + var_names = _list_from_dict(self.var_names) + cell_names = _list_from_dict(self.cell_vars) + free_names = _list_from_dict(self.free_vars, len(cell_names)) + flags = self._get_code_flags() | self.compile_info.flags + bytecode = ''.join([block.get_code() for block in blocks]) + return pycode.PyCode(self.space, + self.argcount, + len(self.var_names), + stack_depth, + flags, + bytecode, + consts_w, + names, + var_names, + self.compile_info.filename, + self.name, + self.first_lineno, + lnotab, + free_names, + cell_names) + + +def _list_from_dict(d, offset=0): + result = [None] * len(d) + for obj, index in d.iteritems(): + result[index - offset] = obj + return result + + +_static_opcode_stack_effects = { + ops.POP_TOP : -1, + ops.ROT_TWO : 0, + ops.ROT_THREE : 0, + ops.ROT_FOUR : 0, + ops.DUP_TOP : 1, + + ops.UNARY_POSITIVE : 0, + ops.UNARY_NEGATIVE : 0, + ops.UNARY_NOT : 0, + ops.UNARY_CONVERT : 0, + ops.UNARY_INVERT : 0, + + ops.LIST_APPEND : -1, + + ops.BINARY_POWER : -1, + ops.BINARY_MULTIPLY : -1, + ops.BINARY_DIVIDE : -1, + ops.BINARY_MODULO : -1, + ops.BINARY_ADD : -1, + ops.BINARY_SUBTRACT : -1, + ops.BINARY_SUBSCR : -1, + ops.BINARY_FLOOR_DIVIDE : -1, + ops.BINARY_TRUE_DIVIDE : -1, + ops.BINARY_LSHIFT : -1, + ops.BINARY_RSHIFT : -1, + ops.BINARY_AND : -1, + ops.BINARY_OR : -1, + ops.BINARY_XOR : -1, + + ops.INPLACE_FLOOR_DIVIDE : -1, + ops.INPLACE_TRUE_DIVIDE : -1, + ops.INPLACE_ADD : -1, + ops.INPLACE_SUBTRACT : -1, + ops.INPLACE_MULTIPLY : -1, + ops.INPLACE_DIVIDE : -1, + ops.INPLACE_MODULO : -1, + ops.INPLACE_POWER : -1, + ops.INPLACE_LSHIFT : -1, + ops.INPLACE_RSHIFT : -1, + ops.INPLACE_AND : -1, + ops.INPLACE_OR : -1, + ops.INPLACE_XOR : -1, + + ops.SLICE+0 : 1, + ops.SLICE+1 : 0, + ops.SLICE+2 : 0, + ops.SLICE+3 : -1, + ops.STORE_SLICE+0 : -2, + ops.STORE_SLICE+1 : -3, + ops.STORE_SLICE+2 : -3, + ops.STORE_SLICE+3 : -4, + ops.DELETE_SLICE+0 : -1, + ops.DELETE_SLICE+1 : -2, + ops.DELETE_SLICE+2 : -2, + ops.DELETE_SLICE+3 : -3, + + ops.STORE_SUBSCR : -2, + ops.DELETE_SUBSCR : -2, + + ops.GET_ITER : 0, + ops.FOR_ITER : 1, + ops.BREAK_LOOP : 0, + ops.CONTINUE_LOOP : 0, + ops.SETUP_LOOP : 0, + + ops.PRINT_EXPR : -1, + ops.PRINT_ITEM : -1, + ops.PRINT_NEWLINE : 0, + ops.PRINT_ITEM_TO : -2, + ops.PRINT_NEWLINE_TO : -1, + + ops.WITH_CLEANUP : -1, + ops.POP_BLOCK : 0, + ops.END_FINALLY : -1, + ops.SETUP_FINALLY : 3, + ops.SETUP_EXCEPT : 3, + + ops.LOAD_LOCALS : 1, + ops.RETURN_VALUE : -1, + ops.EXEC_STMT : -3, + ops.YIELD_VALUE : 0, + ops.BUILD_CLASS : -2, + ops.BUILD_MAP : 1, + ops.COMPARE_OP : -1, + + ops.LOAD_NAME : 1, + ops.STORE_NAME : -1, + ops.DELETE_NAME : 0, + + ops.LOAD_FAST : 1, + ops.STORE_FAST : -1, + ops.DELETE_FAST : 0, + + ops.LOAD_ATTR : 0, + ops.STORE_ATTR : -2, + ops.DELETE_ATTR : -1, + + ops.LOAD_GLOBAL : 1, + ops.STORE_GLOBAL : -1, + ops.DELETE_GLOBAL : 0, + + ops.LOAD_CLOSURE : 1, + ops.LOAD_DEREF : 1, + ops.STORE_DEREF : -1, + + ops.LOAD_CONST : 1, + + ops.IMPORT_STAR : -1, + ops.IMPORT_NAME : 0, + ops.IMPORT_FROM : 1, + + ops.JUMP_FORWARD : 0, + ops.JUMP_ABSOLUTE : 0, + ops.JUMP_IF_TRUE : 0, + ops.JUMP_IF_FALSE : 0, +} + + +def _compute_UNPACK_SEQUENCE(arg): + return arg + 1 + +def _compute_DUP_TOPX(arg): + return arg + +def _compute_BUILD_TUPLE(arg): + return 1 - arg + +def _compute_BUILD_LIST(arg): + return 1 - arg + +def _compute_MAKE_CLOSURE(arg): + return -arg + +def _compute_MAKE_FUNCTION(arg): + return -arg + +def _compute_BUILD_SLICE(arg): + if arg == 3: + return -2 + else: + return -1 + +def _compute_RAISE_VARARGS(arg): + return -arg + +def _num_args(oparg): + return (oparg % 256) + 2 * (oparg / 256) + +def _compute_CALL_FUNCTION(arg): + return _num_args(arg) + +def _compute_CALL_FUNCTION_VAR(arg): + return _num_args(arg) - 1 + +def _compute_CALL_FUNCTION_KW(arg): + return _num_args(arg) - 1 + +def _compute_CALL_FUNCTION_VAR_KW(arg): + return _num_args(arg) - 2 + + +_stack_effect_computers = {} +for name, func in globals().items(): + if name.startswith("_compute_"): + _stack_effect_computers[getattr(ops, name[9:])] = func +for op, value in _static_opcode_stack_effects.iteritems(): + def func(arg, _value=value): + return value + _stack_effect_computers[op] = func +del name, func, op, value + + +def _opcode_stack_effect(op, arg): + if we_are_translated(): + for possible_op in ops.unrolling_op_descs: + if op == possible_op.index: + return _stack_effect_computers[op](arg) + else: + raise AssertionError("unkown opcode: %s" % (op,)) + else: + try: + return _static_opcode_stack_effects[op] + except KeyError: + return _stack_effect_computers[op](arg) + + +class LinenoTableBuilder(object): + + def __init__(self, first_lineno): + self.first = self.current_line = first_lineno + self.current_off = 0 + self.table = [] + + def get_table(self): + return ''.join(self.table) + + def note_lineno_here(self, offset, lineno): + # compute deltas + line = lineno - self.current_line + # Python assumes that lineno always increases with + # increasing bytecode address (lnotab is unsigned char). + # Depending on when SET_LINENO instructions are emitted + # this is not always true. Consider the code: + # a = (1, + # b) + # In the bytecode stream, the assignment to "a" occurs + # after the loading of "b". This works with the C Python + # compiler because it only generates a SET_LINENO instruction + # for the assignment. + if line >= 0: + addr = offset - self.current_off + if not addr and not line: + return + push = self.table.append + while addr > 255: + push(chr(255)) + push(chr(0)) + addr -= 255 + while line > 255: + push(chr(addr)) + push(chr(255)) + line -= 255 + addr = 0 + if addr > 0 or line > 0: + push(chr(addr)) + push(chr(line)) + self.current_line = lineno + self.current_off = offset Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 01:05:48 2009 @@ -0,0 +1,1133 @@ +""" +Generate Python bytecode from a Abstract Syntax Tree. +""" + +from pypy.interpreter.astcompiler import ast2 as ast, assemble, symtable, consts +from pypy.interpreter.pyparser.error import SyntaxError +from pypy.tool import stdlib_opcode as ops + + +def compile_ast(space, module, info): + symbols = symtable.SymtableBuilder(space, module, info) + return TopLevelCodeGenerator(space, module, symbols, info).assemble() + + +name_ops_default = { + ast.Load : ops.LOAD_NAME, + ast.Store : ops.STORE_NAME, + ast.Del : ops.DELETE_NAME +} + +name_ops_fast = { + ast.Load : ops.LOAD_FAST, + ast.Store : ops.STORE_FAST, + ast.Del : ops.DELETE_FAST +} + +name_ops_deref = { + ast.Load : ops.LOAD_DEREF, + ast.Store : ops.STORE_DEREF, +} + +name_ops_global = { + ast.Load : ops.LOAD_GLOBAL, + ast.Store : ops.STORE_GLOBAL, + ast.Del : ops.DELETE_GLOBAL +} + + +unary_operations = { + ast.Invert : ops.UNARY_INVERT, + ast.Not : ops.UNARY_NOT, + ast.UAdd : ops.UNARY_POSITIVE, + ast.USub : ops.UNARY_NEGATIVE +} + +binary_operations = { + ast.Add : ops.BINARY_ADD, + ast.Sub : ops.BINARY_SUBTRACT, + ast.Mult : ops.BINARY_MULTIPLY, + ast.Mod : ops.BINARY_MODULO, + ast.Pow : ops.BINARY_POWER, + ast.LShift : ops.BINARY_LSHIFT, + ast.RShift : ops.BINARY_RSHIFT, + ast.BitOr : ops.BINARY_OR, + ast.BitAnd : ops.BINARY_AND, + ast.BitXor : ops.BINARY_XOR, + ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE +} + +inplace_operations = { + ast.Add : ops.INPLACE_ADD, + ast.Sub : ops.INPLACE_SUBTRACT, + ast.Mult : ops.INPLACE_MULTIPLY, + ast.Mod : ops.INPLACE_MODULO, + ast.Pow : ops.INPLACE_POWER, + ast.LShift : ops.INPLACE_LSHIFT, + ast.RShift : ops.INPLACE_RSHIFT, + ast.BitOr : ops.INPLACE_OR, + ast.BitAnd : ops.INPLACE_AND, + ast.BitXor : ops.INPLACE_XOR, + ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE +} + +compare_operations = { + ast.Eq : 2, + ast.NotEq : 3, + ast.Lt : 0, + ast.LtE : 1, + ast.Gt : 4, + ast.GtE : 5, + ast.In : 6, + ast.NotIn : 7, + ast.Is : 8, + ast.IsNot : 9 +} + +subscr_operations = { + ast.AugLoad : ops.BINARY_SUBSCR, + ast.Load : ops.BINARY_SUBSCR, + ast.AugStore : ops.STORE_SUBSCR, + ast.Store : ops.STORE_SUBSCR, + ast.Del : ops.DELETE_SUBSCR +} + +slice_operations = { + ast.AugLoad : ops.SLICE, + ast.Load : ops.SLICE, + ast.AugStore : ops.STORE_SLICE, + ast.Store : ops.STORE_SLICE, + ast.Del : ops.DELETE_SLICE +} + + +F_BLOCK_LOOP = 0 +F_BLOCK_EXCEPT = 1 +F_BLOCK_FINALLY = 2 +F_BLOCK_FINALLY_END = 3 + + +CONST_NOT_CONST = -1 +CONST_FALSE = 0 +CONST_TRUE = 1 + + +class PythonCodeGenerator(assemble.PythonCodeMaker): + + def __init__(self, space, name, tree, lineno, symbols, compile_info): + self.scope = symbols.find_scope(tree) + assemble.PythonCodeMaker.__init__(self, space, name, lineno, + self.scope, compile_info) + self.symbols = symbols + self.frame_blocks = [] + self.interactive = False + self.temporary_name_counter = 1 + self.done_with_future = False + self._compile(tree) + + def current_temporary_name(self): + name = "_[%i]" % (self.temporary_name_counter,) + self.temporary_name_counter += 1 + return name + + def sub_scope(self, kind, name, node): + generator = kind(self.space, name, node, node.lineno, self.symbols, + self.compile_info) + return generator.assemble() + + def _expr_constant(self, expr): + if isinstance(expr, ast.Num): + return int(self.space.is_true(expr.n)) + elif isinstance(expr, ast.Str): + return int(self.space.is_true(expr.s)) + return CONST_NOT_CONST + + def push_frame_block(self, kind, block): + self.frame_blocks.append((kind, block)) + + def pop_frame_block(self, kind, block): + actual_kind, old_block = self.frame_blocks.pop() + assert actual_kind == kind and old_block is block, \ + "mismatched frame blocks" + + def error(self, msg, node): + raise SyntaxError(msg, node.lineno, node.col_offset, + self.compile_info.filename) + + def name_op(self, identifier, ctx): + scope = self.scope.lookup(identifier) + kind = name_ops_default + container = self.names + if scope == symtable.SCOPE_LOCAL: + if self.scope.optimized: + container = self.var_names + kind = name_ops_fast + elif scope == symtable.SCOPE_FREE: + kind = name_ops_deref + container = self.free_vars + elif scope == symtable.SCOPE_CELL: + kind = name_ops_deref + container = self.cell_vars + elif scope == symtable.SCOPE_GLOBAL_IMPLICIT: + if self.scope.optimized: + kind = name_ops_global + elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: + kind = name_ops_global + try: + op = kind[ctx] + except KeyError: + if kind is names_ops_deref and ctx == ast.Del: + raise SyntaxError("Can't delete variable used in " + "nested scopes: %r" % (identifier,)) + raise AssertionError("Unkown name operation") + self.emit_op_arg(op, self.add_name(container, identifier)) + + def is_docstring(self, node): + return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str) + + def _get_code_flags(self): + return consts.CO_NEWLOCALS + + def _handle_body(self, body): + if body: + start = 0 + if self.is_docstring(body[0]): + start = 1 + body[0].value.walkabout(self) + self.name_op("__doc__", ast.Store) + for i in range(start, len(body)): + body[i].walkabout(self) + return True + else: + return False + + def visit_Module(self, mod): + if not self._handle_body(mod.body): + self.first_lineno = self.lineno = 1 + + def visit_Interactive(self, mod): + self.interactive = True + self.visit_sequence(mod.body) + + def visit_Expression(self, mod): + self.add_none_to_final_return = False + mod.body.walkabout(self) + + def _make_function(self, code, num_defaults=0): + code_index = self.add_const(code) + if code.co_freevars: + for free in code.co_freevars: + free_scope = self.scope.lookup(free) + if free_scope == symtable.SCOPE_CELL: + index = self.cell_vars[free] + else: + index = self.free_vars[free] + self.emit_op_arg(ops.LOAD_CLOSURE, index) + self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars)) + self.emit_op_arg(ops.LOAD_CONST, code_index) + self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults) + else: + self.emit_op_arg(ops.LOAD_CONST, code_index) + self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults) + + def visit_FunctionDef(self, func): + if func.decorators: + self.visit_sequence(func.decorators) + if func.args.defaults: + self.visit_sequence(func.args.defaults) + num_defaults = len(func.args.defaults) + else: + num_defaults = 0 + code = self.sub_scope(FunctionCodeGenerator, func.name, func) + self.update_position(func) + self._make_function(code, num_defaults) + self.name_op(func.name, ast.Store) + + def visit_Lambda(self, lam): + if lam.args.defaults: + self.visit_sequence(lam.args.defaults) + default_count = len(lam.args.defaults) + else: + default_count = 0 + code = self.sub_scope(LambdaCodeGenerator, "", lam) + self.update_position(lam) + self._make_function(code, default_count) + + def visit_ClassDef(self, cls): + self.update_position(cls) + self.load_const(self.space.wrap(cls.name)) + if cls.bases: + bases_count = len(cls.bases) + self.visit_sequence(cls.bases) + else: + bases_count = 0 + self.emit_op_arg(ops.BUILD_TUPLE, bases_count) + code = self.sub_scope(ClassCodeGenerator, cls.name, cls) + self.update_position(cls) + self._make_function(code, 0) + self.emit_op_arg(ops.CALL_FUNCTION, 0) + self.emit_op(ops.BUILD_CLASS) + self.name_op(cls.name, ast.Store) + + def _op_for_augassign(self, op): + if op == ast.Div: + if self.compile_info.flags & consts.CO_FUTURE_DIVISION: + return ops.INPLACE_TRUE_DIVIDE + else: + return ops.INPLACE_FLOOR_DIVIDE + return inplace_operations[op] + + def visit_AugAssign(self, assign): + self.update_position(assign) + target = assign.target + if isinstance(target, ast.Attribute): + attr = ast.Attribute(target.value, target.attr, ast.AugLoad, + target.lineno, target.col_offset) + attr.walkabout(self) + assign.value.walkabout(self) + self.emit_op(self._op_for_augassign(assign.op)) + attr.ctx = ast.AugStore + attr.walkabout(self) + elif isinstance(target, ast.Subscript): + sub = ast.Subscript(target.value, target.slice, ast.AugLoad, + target.lineno, target.col_offset) + sub.walkabout(self) + assign.value.walkabout(self) + self.emit_op(self._op_for_augassign(assign.op)) + sub.ctx = ast.AugStore + sub.walkabout(self) + elif isinstance(target, ast.Name): + self.name_op(target.id, ast.Load) + assign.value.walkabout(self) + self.emit_op(self._op_for_augassign(assign.op)) + self.name_op(target.id, ast.Store) + else: + raise AssertionError("unkown augassign") + + def visit_Assert(self, asrt): + self.update_position(asrt) + end = self.new_block() + asrt.test.walkabout(self) + self.emit_jump(ops.JUMP_IF_TRUE, end) + self.emit_op(ops.POP_TOP) + self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError") + if asrt.msg: + asrt.msg.walkabout(self) + self.emit_op_arg(ops.RAISE_VARARGS, 2) + else: + self.emit_op_arg(ops.RAISE_VARARGS, 1) + self.use_next_block(end) + self.emit_op(ops.POP_TOP) + + def _binop(self, op): + if op == ast.Div: + if self.compile_info.flags & consts.CO_FUTURE_DIVISION: + return ops.BINARY_TRUE_DIVIDE + else: + return ops.BINARY_FLOOR_DIVIDE + return binary_operations[op] + + def visit_BinOp(self, binop): + self.update_position(binop) + binop.left.walkabout(self) + binop.right.walkabout(self) + self.emit_op(self._binop(binop.op)) + + def visit_Return(self, ret): + self.update_position(ret) + if ret.value: + ret.value.walkabout(self) + else: + self.load_const(self.space.w_None) + self.emit_op(ops.RETURN_VALUE) + + def visit_Print(self, pr): + have_dest = bool(pr.dest) + if have_dest: + pr.dest.walkabout(self) + if pr.values: + for value in pr.values: + if have_dest: + self.emit_op(ops.DUP_TOP) + value.walkabout(self) + self.emit_op(ops.ROT_TWO) + self.emit_op(ops.PRINT_ITEM_TO) + else: + value.walkabout(self) + self.emit_op(ops.PRINT_ITEM) + if pr.nl: + if have_dest: + self.emit_op(ops.PRINT_NEWLINE_TO) + else: + self.emit_op(ops.PRINT_NEWLINE) + if have_dest: + self.emit_op(ops.POP_TOP) + + def visit_Delete(self, delete): + self.update_position(delete) + self.visit_sequence(delete.targets) + + def visit_If(self, if_): + self.update_position(if_) + end = self.new_block() + test_constant = self._expr_constant(if_.test) + if test_constant == CONST_FALSE: + if if_.orelse: + self.visit_sequence(if_.orelse) + elif test_constant == CONST_TRUE: + self.visit_sequence(if_.body) + else: + next = self.new_block() + if_.test.walkabout(self) + self.emit_jump(ops.JUMP_IF_FALSE, next) + self.emit_op(ops.POP_TOP) + self.visit_sequence(if_.body) + self.emit_jump(ops.JUMP_FORWARD, end) + self.use_next_block(next) + self.emit_op(ops.POP_TOP) + if if_.orelse: + self.visit_sequence(if_.orelse) + self.use_next_block(end) + + def visit_Break(self, br): + self.update_position(br) + for f_block in self.frame_blocks: + if f_block[0] == F_BLOCK_LOOP: + break + else: + self.error("'break' outside loop", br) + self.emit_op(ops.BREAK_LOOP) + + def visit_Continue(self, cont): + if not self.frame_blocks: + self.error("'continue' outside look", cont) + current_block, block = self.frame_blocks[-1] + if current_block == F_BLOCK_LOOP: + self.emit_jump(ops.JUMP_ABSOLUTE, block, True) + elif current_block == F_BLOCK_EXCEPT or \ + current_block == F_BLOCK_FINALLY: + for i in range(len(self.frame_blocks) - 2, -1, -1): + f_type, block = self.frame_blocks[i] + if f_type == F_BLOCK_LOOP: + self.emit_jump(ops.CONTINUE_LOOP, block, True) + break + if self.frame_blocks[i][0] == F_BLOCK_FINALLY_END: + self.error("'continue' not allowed in 'finally' clause", + cont) + else: + self.error("'continue' outside loop", cont) + elif current_block == F_BLOCK_FINALLY_END: + self.error("'continue' not allowed in 'finally' clause", cont) + + def visit_For(self, fr): + self.update_position(fr) + start = self.new_block() + cleanup = self.new_block() + end = self.new_block() + self.emit_jump(ops.SETUP_LOOP, end) + self.push_frame_block(F_BLOCK_LOOP, start) + fr.iter.walkabout(self) + self.emit_op(ops.GET_ITER) + self.use_next_block(start) + self.emit_jump(ops.FOR_ITER, cleanup) + fr.target.walkabout(self) + self.visit_sequence(fr.body) + self.emit_jump(ops.JUMP_ABSOLUTE, start, True) + self.use_next_block(cleanup) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_LOOP, start) + if fr.orelse: + self.visit_sequence(fr.orelse) + self.use_next_block(end) + + def visit_While(self, wh): + self.update_position(wh) + test_constant = self._expr_constant(wh.test) + if test_constant == CONST_FALSE: + if wh.orelse: + self.visit_sequence(orelse) + else: + end = self.new_block() + if test_constant == CONST_NOT_CONST: + anchor = self.new_block() + self.emit_jump(ops.SETUP_LOOP, end) + loop = self.new_block() + self.push_frame_block(F_BLOCK_LOOP, loop) + self.use_next_block(loop) + if test_constant == CONST_NOT_CONST: + wh.test.walkabout(self) + self.emit_jump(ops.JUMP_IF_FALSE, anchor) + self.emit_op(ops.POP_TOP) + self.visit_sequence(wh.body) + self.emit_jump(ops.JUMP_ABSOLUTE, loop, True) + if test_constant == CONST_NOT_CONST: + self.use_next_block(anchor) + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_LOOP, loop) + if wh.orelse: + self.visit_sequence(wh.orelse) + self.use_next_block(end) + + def visit_TryExcept(self, te): + self.update_position(te) + exc = self.new_block() + otherwise = self.new_block() + end = self.new_block() + self.emit_jump(ops.SETUP_EXCEPT, exc) + body = self.use_next_block() + self.push_frame_block(F_BLOCK_EXCEPT, body) + self.visit_sequence(te.body) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_EXCEPT, body) + self.emit_jump(ops.JUMP_FORWARD, otherwise) + self.use_next_block(exc) + for handler in te.handlers: + self.update_position(handler) + next_except = self.new_block() + if handler.type: + self.emit_op(ops.DUP_TOP) + handler.type.walkabout(self) + self.emit_op_arg(ops.COMPARE_OP, 10) + self.emit_jump(ops.JUMP_IF_FALSE, next_except) + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_TOP) + if handler.name: + handler.name.walkabout(self) + else: + self.emit_op(ops.POP_TOP) + self.emit_op(ops.POP_TOP) + self.visit_sequence(handler.body) + self.emit_jump(ops.JUMP_FORWARD, end) + self.use_next_block(next_except) + if handler.type: + self.emit_op(ops.POP_TOP) + self.emit_op(ops.END_FINALLY) + self.use_next_block(otherwise) + if te.orelse: + self.visit_sequence(te.orelse) + self.use_next_block(end) + + def visit_TryFinally(self, tf): + self.update_position(tf) + end = self.new_block() + self.emit_jump(ops.SETUP_FINALLY, end) + body = self.use_next_block() + self.push_frame_block(F_BLOCK_FINALLY, body) + self.visit_sequence(tf.body) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_FINALLY, body) + self.load_const(self.space.w_None) + self.use_next_block(end) + self.push_frame_block(F_BLOCK_FINALLY_END, end) + self.visit_sequence(tf.finalbody) + self.emit_op(ops.END_FINALLY) + self.pop_frame_block(F_BLOCK_FINALLY_END, end) + + def _import_as(self, alias): + source_name = alias.name + dot = source_name.find(".") + if dot != -1: + start = dot + 1 + while True: + dot = source_name.find(".", start) + if dot == -1: + end = len(source_name) + else: + end = dot + attr = source_name[start:end] + self.emit_op_name(ops.LOAD_ATTR, self.names, attr) + if dot == -1: + break + start = dot + 1 + self.name_op(alias.asname, ast.Store) + + def visit_Import(self, imp): + self.update_position(imp) + for alias in imp.names: + if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: + level = 0 + else: + level = -1 + self.load_const(self.space.wrap(level)) + self.load_const(self.space.w_None) + self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name) + if alias.asname: + self._import_as(alias) + else: + dot = alias.name.find(".") + if dot == -1: + store_name = alias.name + else: + store_name = alias.name[:dot] + self.name_op(store_name, ast.Store) + + def visit_ImportFrom(self, imp): + self.update_position(imp) + space = self.space + if imp.module == "__future__": + if self.done_with_future: + self.error("__future__ statements must appear before other " \ + "imports", imp) + else: + self.done_with_future = True + if imp.level == 0 and \ + not self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: + level = -1 + else: + level = imp.level + self.load_const(space.wrap(level)) + names_w = [space.wrap(alias.name) for alias in imp.names] + self.load_const(space.newtuple(names_w)) + self.emit_op_name(ops.IMPORT_NAME, self.names, imp.module) + if len(imp.names) == 1 and imp.names[0].name == "*": + self.emit_op(ops.IMPORT_STAR) + else: + for alias in imp.names: + self.emit_op_name(ops.IMPORT_FROM, self.names, alias.name) + if alias.asname: + store_name = alias.asname + else: + store_name = alias.name + self.name_op(store_name, ast.Store) + self.emit_op(ops.POP_TOP) + + def visit_Assign(self, assign): + self.update_position(assign) + assign.value.walkabout(self) + duplications = len(assign.targets) - 1 + for i in range(len(assign.targets)): + if i < duplications: + self.emit_op(ops.DUP_TOP) + assign.targets[i].walkabout(self) + + def visit_With(self, wih): + self.update_position(wih) + body_block = self.new_block() + cleanup = self.new_block() + exit_storage = self.current_temporary_name() + if wih.optional_vars: + temp_result = self.current_temporary_name() + wih.context_expr.walkabout(self) + self.emit_op(ops.DUP_TOP) + self.emit_op_name(ops.LOAD_ATTR, self.names, "__exit__") + self.name_op(exit_storage, ast.Store) + self.emit_op_name(ops.LOAD_ATTR, self.names, "__enter__") + self.emit_op_arg(ops.CALL_FUNCTION, 0) + if wih.optional_vars: + self.name_op(temp_result, ast.Store) + else: + self.emit_op(ops.POP_TOP) + self.emit_jump(ops.SETUP_FINALLY, cleanup) + self.use_next_block(body_block) + self.push_frame_block(F_BLOCK_FINALLY, body_block) + if wih.optional_vars: + self.name_op(temp_result, ast.Load) + self.name_op(temp_result, ast.Del) + wih.optional_vars.walkabout(self) + self.visit_sequence(wih.body) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_FINALLY, body_block) + self.load_const(self.space.w_None) + self.use_next_block(cleanup) + self.push_frame_block(F_BLOCK_FINALLY_END, cleanup) + self.name_op(exit_storage, ast.Load) + self.name_op(exit_storage, ast.Del) + self.emit_op(ops.WITH_CLEANUP) + self.emit_op(ops.END_FINALLY) + self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup) + + def visit_Raise(self, rais): + self.update_position(rais) + arg = 0 + if rais.type: + rais.type.walkabout(self) + arg += 1 + if rais.inst: + rais.inst.walkabout(self) + arg += 1 + if rais.tback: + rais.tback.walkabout(self) + arg += 1 + self.emit_op_arg(ops.RAISE_VARARGS, arg) + + def visit_Exec(self, exc): + self.update_position(exc) + exc.body.walkabout(self) + if exc.globals: + exc.globals.walkabout(self) + if exc.locals: + exc.locals.walkabout(self) + else: + self.emit_op(ops.DUP_TOP) + else: + self.load_const(self.space.w_None) + self.emit_op(ops.DUP_TOP) + self.emit_op(ops.EXEC_STMT) + + def visit_Global(self, glob): + # Handled in symbol table building. + pass + + def visit_Pass(self, pas): + self.update_position(pas) + + def visit_Expr(self, expr): + self.update_position(expr) + if self.interactive: + expr.value.walkabout(self) + self.emit_op(ops.PRINT_EXPR) + elif not isinstance(expr.value, ast.Num) and \ + not isinstance(expr.value, ast.Str): + expr.value.walkabout(self) + self.emit_op(ops.POP_TOP) + + def visit_Yield(self, yie): + self.update_position(yie) + if yie.value: + yie.value.walkabout(self) + else: + self.load_const(self.space.w_None) + self.emit_op(ops.YIELD_VALUE) + + def visit_Num(self, num): + self.update_position(num) + self.load_const(num.n) + + def visit_Str(self, string): + self.update_position(string) + self.load_const(string.s) + + def visit_UnaryOp(self, op): + self.update_position(op) + op.operand.walkabout(self) + self.emit_op(unary_operations[op.op]) + + def visit_BoolOp(self, op): + self.update_position(op) + if op.op == ast.And: + instr = ops.JUMP_IF_FALSE + else: + instr = ops.JUMP_IF_TRUE + end = self.new_block() + for value in op.values[:-1]: + value.walkabout(self) + self.emit_jump(instr, end) + self.emit_op(ops.POP_TOP) + op.values[-1].walkabout(self) + self.use_next_block(end) + + def visit_Compare(self, comp): + self.update_position(comp) + comp.left.walkabout(self) + ops_count = len(comp.ops) + if ops_count > 1: + cleanup = self.new_block() + comp.comparators[0].walkabout(self) + for i in range(1, ops_count): + self.emit_op(ops.DUP_TOP) + self.emit_op(ops.ROT_THREE) + op_kind = compare_operations[comp.ops[i - 1]] + self.emit_op_arg(ops.COMPARE_OP, op_kind) + self.emit_jump(ops.JUMP_IF_FALSE, cleanup) + self.emit_op(ops.POP_TOP) + if i < (ops_count - 1): + comp.comparators[i].walkabout(self) + comp.comparators[-1].walkabout(self) + last_kind = compare_operations[comp.ops[-1]] + self.emit_op_arg(ops.COMPARE_OP, last_kind) + if ops_count > 1: + end = self.new_block() + self.emit_jump(ops.JUMP_FORWARD, end) + self.use_next_block(cleanup) + self.emit_op(ops.ROT_TWO) + self.emit_op(ops.POP_TOP) + self.use_next_block(end) + + def visit_IfExp(self, ifexp): + self.update_position(ifexp) + end = self.new_block() + otherwise = self.new_block() + ifexp.test.walkabout(self) + self.emit_jump(ops.JUMP_IF_FALSE, otherwise) + self.emit_op(ops.POP_TOP) + ifexp.body.walkabout(self) + self.emit_jump(ops.JUMP_FORWARD, end) + self.use_next_block(otherwise) + self.emit_op(ops.POP_TOP) + ifexp.orelse.walkabout(self) + self.use_next_block(end) + + def visit_Tuple(self, tup): + self.update_position(tup) + if tup.elts: + elt_count = len(tup.elts) + else: + elt_count = 0 + if tup.ctx == ast.Store: + self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count) + if elt_count: + self.visit_sequence(tup.elts) + if tup.ctx == ast.Load: + self.emit_op_arg(ops.BUILD_TUPLE, elt_count) + + def visit_List(self, l): + self.update_position(l) + if l.elts: + elt_count = len(l.elts) + else: + elt_count = 0 + if l.ctx == ast.Store: + self.emit_op_arg(ops.UNPACK_SEQUENCE, elt_count) + if elt_count: + self.visit_sequence(l.elts) + if l.ctx == ast.Load: + self.emit_op_arg(ops.BUILD_LIST, elt_count) + + def visit_Dict(self, d): + self.update_position(d) + self.emit_op_arg(ops.BUILD_MAP, 0) + if d.values: + for i in range(len(d.values)): + self.emit_op(ops.DUP_TOP) + d.values[i].walkabout(self) + self.emit_op(ops.ROT_TWO) + d.keys[i].walkabout(self) + self.emit_op(ops.STORE_SUBSCR) + + def visit_Name(self, name): + self.update_position(name) + self.name_op(name.id, name.ctx) + + def visit_keyword(self, keyword): + self.load_const(self.space.wrap(keyword.arg)) + keyword.value.walkabout(self) + + def visit_Call(self, call): + self.update_position(call) + call.func.walkabout(self) + arg = 0 + call_type = 0 + if call.args: + arg = len(call.args) + self.visit_sequence(call.args) + if call.keywords: + self.visit_sequence(call.keywords) + arg |= len(call.keywords) << 8 + if call.starargs: + call.starargs.walkabout(self) + call_type |= 1 + if call.kwargs: + call.kwargs.walkabout(self) + call_type |= 2 + if call_type == 0: + op = ops.CALL_FUNCTION + elif call_type == 1: + op = ops.CALL_FUNCTION_VAR + elif call_type == 2: + op = ops.CALL_FUNCTION_KW + elif call_type == 3: + op = ops.CALL_FUNCTION_VAR_KW + self.emit_op_arg(op, arg) + + def _listcomp_generator(self, list_name, gens, gen_index, elt): + start = self.new_block() + skip = self.new_block() + if_cleanup = self.new_block() + anchor = self.new_block() + gen = gens[gen_index] + gen.iter.walkabout(self) + self.emit_op(ops.GET_ITER) + self.use_next_block(start) + self.emit_jump(ops.FOR_ITER, anchor) + self.use_next_block() + gen.target.walkabout(self) + if gen.ifs: + if_count = len(gen.ifs) + for if_ in gen.ifs: + if_.walkabout(self) + self.emit_jump(ops.JUMP_IF_FALSE, if_cleanup) + self.use_next_block() + self.emit_op(ops.POP_TOP) + else: + if_count = 0 + gen_index += 1 + if gen_index < len(gens): + self._listcomp_generator(list_name, gens, gen_index, elt) + else: + self.name_op(list_name, ast.Load) + elt.walkabout(self) + self.emit_op(ops.LIST_APPEND) + self.use_next_block(skip) + for i in range(if_count): + self.emit_op_arg(ops.JUMP_FORWARD, 1) + if i == 0: + self.use_next_block(if_cleanup) + self.emit_op(ops.POP_TOP) + self.emit_jump(ops.JUMP_ABSOLUTE, start, True) + self.use_next_block(anchor) + if gen_index == 1: + self.name_op(list_name, ast.Del) + + def visit_ListComp(self, lc): + self.update_position(lc) + tmp_name = self.current_temporary_name() + self.emit_op_arg(ops.BUILD_LIST, 0) + self.emit_op(ops.DUP_TOP) + self.name_op(tmp_name, ast.Store) + self._listcomp_generator(tmp_name, lc.generators, 0, lc.elt) + + def _genexp_generator(self, generators, gen_index, elt): + start = self.new_block() + skip = self.new_block() + if_cleanup = self.new_block() + anchor = self.new_block() + end = self.new_block() + gen = generators[gen_index] + self.emit_jump(ops.SETUP_LOOP, end) + self.push_frame_block(F_BLOCK_LOOP, start) + if gen_index == 0: + self.argcount = 1 + self.name_op(".0", ast.Load) + else: + gen.iter.walkabout(self) + self.emit_op(ops.GET_ITER) + self.use_next_block(start) + self.emit_jump(ops.FOR_ITER, anchor) + self.use_next_block() + gen.target.walkabout(self) + if gen.ifs: + ifs_count = len(gen.ifs) + for if_ in gen.ifs: + if_.walkabout(self) + self.emit_jump(ops.JUMP_IF_FALSE, if_cleanup) + self.use_next_block() + self.emit_op(ops.POP_TOP) + else: + ifs_count = 0 + gen_index += 1 + if gen_index < len(generators): + self._genexp_generator(generators, gen_index, elt) + else: + elt.walkabout(self) + self.emit_op(ops.YIELD_VALUE) + self.emit_op(ops.POP_TOP) + self.use_next_block(skip) + for i in range(ifs_count): + self.emit_op_arg(ops.JUMP_FORWARD, 1) + if i == 0: + self.use_next_block(if_cleanup) + self.emit_op(ops.POP_TOP) + self.emit_jump(ops.JUMP_ABSOLUTE, start, True) + self.use_next_block(anchor) + self.emit_op(ops.POP_BLOCK) + self.pop_frame_block(F_BLOCK_LOOP, start) + self.use_next_block(end) + + def visit_GeneratorExp(self, genexp): + code = self.sub_scope(GenExpCodeGenerator, "", genexp) + self.update_position(genexp) + self._make_function(code) + genexp.generators[0].iter.walkabout(self) + self.emit_op(ops.GET_ITER) + self.emit_op_arg(ops.CALL_FUNCTION, 1) + + def visit_Attribute(self, attr): + self.update_position(attr) + names = self.names + if attr.ctx != ast.AugStore: + attr.value.walkabout(self) + if attr.ctx == ast.AugLoad: + self.emit_op(ops.DUP_TOP) + self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) + elif attr.ctx == ast.Load: + self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) + elif attr.ctx == ast.AugStore: + self.emit_op(ops.ROT_TWO) + self.emit_op_name(ops.STORE_ATTR, names, attr.attr) + elif attr.ctx == ast.Store: + self.emit_op_name(ops.STORE_ATTR, names, attr.attr) + + def _simple_slice(self, slc, ctx): + slice_offset = 0 + stack_count = 0 + if slc.lower: + slice_offset += 1 + stack_count += 1 + if ctx != ast.AugStore: + slc.lower.walkabout(self) + if slc.upper: + slice_offset += 2 + stack_count += 1 + if ctx != ast.AugStore: + slc.upper.walkabout(self) + if ctx == ast.AugLoad: + if stack_count == 0: + self.emit_op(ops.DUP_TOP) + elif stack_count == 1: + self.emit_op_arg(ops.DUP_TOPX, 2) + elif stack_count == 2: + self.emit_op-arg(ops.DUP_TOPX, 3) + elif ctx == ast.AugStore: + if stack_count == 0: + self.emit_op(ops.ROT_TWO) + elif stack_count == 1: + self.emit_op(ops.ROT_THREE) + elif stack_count == 2: + self.emit_op(ops.ROT_FOUR) + self.emit_op(slice_operations[ctx] + slice_offset) + + def _complex_slice(self, slc, ctx): + if slc.lower: + slc.lower.walkabout(self) + else: + self.load_const(self.space.w_None) + if slc.upper: + slc.upper.walkabout(self) + else: + self.load_const(self.space.w_None) + arg = 2 + if slc.step: + slc.step.walkabout(self) + arg += 1 + self.emit_op_arg(ops.BUILD_SLICE, arg) + + def _nested_slice(self, slc, ctx): + if isinstance(slc, ast.Ellipsis): + self.load_const(self.space.w_Ellipsis) + elif isinstance(slc, ast.Slice): + self._complex_slice(slc, ctx) + elif isinstance(slc, ast.Index): + slc.value.walkabout(self) + else: + raise AssertionError("unkown nested slice type") + + def _compile_slice(self, slc, ctx): + if isinstance(slc, ast.Index): + kind = "index" + if ctx != ast.AugStore: + slc.value.walkabout(self) + elif isinstance(slc, ast.Ellipsis): + kind = "ellipsis" + if ctx != ast.AugStore: + self.load_const(self.space.w_Ellipsis) + elif isinstance(slc, ast.Slice): + kind = "slice" + if not slc.step: + self._simple_slice(slc, ctx) + return + elif ctx != ast.AugStore: + self._complex_slice(slc, ctx) + elif isinstance(slc, ast.ExtSlice): + kind = "extended slice" + if ctx != ast.AugStore: + for dim in slc.dims: + self._nested_slice(dim, ctx) + self.emit_op_arg(ops.BUILD_TUPLE, len(slc.dims)) + else: + raise AssertionError("unkown slice type") + if ctx == ast.AugLoad: + self.emit_op_arg(ops.DUP_TOPX, 2) + elif ctx == ast.AugStore: + self.emit_op(ops.ROT_THREE) + self.emit_op(subscr_operations[ctx]) + + def visit_Subscript(self, sub): + self.update_position(sub) + if sub.ctx != ast.AugStore: + sub.value.walkabout(self) + self._compile_slice(sub.slice, sub.ctx) + + +class TopLevelCodeGenerator(PythonCodeGenerator): + + def __init__(self, space, tree, symbols, compile_info): + PythonCodeGenerator.__init__(self, space, "", tree, -1, + symbols, compile_info) + + def _compile(self, tree): + tree.walkabout(self) + + def _get_code_flags(self): + return 0 + + +class AbstractFunctionCodeGenerator(PythonCodeGenerator): + + def _compile(self, func): + assert isinstance(func, ast.FunctionDef) + if self.is_docstring(func.body[0]): + self.add_const(func.body[0].value.s) + start = 1 + else: + self.add_const(self.space.w_None) + start = 0 + if func.args.args: + self._handle_nested_args(func.args.args) + self.argcount = len(func.args.args) + for i in range(start, len(func.body)): + func.body[i].walkabout(self) + + def _handle_nested_args(self, args): + for i in range(len(args)): + arg = args[i] + if isinstance(arg, ast.Tuple): + self.update_position(arg) + self.name_op(".%i" % (i,), ast.Load) + arg.walkabout(self) + + def _get_code_flags(self): + scope = self.scope + assert isinstance(scope, symtable.FunctionScope) + flags = 0 + if scope.optimized: + flags |= consts.CO_OPTIMIZED + if scope.nested: + flags |= consts.CO_NESTED + if scope.is_generator: + flags |= consts.CO_GENERATOR + if scope.has_variable_arg: + flags |= consts.CO_VARARGS + if scope.has_keywords_arg: + flags |= consts.CO_VARKEYWORDS + if not self.cell_vars and not self.free_vars: + flags |= consts.CO_NOFREE + return PythonCodeGenerator._get_code_flags(self) | flags + + +class FunctionCodeGenerator(AbstractFunctionCodeGenerator): + pass + + +class LambdaCodeGenerator(AbstractFunctionCodeGenerator): + + def _compile(self, lam): + assert isinstance(lam, ast.Lambda) + if lam.args.args: + self._handle_nested_args(lam.args.args) + lam.body.walkabout(self) + self.emit_op(ops.RETURN_VALUE) + + +class GenExpCodeGenerator(AbstractFunctionCodeGenerator): + + def _compile(self, genexp): + assert isinstance(genexp, ast.GeneratorExp) + self.update_position(genexp) + self._genexp_generator(genexp.generators, 0, genexp.elt) + + def _get_code_flags(self): + flags = AbstractFunctionCodeGenerator._get_code_flags(self) + return flags | consts.CO_GENERATOR + + +class ClassCodeGenerator(PythonCodeGenerator): + + def _compile(self, cls): + assert isinstance(cls, ast.ClassDef) + self.lineno = self.first_lineno + self.name_op("__name__", ast.Load) + self.name_op("__module__", ast.Store) + self._handle_body(cls.body) + self.emit_op(ops.LOAD_LOCALS) + self.emit_op(ops.RETURN_VALUE) From benjamin at codespeak.net Thu Jul 16 02:00:26 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 02:00:26 +0200 (CEST) Subject: [pypy-svn] r66267 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090716000026.915FA168518@codespeak.net> Author: benjamin Date: Thu Jul 16 02:00:23 2009 New Revision: 66267 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: add mutation support to AST visitors Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 16 02:00:23 2009 @@ -49,10 +49,18 @@ class ASTNodeVisitor(ASDLVisitor): - def visitType(self, tp): - self.visit(tp.value, tp.name) + def visitModule(self, mod): + dont_touch = set() + for type in mod.dfns: + if isinstance(type.value, asdl.Product) or \ + self.is_simple_sum(type.value): + dont_touch.add(type.name.value) + super(ASTNodeVisitor, self).visitModule(mod, dont_touch) + + def visitType(self, tp, simple): + self.visit(tp.value, tp.name, simple) - def visitSum(self, sum, base): + def visitSum(self, sum, base, simple): if self.is_simple_sum(sum): for i, cons in enumerate(sum.types): self.emit("%s = %i" % (cons.name, i + 1)) @@ -62,10 +70,10 @@ self.emit("pass", 1) self.emit("") for cons in sum.types: - self.visit(cons, base, sum.attributes) + self.visit(cons, base, sum.attributes, simple) self.emit("") - def visitProduct(self, product, name): + def visitProduct(self, product, name, simple): self.emit("class %s(AST):" % (name,)) self.emit("") self.make_constructor(product.fields) @@ -84,13 +92,31 @@ self.emit("def __init__(self):", 1) self.emit("pass", 2) - def visitConstructor(self, cons, base, extra_attributes): + def visitConstructor(self, cons, base, extra_attributes, simple): self.emit("class %s(%s):" % (cons.name, base)) self.emit("") self.make_constructor(cons.fields + extra_attributes) self.emit("") self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) + self.emit("") + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if field.type.value not in asdl.builtin_types and \ + field.type.value not in simple: + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) def visitField(self, field): self.emit("self.%s = %s" % (field.name, field.name), 2) @@ -109,6 +135,10 @@ self.emit("def default_visitor(self, node):", 1) self.emit("raise NodeVisitorNotImplemented", 2) self.emit("") + self.emit("def _mutate_sequence(self, seq):", 1) + self.emit("for i in range(len(seq)):", 2) + self.emit("seq[i] = seq[i].mutate_over(self)", 3) + self.emit("") super(ASTVisitorVisitor, self).visitModule(mod) self.emit("") @@ -185,6 +215,9 @@ def walkabout(self, visitor): raise AssertionError("walkabout() implementation not provided") + def mutate_over(self, visitor): + raise AssertionError("mutate_over() implementation not provided") + class NodeVisitorNotImplemented(Exception): pass From benjamin at codespeak.net Thu Jul 16 02:17:27 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 02:17:27 +0200 (CEST) Subject: [pypy-svn] r66269 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090716001727.E6045169EB7@codespeak.net> Author: benjamin Date: Thu Jul 16 02:17:27 2009 New Revision: 66269 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: make ast nodes extendable, so we can add custom code elsewhere Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 16 02:17:27 2009 @@ -209,9 +209,12 @@ HEAD = """# Generated by tools/asdl_py.py from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter import typedef +from pypy.tool.pairtype import extendabletype class AST(Wrappable): + __metaclass__ = extendabletype + def walkabout(self, visitor): raise AssertionError("walkabout() implementation not provided") From benjamin at codespeak.net Thu Jul 16 02:33:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 02:33:35 +0200 (CEST) Subject: [pypy-svn] r66270 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090716003335.47A04169F14@codespeak.net> Author: benjamin Date: Thu Jul 16 02:33:34 2009 New Revision: 66270 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: return from the default visitor Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Thu Jul 16 02:33:34 2009 @@ -149,11 +149,11 @@ def visitProduct(self, prod, name): self.emit("def visit_%s(self, node):" % (name,), 1) - self.emit("self.default_visitor(node)", 2) + self.emit("return self.default_visitor(node)", 2) def visitConstructor(self, cons, _): self.emit("def visit_%s(self, node):" % (cons.name,), 1) - self.emit("self.default_visitor(node)", 2) + self.emit("return self.default_visitor(node)", 2) class GenericASTVisitorVisitor(ASDLVisitor): From benjamin at codespeak.net Thu Jul 16 02:43:48 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 02:43:48 +0200 (CEST) Subject: [pypy-svn] r66272 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716004348.D3274169F15@codespeak.net> Author: benjamin Date: Thu Jul 16 02:43:48 2009 New Revision: 66272 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: move expr_constant to the util module Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 02:43:48 2009 @@ -107,11 +107,6 @@ F_BLOCK_FINALLY_END = 3 -CONST_NOT_CONST = -1 -CONST_FALSE = 0 -CONST_TRUE = 1 - - class PythonCodeGenerator(assemble.PythonCodeMaker): def __init__(self, space, name, tree, lineno, symbols, compile_info): @@ -135,13 +130,6 @@ self.compile_info) return generator.assemble() - def _expr_constant(self, expr): - if isinstance(expr, ast.Num): - return int(self.space.is_true(expr.n)) - elif isinstance(expr, ast.Str): - return int(self.space.is_true(expr.s)) - return CONST_NOT_CONST - def push_frame_block(self, kind, block): self.frame_blocks.append((kind, block)) @@ -370,11 +358,11 @@ def visit_If(self, if_): self.update_position(if_) end = self.new_block() - test_constant = self._expr_constant(if_.test) - if test_constant == CONST_FALSE: + test_constant = misc.expr_constant(self.space, if_.test) + if test_constant == misc.CONST_FALSE: if if_.orelse: self.visit_sequence(if_.orelse) - elif test_constant == CONST_TRUE: + elif test_constant == misc.CONST_TRUE: self.visit_sequence(if_.body) else: next = self.new_block() @@ -442,25 +430,25 @@ def visit_While(self, wh): self.update_position(wh) - test_constant = self._expr_constant(wh.test) - if test_constant == CONST_FALSE: + test_constant = misc.expr_constant(self.space, wh.test) + if test_constant == misc.CONST_FALSE: if wh.orelse: self.visit_sequence(orelse) else: end = self.new_block() - if test_constant == CONST_NOT_CONST: + if test_constant == misc.CONST_NOT_CONST: anchor = self.new_block() self.emit_jump(ops.SETUP_LOOP, end) loop = self.new_block() self.push_frame_block(F_BLOCK_LOOP, loop) self.use_next_block(loop) - if test_constant == CONST_NOT_CONST: + if test_constant == misc.CONST_NOT_CONST: wh.test.walkabout(self) self.emit_jump(ops.JUMP_IF_FALSE, anchor) self.emit_op(ops.POP_TOP) self.visit_sequence(wh.body) self.emit_jump(ops.JUMP_ABSOLUTE, loop, True) - if test_constant == CONST_NOT_CONST: + if test_constant == misc.CONST_NOT_CONST: self.use_next_block(anchor) self.emit_op(ops.POP_TOP) self.emit_op(ops.POP_BLOCK) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Thu Jul 16 02:43:48 2009 @@ -21,6 +21,18 @@ _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset) +CONST_NOT_CONST = -1 +CONST_FALSE = 0 +CONST_TRUE = 1 + +def expr_constant(space, expr): + if isinstance(expr, ast.Num): + return int(space.is_true(expr.n)) + elif isinstance(expr, ast.Str): + return int(space.is_true(expr.s)) + return CONST_NOT_CONST + + def flatten(tup): elts = [] for elt in tup: From benjamin at codespeak.net Thu Jul 16 03:02:06 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 03:02:06 +0200 (CEST) Subject: [pypy-svn] r66273 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716010206.21805169ECA@codespeak.net> Author: benjamin Date: Thu Jul 16 03:02:04 2009 New Revision: 66273 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: add missing import Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 03:02:04 2009 @@ -2,7 +2,8 @@ Generate Python bytecode from a Abstract Syntax Tree. """ -from pypy.interpreter.astcompiler import ast2 as ast, assemble, symtable, consts +from pypy.interpreter.astcompiler import (ast2 as ast, assemble, symtable, + consts, misc) from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Thu Jul 16 03:02:04 2009 @@ -32,6 +32,11 @@ return int(space.is_true(expr.s)) return CONST_NOT_CONST +def is_constant(expr): + if isinstance(expr, (ast.Num, ast.Str)): + return True + return False + def flatten(tup): elts = [] From arigo at codespeak.net Thu Jul 16 10:32:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 16 Jul 2009 10:32:29 +0200 (CEST) Subject: [pypy-svn] r66275 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090716083229.C25AA169F5E@codespeak.net> Author: arigo Date: Thu Jul 16 10:32:28 2009 New Revision: 66275 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py - copied, changed from r66274, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py Log: Rewrite the test copying test_optimize3.py. Add some tests -- it's much easier now to just write a few extra tests :-) Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (from r66274, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Thu Jul 16 10:32:28 2009 @@ -1,20 +1,17 @@ import py + from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE -from pypy.jit.metainterp.resoperation import rop, ResOperation, opname -from pypy.jit.metainterp.history import ConstAddr, BoxPtr, TreeLoop,\ - ConstInt, BoxInt, BoxObj, ConstObj from pypy.jit.backend.llgraph import runner - -from pypy.jit.metainterp.optimize3 import AbstractOptimization -from pypy.jit.metainterp.optimize3 import optimize_loop, LoopOptimizer,\ - LoopSpecializer, OptimizeGuards, OptimizeVirtuals -from pypy.jit.metainterp.specnode3 import VirtualInstanceSpecNode, \ - NotSpecNode, FixedClassSpecNode +from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, + Const, ConstAddr, TreeLoop, BoxObj) +from pypy.jit.metainterp.optimize import perfect_specialization_finder from pypy.jit.metainterp.test.oparser import parse +# ____________________________________________________________ + def equaloplists(oplist1, oplist2): print '-'*20, 'Comparing lists', '-'*20 for op1, op2 in zip(oplist1, oplist2): @@ -27,18 +24,15 @@ assert op1.opnum == op2.opnum assert len(op1.args) == len(op2.args) for x, y in zip(op1.args, op2.args): - assert x == y # or y == x # for ANY object :-( + assert x == y assert op1.result == op2.result assert op1.descr == op2.descr if op1.suboperations: assert equaloplists(op1.suboperations, op2.suboperations) assert len(oplist1) == len(oplist2) print '-'*57 - #finally: - # Box._extended_display = saved return True - def test_equaloplists(): ops = """ [i0] @@ -55,37 +49,7 @@ py.test.raises(AssertionError, "equaloplists(loop1.operations, loop3.operations)") - -def test_AbstractOptimization(): - - class MyOpt(AbstractOptimization): - def int_add(self, spec, op): - return 'hello world', op - - class MyOpt2(MyOpt): - def handle_default_op(self, spec, op): - return 'default op', op - - def find_nodes_int_add(self, spec, op): - op.found = 42 - - myopt = MyOpt() - myopt2 = MyOpt2() - op = ResOperation(rop.INT_ADD, [], None) - assert myopt.handle_op(None, op) == ('hello world', op) - assert myopt2.handle_op(None, op) == ('hello world', op) - myopt.find_nodes_for_op(None, op) - assert not hasattr(op, 'found') - myopt2.find_nodes_for_op(None, op) - assert op.found == 42 - - op = ResOperation(rop.INT_SUB, [], None) - assert myopt.handle_op(None, op) == op - assert myopt2.handle_op(None, op) == ('default op', op) - myopt2.find_nodes_for_op(None, op) - assert not hasattr(op, 'found') - - +# ____________________________________________________________ class LLtypeMixin(object): type_system = 'lltype' @@ -103,6 +67,7 @@ nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node)) nodesize = cpu.sizeof(NODE) valuedescr = cpu.fielddescrof(NODE, 'value') + nextdescr = cpu.fielddescrof(NODE, 'next') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE)} namespace = locals() @@ -123,418 +88,108 @@ nodebox = BoxObj(ootype.cast_to_object(node)) nodebox2 = BoxObj(ootype.cast_to_object(node)) valuedescr = cpu.fielddescrof(NODE, 'value') + nextdescr = cpu.fielddescrof(NODE, 'next') nodesize = cpu.typedescrof(NODE) cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE)} namespace = locals() -class BaseTestOptimize3(object): +# ____________________________________________________________ - @staticmethod - def newloop(inputargs, operations): - loop = TreeLoop("test") - loop.inputargs = inputargs - loop.operations = operations - return loop +class BaseTestOptimize(object): def parse(self, s, boxkinds=None): return parse(s, self.cpu, self.namespace, type_system=self.type_system, boxkinds=boxkinds) - def optimize(self, lst, optlist=None): - if not isinstance(lst, TreeLoop): - loop = self.parse(lst) - else: - loop = lst - if optlist is None: - optlist = [] - optimize_loop(None, [], loop, self.cpu, - opt=LoopOptimizer(optlist)) - return loop - def assert_equal(self, optimized, expected): equaloplists(optimized.operations, self.parse(expected).operations) + def find_nodes(self, ops, boxkinds=None): + loop = self.parse(ops, boxkinds=boxkinds) + perfect_specialization_finder.find_nodes(loop) + return loop.getboxes(), perfect_specialization_finder.getnode -class CheckConstFold(BaseTestOptimize3): - - def test_constfold(self): - for op in range(rop.INT_ADD, rop._COMPARISON_FIRST): - try: - op = opname[op] - except KeyError: - continue - ops = """ - [] - i1 = %s(3, 2) - jump() - """ % op.lower() - expected = """ - [] - jump() - """ - self.assert_equal(self.optimize(ops), expected) - - def test_constfold_guard(self): + def test_find_nodes_simple(self): ops = """ - [] - i0 = int_add(0, 0) + [i] + i0 = int_sub(i, 1) guard_value(i0, 0) fail(i0) - jump() - """ - expected = """ - [] - jump() + jump(i0) """ - loop = self.optimize(ops, []) - self.assert_equal(loop, expected) - + boxes, getnode = self.find_nodes(ops) + assert getnode(boxes.i) is not getnode(boxes.i0) -class CheckOptimizeGuards(BaseTestOptimize3): - - def test_remove_guard_class(self): + def test_find_nodes_non_escape(self): ops = """ [p0] - guard_class(p0, ConstClass(node_vtable)) - fail() - guard_class(p0, ConstClass(node_vtable)) - fail() + p1 = getfield_gc(p0, descr=nextdescr) + i0 = getfield_gc(p1, descr=valuedescr) + i1 = int_sub(i0, 1) + p2 = getfield_gc(p0, descr=nextdescr) + setfield_gc(p2, i1, descr=valuedescr) jump(p0) """ - expected = """ + boxes, getnode = self.find_nodes(ops) + assert not getnode(boxes.p0).escaped + assert not getnode(boxes.p1).escaped + assert not getnode(boxes.p2).escaped + + def test_find_nodes_escape(self): + ops = """ [p0] - guard_class(p0, ConstClass(node_vtable)) - fail() + p1 = getfield_gc(p0, descr=nextdescr) + p2 = getfield_gc(p1, descr=nextdescr) + i0 = getfield_gc(p2, descr=valuedescr) + i1 = int_sub(i0, 1) + escape(p1) + p3 = getfield_gc(p0, descr=nextdescr) + setfield_gc(p3, i1, descr=valuedescr) + p4 = getfield_gc(p1, descr=nextdescr) + setfield_gc(p4, i1, descr=valuedescr) jump(p0) """ - loop = self.optimize(ops, [OptimizeGuards()]) - self.assert_equal(loop, expected) - + boxes, getnode = self.find_nodes(ops) + assert not getnode(boxes.p0).escaped + assert getnode(boxes.p1).escaped + assert getnode(boxes.p2).escaped # forced by p1 + assert getnode(boxes.p3).escaped # forced because p3 == p1 + assert getnode(boxes.p4).escaped # forced by p1 - def test_remove_consecutive_guard_value_constfold(self): + def test_find_nodes_new(self): ops = """ - [i0] - guard_value(i0, 0) - fail() - i1 = int_add(i0, 1) - guard_value(i1, 1) - fail() - i2 = int_add(i1, 2) - jump(i0) - """ - expected = """ - [i0] - guard_value(i0, 0) + [sum, p1] + guard_class(p1, ConstClass(node_vtable)) fail() - jump(0) - """ - loop = self.parse(ops) - loop.setvalues(i0 = 0, i1 = 1, i2 = 3) - loop = self.optimize(loop, [OptimizeGuards()]) - self.assert_equal(loop, expected) - - -class BaseVirtualTest(BaseTestOptimize3): - - def getloop(self): - raise NotImplementedError - - def check_find_nodes(self, spec, loop): - pass - - def check_intersect_input_and_output(self, spec, loop): - pass - - def check_optimize_loop(self, opt, loop): - pass - - def test_find_nodes(self): - loop = self.getloop() - spec = LoopSpecializer([OptimizeVirtuals()]) - spec._init(loop) - spec.find_nodes() - self.check_find_nodes(spec, loop) - - def test_intersect_input_and_output(self): - loop = self.getloop() - spec = LoopSpecializer([OptimizeVirtuals()]) - spec._init(loop) - spec.find_nodes() - spec.intersect_input_and_output() - self.check_intersect_input_and_output(spec, loop) - - def test_optimize_loop(self): - loop = self.getloop() - opt = LoopOptimizer([OptimizeVirtuals()]) - opt.optimize_loop(loop, self.cpu) - self.check_optimize_loop(opt, loop) - - -class MallocRemoval(BaseVirtualTest): - - def getloop(self): - ops = """ - [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p0, i0, descr=valuedescr) - i1 = getfield_gc(p0, descr=valuedescr) - i2 = int_add(i1, 1) - jump(i2) - """ - loop = self.parse(ops) - loop.setvalues(i0 = 40, i1 = 40, i2 = 41) - return loop - - def test_find_nodes(self): - pass - - def test_intersect_input_and_output(self): - pass - - def check_optimize_loop(self, opt, loop): - expected = """ - [i0] - i2 = int_add(i0, 1) - jump(i2) - """ - self.assert_equal(loop, expected) - - -class VirtualSimpleLoop(BaseVirtualTest): - - def getloop(self): - ops = """ - [i0, p0] - guard_class(p0, ConstClass(node_vtable)) - fail() - i1 = getfield_gc(p0, descr=valuedescr) + i1 = getfield_gc(p1, descr=valuedescr) i2 = int_sub(i1, 1) - i3 = int_add(i0, i1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p1, i2, descr=valuedescr) - jump(i3, p1) - """ - loop = self.parse(ops) - loop.setvalues(i0 = 0, - p0 = self.nodebox.value, - i1 = 20, - i2 = 19, - i3 = 20, - p1 = self.nodebox2.value) - return loop - - def check_find_nodes(self, spec, loop): - b = loop.getboxes() - assert spec.nodes[b.i0] is not spec.nodes[b.i3] - assert spec.nodes[b.p0] is not spec.nodes[b.p1] - assert spec.nodes[b.p0].known_class.source.value == self.node_vtable_adr - assert not spec.nodes[b.p0].escaped - assert spec.nodes[b.p1].known_class.source.value == self.node_vtable_adr - assert not spec.nodes[b.p1].escaped - - assert len(spec.nodes[b.p0].curfields) == 0 - assert spec.nodes[b.p0].origfields[self.valuedescr] is spec.nodes[b.i1] - assert len(spec.nodes[b.p1].origfields) == 0 - assert spec.nodes[b.p1].curfields[self.valuedescr] is spec.nodes[b.i2] - - def check_intersect_input_and_output(self, spec, loop): - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert isinstance(spec_n, VirtualInstanceSpecNode) - assert spec_n.known_class.value == self.node_vtable_adr - assert spec_n.fields[0][0] == self.valuedescr - assert isinstance(spec_n.fields[0][1], NotSpecNode) - - def check_optimize_loop(self, opt, loop): - expected = """ - [i0, i1] - i2 = int_sub(i1, 1) - i3 = int_add(i0, i1) - jump(i3, i2) - """ - self.assert_equal(loop, expected) - - -class VirtualEscape1(BaseVirtualTest): - - def getloop(self): - ops = """ - [sum, n1] - guard_class(n1, ConstClass(node_vtable)) - fail() - escape(n1) - v = getfield_gc(n1, descr=valuedescr) - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - escape(n2) - jump(sum2, n2) - """ - loop = self.parse(ops, boxkinds={'sum': BoxInt, - 'v': BoxInt, - 'n': BoxPtr}) - loop.setvalues(sum = 0, - n1 = self.nodebox.value, - v = 20, - v2 = 19, - sum2 = 20, - n2 = self.nodebox2.value) - return loop - - def check_find_nodes(self, spec, loop): - b = loop.getboxes() - assert spec.nodes[b.n1].known_class.source.value == self.node_vtable_adr - assert spec.nodes[b.n1].escaped - assert spec.nodes[b.n2].known_class.source.value == self.node_vtable_adr - assert spec.nodes[b.n2].escaped - - def check_intersect_input_and_output(self, spec, loop): - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert type(spec_n) is FixedClassSpecNode - assert spec_n.known_class.value == self.node_vtable_adr - - def check_optimize_loop(self, opt, loop): - expected = """ - [sum, n1] - escape(n1) - v = getfield_gc(n1, descr=valuedescr) - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - escape(n2) - jump(sum2, n2) - """ - self.assert_equal(loop, expected) - - -class VirtualEscape2(BaseVirtualTest): - - def getloop(self): - ops = """ - [sum, n1] - guard_class(n1, ConstClass(node_vtable)) - fail() - escape(n1) - v = getfield_gc(n1, descr=valuedescr) - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - escape(n1) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - jump(sum2, n2) - """ - loop = self.parse(ops, boxkinds={'sum': BoxInt, - 'v': BoxInt, - 'n': BoxPtr}) - loop.setvalues(sum = 0, - n1 = self.nodebox.value, - v = 20, - v2 = 19, - sum2 = 20, - n2 = self.nodebox2.value) - return loop - - def check_find_nodes(self, spec, loop): - b = loop.getboxes() - assert spec.nodes[b.n1].known_class.source.value == self.node_vtable_adr - assert spec.nodes[b.n1].escaped - assert spec.nodes[b.n2].known_class.source.value == self.node_vtable_adr - - def check_intersect_input_and_output(self, spec, loop): - b = loop.getboxes() - assert spec.nodes[b.n2].escaped - assert len(spec.specnodes) == 2 - spec_sum, spec_n = spec.specnodes - assert isinstance(spec_sum, NotSpecNode) - assert type(spec_n) is FixedClassSpecNode - assert spec_n.known_class.value == self.node_vtable_adr - - def check_optimize_loop(self, opt, loop): - expected = """ - [sum, n1] - escape(n1) - v = getfield_gc(n1, descr=valuedescr) - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - escape(n1) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - jump(sum2, n2) - """ - self.assert_equal(loop, expected) - - -class RebuildOps(BaseVirtualTest): - - def getloop(self): - ops = """ - [sum, n1] - guard_class(n1, ConstClass(node_vtable)) - fail() - v = getfield_gc(n1, descr=valuedescr) - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - guard_true(v2) - fail(sum2, n2) - jump(sum2, n2) - """ - loop = self.parse(ops, boxkinds={'sum': BoxInt, - 'v': BoxInt, - 'n': BoxPtr}) - loop.setvalues(sum = 0, - n1 = self.nodebox.value, - v = 20, - v2 = 19, - sum2 = 20, - n2 = self.nodebox2.value) - return loop - - def test_find_nodes(self): - pass - - def test_intersect_input_and_output(self): - pass - - def check_optimize_loop(self, opt, loop): - expected = """ - [sum, v] - v2 = int_sub(v, 1) - sum2 = int_add(sum, v) - guard_true(v2) - n2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(n2, v2, descr=valuedescr) - fail(sum2, n2) - jump(sum2, v2) - """ - self.assert_equal(loop, expected) - - - + sum2 = int_add(sum, i1) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p2, descr=nextdescr) + jump(sum2, p2) + """ + boxes, getnode = self.find_nodes(ops, boxkinds={'sum': BoxInt, + 'sum2': BoxInt}) + assert getnode(boxes.sum) is not getnode(boxes.sum2) + assert getnode(boxes.p1) is not getnode(boxes.p2) + + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert not boxp1.escaped + assert not boxp2.escaped + + assert not boxp1.curfields + assert boxp1.origfields[self.valuedescr] is getnode(boxes.i1) + assert not boxp2.origfields + assert boxp2.curfields[self.nextdescr] is boxp2 -def create_tests(ns): - for name, value in ns.items(): - if (isinstance(value, type) and - issubclass(value, BaseTestOptimize3) and - not name.startswith('Base')): - - src = py.code.Source(""" - class Test%(name)sLLtype(%(name)s, LLtypeMixin, BaseTestOptimize3): - pass - class Test%(name)sOOtype(%(name)s, OOtypeMixin, BaseTestOptimize3): - pass - """ % {'name': name}) - exec src.compile() in ns - +class TestLLtype(BaseTestOptimize, LLtypeMixin): + pass -create_tests(globals()) +class TestOOtype(BaseTestOptimize, OOtypeMixin): + pass From arigo at codespeak.net Thu Jul 16 10:39:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 16 Jul 2009 10:39:33 +0200 (CEST) Subject: [pypy-svn] r66276 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090716083933.31513169F3F@codespeak.net> Author: arigo Date: Thu Jul 16 10:39:32 2009 New Revision: 66276 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Rename 'startbox' into 'fromstart' as it is just a flag. Add tests for it. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Thu Jul 16 10:39:32 2009 @@ -32,9 +32,9 @@ curfields = None # optimization; equivalent to an empty dict dependencies = None - def __init__(self, escaped, startbox=False): + def __init__(self, escaped, fromstart=False): self.escaped = escaped - self.startbox = startbox + self.fromstart = fromstart def add_escape_dependency(self, other): assert not self.escaped @@ -54,8 +54,8 @@ def __repr__(self): flags = '' - if self.escaped: flags += 'e' - if self.startbox: flags += 's' + if self.escaped: flags += 'e' + if self.fromstart: flags += 's' return "" % (flags,) @@ -74,7 +74,7 @@ def find_nodes(self, loop): self.clear() for box in loop.inputargs: - self.nodes[box] = InstanceNode(escaped=False, startbox=True) + self.nodes[box] = InstanceNode(escaped=False, fromstart=True) # for op in loop.operations: opnum = op.opnum @@ -117,7 +117,8 @@ elif instnode.origfields is not None and field in instnode.origfields: fieldnode = instnode.origfields[field] else: - fieldnode = InstanceNode(escaped=False, startbox=instnode.startbox) + fieldnode = InstanceNode(escaped=False, + fromstart=instnode.fromstart) instnode.add_escape_dependency(fieldnode) if instnode.origfields is None: instnode.origfields = av_newdict() @@ -133,6 +134,9 @@ def find_nodes_JUMP(self, op): pass # prevent the default handling + def find_nodes_FAIL(self, op): + pass # prevent the default handling + find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') perfect_specialization_finder = PerfectSpecializationFinder() Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Thu Jul 16 10:39:32 2009 @@ -121,7 +121,8 @@ jump(i0) """ boxes, getnode = self.find_nodes(ops) - assert getnode(boxes.i) is not getnode(boxes.i0) + assert getnode(boxes.i).fromstart + assert not getnode(boxes.i0).fromstart def test_find_nodes_non_escape(self): ops = """ @@ -137,6 +138,9 @@ assert not getnode(boxes.p0).escaped assert not getnode(boxes.p1).escaped assert not getnode(boxes.p2).escaped + assert getnode(boxes.p0).fromstart + assert getnode(boxes.p1).fromstart + assert getnode(boxes.p2).fromstart def test_find_nodes_escape(self): ops = """ @@ -158,6 +162,11 @@ assert getnode(boxes.p2).escaped # forced by p1 assert getnode(boxes.p3).escaped # forced because p3 == p1 assert getnode(boxes.p4).escaped # forced by p1 + assert getnode(boxes.p0).fromstart + assert getnode(boxes.p1).fromstart + assert getnode(boxes.p2).fromstart + assert getnode(boxes.p3).fromstart + assert not getnode(boxes.p4).fromstart def test_find_nodes_new(self): ops = """ @@ -187,6 +196,9 @@ assert not boxp2.origfields assert boxp2.curfields[self.nextdescr] is boxp2 + assert boxp1.fromstart + assert not boxp2.fromstart + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From antocuni at codespeak.net Thu Jul 16 11:36:22 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 16 Jul 2009 11:36:22 +0200 (CEST) Subject: [pypy-svn] r66277 - pypy/branch/pyjitpl5/pypy/jit/backend/cli Message-ID: <20090716093622.4A091169F2D@codespeak.net> Author: antocuni Date: Thu Jul 16 11:36:20 2009 New Revision: 66277 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Log: fix cli tests: they were failing because render_op was unable to automatically generate this method Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Thu Jul 16 11:36:20 2009 @@ -497,6 +497,9 @@ self.il.Emit(OpCodes.Cgt_Un) self.store_result(op) + def emit_op_subclassof(self, op): + raise NotImplementedError + def emit_op_ooidentityhash(self, op): raise NotImplementedError From fijal at codespeak.net Thu Jul 16 13:49:10 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 16 Jul 2009 13:49:10 +0200 (CEST) Subject: [pypy-svn] r66278 - in pypy/branch/pyjitpl5/pypy/module/micronumpy: . test Message-ID: <20090716114910.404E81683BA@codespeak.net> Author: fijal Date: Thu Jul 16 13:49:08 2009 New Revision: 66278 Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Log: Support a couple more operations Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Thu Jul 16 13:49:08 2009 @@ -12,12 +12,31 @@ self.storage = [0] * dim[0] def descr_getitem(self, space, index): - return space.wrap(self.storage[index]) + try: + return space.wrap(self.storage[index]) + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) descr_getitem.unwrap_spec = ['self', ObjSpace, int] + def descr_setitem(self, space, index, value): + try: + self.storage[index] = value + except IndexError: + raise OperationError(space.w_IndexError, + space.wrap("list index out of range")) + return space.w_None + descr_setitem.unwrap_spec = ['self', ObjSpace, int, int] + + def descr_len(self, space): + return space.wrap(len(self.storage)) + descr_len.unwrap_spec = ['self', ObjSpace] + NumArray.typedef = TypeDef( 'NumArray', __getitem__ = interp2app(NumArray.descr_getitem), + __setitem__ = interp2app(NumArray.descr_setitem), + __len__ = interp2app(NumArray.descr_len), ) def unpack_dim(space, w_dim): Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Thu Jul 16 13:49:08 2009 @@ -10,3 +10,14 @@ ar = zeros(3, dtype=int) assert ar[0] == 0 + def test_setitem_getitem(self): + from numpy import zeros + ar = zeros(8, dtype=int) + assert ar[0] == 0 + ar[1] = 3 + assert ar[1] == 3 + raises((TypeError, ValueError), ar.__getitem__, 'xyz') + raises(IndexError, ar.__getitem__, 38) + assert ar[-2] == 0 + assert ar[-7] == 3 + assert len(ar) == 8 From fijal at codespeak.net Thu Jul 16 13:51:46 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 16 Jul 2009 13:51:46 +0200 (CEST) Subject: [pypy-svn] r66279 - pypy/branch/pyjitpl5/pypy/doc/config Message-ID: <20090716115146.2FD121683D7@codespeak.net> Author: fijal Date: Thu Jul 16 13:51:45 2009 New Revision: 66279 Added: pypy/branch/pyjitpl5/pypy/doc/config/objspace.usemodules.micronumpy.txt (contents, props changed) Log: Provide docstring for this module Added: pypy/branch/pyjitpl5/pypy/doc/config/objspace.usemodules.micronumpy.txt ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/doc/config/objspace.usemodules.micronumpy.txt Thu Jul 16 13:51:45 2009 @@ -0,0 +1,3 @@ +Use the micronumpy module. +This module provides a very basic numpy-like interface. Major use-case +is to show how jit scales for other code. From fijal at codespeak.net Thu Jul 16 14:09:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 16 Jul 2009 14:09:32 +0200 (CEST) Subject: [pypy-svn] r66280 - pypy/branch/pyjitpl5/pypy/module/pypyjit Message-ID: <20090716120932.0C6FC168557@codespeak.net> Author: fijal Date: Thu Jul 16 14:09:31 2009 New Revision: 66280 Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py Log: Enable micronumpy to be seen by the jit Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py Thu Jul 16 14:09:31 2009 @@ -46,8 +46,10 @@ if mod.startswith('pypy.interpreter.pyparser.'): return False if mod.startswith('pypy.module.'): - if not mod.startswith('pypy.module.pypyjit.'): + if (not mod.startswith('pypy.module.pypyjit.') or + not mod.startswith('pypy.module.micronumpy.')): return False + if mod.startswith('pypy.translator.'): return False # string builder interface From antocuni at codespeak.net Thu Jul 16 15:13:46 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 16 Jul 2009 15:13:46 +0200 (CEST) Subject: [pypy-svn] r66281 - in pypy/branch/pyjitpl5/pypy/jit: backend backend/cli backend/llgraph metainterp metainterp/test Message-ID: <20090716131346.E8389169EBC@codespeak.net> Author: antocuni Date: Thu Jul 16 15:13:45 2009 New Revision: 66281 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/model.py pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: - hack test_subclassof until it really emits a subclassof operation and can be run by zrpy_* tests - move do_subclassof to executor.py, as its implementation is backend indipendent - implement llimpl.op_subclassof for the llgraph backend - implement emit_op_subclassof for the cli backend Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Thu Jul 16 15:13:45 2009 @@ -20,6 +20,7 @@ DelegateHolder = CLR.pypy.runtime.DelegateHolder InputArgs = CLR.pypy.runtime.InputArgs ListOfVoid = CLR.pypy.runtime.ListOfVoid +Utils = CLR.pypy.runtime.Utils cVoid = ootype.nullruntimeclass @@ -498,7 +499,12 @@ self.store_result(op) def emit_op_subclassof(self, op): - raise NotImplementedError + clitype_utils = dotnet.typeof(Utils) + methinfo = clitype_utils.GetMethod('SubclassOf') + op.args[0].load(self) + op.args[1].load(self) + self.il.Emit(OpCodes.Call, methinfo) + self.store_result(op) def emit_op_ooidentityhash(self, op): raise NotImplementedError Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Thu Jul 16 15:13:45 2009 @@ -768,8 +768,10 @@ inst = ootype.cast_from_object(ootype.ROOT, obj) return ootype.instanceof(inst, typedescr.TYPE) - def op_subclassof(self, *args): - raise NotImplementedError + def op_subclassof(self, _, obj1, obj2): + cls1 = ootype.cast_from_object(ootype.Class, obj1) + cls2 = ootype.cast_from_object(ootype.Class, obj2) + return ootype.subclassof(cls1, cls2) def _cast_exception(self, exception): return ootype.cast_from_object(ootype.Class, exception) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Thu Jul 16 15:13:45 2009 @@ -469,14 +469,6 @@ assert len(args) == 1 return typedescr.instanceof(args[0]) - def do_subclassof(self, args, descr): - assert len(args) == 2 - box1, box2 = args - cls1 = ootype.cast_from_object(ootype.Class, box1.getobj()) - cls2 = ootype.cast_from_object(ootype.Class, box2.getobj()) - res = ootype.subclassof(cls1, cls2) - return history.BoxInt(res) - def do_getfield_gc(self, args, fielddescr): assert isinstance(fielddescr, FieldDescr) return fielddescr.getfield(args[0]) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/model.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/model.py Thu Jul 16 15:13:45 2009 @@ -171,6 +171,3 @@ def do_instanceof(self, args, descr=None): raise NotImplementedError - - def do_subclassof(self, args, descr=None): - raise NotImplementedError Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py Thu Jul 16 15:13:45 2009 @@ -149,6 +149,16 @@ obj = args[0].getobj() return ConstInt(ootype.ooidentityhash(obj)) + +def do_subclassof(self, args, descr=None): + assert len(args) == 2 + box1, box2 = args + cls1 = ootype.cast_from_object(ootype.Class, box1.getobj()) + cls2 = ootype.cast_from_object(ootype.Class, box2.getobj()) + res = ootype.subclassof(cls1, cls2) + return BoxInt(res) + + # ---------- # the following operations just delegate to the cpu: Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Thu Jul 16 15:13:45 2009 @@ -782,22 +782,38 @@ def test_subclassof(self): + from pypy.jit.metainterp import simple_optimize A = ootype.Instance("A", ootype.ROOT) B = ootype.Instance("B", A) clsA = ootype.runtimeClass(A) clsB = ootype.runtimeClass(B) - def f(n): - if n: - obj = ootype.ooupcast(A, ootype.new(B)) + myjitdriver = JitDriver(greens = [], reds = ['n', 'flag', 'res']) + + def getcls(flag): + if flag: + return clsA else: - obj = ootype.new(A) - cls = ootype.classof(obj) - return ootype.subclassof(cls, clsB) + return clsB - res = self.interp_operations(f, [True]) - assert res - res = self.interp_operations(f, [False]) + def f(flag, n): + res = True + while n > -100: + myjitdriver.can_enter_jit(n=n, flag=flag, res=res) + myjitdriver.jit_merge_point(n=n, flag=flag, res=res) + cls = getcls(flag) + n -= 1 + res = ootype.subclassof(cls, clsB) + return res + + res = self.meta_interp(f, [1, 100], + policy=StopAtXPolicy(getcls), + optimizer=simple_optimize) assert not res + + res = self.meta_interp(f, [0, 100], + policy=StopAtXPolicy(getcls), + optimizer=simple_optimize) + assert res From fijal at codespeak.net Thu Jul 16 15:22:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 16 Jul 2009 15:22:14 +0200 (CEST) Subject: [pypy-svn] r66282 - in pypy/branch/pyjitpl5/pypy/module/micronumpy: . test Message-ID: <20090716132214.34F31168459@codespeak.net> Author: fijal Date: Thu Jul 16 15:22:13 2009 New Revision: 66282 Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/ufunc.py (contents, props changed) Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Log: implement minimum Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/__init__.py Thu Jul 16 15:22:13 2009 @@ -7,6 +7,7 @@ interpleveldefs = { 'zeros' : 'numarray.zeros', + 'minimum' : 'ufunc.minimum', } appleveldefs = {} Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Thu Jul 16 15:22:13 2009 @@ -21,3 +21,21 @@ assert ar[-2] == 0 assert ar[-7] == 3 assert len(ar) == 8 + + def test_minimum(self): + from numpy import zeros, minimum + ar = zeros(5, dtype=int) + ar2 = zeros(5, dtype=int) + ar[0] = 3 + ar[1] = -3 + ar[2] = 8 + ar2[3] = -1 + ar2[4] = 8 + x = minimum(ar, ar2) + assert x[0] == 0 + assert x[1] == -3 + assert x[2] == 0 + assert x[3] == -1 + assert x[4] == 0 + assert len(x) == 5 + raises(ValueError, minimum, ar, zeros(3, dtype=int)) Added: pypy/branch/pyjitpl5/pypy/module/micronumpy/ufunc.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/ufunc.py Thu Jul 16 15:22:13 2009 @@ -0,0 +1,22 @@ + +from numarray import NumArray +from pypy.interpreter.baseobjspace import ObjSpace, W_Root +from pypy.interpreter.error import OperationError + +def minimum(space, w_a, w_b): + if not isinstance(w_a, NumArray) or not isinstance(w_b, NumArray): + raise OperationError(space.w_TypeError, + space.wrap("expecting NumArrat object")) + if w_a.dim != w_b.dim: + raise OperationError(space.w_ValueError, + space.wrap("minimum of arrays of different length")) + res = NumArray(space, w_a.dim, 'i') + for i in range(len(w_a.storage)): + one = w_a.storage[i] + two = w_b.storage[i] + if one < two: + res.storage[i] = one + else: + res.storage[i] = two + return space.wrap(res) +minimum.unwrap_spec = [ObjSpace, W_Root, W_Root] From fijal at codespeak.net Thu Jul 16 15:24:05 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 16 Jul 2009 15:24:05 +0200 (CEST) Subject: [pypy-svn] r66283 - pypy/branch/pyjitpl5/pypy/module/pypyjit Message-ID: <20090716132405.F15F1168541@codespeak.net> Author: fijal Date: Thu Jul 16 15:24:05 2009 New Revision: 66283 Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py Log: usuall problems with negating boolean properties... Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/pyjitpl5/pypy/module/pypyjit/policy.py Thu Jul 16 15:24:05 2009 @@ -46,7 +46,7 @@ if mod.startswith('pypy.interpreter.pyparser.'): return False if mod.startswith('pypy.module.'): - if (not mod.startswith('pypy.module.pypyjit.') or + if (not mod.startswith('pypy.module.pypyjit.') and not mod.startswith('pypy.module.micronumpy.')): return False From benjamin at codespeak.net Thu Jul 16 15:35:23 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 15:35:23 +0200 (CEST) Subject: [pypy-svn] r66285 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716133523.70889169E30@codespeak.net> Author: benjamin Date: Thu Jul 16 15:35:22 2009 New Revision: 66285 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: fix import Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Thu Jul 16 15:35:22 2009 @@ -1,5 +1,5 @@ from pypy.interpreter import gateway -from pypy.interpreter.astcompiler import ast +from pypy.interpreter.astcompiler import ast2 as ast app = gateway.applevel(""" From benjamin at codespeak.net Thu Jul 16 16:06:32 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 16:06:32 +0200 (CEST) Subject: [pypy-svn] r66289 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090716140632.9E5A0169F1A@codespeak.net> Author: benjamin Date: Thu Jul 16 16:06:31 2009 New Revision: 66289 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: fix bug with yield and return detection Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Thu Jul 16 16:06:31 2009 @@ -193,10 +193,11 @@ self.is_generator = True def note_return(self, ret): - if self.is_generator and ret.value: - raise SyntaxError("return with value in generator", ret.lineno, - ret.col_offset) - self.return_with_value = True + if ret.value: + if self.is_generator: + raise SyntaxError("return with value in generator", ret.lineno, + ret.col_offset) + self.return_with_value = True def note_exec(self, exc): Scope.note_exec(self, exc) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Thu Jul 16 16:06:31 2009 @@ -312,6 +312,7 @@ input = "def f():\n " + input exc = py.test.raises(SyntaxError, self.func_scope, input).value assert exc.msg == "return with value in generator" + scp = self.func_scope("def f():\n return\n yield x") def test_return(self): for input in ("class x: return", "return"): From benjamin at codespeak.net Thu Jul 16 16:06:55 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 16:06:55 +0200 (CEST) Subject: [pypy-svn] r66290 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090716140655.797AC169F1A@codespeak.net> Author: benjamin Date: Thu Jul 16 16:06:55 2009 New Revision: 66290 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: any exec stops optimization Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Thu Jul 16 16:06:55 2009 @@ -298,7 +298,7 @@ assert scp.has_exec for line in ("exec 'hi' in g", "exec 'hi' in g, h"): scp = self.func_scope("def f(): " + line) - assert scp.optimized + assert not scp.optimized assert scp.bare_exec is None assert scp.has_exec From benjamin at codespeak.net Thu Jul 16 16:42:30 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 16:42:30 +0200 (CEST) Subject: [pypy-svn] r66291 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716144230.99162168459@codespeak.net> Author: benjamin Date: Thu Jul 16 16:42:29 2009 New Revision: 66291 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: implement CALL_LIKELY_BUILTIN and CALL_METHOD Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Thu Jul 16 16:42:29 2009 @@ -402,6 +402,8 @@ ops.BUILD_MAP : 1, ops.COMPARE_OP : -1, + ops.LOOKUP_METHOD : 1, + ops.LOAD_NAME : 1, ops.STORE_NAME : -1, ops.DELETE_NAME : 0, @@ -477,6 +479,12 @@ def _compute_CALL_FUNCTION_VAR_KW(arg): return _num_args(arg) - 2 +def _compute_CALL_LIKELY_BUILTIN(arg): + return -(arg & 0xFF) + 1 + +def _compute_CALL_METHOD(arg): + return -arg - 1 + _stack_effect_computers = {} for name, func in globals().items(): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 16:42:29 2009 @@ -6,6 +6,7 @@ consts, misc) from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops +from pypy.module.__builtin__.__init__ import BUILTIN_TO_INDEX def compile_ast(space, module, info): @@ -794,6 +795,9 @@ def visit_Call(self, call): self.update_position(call) + if self._optimize_builtin_call(call) or \ + self._optimize_method_call(call): + return call.func.walkabout(self) arg = 0 call_type = 0 @@ -819,6 +823,43 @@ op = ops.CALL_FUNCTION_VAR_KW self.emit_op_arg(op, arg) + def _call_has_simple_args(self, call): + return not call.starargs and not call.kwargs and not call.keywords + + def _optimize_builtin_call(self, call): + if not self.space.config.objspace.opcodes.CALL_LIKELY_BUILTIN or \ + not self._call_has_simple_args(call) or \ + not isinstance(call.func, ast.Name): + return False + name_scope = self.scope.lookup(call.func.id) + if name_scope == symtable.SCOPE_GLOBAL_IMPLICIT or \ + name_scope == symtable.SCOPE_UNKNOWN: + builtin_index = BUILTIN_TO_INDEX.get(call.func.id, -1) + if builtin_index != -1: + if call.args: + args_count = len(call.args) + self.visit_sequence(call.args) + else: + args_count = 0 + arg = builtin_index << 8 | args_count + self.emit_op_arg(ops.CALL_LIKELY_BUILTIN, arg) + return True + return False + + def _optimize_method_call(self, call): + if not self.space.config.objspace.opcodes.CALL_METHOD or \ + not self._call_has_simple_args(call) or \ + not isinstance(call.func, ast.Attribute): + return False + call.func.value.walkabout(self) + self.emit_op_name(ops.LOOKUP_METHOD, self.names, call.func.attr) + if call.args: + self.visit_sequence(call.args) + arg_count = len(call.args) + else: + arg_count = 0 + self.emit_op_arg(ops.CALL_METHOD, arg_count) + def _listcomp_generator(self, list_name, gens, gen_index, elt): start = self.new_block() skip = self.new_block() From cfbolz at codespeak.net Thu Jul 16 16:47:29 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 16 Jul 2009 16:47:29 +0200 (CEST) Subject: [pypy-svn] r66292 - in pypy/trunk/pypy/rlib/rsre: . test Message-ID: <20090716144729.552A0169E88@codespeak.net> Author: cfbolz Date: Thu Jul 16 16:47:28 2009 New Revision: 66292 Modified: pypy/trunk/pypy/rlib/rsre/rsre.py pypy/trunk/pypy/rlib/rsre/test/test_rsre.py Log: Make rsre more generally useful (patch by Trundle, thanks a lot) Modified: pypy/trunk/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/trunk/pypy/rlib/rsre/rsre.py (original) +++ pypy/trunk/pypy/rlib/rsre/rsre.py Thu Jul 16 16:47:28 2009 @@ -12,6 +12,7 @@ rsre_core_filename = rsre_core_filename[:-1] rsre_core_filename = os.path.abspath(rsre_core_filename) del rsre_core +from pypy.rlib.rsre.rsre_char import getlower def insert_sre_methods(locals, name): """A hack that inserts the SRE entry point methods into the 'locals' @@ -58,3 +59,6 @@ def get_char_ord(self, p): return ord(self.string[p]) + + def lower(self, char_ord): + return getlower(char_ord, 0) Modified: pypy/trunk/pypy/rlib/rsre/test/test_rsre.py ============================================================================== --- pypy/trunk/pypy/rlib/rsre/test/test_rsre.py (original) +++ pypy/trunk/pypy/rlib/rsre/test/test_rsre.py Thu Jul 16 16:47:28 2009 @@ -80,6 +80,11 @@ def test_getlower(): assert rsre_char.getlower(ord("A"), 0) == ord("a") +def test_SimpleStringState(): + state = SimpleStringState("A", 0, -1) + assert state.get_char_ord(0) == ord("A") + assert state.lower(state.get_char_ord(0)) == ord("a") + def test_get_byte_array(): if sys.byteorder == "big": if rsre_char.CODESIZE == 2: From benjamin at codespeak.net Thu Jul 16 16:55:05 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 16:55:05 +0200 (CEST) Subject: [pypy-svn] r66293 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716145505.8F832169ED3@codespeak.net> Author: benjamin Date: Thu Jul 16 16:55:02 2009 New Revision: 66293 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: fix division handling Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 16:55:02 2009 @@ -56,7 +56,7 @@ ast.BitOr : ops.BINARY_OR, ast.BitAnd : ops.BINARY_AND, ast.BitXor : ops.BINARY_XOR, - ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE + ast.FloorDiv : ops.BINARY_FLOOR_DIVIDE } inplace_operations = { @@ -264,7 +264,7 @@ if self.compile_info.flags & consts.CO_FUTURE_DIVISION: return ops.INPLACE_TRUE_DIVIDE else: - return ops.INPLACE_FLOOR_DIVIDE + return ops.INPLACE_DIVIDE return inplace_operations[op] def visit_AugAssign(self, assign): @@ -314,7 +314,7 @@ if self.compile_info.flags & consts.CO_FUTURE_DIVISION: return ops.BINARY_TRUE_DIVIDE else: - return ops.BINARY_FLOOR_DIVIDE + return ops.BINARY_DIVIDE return binary_operations[op] def visit_BinOp(self, binop): From benjamin at codespeak.net Thu Jul 16 17:37:48 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 17:37:48 +0200 (CEST) Subject: [pypy-svn] r66294 - in pypy/branch/parser-compiler/pypy: interpreter interpreter/astcompiler interpreter/pyparser interpreter/pyparser/test module/__builtin__ Message-ID: <20090716153748.3FB03169E30@codespeak.net> Author: benjamin Date: Thu Jul 16 17:37:47 2009 New Revision: 66294 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/consts.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py pypy/branch/parser-compiler/pypy/module/__builtin__/compiling.py Log: indicate with a parser flag if the source is unicode This also moves compiling to astcompiler.consts. Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/consts.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/consts.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/consts.py Thu Jul 16 17:37:47 2009 @@ -22,3 +22,6 @@ CO_FUTURE_DIVISION = 0x2000 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 CO_FUTURE_WITH_STATEMENT = 0x8000 + +PyCF_SOURCE_IS_UTF8 = 0x0100 +PyCF_DONT_IMPLY_DEDENT = 0x0200 Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Thu Jul 16 17:37:47 2009 @@ -4,7 +4,7 @@ """ import sys -from codeop import PyCF_DONT_IMPLY_DEDENT +from pypy.interpreter.astcompiler.consts import PyCF_DONT_IMPLY_DEDENT from pypy.interpreter.error import OperationError class AbstractCompiler(object): Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Thu Jul 16 17:37:47 2009 @@ -1,8 +1,7 @@ -import codeop from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.interpreter.pyparser import parser, pytokenizer, pygram, error -from pypy.interpreter.astcompiler.consts import CO_FUTURE_WITH_STATEMENT +from pypy.interpreter.astcompiler import consts _recode_to_utf8 = gateway.applevel(r''' @@ -82,13 +81,18 @@ """Parse a python source according to goal""" # Detect source encoding. enc = None - if textsrc[:3] == '\xEF\xBB\xBF': + if textsrc.startswith("\xEF\xBB\xBF"): textsrc = textsrc[3:] enc = 'utf-8' - # check that there is no explicit encoding declared + # If an encoding is explicitly given check that it is utf-8. decl_enc = _check_for_encoding(textsrc) - if decl_enc is not None: - raise SyntaxError("encoding declaration in Unicode string") + if decl_enc and decl_enc != "utf-8": + raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie") + elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8: + enc = 'utf-8' + + if _check_for_encoding(textsrc) is not None: + raise error.SyntaxError("coding declaration in unicode string") else: enc = _normalize_encoding(_check_for_encoding(textsrc)) if enc is not None and enc not in ('utf-8', 'iso-8859-1'): @@ -104,7 +108,7 @@ raise flags = compile_info.flags - if flags & CO_FUTURE_WITH_STATEMENT: + if flags & consts.CO_FUTURE_WITH_STATEMENT: self.grammar = pygram.python_grammar else: self.grammar = pygram.python_grammar_no_with_statement @@ -112,7 +116,7 @@ if source_lines and not source_lines[-1].endswith("\n"): source_lines[-1] += '\n' if textsrc and textsrc[-1] == "\n": - flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT + flags &= ~consts.PyCF_DONT_IMPLY_DEDENT self.prepare(_targets[compile_info.mode]) try: Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Thu Jul 16 17:37:47 2009 @@ -1,10 +1,10 @@ -import codeop from pypy.interpreter.pyparser import automata from pypy.interpreter.pyparser.pygram import tokens from pypy.interpreter.pyparser.pytoken import python_opmap from pypy.interpreter.pyparser.error import TokenError, TokenIndentationError from pypy.interpreter.pyparser.pytokenize import tabsize, whiteSpaceDFA, \ triple_quoted, endDFAs, single_quoted, pseudoDFA +from pypy.interpreter.astcompiler import consts NAMECHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_' NUMCHARS = '0123456789' @@ -231,7 +231,7 @@ pos = pos + 1 lnum -= 1 - if not (flags & codeop.PyCF_DONT_IMPLY_DEDENT): + if not (flags & consts.PyCF_DONT_IMPLY_DEDENT): if token_list and token_list[-1][0] != tokens.NEWLINE: tok = (tokens.NEWLINE, '', lnum, 0, '\n') token_list.append(tok) Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Thu Jul 16 17:37:47 2009 @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import codeop import py from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError, IndentationError +from pypy.interpreter.astcompiler import consts class TestPythonParser: @@ -22,7 +22,7 @@ def test_dont_imply_dedent(self): info = pyparse.CompileInfo("", "single", - codeop.PyCF_DONT_IMPLY_DEDENT) + consts.PyCF_DONT_IMPLY_DEDENT) self.parse('if 1:\n x\n', info=info) self.parse('x = 5 ', info=info) @@ -42,6 +42,17 @@ input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") tree = self.parse(input, info=info) assert info.encoding == "utf-7" + input = "\xEF\xBB\xBF# coding: utf-8\nx" + self.parse(input, info=info) + assert info.encoding == "utf-8" + input = "# coding: utf-8\nx" + info.flags |= consts.PyCF_SOURCE_IS_UTF8 + exc = py.test.raises(SyntaxError, self.parse, input, info=info).value + info.flags &= ~consts.PyCF_SOURCE_IS_UTF8 + assert exc.msg == "coding declaration in unicode string" + input = "\xEF\xBB\xBF# coding: latin-1\nx" + exc = py.test.raises(SyntaxError, self.parse, input).value + assert exc.msg == "UTF-8 BOM with non-utf8 coding cookie" input = "# coding: not-here" exc = py.test.raises(SyntaxError, self.parse, input).value assert exc.msg == "Unknown encoding: not-here" Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/compiling.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/compiling.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/compiling.py Thu Jul 16 17:37:47 2009 @@ -4,7 +4,8 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.baseobjspace import W_Root, ObjSpace -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError +from pypy.interpreter.astcompiler import consts from pypy.interpreter.gateway import NoneNotWrapped def compile(space, w_source, filename, mode, flags=0, dont_inherit=0): @@ -21,11 +22,11 @@ in addition to any features explicitly specified. """ if space.is_true(space.isinstance(w_source, space.w_unicode)): - # hack: encode the unicode string as UTF-8 and attach - # a BOM at the start - w_source = space.call_method(w_source, 'encode', space.wrap('utf-8')) - str_ = space.str_w(w_source) - str_ = '\xEF\xBB\xBF' + str_ + w_utf_8_source = space.call_method(w_source, "encode", + space.wrap("utf-8")) + str_ = space.str_w(w_utf_8_source) + # This flag tells the parser to reject any coding cookies it sees. + flags |= consts.PyCF_SOURCE_IS_UTF8 else: str_ = space.str_w(w_source) From benjamin at codespeak.net Thu Jul 16 17:48:43 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 17:48:43 +0200 (CEST) Subject: [pypy-svn] r66295 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716154843.44899168066@codespeak.net> Author: benjamin Date: Thu Jul 16 17:48:42 2009 New Revision: 66295 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: when there is no module name, the interpreter should get an empty string Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 17:48:42 2009 @@ -569,7 +569,11 @@ self.load_const(space.wrap(level)) names_w = [space.wrap(alias.name) for alias in imp.names] self.load_const(space.newtuple(names_w)) - self.emit_op_name(ops.IMPORT_NAME, self.names, imp.module) + if imp.module: + mod_name = imp.module + else: + mod_name = "" + self.emit_op_name(ops.IMPORT_NAME, self.names, mod_name) if len(imp.names) == 1 and imp.names[0].name == "*": self.emit_op(ops.IMPORT_STAR) else: From benjamin at codespeak.net Thu Jul 16 18:27:39 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 18:27:39 +0200 (CEST) Subject: [pypy-svn] r66296 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090716162739.A8A82169E7D@codespeak.net> Author: benjamin Date: Thu Jul 16 18:27:38 2009 New Revision: 66296 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: don't regular call code if we use CALL_METHOD Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 16 18:27:38 2009 @@ -863,6 +863,7 @@ else: arg_count = 0 self.emit_op_arg(ops.CALL_METHOD, arg_count) + return True def _listcomp_generator(self, list_name, gens, gen_index, elt): start = self.new_block() From benjamin at codespeak.net Thu Jul 16 19:12:54 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 16 Jul 2009 19:12:54 +0200 (CEST) Subject: [pypy-svn] r66297 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090716171254.1771F169E0D@codespeak.net> Author: benjamin Date: Thu Jul 16 19:12:54 2009 New Revision: 66297 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: compile_info is not optional Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Thu Jul 16 19:12:54 2009 @@ -77,7 +77,7 @@ parser.Parser.__init__(self, grammar) self.space = space - def parse_source(self, textsrc, compile_info=None): + def parse_source(self, textsrc, compile_info): """Parse a python source according to goal""" # Detect source encoding. enc = None From benjamin at codespeak.net Fri Jul 17 00:53:01 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 00:53:01 +0200 (CEST) Subject: [pypy-svn] r66298 - in pypy/branch/parser-compiler/pypy: config module/parser module/parser/test module/recparser Message-ID: <20090716225301.73E46169F12@codespeak.net> Author: benjamin Date: Fri Jul 17 00:53:00 2009 New Revision: 66298 Added: pypy/branch/parser-compiler/pypy/module/parser/ pypy/branch/parser-compiler/pypy/module/parser/__init__.py (contents, props changed) pypy/branch/parser-compiler/pypy/module/parser/pyparser.py (contents, props changed) pypy/branch/parser-compiler/pypy/module/parser/test/ pypy/branch/parser-compiler/pypy/module/parser/test/__init__.py (contents, props changed) pypy/branch/parser-compiler/pypy/module/parser/test/test_parser.py (contents, props changed) Removed: pypy/branch/parser-compiler/pypy/module/recparser/ Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py Log: rip out the old recparser module and replace it with a simple cpython like one Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/parser-compiler/pypy/config/pypyoption.py (original) +++ pypy/branch/parser-compiler/pypy/config/pypyoption.py Fri Jul 17 00:53:00 2009 @@ -18,7 +18,7 @@ default_modules.update(dict.fromkeys( ["_codecs", "gc", "_weakref", "marshal", "errno", "math", "_sre", "_pickle_support", "operator", - "recparser", "symbol", "_random", "__pypy__"])) + "parser", "symbol", "_random", "__pypy__"])) # --allworkingmodules Added: pypy/branch/parser-compiler/pypy/module/parser/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/parser/__init__.py Fri Jul 17 00:53:00 2009 @@ -0,0 +1,28 @@ +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + """The builtin parser module.""" + + applevel_name = 'parser' + + appleveldefs = {} + + interpleveldefs = { + '__name__' : '(space.wrap("parser"))', + '__doc__' : '(space.wrap("parser module"))', + + 'suite' : 'pyparser.suite', + 'expr' : 'pyparser.expr', + 'issuite' : 'pyparser.issuite', + 'isexpr' : 'pyparser.isexpr', + 'STType' : 'pyparser.STType', + 'ast2tuple' : 'pyparser.st2tuple', + 'st2tuple' : 'pyparser.st2tuple', + 'ast2list' : 'pyparser.st2list', + 'ast2tuple' : 'pyparser.st2tuple', + 'ASTType' : 'pyparser.STType', + 'sequence2st' : 'pyparser.sequence2st', + 'compilest' : 'pyparser.compilest', + 'compileast' : 'pyparser.compileast' + } Added: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Fri Jul 17 00:53:00 2009 @@ -0,0 +1,108 @@ +from pypy.interpreter.baseobjspace import ObjSpace, Wrappable +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.argument import Arguments +from pypy.interpreter.error import OperationError +from pypy.interpreter.pyparser import pyparse, pygram, error +from pypy.interpreter.astcompiler.astbuilder import ast_from_node +from pypy.interpreter.astcompiler.codegen import compile_ast + + +class STType(Wrappable): + + def __init__(self, tree, mode): + self.tree = tree + self.mode = mode + + def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column): + if node.children is not None: + seq_w = [None]*(1 + len(node.children)) + seq_w[0] = space.wrap(node.type) + for i in range(1, len(node.children) + 1): + seq_w[i] = self._build_app_tree(space, node.children[i - 1], + seq_maker, with_lineno, + with_column) + else: + seq_w = [space.wrap(node.type), space.wrap(node.value)] + if with_lineno: + seq_w.append(space.wrap(node.lineno)) + if with_column: + seq_w.append(space.wrap(node.column)) + return seq_maker(seq_w) + + def descr_issuite(self, space): + return space.wrap(self.tree.type == pygram.syms.file_input) + descr_issuite.unwrap_spec = ["self", ObjSpace] + + def descr_isexpr(self, space): + return space.wrap(self.tree.type == pygram.syms.eval_input) + descr_isexpr.unwrap_spec = ["self", ObjSpace] + + def descr_totuple(self, space, with_lineno=False, with_offset=False): + return self._build_app_tree(space, self.tree, space.newtuple, + with_lineno, with_offset) + descr_totuple.unwrap_spec = ["self", ObjSpace, bool, bool] + + def descr_tolist(self, space, with_lineno=False, with_offset=False): + return self._build_app_tree(space, self.tree, space.newlist, + with_lineno, with_offset) + descr_tolist.unwrap_spec = ["self", ObjSpace, bool, bool] + + def descr_compile(self, space, filename=""): + info = pyparse.CompileInfo(filename, self.mode) + ast = ast_from_node(space, self.tree, info) + return space.wrap(compile_ast(space, ast, info)) + descr_compile.unwrap_spec = ["self", ObjSpace, str] + +STType.typedef = TypeDef("parser.st", + issuite=interp2app(STType.descr_issuite), + isexpr=interp2app(STType.descr_isexpr), + totuple=interp2app(STType.descr_totuple), + tolist=interp2app(STType.descr_tolist), + compile=interp2app(STType.descr_compile) +) + + +def parse_python(space, source, mode): + info = pyparse.CompileInfo("", mode) + parser = pyparse.PythonParser(space) + try: + tree = parser.parse_source(source, info) + except error.IndentationError, e: + raise OperationError(space.w_IndentationError, + e.wrap_info(space, "")) + except error.SyntaxError, e: + raise OperationError(space.w_SyntaxError, + e.wrap_info(space, "")) + return space.wrap(STType(tree, mode)) + + +def suite(space, source): + return parse_python(space, source, 'exec') +suite.unwrap_spec = [ObjSpace, str] + + +def expr(space, source): + return parse_python(space, source, 'eval') +expr.unwrap_spec = [ObjSpace, str] + + +def isexpr(space, st): + return space.call_method(st, "isexpr") +isexpr.unwrap_spec = [ObjSpace, STType] + +def issuite(space, st): + return space.call_method(st, "issuite") +issuite.unwrap_spec = [ObjSpace, STType] + +def st2tuple(space, st, __args__): + return space.call_args(space.getattr(st, space.wrap("totuple")), __args__) +st2tuple.unwrap_spec = [ObjSpace, STType, Arguments] + +def st2list(space, st, __args__): + return space.call_args(space.getattr(st, space.wrap("tolist")), __args__) +st2list.unwrap_spec = [ObjSpace, STType, Arguments] + +def compilest(space, st, __args__): + return space.call_args(space.getattr(st, space.wrap("compile")), __args__) +compilest.unwrap_spec = [ObjSpace, STType, Arguments] Added: pypy/branch/parser-compiler/pypy/module/parser/test/__init__.py ============================================================================== Added: pypy/branch/parser-compiler/pypy/module/parser/test/test_parser.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/parser/test/test_parser.py Fri Jul 17 00:53:00 2009 @@ -0,0 +1,57 @@ +from pypy.conftest import gettestobjspace + +def setup_module(mod): + mod.space = gettestobjspace(usemodules=["parser"]) + +class ParserModuleTest: + + def setup_class(cls): + cls.space = space + cls.w_m = space.appexec([], """(): + import parser + return parser""") + + +class AppTestParser(ParserModuleTest): + + def test_suite(self): + s = self.m.suite("x = 4") + assert isinstance(s, self.m.STType) + assert self.m.issuite(s) + assert s.issuite() + assert not self.m.isexpr(s) + assert not s.isexpr() + + def test_expr(self): + s = self.m.expr("x") + assert isinstance(s, self.m.STType) + assert self.m.isexpr(s) + assert s.isexpr() + assert not self.m.issuite(s) + assert not s.issuite() + + def test_totuple_and_tolist(self): + for meth, tp in (("totuple", tuple), ("tolist", list)): + s = self.m.suite("x = 4") + seq = getattr(s, meth)() + assert isinstance(seq, tp) + assert len(seq) == 4 + assert seq[0] == 286 + assert len(seq[2]) == 2 + assert len(seq[3]) == 2 + assert seq[2][0] == 4 + assert seq[3][0] == 0 + seq = getattr(s, meth)(True) + assert len(seq[2]) == 3 + assert seq[2][2] == 1 + seq = getattr(s, meth)(True, True) + assert len(seq[2]) == 4 + assert seq[2][2] == 1 + assert seq[2][3] == 0 + + def test_compile(self): + import types + for code in (self.m.suite("x = 4").compile(), + self.m.compilest(self.m.suite("x = 4"))): + assert isinstance(code, types.CodeType) + assert code.co_filename == "" From benjamin at codespeak.net Fri Jul 17 05:13:10 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 05:13:10 +0200 (CEST) Subject: [pypy-svn] r66300 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090717031310.A6314169EAC@codespeak.net> Author: benjamin Date: Fri Jul 17 05:13:09 2009 New Revision: 66300 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: refactor scope optimization slightly Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 05:13:09 2009 @@ -149,7 +149,7 @@ kind = name_ops_default container = self.names if scope == symtable.SCOPE_LOCAL: - if self.scope.optimized: + if self.scope.can_be_optimized: container = self.var_names kind = name_ops_fast elif scope == symtable.SCOPE_FREE: @@ -159,7 +159,7 @@ kind = name_ops_deref container = self.cell_vars elif scope == symtable.SCOPE_GLOBAL_IMPLICIT: - if self.scope.optimized: + if self.scope.locals_fully_known: kind = name_ops_global elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: kind = name_ops_global @@ -1115,7 +1115,7 @@ scope = self.scope assert isinstance(scope, symtable.FunctionScope) flags = 0 - if scope.optimized: + if scope.locals_fully_known: flags |= consts.CO_OPTIMIZED if scope.nested: flags |= consts.CO_NESTED Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 17 05:13:09 2009 @@ -24,11 +24,13 @@ class Scope(object): - def __init__(self, node, name, optimized): + can_be_optimized = False + + def __init__(self, node, name): self.node = node self.parent = None self.name = name - self.optimized = optimized + self.locals_fully_known = False self.symbols = None self.roles = {} self.varnames = [] @@ -172,16 +174,19 @@ class ModuleScope(Scope): def __init__(self, module): - Scope.__init__(self, module, "top", False) + Scope.__init__(self, module, "top") class FunctionScope(Scope): + can_be_optimized = True + def __init__(self, func, name): - Scope.__init__(self, func, name, True) + Scope.__init__(self, func, name) self.has_variable_arg = False self.has_keywords_arg = False self.is_generator = False + self.optimized = True self.return_with_value = False self.import_star = None self.bare_exec = None @@ -251,7 +256,7 @@ trailer = "is a nested function" raise SyntaxError(err % (self.name, trailer), node.lineno, node.col_offset) - self.optimized = self.optimized and not self.has_exec + self.locals_fully_known = self.optimized and not self.has_exec class ClassScope(Scope): @@ -259,7 +264,7 @@ _hide_bound_from_nested_scopes = True def __init__(self, clsdef): - Scope.__init__(self, clsdef, clsdef.name, False) + Scope.__init__(self, clsdef, clsdef.name) def mangle(self, name): return misc.mangle(name, self.name) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Fri Jul 17 05:13:09 2009 @@ -55,12 +55,12 @@ def test_toplevel(self): scp = self.mod_scope("x = 4") assert scp.lookup("x") == symtable.SCOPE_LOCAL - assert not scp.optimized + assert not scp.locals_fully_known scp = self.mod_scope("x = 4", "single") - assert not scp.optimized + assert not scp.locals_fully_known assert scp.lookup("x") == symtable.SCOPE_LOCAL scp = self.mod_scope("x*4*6", "eval") - assert not scp.optimized + assert not scp.locals_fully_known assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT def test_duplicate_argument(self): @@ -253,9 +253,9 @@ assert exc.msg == "name 'x' is both local and global" def test_optimization(self): - assert not self.mod_scope("").optimized - assert not self.class_scope("class x: pass").optimized - assert self.func_scope("def f(): pass").optimized + assert not self.mod_scope("").can_be_optimized + assert not self.class_scope("class x: pass").can_be_optimized + assert self.func_scope("def f(): pass").can_be_optimized def test_unoptimization_with_nested_scopes(self): table = ( @@ -294,11 +294,13 @@ self.mod_scope("exec 'hi'") scp = self.func_scope("def f(): exec 'hi'") assert not scp.optimized + assert not scp.locals_fully_known assert isinstance(scp.bare_exec, ast.Exec) assert scp.has_exec for line in ("exec 'hi' in g", "exec 'hi' in g, h"): scp = self.func_scope("def f(): " + line) - assert not scp.optimized + assert scp.optimized + assert not scp.locals_fully_known assert scp.bare_exec is None assert scp.has_exec From antocuni at codespeak.net Fri Jul 17 12:41:52 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 17 Jul 2009 12:41:52 +0200 (CEST) Subject: [pypy-svn] r66301 - in pypy/branch/pyjitpl5/pypy/translator/cli: . src test Message-ID: <20090717104152.61103169EA9@codespeak.net> Author: antocuni Date: Fri Jul 17 12:41:51 2009 New Revision: 66301 Modified: pypy/branch/pyjitpl5/pypy/translator/cli/metavm.py pypy/branch/pyjitpl5/pypy/translator/cli/opcodes.py pypy/branch/pyjitpl5/pypy/translator/cli/src/pypylib.cs pypy/branch/pyjitpl5/pypy/translator/cli/test/test_runtest.py Log: add (partial) support for debug_print to gencli Modified: pypy/branch/pyjitpl5/pypy/translator/cli/metavm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/metavm.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/metavm.py Fri Jul 17 12:41:51 2009 @@ -250,6 +250,26 @@ generator.ilasm.store_static_field(cts_type, desc) +class _DebugPrint(MicroInstruction): + def render(self, generator, op): + MAXARGS = 4 + if len(op.args) > MAXARGS: + generator.db.genoo.log.WARNING('debug_print supported only up to ' + '%d arguments (got %d)' % (MAXARGS, len(op.args))) + return + signature = ', '.join(['object'] * len(op.args)) + + for arg in op.args: + generator.load(arg) + TYPE = arg.concretetype + if not isinstance(TYPE, ootype.OOType): + # assume it's a primitive type, needs boxing + boxtype = generator.cts.lltype_to_cts(TYPE) + generator.ilasm.opcode('box', boxtype) + + generator.ilasm.call('void [pypylib]pypy.runtime.Utils::debug_print(%s)' % signature) + + OOTYPE_TO_MNEMONIC = { ootype.Bool: 'i1', ootype.Char: 'i2', @@ -283,3 +303,4 @@ GetStaticField = _GetStaticField() SetStaticField = _SetStaticField() CastPrimitive = _CastPrimitive() +DebugPrint = _DebugPrint() Modified: pypy/branch/pyjitpl5/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/opcodes.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/opcodes.py Fri Jul 17 12:41:51 2009 @@ -1,7 +1,7 @@ from pypy.translator.cli.metavm import Call, CallMethod, \ IndirectCall, GetField, SetField, DownCast, NewCustomDict,\ MapException, Box, Unbox, NewArray, GetArrayElem, SetArrayElem,\ - TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField + TypeOf, CastPrimitive, EventHandler, GetStaticField, SetStaticField, DebugPrint from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult, InstructionList,\ New, RuntimeNew, CastTo, PushPrimitive, OOString, OOUnicode, OONewArray from pypy.translator.cli.cts import WEAKREF @@ -75,7 +75,7 @@ 'gc_set_max_heap_size': Ignore, 'resume_point': Ignore, 'debug_assert': Ignore, - 'debug_print': Ignore, + 'debug_print': [DebugPrint], 'debug_fatalerror': [PushAllArgs, 'call void [pypylib]pypy.runtime.Utils::debug_fatalerror(string)'], 'keepalive': Ignore, 'is_early_constant': [PushPrimitive(ootype.Bool, False)], Modified: pypy/branch/pyjitpl5/pypy/translator/cli/src/pypylib.cs ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/src/pypylib.cs (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/src/pypylib.cs Fri Jul 17 12:41:51 2009 @@ -376,6 +376,26 @@ throw new Exception("debug_fatalerror: " + msg); } + public static void debug_print(object a) + { + Console.Error.WriteLine(a); + } + + public static void debug_print(object a, object b) + { + Console.Error.WriteLine("{0} {1}", a, b); + } + + public static void debug_print(object a, object b, object c) + { + Console.Error.WriteLine("{0} {1} {2}", a, b, c); + } + + public static void debug_print(object a, object b, object c, object d) + { + Console.Error.WriteLine("{0} {1} {2} {3}", a, b, c, d); + } + public static DynamicMethod CreateDynamicMethod(string name, Type res, Type[] args) { return new DynamicMethod(name, res, args, typeof(Utils).Module); Modified: pypy/branch/pyjitpl5/pypy/translator/cli/test/test_runtest.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/test/test_runtest.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/test/test_runtest.py Fri Jul 17 12:41:51 2009 @@ -19,6 +19,26 @@ def test_input_string(self): def fn(s): return len(s) - res = self.interpret(fn, ["hello"]) assert res == 5 + + def test_debug_print(self): + from pypy.rlib.debug import debug_print + def fn(s): + debug_print('Hello world', 42) + return s + func = self._compile(fn, [42]) + stdout, stderr, retval = func.run(42) + assert retval == 0 + assert stdout == '42\n' + assert stderr == 'Hello world 42\n' + + def fn(s): + # too many arguments, ignore it + debug_print('Hello world', 42, 43, 44, 45, 46, 47, 48) + return s + func = self._compile(fn, [42]) + stdout, stderr, retval = func.run(42) + assert retval == 0 + assert stdout == '42\n' + assert stderr == '' From fijal at codespeak.net Fri Jul 17 12:43:39 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 17 Jul 2009 12:43:39 +0200 (CEST) Subject: [pypy-svn] r66302 - pypy/branch/pyjitpl5/pypy/module/micronumpy Message-ID: <20090717104339.CC148169F19@codespeak.net> Author: fijal Date: Fri Jul 17 12:43:39 2009 New Revision: 66302 Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Log: store space as a field instead of passing it as an arg. This makes fastcalls possible. Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Fri Jul 17 12:43:39 2009 @@ -7,30 +7,33 @@ class NumArray(Wrappable): def __init__(self, space, dim, dtype): self.dim = dim + self.space = space # ignore dtype for now assert len(dim) == 1 self.storage = [0] * dim[0] - def descr_getitem(self, space, index): + def descr_getitem(self, index): + space = self.space try: return space.wrap(self.storage[index]) except IndexError: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) - descr_getitem.unwrap_spec = ['self', ObjSpace, int] + descr_getitem.unwrap_spec = ['self', int] - def descr_setitem(self, space, index, value): + def descr_setitem(self, index, value): + space = self.space try: self.storage[index] = value except IndexError: raise OperationError(space.w_IndexError, space.wrap("list index out of range")) return space.w_None - descr_setitem.unwrap_spec = ['self', ObjSpace, int, int] + descr_setitem.unwrap_spec = ['self', int, int] - def descr_len(self, space): - return space.wrap(len(self.storage)) - descr_len.unwrap_spec = ['self', ObjSpace] + def descr_len(self): + return self.space.wrap(len(self.storage)) + descr_len.unwrap_spec = ['self'] NumArray.typedef = TypeDef( 'NumArray', From fijal at codespeak.net Fri Jul 17 13:08:23 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 17 Jul 2009 13:08:23 +0200 (CEST) Subject: [pypy-svn] r66303 - pypy/branch/pyjitpl5/pypy/module/micronumpy Message-ID: <20090717110823.C21D4169F4A@codespeak.net> Author: fijal Date: Fri Jul 17 13:08:22 2009 New Revision: 66303 Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Log: make sure the storage is a non-resizable array Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Fri Jul 17 13:08:22 2009 @@ -3,6 +3,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, NoneNotWrapped +from pypy.rlib.debug import make_sure_not_resized class NumArray(Wrappable): def __init__(self, space, dim, dtype): @@ -11,6 +12,7 @@ # ignore dtype for now assert len(dim) == 1 self.storage = [0] * dim[0] + make_sure_not_resized(self.storage) def descr_getitem(self, index): space = self.space From benjamin at codespeak.net Fri Jul 17 15:37:08 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 15:37:08 +0200 (CEST) Subject: [pypy-svn] r66305 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090717133708.C7477169F67@codespeak.net> Author: benjamin Date: Fri Jul 17 15:37:07 2009 New Revision: 66305 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: fix a subtle bug in the print statement Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 15:37:07 2009 @@ -350,7 +350,7 @@ self.emit_op(ops.PRINT_NEWLINE_TO) else: self.emit_op(ops.PRINT_NEWLINE) - if have_dest: + elif have_dest: self.emit_op(ops.POP_TOP) def visit_Delete(self, delete): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Fri Jul 17 15:37:07 2009 @@ -694,3 +694,15 @@ l.append(x) """ self.simple_test(source, 'l', [1, 2]) + + +class AppTestPrint: + + def test_print_to(self): + exec """from StringIO import StringIO +s = StringIO() +print >> s, "hi", "lovely!" +assert s.getvalue() == "hi lovely!\\n" +s = StringIO() +print >> s, "hi", "lovely!", +assert s.getvalue() == "hi lovely!\"""" in {} From arigo at codespeak.net Fri Jul 17 15:45:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 15:45:06 +0200 (CEST) Subject: [pypy-svn] r66306 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090717134506.43995169E41@codespeak.net> Author: arigo Date: Fri Jul 17 15:45:05 2009 New Revision: 66306 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Track GUARD_CLASS again. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Fri Jul 17 15:45:05 2009 @@ -31,6 +31,8 @@ origfields = None # optimization; equivalent to an empty dict curfields = None # optimization; equivalent to an empty dict dependencies = None + origclass = False + curclass = None def __init__(self, escaped, fromstart=False): self.escaped = escaped @@ -92,7 +94,9 @@ self.getnode(box).mark_escaped() def find_nodes_NEW_WITH_VTABLE(self, op): - self.nodes[op.result] = InstanceNode(escaped=False) + instnode = InstanceNode(escaped=False) + instnode.curclass = op.args[0] + self.nodes[op.result] = instnode def find_nodes_SETFIELD_GC(self, op): instnode = self.getnode(op.args[0]) @@ -129,7 +133,15 @@ self.find_nodes_GETFIELD_GC(op) def find_nodes_GUARD_CLASS(self, op): - pass # prevent the default handling + instbox = op.args[0] + try: + instnode = self.nodes[instbox] + except KeyError: + instnode = self.nodes[instbox] = InstanceNode(escaped=True) + else: + if instnode.fromstart: + instnode.origclass = True + instnode.curclass = op.args[1] def find_nodes_JUMP(self, op): pass # prevent the default handling Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 15:45:05 2009 @@ -168,6 +168,22 @@ assert getnode(boxes.p3).fromstart assert not getnode(boxes.p4).fromstart + def test_find_nodes_guard_class(self): + ops = """ + [p1] + p2 = getfield_gc(p1, descr=nextdescr) + guard_class(p2, ConstClass(node_vtable)) + fail() + jump(p1) + """ + boxes, getnode = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert not boxp1.origclass + assert boxp1.curclass is None + assert boxp2.origclass + assert boxp2.curclass.value == self.node_vtable_adr + def test_find_nodes_new(self): ops = """ [sum, p1] @@ -199,6 +215,11 @@ assert boxp1.fromstart assert not boxp2.fromstart + assert boxp1.origclass + assert boxp1.curclass.value == self.node_vtable_adr + assert not boxp2.origclass + assert boxp2.curclass.value == self.node_vtable_adr + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From arigo at codespeak.net Fri Jul 17 16:46:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 16:46:33 +0200 (CEST) Subject: [pypy-svn] r66307 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090717144633.766EE169F5E@codespeak.net> Author: arigo Date: Fri Jul 17 16:46:32 2009 New Revision: 66307 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Start support for the next phase, namely intersect(). Support for tracking which boxes know their class. Write more tests about that. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Fri Jul 17 16:46:32 2009 @@ -1,6 +1,6 @@ from pypy.rlib.objectmodel import r_dict from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.metainterp import resoperation +from pypy.jit.metainterp import resoperation, specnode from pypy.jit.metainterp.history import AbstractValue @@ -31,8 +31,7 @@ origfields = None # optimization; equivalent to an empty dict curfields = None # optimization; equivalent to an empty dict dependencies = None - origclass = False - curclass = None + knownclsbox = None def __init__(self, escaped, fromstart=False): self.escaped = escaped @@ -62,21 +61,21 @@ class PerfectSpecializationFinder(object): + node_escaped = InstanceNode(escaped=True) def __init__(self): self.nodes = {} # Box -> InstanceNode - self.node_escaped = InstanceNode(escaped=True) - - def clear(self): - self.nodes.clear() def getnode(self, box): return self.nodes.get(box, self.node_escaped) def find_nodes(self, loop): - self.clear() + inputnodes = [] for box in loop.inputargs: - self.nodes[box] = InstanceNode(escaped=False, fromstart=True) + instnode = InstanceNode(escaped=False, fromstart=True) + inputnodes.append(instnode) + self.nodes[box] = instnode + self.inputnodes = inputnodes # for op in loop.operations: opnum = op.opnum @@ -95,7 +94,7 @@ def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode(escaped=False) - instnode.curclass = op.args[0] + instnode.knownclsbox = op.args[0] self.nodes[op.result] = instnode def find_nodes_SETFIELD_GC(self, op): @@ -134,22 +133,40 @@ def find_nodes_GUARD_CLASS(self, op): instbox = op.args[0] + clsbox = op.args[1] try: instnode = self.nodes[instbox] except KeyError: instnode = self.nodes[instbox] = InstanceNode(escaped=True) - else: - if instnode.fromstart: - instnode.origclass = True - instnode.curclass = op.args[1] + assert instnode is not self.node_escaped + assert (instnode.knownclsbox is None or + instnode.knownclsbox.equals(clsbox)) + instnode.knownclsbox = clsbox def find_nodes_JUMP(self, op): - pass # prevent the default handling - - def find_nodes_FAIL(self, op): - pass # prevent the default handling + """Build the list of specnodes based on the result + computed by this PerfectSpecializationFinder. + """ + specnodes = [] + assert len(self.inputnodes) == len(op.args) + for i in range(len(op.args)): + inputnode = self.inputnodes[i] + specnodes.append(self.intersect(inputnode, op.args[i])) + self.specnodes = specnodes + + def intersect(self, inputnode, exitbox): + assert inputnode.fromstart + exitnode = self.getnode(exitbox) + if inputnode.escaped or exitnode.escaped or exitnode.fromstart: + if (inputnode.knownclsbox is not None and + exitnode.knownclsbox is not None and + inputnode.knownclsbox.equals(exitnode.knownclsbox)): + return specnode.FixedClassSpecNode(inputnode.knownclsbox) + else: + return specnode.prebuiltNotSpecNode + else: + xxx #... find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') -perfect_specialization_finder = PerfectSpecializationFinder() # ____________________________________________________________ Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Fri Jul 17 16:46:32 2009 @@ -1,369 +1,25 @@ -from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp import executor + class SpecNode(object): __slots__ = () - def expand_boxlist(self, instnode, newboxlist): - newboxlist.append(instnode.source) - def extract_runtime_data(self, cpu, valuebox, resultlist): resultlist.append(valuebox) - def adapt_to(self, instnode, offsets): - instnode.escaped = True - - def mutate_nodes(self, instnode): - raise NotImplementedError - - def equals(self, other): - raise NotImplementedError - - def matches(self, other): - raise NotImplementedError - - def compute_number_of_nodes(self): - raise NotImplementedError class NotSpecNode(SpecNode): __slots__ = () - def mutate_nodes(self, instnode): - instnode.cursize = -1 - def equals(self, other): - if type(other) is NotSpecNode: - return True - return False - - def matches(self, other): - # NotSpecNode matches everything - return True - - def compute_number_of_nodes(self): - return 1 - -class MatchEverythingSpecNode(SpecNode): - - def compute_number_of_nodes(self): - return 0 - -#class SpecNodeWithBox(NotSpecNode): -# # XXX what is this class used for? -# def __init__(self, box): -# self.box = box - -# def equals(self, other): -# if type(other) is SpecNodeWithBox: -# return True -# return False +prebuiltNotSpecNode = NotSpecNode() + class FixedClassSpecNode(SpecNode): def __init__(self, known_class): self.known_class = known_class - def mutate_nodes(self, instnode): - from pypy.jit.metainterp.optimize import InstanceNode - - if instnode.cls is None: - instnode.cls = InstanceNode(self.known_class, const=True) - else: - assert instnode.cls.source.equals(self.known_class) - - def equals(self, other): - if type(other) is not FixedClassSpecNode: - return False - else: - assert isinstance(other, FixedClassSpecNode) # make annotator happy - return self.known_class.equals(other.known_class) - - def matches(self, instnode): - if instnode.cls is None: - return False - return instnode.cls.source.equals(self.known_class) - - def compute_number_of_nodes(self): - return 1 - -##class FixedListSpecNode(FixedClassSpecNode): - -## def equals(self, other): -## if type(other) is not FixedListSpecNode: -## return False -## else: -## assert isinstance(other, FixedListSpecNode) # make annotator happy -## return self.known_class.equals(other.known_class) -class SpecNodeWithFields(FixedClassSpecNode): +class VirtualInstanceSpecNode(FixedClassSpecNode): def __init__(self, known_class, fields): FixedClassSpecNode.__init__(self, known_class) self.fields = fields - - def mutate_nodes(self, instnode): - from pypy.jit.metainterp.optimize import av_eq, av_hash - from pypy.rlib.objectmodel import r_dict - - FixedClassSpecNode.mutate_nodes(self, instnode) - curfields = r_dict(av_eq, av_hash) - for ofs, subspecnode in self.fields: - if not isinstance(subspecnode, MatchEverythingSpecNode): - subinstnode = instnode.origfields[ofs] - # should really be there - subspecnode.mutate_nodes(subinstnode) - curfields[ofs] = subinstnode - instnode.curfields = curfields - - def equals(self, other): - if not isinstance(other, SpecNodeWithFields): - return False - if not self.known_class.equals(other.known_class): - return False - elif len(self.fields) != len(other.fields): - return False - else: - for i in range(len(self.fields)): - key, value = self.fields[i] - otherkey, othervalue = other.fields[i] - if key != otherkey: - return False - if not value.equals(othervalue): - return False - return True - - def matches(self, instnode): - # XXX think about details of virtual vs virtualizable - if not FixedClassSpecNode.matches(self, instnode): - return False - for key, value in self.fields: - if key not in instnode.curfields: - return False - if value is not None and not value.matches(instnode.curfields[key]): - return False - return True - - def expand_boxlist(self, instnode, newboxlist): - for ofs, subspecnode in self.fields: - if not isinstance(subspecnode, MatchEverythingSpecNode): - subinstnode = instnode.curfields[ofs] # should really be there - subspecnode.expand_boxlist(subinstnode, newboxlist) - - def extract_runtime_data(self, cpu, valuebox, resultlist): - for ofs, subspecnode in self.fields: - from pypy.jit.metainterp.history import AbstractDescr - assert isinstance(ofs, AbstractDescr) - if not isinstance(subspecnode, MatchEverythingSpecNode): - fieldbox = executor.execute(cpu, rop.GETFIELD_GC, - [valuebox], ofs) - subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) - - def adapt_to(self, instnode, offsets): - for ofs, subspecnode in self.fields: - subspecnode.adapt_to(instnode.curfields[ofs], offsets) - - def compute_number_of_nodes(self): - counter = 0 - for ofs, subspecnode in self.fields: - counter += subspecnode.compute_number_of_nodes() - return counter - -class VirtualizedSpecNode(SpecNodeWithFields): - - def equals(self, other): - if not self.known_class.equals(other.known_class): - return False - assert len(self.fields) == len(other.fields) - for i in range(len(self.fields)): - if (isinstance(self.fields[i][1], MatchEverythingSpecNode) or - isinstance(other.fields[i][1], MatchEverythingSpecNode)): - continue - assert self.fields[i][0].equals(other.fields[i][0]) - if not self.fields[i][1].equals(other.fields[i][1]): - return False - return True - - def matches(self, instnode): - for key, value in self.fields: - if not isinstance(value, MatchEverythingSpecNode): - if key not in instnode.curfields: - return False - if value is not None and not value.matches(instnode.curfields[key]): - return False - return True - - def expand_boxlist(self, instnode, newboxlist): - newboxlist.append(instnode.source) - SpecNodeWithFields.expand_boxlist(self, instnode, newboxlist) - - def extract_runtime_data(self, cpu, valuebox, resultlist): - resultlist.append(valuebox) - SpecNodeWithFields.extract_runtime_data(self, cpu, valuebox, resultlist) - - def adapt_to(self, instnode, offsets_relative_to): - instnode.escaped = True - fields = [] - offsets_so_far = 0 - for ofs, subspecnode in self.fields: - if isinstance(subspecnode, MatchEverythingSpecNode): - node = None - if ofs in instnode.curfields: - node = instnode.curfields[ofs] - orignode = instnode.origfields[ofs] - subspecnode = orignode.intersect(node, {}) - elif ofs in instnode.origfields: - node = instnode.origfields[ofs] - subspecnode = node.intersect(node, {}) - orignode = node - if node is not None: - subspecnode.mutate_nodes(orignode) - offsets_relative_to.append((subspecnode, ofs, instnode, - offsets_so_far, orignode)) - else: - subspecnode.adapt_to(instnode.curfields[ofs], - offsets_relative_to) - offsets_so_far += subspecnode.compute_number_of_nodes() - fields.append((ofs, subspecnode)) - - self.fields = fields - -# class DelayedSpecNode(VirtualizedSpecNode): - -# def expand_boxlist(self, instnode, newboxlist, oplist): -# from pypy.jit.metainterp.history import AbstractDescr -# newboxlist.append(instnode.source) -# for ofs, subspecnode in self.fields: -# assert isinstance(subspecnode, SpecNodeWithBox) -# if oplist is None: -# instnode.cleanfields[ofs] = instnode.origfields[ofs] -# newboxlist.append(instnode.curfields[ofs].source) -# else: -# if ofs in instnode.cleanfields: -# newboxlist.append(instnode.cleanfields[ofs].source) -# else: -# box = subspecnode.box.clonebox() -# assert isinstance(ofs, AbstractDescr) -# oplist.append(ResOperation(rop.GETFIELD_GC, -# [instnode.source], box, ofs)) -# newboxlist.append(box) - -# class DelayedFixedListSpecNode(DelayedSpecNode): - -# def expand_boxlist(self, instnode, newboxlist, oplist): -# from pypy.jit.metainterp.history import ResOperation -# from pypy.jit.metainterp.resoperation import rop -# from pypy.jit.metainterp.optimize import FixedList - -# newboxlist.append(instnode.source) -# cls = self.known_class -# assert isinstance(cls, FixedList) -# arraydescr = cls.arraydescr -# for ofs, subspecnode in self.fields: -# assert isinstance(subspecnode, SpecNodeWithBox) -# if oplist is None: -# instnode.cleanfields[ofs] = instnode.origfields[ofs] -# newboxlist.append(instnode.curfields[ofs].source) -# else: -# if ofs in instnode.cleanfields: -# newboxlist.append(instnode.cleanfields[ofs].source) -# else: -# box = subspecnode.box.clonebox() -# oplist.append(ResOperation(rop.GETARRAYITEM_GC, -# [instnode.source, ofs], box, arraydescr)) -# newboxlist.append(box) - -# def extract_runtime_data(self, cpu, valuebox, resultlist): -# from pypy.jit.metainterp.resoperation import rop -# from pypy.jit.metainterp.optimize import FixedList -# from pypy.jit.metainterp.history import check_descr - -# resultlist.append(valuebox) -# cls = self.known_class -# assert isinstance(cls, FixedList) -# arraydescr = cls.arraydescr -# check_descr(arraydescr) -# for ofs, subspecnode in self.fields: -# fieldbox = executor.execute(cpu, rop.GETARRAYITEM_GC, -# [valuebox, ofs], arraydescr) -# subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) - -class VirtualizableSpecNode(VirtualizedSpecNode): - - def equals(self, other): - if not isinstance(other, VirtualizableSpecNode): - return False - return VirtualizedSpecNode.equals(self, other) - - def adapt_to(self, instnode, offsets): - instnode.virtualized = True - VirtualizedSpecNode.adapt_to(self, instnode, offsets) - -class VirtualizableListSpecNode(VirtualizedSpecNode): - - def equals(self, other): - if not isinstance(other, VirtualizableListSpecNode): - return False - return VirtualizedSpecNode.equals(self, other) - - def extract_runtime_data(self, cpu, valuebox, resultlist): - from pypy.jit.metainterp.resoperation import rop - from pypy.jit.metainterp.optimize import FixedList - from pypy.jit.metainterp.history import check_descr - - resultlist.append(valuebox) - cls = self.known_class - assert isinstance(cls, FixedList) - arraydescr = cls.arraydescr - check_descr(arraydescr) - for ofs, subspecnode in self.fields: - if not isinstance(subspecnode, MatchEverythingSpecNode): - fieldbox = executor.execute(cpu, rop.GETARRAYITEM_GC, - [valuebox, ofs], arraydescr) - subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) - - def adapt_to(self, instnode, offsets): - instnode.virtualized = True - VirtualizedSpecNode.adapt_to(self, instnode, offsets) - -class VirtualSpecNode(SpecNodeWithFields): - - def adapt_to(self, instnode, offsets): - instnode.virtual = True - SpecNodeWithFields.adapt_to(self, instnode, offsets) - - def mutate_nodes(self, instnode): - SpecNodeWithFields.mutate_nodes(self, instnode) - instnode.virtual = True - -class VirtualInstanceSpecNode(VirtualSpecNode): - - def equals(self, other): - if not isinstance(other, VirtualInstanceSpecNode): - return False - return SpecNodeWithFields.equals(self, other) - -class VirtualFixedListSpecNode(VirtualSpecNode): - - def __init__(self, known_class, fields, known_length): - VirtualSpecNode.__init__(self, known_class, fields) - self.known_length = known_length - - def mutate_nodes(self, instnode): - VirtualSpecNode.mutate_nodes(self, instnode) - instnode.cursize = self.known_length - - def equals(self, other): - if not isinstance(other, VirtualFixedListSpecNode): - return False - return SpecNodeWithFields.equals(self, other) - - def extract_runtime_data(self, cpu, valuebox, resultlist): - from pypy.jit.metainterp.resoperation import rop - from pypy.jit.metainterp.optimize import FixedList - from pypy.jit.metainterp.history import check_descr - cls = self.known_class - assert isinstance(cls, FixedList) - arraydescr = cls.arraydescr - check_descr(arraydescr) - for ofs, subspecnode in self.fields: - fieldbox = executor.execute(cpu, rop.GETARRAYITEM_GC, - [valuebox, ofs], arraydescr) - subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 16:46:32 2009 @@ -7,7 +7,9 @@ from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, ConstAddr, TreeLoop, BoxObj) -from pypy.jit.metainterp.optimize import perfect_specialization_finder +from pypy.jit.metainterp.optimize import PerfectSpecializationFinder +from pypy.jit.metainterp.specnode import NotSpecNode, FixedClassSpecNode +from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.test.oparser import parse # ____________________________________________________________ @@ -56,12 +58,16 @@ node_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) node_vtable_adr = llmemory.cast_ptr_to_adr(node_vtable) + node_vtable2 = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2) cpu = runner.LLtypeCPU(None) NODE = lltype.GcForwardReference() NODE.become(lltype.GcStruct('NODE', ('parent', OBJECT), ('value', lltype.Signed), ('next', lltype.Ptr(NODE)))) + NODE2 = lltype.GcStruct('NODE2', ('parent', NODE), + ('other', lltype.Ptr(NODE))) node = lltype.malloc(NODE) nodebox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node)) nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node)) @@ -69,7 +75,8 @@ valuedescr = cpu.fielddescrof(NODE, 'value') nextdescr = cpu.fielddescrof(NODE, 'next') - cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE)} + cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), + cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2)} namespace = locals() class OOtypeMixin(object): @@ -80,9 +87,12 @@ NODE = ootype.Instance('NODE', ootype.ROOT, {}) NODE._add_fields({'value': ootype.Signed, 'next': NODE}) + NODE2 = ootype.Instance('NODE2', NODE, {'other': NODE}) node_vtable = ootype.runtimeClass(NODE) node_vtable_adr = ootype.cast_to_object(node_vtable) + node_vtable2 = ootype.runtimeClass(NODE2) + node_vtable_adr2 = ootype.cast_to_object(node_vtable2) node = ootype.new(NODE) nodebox = BoxObj(ootype.cast_to_object(node)) @@ -91,7 +101,8 @@ nextdescr = cpu.fielddescrof(NODE, 'next') nodesize = cpu.typedescrof(NODE) - cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE)} + cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), + node_vtable_adr2: cpu.typedescrof(NODE2)} namespace = locals() # ____________________________________________________________ @@ -109,8 +120,11 @@ def find_nodes(self, ops, boxkinds=None): loop = self.parse(ops, boxkinds=boxkinds) + perfect_specialization_finder = PerfectSpecializationFinder() perfect_specialization_finder.find_nodes(loop) - return loop.getboxes(), perfect_specialization_finder.getnode + return (loop.getboxes(), + perfect_specialization_finder.getnode, + perfect_specialization_finder.specnodes) def test_find_nodes_simple(self): ops = """ @@ -120,9 +134,10 @@ fail(i0) jump(i0) """ - boxes, getnode = self.find_nodes(ops) + boxes, getnode, specnodes = self.find_nodes(ops) assert getnode(boxes.i).fromstart assert not getnode(boxes.i0).fromstart + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_non_escape(self): ops = """ @@ -134,13 +149,14 @@ setfield_gc(p2, i1, descr=valuedescr) jump(p0) """ - boxes, getnode = self.find_nodes(ops) + boxes, getnode, specnodes = self.find_nodes(ops) assert not getnode(boxes.p0).escaped assert not getnode(boxes.p1).escaped assert not getnode(boxes.p2).escaped assert getnode(boxes.p0).fromstart assert getnode(boxes.p1).fromstart assert getnode(boxes.p2).fromstart + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_escape(self): ops = """ @@ -156,7 +172,7 @@ setfield_gc(p4, i1, descr=valuedescr) jump(p0) """ - boxes, getnode = self.find_nodes(ops) + boxes, getnode, specnodes = self.find_nodes(ops) assert not getnode(boxes.p0).escaped assert getnode(boxes.p1).escaped assert getnode(boxes.p2).escaped # forced by p1 @@ -167,8 +183,21 @@ assert getnode(boxes.p2).fromstart assert getnode(boxes.p3).fromstart assert not getnode(boxes.p4).fromstart + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] - def test_find_nodes_guard_class(self): + def test_find_nodes_guard_class_1(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + jump(p1) + """ + boxes, getnode, specnodes = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + assert boxp1.knownclsbox.value == self.node_vtable_adr + assert [sn.__class__ for sn in specnodes] == [FixedClassSpecNode] + + def test_find_nodes_guard_class_2(self): ops = """ [p1] p2 = getfield_gc(p1, descr=nextdescr) @@ -176,15 +205,79 @@ fail() jump(p1) """ - boxes, getnode = self.find_nodes(ops) + boxes, getnode, specnodes = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert boxp1.knownclsbox is None + assert boxp2.knownclsbox.value == self.node_vtable_adr + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] + + def test_find_nodes_guard_class_outonly(self): + ops = """ + [p1] + p2 = escape() + guard_class(p2, ConstClass(node_vtable)) + fail() + jump(p2) + """ + boxes, getnode, specnodes = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert boxp1.knownclsbox is None + assert boxp2.knownclsbox.value == self.node_vtable_adr + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] + + def test_find_nodes_guard_class_inonly(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + p2 = escape() + jump(p2) + """ + boxes, getnode, specnodes = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert boxp1.knownclsbox.value == self.node_vtable_adr + assert boxp2.knownclsbox is None + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] + + def test_find_nodes_guard_class_inout(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + p2 = escape() + guard_class(p2, ConstClass(node_vtable)) + fail() + jump(p2) + """ + boxes, getnode, specnodes = self.find_nodes(ops) boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) - assert not boxp1.origclass - assert boxp1.curclass is None - assert boxp2.origclass - assert boxp2.curclass.value == self.node_vtable_adr + assert boxp1.knownclsbox.value == self.node_vtable_adr + assert boxp2.knownclsbox.value == self.node_vtable_adr + assert [sn.__class__ for sn in specnodes] == [FixedClassSpecNode] + + def test_find_nodes_guard_class_mismatch(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + p2 = escape() + guard_class(p2, ConstClass(node_vtable2)) + fail() + jump(p2) + """ + boxes, getnode, specnodes = self.find_nodes(ops) + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert boxp1.knownclsbox.value == self.node_vtable_adr + assert boxp2.knownclsbox.value == self.node_vtable_adr2 + assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_new(self): + py.test.skip("VirtualInstanceSpecNode in-progress") ops = """ [sum, p1] guard_class(p1, ConstClass(node_vtable)) @@ -197,8 +290,8 @@ setfield_gc(p2, p2, descr=nextdescr) jump(sum2, p2) """ - boxes, getnode = self.find_nodes(ops, boxkinds={'sum': BoxInt, - 'sum2': BoxInt}) + boxes, getnode, specnodes = self.find_nodes( + ops, boxkinds={'sum': BoxInt, 'sum2': BoxInt}) assert getnode(boxes.sum) is not getnode(boxes.sum2) assert getnode(boxes.p1) is not getnode(boxes.p2) @@ -215,10 +308,11 @@ assert boxp1.fromstart assert not boxp2.fromstart - assert boxp1.origclass - assert boxp1.curclass.value == self.node_vtable_adr - assert not boxp2.origclass - assert boxp2.curclass.value == self.node_vtable_adr + assert boxp1.knownclsbox.value == self.node_vtable_adr + assert boxp2.knownclsbox.value == self.node_vtable_adr + + assert [sn.__class__ for sn in specnodes] == [NotSpecNode, + VirtualInstanceSpecNode] class TestLLtype(BaseTestOptimize, LLtypeMixin): From arigo at codespeak.net Fri Jul 17 17:07:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 17:07:24 +0200 (CEST) Subject: [pypy-svn] r66308 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090717150724.BAF18169F61@codespeak.net> Author: arigo Date: Fri Jul 17 17:07:23 2009 New Revision: 66308 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: VirtualInstanceSpecNode. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Fri Jul 17 17:07:23 2009 @@ -1,6 +1,9 @@ from pypy.rlib.objectmodel import r_dict from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.metainterp import resoperation, specnode +from pypy.jit.metainterp import resoperation +from pypy.jit.metainterp.specnode import prebuiltNotSpecNode +from pypy.jit.metainterp.specnode import FixedClassSpecNode +from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.history import AbstractValue @@ -157,16 +160,55 @@ def intersect(self, inputnode, exitbox): assert inputnode.fromstart exitnode = self.getnode(exitbox) + if exitnode.knownclsbox is None: + return prebuiltNotSpecNode # no known class at exit + if (inputnode.knownclsbox is not None and + not inputnode.knownclsbox.equals(exitnode.knownclsbox)): + return prebuiltNotSpecNode # mismatched known class at exit + # + # for the sequel, we know that the class is known and matches if inputnode.escaped or exitnode.escaped or exitnode.fromstart: - if (inputnode.knownclsbox is not None and - exitnode.knownclsbox is not None and - inputnode.knownclsbox.equals(exitnode.knownclsbox)): - return specnode.FixedClassSpecNode(inputnode.knownclsbox) - else: - return specnode.prebuiltNotSpecNode + if inputnode.knownclsbox is None: + return prebuiltNotSpecNode # class not needed at input + return FixedClassSpecNode(exitnode.knownclsbox) else: - xxx #... + fields = [] + d = exitnode.curfields + lst = d.keys() + sort_descrs(lst) + for ofs in lst: + if ofs not in inputnode.origfields: + # field stored at exit, but not read at input. Must + # still be allocated, otherwise it will be incorrectly + # uninitialized after a guard failure. + inputnode.origfields[ofs] = InstanceNode(escaped=False, + fromstart=True) + specnode = self.intersect(inputnode.origfields[ofs], d[ofs]) + fields.append((ofs, specnode)) + return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') # ____________________________________________________________ + +def partition(array, left, right): + last_item = array[right] + pivot = last_item.sort_key() + storeindex = left + for i in range(left, right): + if array[i].sort_key() <= pivot: + array[i], array[storeindex] = array[storeindex], array[i] + storeindex += 1 + # Move pivot to its final place + array[storeindex], array[right] = last_item, array[storeindex] + return storeindex + +def quicksort(array, left, right): + # sort array[left:right+1] (i.e. bounds included) + if right > left: + pivotnewindex = partition(array, left, right) + quicksort(array, left, pivotnewindex - 1) + quicksort(array, pivotnewindex + 1, right) + +def sort_descrs(lst): + quicksort(lst, 0, len(lst)-1) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 17:07:23 2009 @@ -1,4 +1,4 @@ -import py +import py, random from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype @@ -6,8 +6,10 @@ from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, - Const, ConstAddr, TreeLoop, BoxObj) + Const, ConstAddr, TreeLoop, BoxObj, + AbstractDescr) from pypy.jit.metainterp.optimize import PerfectSpecializationFinder +from pypy.jit.metainterp.optimize import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.test.oparser import parse @@ -51,6 +53,18 @@ py.test.raises(AssertionError, "equaloplists(loop1.operations, loop3.operations)") +def test_sort_descrs(): + class PseudoDescr(AbstractDescr): + def __init__(self, n): + self.n = n + def sort_key(self): + return self.n + lst = [PseudoDescr(2), PseudoDescr(3), PseudoDescr(6)] + lst2 = lst[:] + random.shuffle(lst2) + sort_descrs(lst2) + assert lst2 == lst + # ____________________________________________________________ class LLtypeMixin(object): @@ -277,7 +291,6 @@ assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_new(self): - py.test.skip("VirtualInstanceSpecNode in-progress") ops = """ [sum, p1] guard_class(p1, ConstClass(node_vtable)) From arigo at codespeak.net Fri Jul 17 18:15:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 18:15:11 +0200 (CEST) Subject: [pypy-svn] r66309 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090717161511.E19721683F2@codespeak.net> Author: arigo Date: Fri Jul 17 18:15:10 2009 New Revision: 66309 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: More tests about new_with_vtable. Fix some code about intersect(). Skipped test about an infinite loop. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Fri Jul 17 18:15:10 2009 @@ -58,8 +58,9 @@ def __repr__(self): flags = '' - if self.escaped: flags += 'e' - if self.fromstart: flags += 's' + if self.escaped: flags += 'e' + if self.fromstart: flags += 's' + if self.knownclsbox: flags += 'c' return "" % (flags,) @@ -154,12 +155,12 @@ assert len(self.inputnodes) == len(op.args) for i in range(len(op.args)): inputnode = self.inputnodes[i] - specnodes.append(self.intersect(inputnode, op.args[i])) + exitnode = self.getnode(op.args[i]) + specnodes.append(self.intersect(inputnode, exitnode)) self.specnodes = specnodes - def intersect(self, inputnode, exitbox): + def intersect(self, inputnode, exitnode): assert inputnode.fromstart - exitnode = self.getnode(exitbox) if exitnode.knownclsbox is None: return prebuiltNotSpecNode # no known class at exit if (inputnode.knownclsbox is not None and @@ -171,21 +172,25 @@ if inputnode.knownclsbox is None: return prebuiltNotSpecNode # class not needed at input return FixedClassSpecNode(exitnode.knownclsbox) - else: - fields = [] - d = exitnode.curfields + # + fields = [] + d = exitnode.curfields + if d is not None: + if inputnode.origfields is None: + inputnode.origfields = av_newdict() lst = d.keys() sort_descrs(lst) for ofs in lst: - if ofs not in inputnode.origfields: + try: + node = inputnode.origfields[ofs] + except KeyError: # field stored at exit, but not read at input. Must # still be allocated, otherwise it will be incorrectly # uninitialized after a guard failure. - inputnode.origfields[ofs] = InstanceNode(escaped=False, - fromstart=True) - specnode = self.intersect(inputnode.origfields[ofs], d[ofs]) + node = InstanceNode(escaped=False, fromstart=True) + specnode = self.intersect(node, d[ofs]) fields.append((ofs, specnode)) - return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) + return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Fri Jul 17 18:15:10 2009 @@ -10,6 +10,9 @@ class NotSpecNode(SpecNode): __slots__ = () + def _equals(self, other): # for tests only + return type(other) is NotSpecNode + prebuiltNotSpecNode = NotSpecNode() @@ -18,8 +21,21 @@ def __init__(self, known_class): self.known_class = known_class + def _equals(self, other): # for tests only + return (type(other) is FixedClassSpecNode and + self.known_class.equals(other.known_class)) + class VirtualInstanceSpecNode(FixedClassSpecNode): def __init__(self, known_class, fields): FixedClassSpecNode.__init__(self, known_class) self.fields = fields + + def _equals(self, other): # for tests only + ok = (type(other) is VirtualInstanceSpecNode and + self.known_class.equals(other.known_class) and + len(self.fields) == len(other.fields)) + if ok: + for (o1, s1), (o2, s2) in zip(self.fields, other.fields): + ok = ok and o1 == o2 and s1._equals(s2) + return ok Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 18:15:10 2009 @@ -7,10 +7,11 @@ from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, ConstAddr, TreeLoop, BoxObj, - AbstractDescr) + ConstObj, AbstractDescr) from pypy.jit.metainterp.optimize import PerfectSpecializationFinder from pypy.jit.metainterp.optimize import sort_descrs -from pypy.jit.metainterp.specnode import NotSpecNode, FixedClassSpecNode +from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode +from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.test.oparser import parse @@ -86,6 +87,7 @@ nodebox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node)) nodebox2 = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, node)) nodesize = cpu.sizeof(NODE) + nodesize2 = cpu.sizeof(NODE2) valuedescr = cpu.fielddescrof(NODE, 'value') nextdescr = cpu.fielddescrof(NODE, 'next') @@ -114,6 +116,7 @@ valuedescr = cpu.fielddescrof(NODE, 'value') nextdescr = cpu.fielddescrof(NODE, 'next') nodesize = cpu.typedescrof(NODE) + nodesize2 = cpu.typedescrof(NODE2) cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), node_vtable_adr2: cpu.typedescrof(NODE2)} @@ -132,13 +135,39 @@ equaloplists(optimized.operations, self.parse(expected).operations) - def find_nodes(self, ops, boxkinds=None): + def assert_specnodes(self, specnodes, text): + # + def constclass(cls_vtable): + if self.type_system == 'lltype': + return ConstAddr(llmemory.cast_ptr_to_adr(cls_vtable), + self.cpu) + else: + return ConstObj(ootype.cast_to_object(cls_vtable)) + def makeFixed(cls_vtable): + return FixedClassSpecNode(constclass(cls_vtable)) + def makeVirtual(cls_vtable, **kwds_fields): + fields = [] + for key, value in kwds_fields.items(): + fields.append((self.namespace[key], value)) + fields.sort(key = lambda (x, _): x.sort_key()) + return VirtualInstanceSpecNode(constclass(cls_vtable), fields) + # + context = {'Not': prebuiltNotSpecNode, + 'Fixed': makeFixed, + 'Virtual': makeVirtual} + lst = eval('[' + text + ']', self.namespace, context) + assert len(specnodes) == len(lst) + for x, y in zip(specnodes, lst): + assert x._equals(y) + return True + + def find_nodes(self, ops, spectext, boxkinds=None): loop = self.parse(ops, boxkinds=boxkinds) perfect_specialization_finder = PerfectSpecializationFinder() perfect_specialization_finder.find_nodes(loop) - return (loop.getboxes(), - perfect_specialization_finder.getnode, - perfect_specialization_finder.specnodes) + assert self.assert_specnodes(perfect_specialization_finder.specnodes, + spectext) + return (loop.getboxes(), perfect_specialization_finder.getnode) def test_find_nodes_simple(self): ops = """ @@ -148,10 +177,9 @@ fail(i0) jump(i0) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') assert getnode(boxes.i).fromstart assert not getnode(boxes.i0).fromstart - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_non_escape(self): ops = """ @@ -163,14 +191,13 @@ setfield_gc(p2, i1, descr=valuedescr) jump(p0) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') assert not getnode(boxes.p0).escaped assert not getnode(boxes.p1).escaped assert not getnode(boxes.p2).escaped assert getnode(boxes.p0).fromstart assert getnode(boxes.p1).fromstart assert getnode(boxes.p2).fromstart - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_escape(self): ops = """ @@ -186,7 +213,7 @@ setfield_gc(p4, i1, descr=valuedescr) jump(p0) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') assert not getnode(boxes.p0).escaped assert getnode(boxes.p1).escaped assert getnode(boxes.p2).escaped # forced by p1 @@ -197,7 +224,6 @@ assert getnode(boxes.p2).fromstart assert getnode(boxes.p3).fromstart assert not getnode(boxes.p4).fromstart - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_guard_class_1(self): ops = """ @@ -206,10 +232,9 @@ fail() jump(p1) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Fixed(node_vtable)') boxp1 = getnode(boxes.p1) assert boxp1.knownclsbox.value == self.node_vtable_adr - assert [sn.__class__ for sn in specnodes] == [FixedClassSpecNode] def test_find_nodes_guard_class_2(self): ops = """ @@ -219,12 +244,11 @@ fail() jump(p1) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) assert boxp1.knownclsbox is None assert boxp2.knownclsbox.value == self.node_vtable_adr - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_guard_class_outonly(self): ops = """ @@ -234,12 +258,11 @@ fail() jump(p2) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) assert boxp1.knownclsbox is None assert boxp2.knownclsbox.value == self.node_vtable_adr - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_guard_class_inonly(self): ops = """ @@ -249,12 +272,11 @@ p2 = escape() jump(p2) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox is None - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] def test_find_nodes_guard_class_inout(self): ops = """ @@ -266,12 +288,11 @@ fail() jump(p2) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Fixed(node_vtable)') boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox.value == self.node_vtable_adr - assert [sn.__class__ for sn in specnodes] == [FixedClassSpecNode] def test_find_nodes_guard_class_mismatch(self): ops = """ @@ -283,14 +304,52 @@ fail() jump(p2) """ - boxes, getnode, specnodes = self.find_nodes(ops) + boxes, getnode = self.find_nodes(ops, 'Not') boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox.value == self.node_vtable_adr2 - assert [sn.__class__ for sn in specnodes] == [NotSpecNode] - def test_find_nodes_new(self): + def test_find_nodes_new_1(self): + ops = """ + [p1] + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + jump(p2) + """ + boxes, getnode = self.find_nodes(ops, 'Virtual(node_vtable)') + + boxp1 = getnode(boxes.p1) + boxp2 = getnode(boxes.p2) + assert not boxp1.escaped + assert not boxp2.escaped + + assert not boxp1.origfields + assert not boxp1.curfields + assert not boxp2.origfields + assert not boxp2.curfields + + assert boxp1.fromstart + assert not boxp2.fromstart + + assert boxp1.knownclsbox is None + assert boxp2.knownclsbox.value == self.node_vtable_adr + + def test_find_nodes_new_2(self): + ops = """ + [i1, p1] + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p3 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + setfield_gc(p2, p3, descr=nextdescr) + setfield_gc(p3, i1, descr=valuedescr) + jump(i1, p2) + """ + self.find_nodes(ops, + '''Not, + Virtual(node_vtable, + nextdescr=Virtual(node_vtable2, + valuedescr=Not))''') + + def test_find_nodes_new_3(self): ops = """ [sum, p1] guard_class(p1, ConstClass(node_vtable)) @@ -300,32 +359,61 @@ sum2 = int_add(sum, i1) p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p2, i2, descr=valuedescr) - setfield_gc(p2, p2, descr=nextdescr) + p3 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + setfield_gc(p2, p3, descr=nextdescr) jump(sum2, p2) """ - boxes, getnode, specnodes = self.find_nodes( - ops, boxkinds={'sum': BoxInt, 'sum2': BoxInt}) + boxes, getnode = self.find_nodes( + ops, + '''Not, + Virtual(node_vtable, + valuedescr=Not, + nextdescr=Virtual(node_vtable2))''', + boxkinds={'sum': BoxInt, 'sum2': BoxInt}) assert getnode(boxes.sum) is not getnode(boxes.sum2) assert getnode(boxes.p1) is not getnode(boxes.p2) boxp1 = getnode(boxes.p1) boxp2 = getnode(boxes.p2) + boxp3 = getnode(boxes.p3) assert not boxp1.escaped assert not boxp2.escaped + assert not boxp3.escaped assert not boxp1.curfields assert boxp1.origfields[self.valuedescr] is getnode(boxes.i1) assert not boxp2.origfields - assert boxp2.curfields[self.nextdescr] is boxp2 + assert boxp2.curfields[self.nextdescr] is boxp3 assert boxp1.fromstart assert not boxp2.fromstart + assert not boxp3.fromstart assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox.value == self.node_vtable_adr + assert boxp3.knownclsbox.value == self.node_vtable_adr2 - assert [sn.__class__ for sn in specnodes] == [NotSpecNode, - VirtualInstanceSpecNode] + def test_find_nodes_new_aliasing(self): + py.test.skip("infinite loop") + ops = """ + [sum, p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + p3 = getfield_gc(p1, descr=nextdescr) + guard_class(p3, ConstClass(node_vtable)) + fail() + i1 = getfield_gc(p1, descr=valuedescr) + i2 = int_sub(i1, 1) + sum2 = int_add(sum, i1) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p2, descr=nextdescr) + jump(sum2, p2) + """ + # the issue is the loop involving p2, which cannot be represented + # with SpecNodes so far + self.find_nodes(ops, 'Not, Not', + boxkinds={'sum': BoxInt, 'sum2': BoxInt}) class TestLLtype(BaseTestOptimize, LLtypeMixin): From arigo at codespeak.net Fri Jul 17 18:18:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 18:18:45 +0200 (CEST) Subject: [pypy-svn] r66311 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090717161845.24E26169F19@codespeak.net> Author: arigo Date: Fri Jul 17 18:18:44 2009 New Revision: 66311 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Another test showing the issue. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 18:18:44 2009 @@ -393,7 +393,7 @@ assert boxp2.knownclsbox.value == self.node_vtable_adr assert boxp3.knownclsbox.value == self.node_vtable_adr2 - def test_find_nodes_new_aliasing(self): + def test_find_nodes_new_aliasing_1(self): py.test.skip("infinite loop") ops = """ [sum, p1] @@ -415,6 +415,18 @@ self.find_nodes(ops, 'Not, Not', boxkinds={'sum': BoxInt, 'sum2': BoxInt}) + def test_find_nodes_new_aliasing_2(self): + py.test.skip("in-progress") + ops = """ + [p1, p2] + escape(p2) + p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + jump(p3, p3) + """ + # both p1 and p2 must be NotSpecNodes; it's not possible to pass + # in p1 a Virtual and not in p2, as they both come from the same p3. + self.find_nodes(ops, 'Not, Not') + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From arigo at codespeak.net Fri Jul 17 18:25:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 17 Jul 2009 18:25:09 +0200 (CEST) Subject: [pypy-svn] r66312 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090717162509.DBAA0169F2C@codespeak.net> Author: arigo Date: Fri Jul 17 18:25:09 2009 New Revision: 66312 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Clarify this comment. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Fri Jul 17 18:25:09 2009 @@ -410,7 +410,7 @@ setfield_gc(p2, p2, descr=nextdescr) jump(sum2, p2) """ - # the issue is the loop involving p2, which cannot be represented + # the issue is the cycle "p2->p2", which cannot be represented # with SpecNodes so far self.find_nodes(ops, 'Not, Not', boxkinds={'sum': BoxInt, 'sum2': BoxInt}) From benjamin at codespeak.net Fri Jul 17 20:35:28 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 20:35:28 +0200 (CEST) Subject: [pypy-svn] r66319 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717183528.55F60169F5E@codespeak.net> Author: benjamin Date: Fri Jul 17 20:35:27 2009 New Revision: 66319 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: generate temporary variables on a scope by scope basis Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 20:35:27 2009 @@ -125,6 +125,7 @@ def current_temporary_name(self): name = "_[%i]" % (self.temporary_name_counter,) self.temporary_name_counter += 1 + assert self.scope.lookup(name) != symtable.SCOPE_UNKNOWN return name def sub_scope(self, kind, name, node): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 17 20:35:27 2009 @@ -36,6 +36,7 @@ self.varnames = [] self.children = [] self.free_vars = [] + self.temp_name_counter = 1 self.has_exec = False self.has_free = False self.child_has_free = False @@ -47,6 +48,10 @@ def lookup_role(self, name): return self.roles.get(self.mangle(name), SYM_BLANK) + def new_temporary_name(self): + self.note_symbol("_[%i]" % (self.temp_name_counter,), SYM_ASSIGNED) + self.temp_name_counter += 1 + def note_symbol(self, identifier, role): mangled = self.mangle(identifier) new_role = role @@ -279,7 +284,6 @@ self.scopes = {} self.scope = None self.stack = [] - self.tmp_name_counter = 1 top = ModuleScope(module) self.globs = top.roles self.push_scope(top) @@ -314,10 +318,6 @@ name = ".%i" % (pos,) self.note_symbol(name, SYM_PARAM) - def new_temporary_name(self): - self.note_symbol("_[%i]" % (self.tmp_name_counter,), SYM_ASSIGNED) - self.tmp_name_counter += 1 - def note_symbol(self, identifier, role): mangled = self.scope.note_symbol(identifier, role) if role & SYM_GLOBAL: @@ -407,13 +407,13 @@ self.pop_scope() def visit_ListComp(self, lc): - self.new_temporary_name() + self.scope.new_temporary_name() ast.GenericASTVisitor.visit_ListComp(self, lc) def visit_With(self, wih): - self.new_temporary_name() + self.scope.new_temporary_name() if wih.optional_vars: - self.new_temporary_name() + self.scope.new_temporary_name() ast.GenericASTVisitor.visit_With(self, wih) def visit_arguments(self, arguments): From benjamin at codespeak.net Fri Jul 17 20:40:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 20:40:15 +0200 (CEST) Subject: [pypy-svn] r66320 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090717184015.CA602169F69@codespeak.net> Author: benjamin Date: Fri Jul 17 20:40:15 2009 New Revision: 66320 Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py Log: typo Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/__init__.py Fri Jul 17 20:40:15 2009 @@ -24,5 +24,5 @@ 'ASTType' : 'pyparser.STType', 'sequence2st' : 'pyparser.sequence2st', 'compilest' : 'pyparser.compilest', - 'compileast' : 'pyparser.compileast' + 'compileast' : 'pyparser.compilest' } From benjamin at codespeak.net Fri Jul 17 20:47:01 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 20:47:01 +0200 (CEST) Subject: [pypy-svn] r66321 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090717184701.59908169F69@codespeak.net> Author: benjamin Date: Fri Jul 17 20:47:00 2009 New Revision: 66321 Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py Log: remove unimplemented function Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/__init__.py Fri Jul 17 20:47:00 2009 @@ -22,7 +22,6 @@ 'ast2list' : 'pyparser.st2list', 'ast2tuple' : 'pyparser.st2tuple', 'ASTType' : 'pyparser.STType', - 'sequence2st' : 'pyparser.sequence2st', 'compilest' : 'pyparser.compilest', 'compileast' : 'pyparser.compilest' } From benjamin at codespeak.net Fri Jul 17 21:04:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 21:04:25 +0200 (CEST) Subject: [pypy-svn] r66322 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090717190425.89B953180F4@codespeak.net> Author: benjamin Date: Fri Jul 17 21:04:23 2009 New Revision: 66322 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: fix lambdas with arguments Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 21:04:23 2009 @@ -1141,6 +1141,7 @@ assert isinstance(lam, ast.Lambda) if lam.args.args: self._handle_nested_args(lam.args.args) + self.argcount = len(lam.args.args) lam.body.walkabout(self) self.emit_op(ops.RETURN_VALUE) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Fri Jul 17 21:04:23 2009 @@ -695,6 +695,9 @@ """ self.simple_test(source, 'l', [1, 2]) + def test_lambda(self): + yield self.st, "y = lambda x: x", "y(4)", 4 + class AppTestPrint: From benjamin at codespeak.net Fri Jul 17 21:31:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 21:31:15 +0200 (CEST) Subject: [pypy-svn] r66323 - pypy/trunk/pypy/translator/c/test Message-ID: <20090717193115.D4E3F16845E@codespeak.net> Author: benjamin Date: Fri Jul 17 21:31:14 2009 New Revision: 66323 Modified: pypy/trunk/pypy/translator/c/test/test_genc.py Log: use the offical py.test option entry Modified: pypy/trunk/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_genc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_genc.py Fri Jul 17 21:31:14 2009 @@ -12,7 +12,6 @@ from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation -from pypy import conftest def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): @@ -24,7 +23,7 @@ # XXX fish t.driver.config.translation.countmallocs = True compiled_fn = t.compile_c() - if conftest.option.view: + if py.test.config.option.view: t.view() malloc_counters = t.driver.cbuilder.get_malloc_counters() def checking_fn(*args, **kwds): @@ -403,6 +402,6 @@ t = Translation(f, [], backend="c") t.annotate() compiled_fn = t.compile_c() - if conftest.option.view: + if py.test.config.option.view: t.view() assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read() From benjamin at codespeak.net Fri Jul 17 21:43:51 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 21:43:51 +0200 (CEST) Subject: [pypy-svn] r66324 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090717194351.703AC1684BE@codespeak.net> Author: benjamin Date: Fri Jul 17 21:43:49 2009 New Revision: 66324 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Log: don't complain when the annotator calls this method multiple times Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Fri Jul 17 21:43:49 2009 @@ -32,9 +32,12 @@ def _freeze_(self): # Remove some attributes not used in parsing. - del self.symbol_to_label - del self.symbol_names - del self.symbol_ids + try: + del self.symbol_to_label + del self.symbol_names + del self.symbol_ids + except AttributeError: + pass return True From benjamin at codespeak.net Fri Jul 17 21:47:29 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 21:47:29 +0200 (CEST) Subject: [pypy-svn] r66325 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090717194729.6A4D7168518@codespeak.net> Author: benjamin Date: Fri Jul 17 21:47:28 2009 New Revision: 66325 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: help the annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Fri Jul 17 21:47:28 2009 @@ -119,6 +119,7 @@ flags &= ~consts.PyCF_DONT_IMPLY_DEDENT self.prepare(_targets[compile_info.mode]) + tp = 0 try: tokens = pytokenizer.generate_tokens(source_lines, flags) for tp, value, lineno, column, line in tokens: From benjamin at codespeak.net Fri Jul 17 21:56:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 21:56:03 +0200 (CEST) Subject: [pypy-svn] r66326 - in pypy/branch/parser-compiler/pypy: annotation annotation/test doc/jit interpreter lang/smalltalk lang/smalltalk/tool module/__builtin__ module/__builtin__/test module/mmap/test module/posix objspace/std objspace/std/test rlib rlib/rsre rlib/rsre/test rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/module rpython/module/test rpython/test rpython/tool rpython/tool/test translator translator/backendopt/test translator/c translator/c/test translator/cli/src translator/goal translator/jvm translator/platform translator/platform/test Message-ID: <20090717195603.91B4F169E2A@codespeak.net> Author: benjamin Date: Fri Jul 17 21:55:59 2009 New Revision: 66326 Added: pypy/branch/parser-compiler/pypy/rlib/bitmanipulation.py - copied unchanged from r66325, pypy/trunk/pypy/rlib/bitmanipulation.py pypy/branch/parser-compiler/pypy/rlib/test/test_bitmanipulation.py - copied unchanged from r66325, pypy/trunk/pypy/rlib/test/test_bitmanipulation.py pypy/branch/parser-compiler/pypy/translator/c/dlltool.py - copied unchanged from r66325, pypy/trunk/pypy/translator/c/dlltool.py pypy/branch/parser-compiler/pypy/translator/c/test/test_dlltool.py - copied unchanged from r66325, pypy/trunk/pypy/translator/c/test/test_dlltool.py pypy/branch/parser-compiler/pypy/translator/goal/sharedpypy.py - copied unchanged from r66325, pypy/trunk/pypy/translator/goal/sharedpypy.py Removed: pypy/branch/parser-compiler/pypy/lang/smalltalk/tool/bitmanipulation.py pypy/branch/parser-compiler/pypy/lang/smalltalk/tool/test_bitmanipulation.py Modified: pypy/branch/parser-compiler/pypy/annotation/specialize.py pypy/branch/parser-compiler/pypy/annotation/test/test_annrpython.py pypy/branch/parser-compiler/pypy/annotation/unaryop.py pypy/branch/parser-compiler/pypy/doc/jit/pyjitpl5.txt pypy/branch/parser-compiler/pypy/interpreter/pyopcode.py pypy/branch/parser-compiler/pypy/lang/smalltalk/model.py pypy/branch/parser-compiler/pypy/lang/smalltalk/squeakimage.py pypy/branch/parser-compiler/pypy/module/__builtin__/functional.py pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_classobj.py pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_functional.py pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_range.py pypy/branch/parser-compiler/pypy/module/mmap/test/test_mmap.py pypy/branch/parser-compiler/pypy/module/posix/__init__.py pypy/branch/parser-compiler/pypy/objspace/std/dictmultiobject.py pypy/branch/parser-compiler/pypy/objspace/std/dictobject.py pypy/branch/parser-compiler/pypy/objspace/std/fake.py pypy/branch/parser-compiler/pypy/objspace/std/stringobject.py pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictobject.py pypy/branch/parser-compiler/pypy/objspace/std/test/test_stringobject.py pypy/branch/parser-compiler/pypy/objspace/std/test/test_typeobject.py pypy/branch/parser-compiler/pypy/objspace/std/typeobject.py pypy/branch/parser-compiler/pypy/rlib/libffi.py pypy/branch/parser-compiler/pypy/rlib/rsre/rsre.py pypy/branch/parser-compiler/pypy/rlib/rsre/test/test_rsre.py pypy/branch/parser-compiler/pypy/rlib/runicode.py pypy/branch/parser-compiler/pypy/rlib/streamio.py pypy/branch/parser-compiler/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/parser-compiler/pypy/rpython/lltypesystem/rstr.py pypy/branch/parser-compiler/pypy/rpython/lltypesystem/test/test_ll2ctypes.py pypy/branch/parser-compiler/pypy/rpython/module/ll_os.py pypy/branch/parser-compiler/pypy/rpython/module/test/test_ll_os.py pypy/branch/parser-compiler/pypy/rpython/rstr.py pypy/branch/parser-compiler/pypy/rpython/test/test_rfloat.py pypy/branch/parser-compiler/pypy/rpython/test/test_rstr.py pypy/branch/parser-compiler/pypy/rpython/test/test_runicode.py pypy/branch/parser-compiler/pypy/rpython/tool/rffi_platform.py pypy/branch/parser-compiler/pypy/rpython/tool/test/test_rffi_platform.py pypy/branch/parser-compiler/pypy/translator/backendopt/test/test_merge_if_blocks.py pypy/branch/parser-compiler/pypy/translator/c/genc.py pypy/branch/parser-compiler/pypy/translator/c/node.py pypy/branch/parser-compiler/pypy/translator/c/test/test_genc.py pypy/branch/parser-compiler/pypy/translator/c/test/test_newgc.py pypy/branch/parser-compiler/pypy/translator/c/test/test_standalone.py pypy/branch/parser-compiler/pypy/translator/cli/src/ll_os.cs pypy/branch/parser-compiler/pypy/translator/driver.py pypy/branch/parser-compiler/pypy/translator/goal/targetpypystandalone.py pypy/branch/parser-compiler/pypy/translator/goal/translate.py pypy/branch/parser-compiler/pypy/translator/jvm/genjvm.py pypy/branch/parser-compiler/pypy/translator/platform/__init__.py pypy/branch/parser-compiler/pypy/translator/platform/darwin.py pypy/branch/parser-compiler/pypy/translator/platform/linux.py pypy/branch/parser-compiler/pypy/translator/platform/test/test_maemo.py pypy/branch/parser-compiler/pypy/translator/platform/test/test_platform.py pypy/branch/parser-compiler/pypy/translator/platform/windows.py Log: merge from the trunk -r65588:66325 Modified: pypy/branch/parser-compiler/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/annotation/specialize.py (original) +++ pypy/branch/parser-compiler/pypy/annotation/specialize.py Fri Jul 17 21:55:59 2009 @@ -123,10 +123,10 @@ example_args, example_value = self.table.iteritems().next() nbargs = len(example_args) # list of sets of possible argument values -- one set per argument index - sets = [{} for i in range(nbargs)] + sets = [set() for i in range(nbargs)] for args in self.table: for i in range(nbargs): - sets[i][args[i]] = True + sets[i].add(args[i]) bookkeeper = self.funcdesc.bookkeeper annotator = bookkeeper.annotator @@ -233,8 +233,8 @@ # schedule this new graph for being annotated args_s = [] - for set in sets: - values_s = [bookkeeper.immutablevalue(x) for x in set] + for arg_types in sets: + values_s = [bookkeeper.immutablevalue(x) for x in arg_types] args_s.append(unionof(*values_s)) annotator.addpendinggraph(self.graph, args_s) Modified: pypy/branch/parser-compiler/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/parser-compiler/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/parser-compiler/pypy/annotation/test/test_annrpython.py Fri Jul 17 21:55:59 2009 @@ -425,6 +425,14 @@ s_meth = s_example.getattr(iv(methname)) assert isinstance(s_constmeth, annmodel.SomeBuiltin) + def test_str_splitlines(self): + a = self.RPythonAnnotator() + def f(a_str): + return a_str.splitlines() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeList) + assert s.listdef.listitem.resized + def test_simple_slicing(self): a = self.RPythonAnnotator() s = a.build_types(snippet.simple_slice, [list]) @@ -3068,7 +3076,6 @@ a = self.RPythonAnnotator() a.translator.config.translation.list_comprehension_operations = True py.test.raises(TooLateForChange, a.build_types, fn, [int]) - def test_listitem_never_resize(self): from pypy.rlib.debug import check_annotation Modified: pypy/branch/parser-compiler/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/parser-compiler/pypy/annotation/unaryop.py (original) +++ pypy/branch/parser-compiler/pypy/annotation/unaryop.py Fri Jul 17 21:55:59 2009 @@ -521,6 +521,13 @@ def method_lower(str): return SomeString() + def method_splitlines(str, s_keep_newlines=None): + s_list = getbookkeeper().newlist(str.basestringclass()) + # Force the list to be resizable because ll_splitlines doesn't + # preallocate the list. + s_list.listdef.listitem.resize() + return s_list + def method_decode(str, s_enc): if not s_enc.is_constant(): raise TypeError("Non-constant encoding not supported") Modified: pypy/branch/parser-compiler/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/branch/parser-compiler/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/branch/parser-compiler/pypy/doc/jit/pyjitpl5.txt Fri Jul 17 21:55:59 2009 @@ -17,3 +17,5 @@ * http://morepypy.blogspot.com/2009/03/jit-bit-of-look-inside.html * http://morepypy.blogspot.com/2009/03/good-news-everyone.html * http://morepypy.blogspot.com/2009/04/roadmap-for-jit.html +* http://morepypy.blogspot.com/2009/04/4-weeks-of-gdb.html +* http://morepypy.blogspot.com/2009/05/icooolps-submissions.html Modified: pypy/branch/parser-compiler/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyopcode.py Fri Jul 17 21:55:59 2009 @@ -1,7 +1,7 @@ """ Implementation of a part of the standard Python opcodes. -The rest, dealing with variables in optimized ways, is in -pyfastscope.py and pynestedscope.py. + +The rest, dealing with variables in optimized ways, is in nestedscope.py. """ import sys @@ -304,9 +304,9 @@ ################################################################ ## Implementation of the "operational" opcodes - ## See also pyfastscope.py and pynestedscope.py for the rest. + ## See also nestedscope.py for the rest. ## - + # the 'self' argument of opcode implementations is called 'f' # for historical reasons Modified: pypy/branch/parser-compiler/pypy/lang/smalltalk/model.py ============================================================================== --- pypy/branch/parser-compiler/pypy/lang/smalltalk/model.py (original) +++ pypy/branch/parser-compiler/pypy/lang/smalltalk/model.py Fri Jul 17 21:55:59 2009 @@ -20,7 +20,7 @@ from pypy.lang.smalltalk import constants, error from pypy.tool.pairtype import extendabletype from pypy.rlib.objectmodel import instantiate -from pypy.lang.smalltalk.tool.bitmanipulation import splitter +from pypy.rlib.bitmanipulation import splitter class W_Object(object): """Root of Squeak model, abstract.""" Modified: pypy/branch/parser-compiler/pypy/lang/smalltalk/squeakimage.py ============================================================================== --- pypy/branch/parser-compiler/pypy/lang/smalltalk/squeakimage.py (original) +++ pypy/branch/parser-compiler/pypy/lang/smalltalk/squeakimage.py Fri Jul 17 21:55:59 2009 @@ -3,7 +3,7 @@ from pypy.lang.smalltalk import constants from pypy.lang.smalltalk import model from pypy.rlib import objectmodel -from pypy.lang.smalltalk.tool.bitmanipulation import splitter +from pypy.rlib.bitmanipulation import splitter def chrs2int(b): assert len(b) == 4 Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/functional.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/functional.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/functional.py Fri Jul 17 21:55:59 2009 @@ -61,33 +61,27 @@ get a list in decending order.""" try: - # save duplication by redirecting every error to applevel - x = space.int_w(w_x) + x = space.int_w(space.int(w_x)) if space.is_w(w_y, space.w_None): start, stop = 0, x else: - start, stop = x, space.int_w(w_y) - step = space.int_w(w_step) + start, stop = x, space.int_w(space.int(w_y)) + step = space.int_w(space.int(w_step)) howmany = get_len_of_range(start, stop, step) - except OperationError, e: - if not e.match(space, space.w_TypeError): - pass - else: - raise - except (ValueError, OverflowError): - pass - else: - if (space.config.objspace.std.withmultilist or - space.config.objspace.std.withrangelist): - return range_withspecialized_implementation(space, start, - step, howmany) - res_w = [None] * howmany - v = start - for idx in range(howmany): - res_w[idx] = space.wrap(v) - v += step - return space.newlist(res_w) - return range_fallback(space, w_x, w_y, w_step) + except (ValueError, OverflowError, OperationError): + # save duplication by redirecting every error to applevel + return range_fallback(space, w_x, w_y, w_step) + + if (space.config.objspace.std.withmultilist or + space.config.objspace.std.withrangelist): + return range_withspecialized_implementation(space, start, + step, howmany) + res_w = [None] * howmany + v = start + for idx in range(howmany): + res_w[idx] = space.wrap(v) + v += step + return space.newlist(res_w) range_int = range range_int.unwrap_spec = [ObjSpace, W_Root, W_Root, W_Root] del range # don't hide the builtin one @@ -207,7 +201,7 @@ def _toint(space, w_obj): # trying to support float arguments, just because CPython still does try: - return space.int_w(w_obj) + return space.int_w(space.int(w_obj)) except OperationError, e: if space.is_true(space.isinstance(w_obj, space.w_float)): return space.int_w(space.int(w_obj)) Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_classobj.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_classobj.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_classobj.py Fri Jul 17 21:55:59 2009 @@ -505,6 +505,7 @@ raises(TypeError, cmp, a, b) def test_hash(self): + import sys class A: pass hash(A()) # does not crash @@ -528,11 +529,12 @@ return 1 a = A() raises(TypeError, hash, a) + bigint = sys.maxint + 1 class A: # can return long def __hash__(self): - return long(2**31) + return long(bigint) a = A() - assert hash(a) == -2147483648 + assert hash(a) == -bigint def test_index(self): import sys Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_functional.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_functional.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_functional.py Fri Jul 17 21:55:59 2009 @@ -117,6 +117,25 @@ # test again, to make sure that xrange() is not its own iterator assert iter(x).next() == 2 + def test_xrange_object_with___int__(self): + class A(object): + def __int__(self): + return 5 + + assert list(xrange(A())) == [0, 1, 2, 3, 4] + assert list(xrange(0, A())) == [0, 1, 2, 3, 4] + assert list(xrange(0, 10, A())) == [0, 5] + + def test_xrange_float(self): + assert list(xrange(0.1, 2.0, 1.1)) == [0, 1] + + def test_xrange_long(self): + import sys + a = long(10 * sys.maxint) + raises(OverflowError, xrange, a) + raises(OverflowError, xrange, 0, a) + raises(OverflowError, xrange, 0, 1, a) + class AppTestReversed: def test_reversed(self): r = reversed("hello") Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_range.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_range.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_range.py Fri Jul 17 21:55:59 2009 @@ -17,7 +17,6 @@ def test_range_negstartisstop(self): assert range(-1, -1) == [] - def test_range_zero(self): assert range(0) == [] @@ -60,10 +59,34 @@ def test_range_zerostep(self): raises(ValueError, range, 1, 5, 0) - def DONT_test_range_float(self): - "How CPython does it - UGLY, ignored for now." + def test_range_float(self): + "How CPython does it - UGLY." assert range(0.1, 2.0, 1.1) == [0, 1] def test_range_wrong_type(self): raises(TypeError, range, "42") + def test_range_object_with___int__(self): + class A(object): + def __int__(self): + return 5 + + assert range(A()) == [0, 1, 2, 3, 4] + assert range(0, A()) == [0, 1, 2, 3, 4] + assert range(0, 10, A()) == [0, 5] + + def test_range_long(self): + import sys + assert range(-2**100) == [] + assert range(0, -2**100) == [] + assert range(0, 2**100, -1) == [] + assert range(0, 2**100, -1) == [] + + a = long(10 * sys.maxint) + b = long(100 * sys.maxint) + c = long(50 * sys.maxint) + + assert range(a, a+2) == [a, a+1] + assert range(a+2, a, -1L) == [a+2, a+1] + assert range(a+4, a, -2) == [a+4, a+2] + Modified: pypy/branch/parser-compiler/pypy/module/mmap/test/test_mmap.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/mmap/test/test_mmap.py (original) +++ pypy/branch/parser-compiler/pypy/module/mmap/test/test_mmap.py Fri Jul 17 21:55:59 2009 @@ -42,7 +42,7 @@ raises(TypeError, mmap, 0, 1, 2, 3, 4, 5) raises(TypeError, mmap, 0, 1, 2, 3, "foo", 5) raises(TypeError, mmap, 0, 1, foo="foo") - raises(TypeError, mmap, 0, -1) + raises((TypeError, OverflowError), mmap, 0, -1) raises(OverflowError, mmap, 0, sys.maxint ** 3) raises(ValueError, mmap, 0, 1, flags=2, access=3) raises(ValueError, mmap, 0, 1, access=123) @@ -410,7 +410,7 @@ def fn(m): m *= 1 # but it raises((SystemError, TypeError), fn, m) # doesn't def fn(): 1 * m # make much sense - raises(TypeError, fn) + raises((SystemError, TypeError), fn) m.close() f.close() # Modified: pypy/branch/parser-compiler/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/posix/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/module/posix/__init__.py Fri Jul 17 21:55:59 2009 @@ -120,6 +120,14 @@ if hasattr(os, name): interpleveldefs[name] = 'interp_posix.' + name + def __init__(self, space, w_name): + backend = space.config.translation.backend + # the Win32 urandom implementation isn't going to translate on JVM or CLI + # so we have to remove it + if backend == 'cli' or backend == 'jvm': + del self.interpleveldefs['urandom'] + MixedModule.__init__(self, space, w_name) + def startup(self, space): from pypy.module.posix import interp_posix interp_posix.get(space).startup(space) Modified: pypy/branch/parser-compiler/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/dictmultiobject.py Fri Jul 17 21:55:59 2009 @@ -1049,6 +1049,7 @@ w_self.implementation = SharedDictImplementation(space) else: w_self.implementation = space.emptydictimpl + w_self.space = space def initialize_content(w_self, list_pairs_w): impl = w_self.implementation @@ -1056,6 +1057,11 @@ impl = impl.setitem(w_k, w_v) w_self.implementation = impl + def initialize_from_strdict_shared(w_self, strdict): + impl = StrDictImplementation(w_self.space) + impl.content = strdict + w_self.implementation = impl + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%s)" % (w_self.__class__.__name__, w_self.implementation) Modified: pypy/branch/parser-compiler/pypy/objspace/std/dictobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/dictobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/dictobject.py Fri Jul 17 21:55:59 2009 @@ -13,11 +13,18 @@ w_self.content = r_dict(space.eq_w, space.hash_w) else: w_self.content = w_otherdict.content.copy() + w_self.space = space def initialize_content(w_self, list_pairs_w): for w_k, w_v in list_pairs_w: w_self.content[w_k] = w_v + def initialize_from_strdict_shared(w_self, strdict): + # XXX the stuff below is slightly broken, as the dict is not really shared + # this would be very very annoying to fix with non-multidicts + for key, w_value in strdict.items(): + w_self.content[w_self.space.wrap(key)] = w_value + def __repr__(w_self): """ representation for debugging purposes """ return "%s(%s)" % (w_self.__class__.__name__, w_self.content) Modified: pypy/branch/parser-compiler/pypy/objspace/std/fake.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/fake.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/fake.py Fri Jul 17 21:55:59 2009 @@ -10,11 +10,10 @@ # this file automatically generates non-reimplementations of CPython # types that we do not yet implement in the standard object space -# (files being the biggy) def fake_object(space, x): - if isinstance(x, file): + if isinstance(x, file): debug_print("fake-wrapping interp file %s" % x) if isinstance(x, type): ft = fake_type(x) @@ -61,10 +60,9 @@ def really_build_fake_type(cpy_type): "NOT_RPYTHON (not remotely so!)." - #assert not issubclass(cpy_type, file), cpy_type debug_print('faking %r'%(cpy_type,)) kw = {} - + if cpy_type.__name__ == 'SRE_Pattern': import re import __builtin__ @@ -92,7 +90,7 @@ for (key, w_value) in kwds_w.items(): kwds[key] = space.unwrap(w_value) try: - r = apply(cpy_type.__new__, [cpy_type]+args, kwds) + r = cpy_type.__new__(*[cpy_type]+args, **kwds) except: wrap_exception(space) raise @@ -118,11 +116,7 @@ w_self.val = val def unwrap(w_self, space): return w_self.val - - # cannot write to W_Fake.__name__ in Python 2.2! - W_Fake = type(W_Object)('W_Fake%s'%(cpy_type.__name__.capitalize()), - (W_Object,), - dict(W_Fake.__dict__.items())) + W_Fake.__name__ = 'W_Fake%s'%(cpy_type.__name__.capitalize()) W_Fake.typedef.fakedcpytype = cpy_type return W_Fake @@ -159,7 +153,7 @@ def getfastscope(self): raise OperationError(self.space.w_TypeError, - self.space.wrap("cannot get fastscope of a CPythonFakeFrame")) + self.space.wrap("cannot get fastscope of a CPythonFakeFrame")) def run(self): code = self.fakecode assert isinstance(code, CPythonFakeCode) Modified: pypy/branch/parser-compiler/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/stringobject.py Fri Jul 17 21:55:59 2009 @@ -701,10 +701,9 @@ def str_splitlines__String_ANY(space, w_self, w_keepends): - data = w_self._value u_keepends = space.int_w(w_keepends) # truth value, but type checked + data = w_self._value selflen = len(data) - strs_w = [] i = j = 0 while i < selflen: @@ -723,7 +722,6 @@ if j < selflen: strs_w.append(sliced(space, data, j, len(data), w_self)) - return space.newlist(strs_w) def str_zfill__String_ANY(space, w_self, w_width): Modified: pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictmultiobject.py Fri Jul 17 21:55:59 2009 @@ -12,6 +12,17 @@ def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + def test_initialize_from_strdict_really_shared(self): + space = self.space + w = space.wrap + d = {"a": w(1), "b": w(2)} + w_d = space.DictObjectCls(space) + w_d.initialize_from_strdict_shared(d) + assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) + assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) + d["c"] = w(41) + assert self.space.eq_w(space.getitem(w_d, w("c")), w(41)) + class AppTest_DictMultiObject(test_dictobject.AppTest_DictObject): def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) Modified: pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/test/test_dictobject.py Fri Jul 17 21:55:59 2009 @@ -123,6 +123,16 @@ assert self.space.eq_w(space.call_function(get, w("33")), w(None)) assert self.space.eq_w(space.call_function(get, w("33"), w(44)), w(44)) + def test_initialize_from_strdict_shared(self): + space = self.space + w = space.wrap + d = {"a": w(1), "b": w(2)} + w_d = space.DictObjectCls(space) + w_d.initialize_from_strdict_shared(d) + assert self.space.eq_w(space.getitem(w_d, w("a")), w(1)) + assert self.space.eq_w(space.getitem(w_d, w("b")), w(2)) + + class AppTest_DictObject: Modified: pypy/branch/parser-compiler/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/test/test_stringobject.py Fri Jul 17 21:55:59 2009 @@ -723,6 +723,9 @@ raises(TypeError, len, iter(iterable)) def test_overflow_replace(self): + import sys + if sys.maxint > 2**31-1: + skip("Wrong platform") x = "A" * (2**16) raises(OverflowError, x.replace, '', x) Modified: pypy/branch/parser-compiler/pypy/objspace/std/test/test_typeobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/test/test_typeobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/test/test_typeobject.py Fri Jul 17 21:55:59 2009 @@ -915,14 +915,19 @@ return 0 raises(TypeError, X) +class AppTestWithMultidictTypes: + def setup_class(cls): + cls.space = gettestobjspace(**{"objspace.std.withmultidict": True}) + def test_dictproxy_is_updated(self): - skip("fix me") class A(object): x = 1 d = A.__dict__ assert d["x"] == 1 A.y = 2 assert d["y"] == 2 + assert ("x", 1) in d.items() + assert ("y", 2) in d.items() class AppTestMutableBuiltintypes: Modified: pypy/branch/parser-compiler/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/parser-compiler/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/parser-compiler/pypy/objspace/std/typeobject.py Fri Jul 17 21:55:59 2009 @@ -246,14 +246,8 @@ if w_self.lazyloaders: w_self._freeze_() # force un-lazification space = w_self.space - dictspec = [] - for key, w_value in w_self.dict_w.items(): - dictspec.append((space.wrap(key), w_value)) - # speed hack: instantiate a dict object cls directly - # NB: cannot use newdict, because that could return something else - # than an instance of DictObjectCls newdic = space.DictObjectCls(space) - newdic.initialize_content(dictspec) + newdic.initialize_from_strdict_shared(w_self.dict_w) return W_DictProxyObject(newdic) def unwrap(w_self, space): Modified: pypy/branch/parser-compiler/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rlib/libffi.py (original) +++ pypy/branch/parser-compiler/pypy/rlib/libffi.py Fri Jul 17 21:55:59 2009 @@ -255,6 +255,8 @@ pass # No check libc_name = ctypes.util.find_library('c') + assert libc_name is not None, "Cannot find C library, ctypes.util.find_library('c') returned None" + def get_libc_name(): return libc_name Modified: pypy/branch/parser-compiler/pypy/rlib/rsre/rsre.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rlib/rsre/rsre.py (original) +++ pypy/branch/parser-compiler/pypy/rlib/rsre/rsre.py Fri Jul 17 21:55:59 2009 @@ -12,6 +12,7 @@ rsre_core_filename = rsre_core_filename[:-1] rsre_core_filename = os.path.abspath(rsre_core_filename) del rsre_core +from pypy.rlib.rsre.rsre_char import getlower def insert_sre_methods(locals, name): """A hack that inserts the SRE entry point methods into the 'locals' @@ -58,3 +59,6 @@ def get_char_ord(self, p): return ord(self.string[p]) + + def lower(self, char_ord): + return getlower(char_ord, 0) Modified: pypy/branch/parser-compiler/pypy/rlib/rsre/test/test_rsre.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rlib/rsre/test/test_rsre.py (original) +++ pypy/branch/parser-compiler/pypy/rlib/rsre/test/test_rsre.py Fri Jul 17 21:55:59 2009 @@ -80,6 +80,11 @@ def test_getlower(): assert rsre_char.getlower(ord("A"), 0) == ord("a") +def test_SimpleStringState(): + state = SimpleStringState("A", 0, -1) + assert state.get_char_ord(0) == ord("A") + assert state.lower(state.get_char_ord(0)) == ord("a") + def test_get_byte_array(): if sys.byteorder == "big": if rsre_char.CODESIZE == 2: Modified: pypy/branch/parser-compiler/pypy/rlib/runicode.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rlib/runicode.py (original) +++ pypy/branch/parser-compiler/pypy/rlib/runicode.py Fri Jul 17 21:55:59 2009 @@ -1,5 +1,5 @@ import sys -from pypy.lang.smalltalk.tool.bitmanipulation import splitter +from pypy.rlib.bitmanipulation import splitter from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.objectmodel import we_are_translated Modified: pypy/branch/parser-compiler/pypy/rlib/streamio.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rlib/streamio.py (original) +++ pypy/branch/parser-compiler/pypy/rlib/streamio.py Fri Jul 17 21:55:59 2009 @@ -176,6 +176,10 @@ compilation_info=_eci) SetEndOfFile = rffi.llexternal('SetEndOfFile', [rffi.LONG], rwin32.BOOL, compilation_info=_eci) + + # HACK: These implementations are specific to MSVCRT and the C backend. + # When generating on CLI or JVM, these are patched out. + # See PyPyTarget.target() in targetpypystandalone.py def _setfd_binary(fd): _setmode(fd, os.O_BINARY) Modified: pypy/branch/parser-compiler/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/lltypesystem/ll2ctypes.py Fri Jul 17 21:55:59 2009 @@ -19,6 +19,7 @@ from pypy.rpython.lltypesystem.rclass import OBJECT from pypy.rpython.annlowlevel import base_ptr_lltype from pypy.rpython import raddress +from pypy.translator.platform import platform def uaddressof(obj): return fixid(ctypes.addressof(obj)) @@ -813,22 +814,46 @@ cfunc = get_on_lib(ctypes.windll.kernel32, funcname) else: cfunc = None + not_found = [] for libname in libraries: - libpath = ctypes.util.find_library(libname) - if not libpath and os.path.isabs(libname): - libpath = libname + libpath = None + ext = platform.so_ext + prefixes = platform.so_prefixes + for dir in eci.library_dirs: + if libpath: + break + for prefix in prefixes: + tryfile = os.path.join(dir, prefix + libname + '.' + ext) + if os.path.isfile(tryfile): + libpath = tryfile + break + if not libpath: + libpath = ctypes.util.find_library(libname) + if not libpath and os.path.isabs(libname): + libpath = libname if libpath: dllclass = getattr(ctypes, calling_conv + 'dll') - # urgh, cannot pass the flag to dllclass.LoadLibrary - clib = dllclass._dlltype(libpath, ctypes.RTLD_GLOBAL) + # on ie slackware there was need for RTLD_GLOBAL here. + # this breaks a lot of things, since passing RTLD_GLOBAL + # creates symbol conflicts on C level. + clib = dllclass.LoadLibrary(libpath) cfunc = get_on_lib(clib, funcname) if cfunc is not None: break + else: + not_found.append(libname) if cfunc is None: # function name not found in any of the libraries if not libraries: place = 'the standard C library (missing libraries=...?)' + elif len(not_found) == len(libraries): + if len(not_found) == 1: + raise NotImplementedError( + 'cannot find the library %r' % (not_found[0],)) + else: + raise NotImplementedError( + 'cannot find any of the libraries %r' % (not_found,)) elif len(libraries) == 1: place = 'library %r' % (libraries[0],) else: Modified: pypy/branch/parser-compiler/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/lltypesystem/rstr.py Fri Jul 17 21:55:59 2009 @@ -674,6 +674,7 @@ s1.copy_contents(s1, newstr, 0, 0, newlen) return newstr + def ll_split_chr(LIST, s, c): chars = s.chars strlen = len(chars) Modified: pypy/branch/parser-compiler/pypy/rpython/lltypesystem/test/test_ll2ctypes.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/lltypesystem/test/test_ll2ctypes.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/lltypesystem/test/test_ll2ctypes.py Fri Jul 17 21:55:59 2009 @@ -15,6 +15,7 @@ from pypy.rpython.test.test_llinterp import interpret from pypy.annotation.annrpython import RPythonAnnotator from pypy.rpython.rtyper import RPythonTyper +from pypy.tool.udir import udir class TestLL2Ctypes(object): @@ -994,3 +995,46 @@ v2 = ctypes2lltype(llmemory.GCREF, ctypes.c_void_p(1235)) assert v2 != v +class TestPlatform(object): + def test_lib_on_libpaths(self): + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo(export_symbols=['f']) + so = platform.compile([c_file], eci, standalone=False) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 + + def test_prefix(self): + + if sys.platform != 'linux2': + py.test.skip("Not supported") + + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + tmpdir = udir.join('lib_on_libppaths_prefix') + tmpdir.ensure(dir=1) + c_file = tmpdir.join('c_file.c') + c_file.write('int f(int a, int b) { return (a + b); }') + eci = ExternalCompilationInfo() + so = platform.compile([c_file], eci, standalone=False) + sopath = py.path.local(so) + sopath.move(sopath.dirpath().join('libc_file.so')) + eci = ExternalCompilationInfo( + libraries = ['c_file'], + library_dirs = [str(so.dirpath())] + ) + f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT, + compilation_info=eci) + assert f(3, 4) == 7 + Modified: pypy/branch/parser-compiler/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/module/ll_os.py Fri Jul 17 21:55:59 2009 @@ -88,10 +88,31 @@ def __init__(self): self.configure(CConfig) + # on some platforms, e.g. OS X Leopard, the following constants which + # may be defined in pyconfig.h triggers "legacy" behaviour for functions + # like setpgrp(): + # + # _POSIX_C_SOURCE 200112L + # _XOPEN_SOURCE 600 + # _DARWIN_C_SOURCE 1 + # + # since the translation currently includes pyconfig.h, the checkcompiles + # call below include the pyconfig.h file so that the same behaviour is + # present in both the check and the final translation... + if hasattr(os, 'getpgrp'): - self.GETPGRP_HAVE_ARG = platform.checkcompiles("getpgrp(0)", "#include ") + self.GETPGRP_HAVE_ARG = platform.checkcompiles( + "getpgrp(0)", + '#include "pyconfig.h"\n#include ', + [platform.get_python_include_dir()] + ) + if hasattr(os, 'setpgrp'): - self.SETPGRP_HAVE_ARG = platform.checkcompiles("setpgrp(0,0)", "#include ") + self.SETPGRP_HAVE_ARG = platform.checkcompiles( + "setpgrp(0,0)", + '#include "pyconfig.h"\n#include ', + [platform.get_python_include_dir()] + ) # we need an indirection via c functions to get macro calls working on llvm XXX still? if hasattr(os, 'WCOREDUMP'): Modified: pypy/branch/parser-compiler/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/module/test/test_ll_os.py Fri Jul 17 21:55:59 2009 @@ -84,7 +84,13 @@ ll_execve = getllimpl(os.execve) - def run_execve(program, env): + def run_execve(program, args=None, env=None, do_path_lookup=False): + if args is None: + args = [program] + else: + args = [program] + args + if env is None: + env = {} # we cannot directly call ll_execve() because it replaces the # current process. fd_read, fd_write = os.pipe() @@ -94,7 +100,10 @@ os.close(fd_read) os.dup2(fd_write, 1) # stdout os.close(fd_write) - ll_execve(program, [program], env) + if do_path_lookup: + os.execvp(program, args) + else: + ll_execve(program, args, env) assert 0, "should not arrive here" else: # in the parent @@ -109,15 +118,17 @@ return status, ''.join(child_stdout) # Test exit status and code - result, child_stdout = run_execve("/bin/true", {}) + result, child_stdout = run_execve("/usr/bin/which", ["true"], do_path_lookup=True) + result, child_stdout = run_execve(child_stdout.strip()) # /bin/true or /usr/bin/true assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) == 0 - result, child_stdout = run_execve("/bin/false", {}) + result, child_stdout = run_execve("/usr/bin/which", ["false"], do_path_lookup=True) + result, child_stdout = run_execve(child_stdout.strip()) # /bin/false or /usr/bin/false assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) == 1 # Test environment - result, child_stdout = run_execve("/usr/bin/env", EXECVE_ENV) + result, child_stdout = run_execve("/usr/bin/env", env=EXECVE_ENV) assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) == 0 assert dict([line.split('=') for line in child_stdout.splitlines()]) == EXECVE_ENV Modified: pypy/branch/parser-compiler/pypy/rpython/rstr.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/rstr.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/rstr.py Fri Jul 17 21:55:59 2009 @@ -7,7 +7,7 @@ from pypy.rpython.rtuple import AbstractTupleRepr from pypy.rpython import rint from pypy.rpython.lltypesystem.lltype import Signed, Bool, Void, UniChar,\ - cast_primitive + cast_primitive, typeOf class AbstractStringRepr(Repr): pass @@ -213,6 +213,20 @@ raise TyperError("sep.join() of non-string list: %r" % r_lst) return hop.gendirectcall(llfn, v_str, v_length, v_items) + def rtype_method_splitlines(self, hop): + rstr = hop.args_r[0].repr + if hop.nb_args == 2: + args = hop.inputargs(rstr.repr, Bool) + else: + args = [hop.inputarg(rstr.repr, 0), hop.inputconst(Bool, False)] + try: + list_type = hop.r_result.lowleveltype.TO + except AttributeError: + list_type = hop.r_result.lowleveltype + cLIST = hop.inputconst(Void, list_type) + hop.exception_cannot_occur() + return hop.gendirectcall(self.ll.ll_splitlines, cLIST, *args) + def rtype_method_split(self, hop): rstr = hop.args_r[0].repr v_str, v_chr = hop.inputargs(rstr.repr, rstr.char_repr) @@ -718,3 +732,36 @@ raise ValueError return parts_to_float(sign, before_point, after_point, exponent) + + def ll_splitlines(cls, LIST, ll_str, keep_newlines): + from pypy.rpython.annlowlevel import hlstr + s = hlstr(ll_str) + STR = typeOf(ll_str) + strlen = len(s) + i = 0 + j = 0 + # The annotator makes sure this list is resizable. + res = LIST.ll_newlist(0) + while j < strlen: + while i < strlen and s[i] != '\n' and s[i] != '\r': + i += 1 + eol = i + if i < strlen: + if s[i] == '\r' and i + 1 < strlen and s[i + 1] == '\n': + i += 2 + else: + i += 1 + if keep_newlines: + eol = i + list_length = res.ll_length() + res._ll_resize_ge(list_length + 1) + item = cls.ll_stringslice_startstop(ll_str, j, eol) + res.ll_setitem_fast(list_length, item) + j = i + if j < strlen: + list_length = res.ll_length() + res._ll_resize_ge(list_length + 1) + item = cls.ll_stringslice_startstop(ll_str, j, strlen) + res.ll_setitem_fast(list_length, item) + return res + ll_splitlines = classmethod(ll_splitlines) Modified: pypy/branch/parser-compiler/pypy/rpython/test/test_rfloat.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/test/test_rfloat.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/test/test_rfloat.py Fri Jul 17 21:55:59 2009 @@ -2,7 +2,7 @@ from pypy.translator.translator import TranslationContext from pypy.rpython.test import snippet from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.rlib.rarithmetic import r_uint, r_longlong, r_singlefloat,\ +from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_singlefloat,\ isnan, isinf class TestSnippet(object): @@ -76,12 +76,17 @@ assert res == fn(2.34) def test_longlong_conversion(self): + import sys def fn(f): return r_longlong(f) res = self.interpret(fn, [1.0]) assert res == 1 - assert self.is_of_type(res, r_longlong) + # r_longlong is int on a 64 bit system + if sys.maxint == 2**63 - 1: + assert self.is_of_type(res, int) + else: + assert self.is_of_type(res, r_longlong) res = self.interpret(fn, [2.34]) assert res == fn(2.34) big = float(0x7fffffffffffffff) Modified: pypy/branch/parser-compiler/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/test/test_rstr.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/test/test_rstr.py Fri Jul 17 21:55:59 2009 @@ -554,6 +554,21 @@ res = self.ll_to_string(self.interpret(dummy, [])) assert res == expected + def test_splitlines(self): + const = self.const + def f(i, newlines): + s = [const(''), const("\n"), const("\n\n"), const("hi\n"), + const("random data\r\n"), const("\r\n"), const("\rdata")] + test_string = s[i] + if newlines: + return len(test_string.splitlines(True)) + else: + return len(test_string.splitlines()) + for newlines in (True, False): + for i in xrange(5): + res = self.interpret(f, [i, newlines]) + assert res == f(i, newlines) + def test_split(self): const = self.const def fn(i): Modified: pypy/branch/parser-compiler/pypy/rpython/test/test_runicode.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/test/test_runicode.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/test/test_runicode.py Fri Jul 17 21:55:59 2009 @@ -200,6 +200,7 @@ test_char_isxxx = unsupported test_upper = unsupported test_lower = unsupported + test_splitlines = unsupported test_strformat = unsupported test_strformat_instance = unsupported test_strformat_nontuple = unsupported Modified: pypy/branch/parser-compiler/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/tool/rffi_platform.py Fri Jul 17 21:55:59 2009 @@ -14,9 +14,12 @@ # # Helpers for simple cases -def eci_from_header(c_header_source): +def eci_from_header(c_header_source, include_dirs=None): + if include_dirs is None: + include_dirs = [] return ExternalCompilationInfo( - pre_include_bits=[c_header_source] + pre_include_bits=[c_header_source], + include_dirs=include_dirs ) def getstruct(name, c_header_source, interesting_fields): @@ -43,9 +46,9 @@ DEFINED = Defined(macro) return configure(CConfig)['DEFINED'] -def has(name, c_header_source): +def has(name, c_header_source, include_dirs=None): class CConfig: - _compilation_info_ = eci_from_header(c_header_source) + _compilation_info_ = eci_from_header(c_header_source, include_dirs) HAS = Has(name) return configure(CConfig)['HAS'] @@ -57,9 +60,9 @@ WORKS = Works() configure(CConfig) -def checkcompiles(expression, c_header_source): +def checkcompiles(expression, c_header_source, include_dirs=None): """Check if expression compiles. If not, returns False""" - return has(expression, c_header_source) + return has(expression, c_header_source, include_dirs) def sizeof(name, eci, **kwds): class CConfig: Modified: pypy/branch/parser-compiler/pypy/rpython/tool/test/test_rffi_platform.py ============================================================================== --- pypy/branch/parser-compiler/pypy/rpython/tool/test/test_rffi_platform.py (original) +++ pypy/branch/parser-compiler/pypy/rpython/tool/test/test_rffi_platform.py Fri Jul 17 21:55:59 2009 @@ -4,6 +4,7 @@ from pypy.rpython.lltypesystem import rffi from pypy.tool.udir import udir from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.translator.platform import platform def import_ctypes(): try: @@ -234,3 +235,24 @@ a = rffi_platform.memory_alignment() print a assert a % struct.calcsize("P") == 0 + +def test_external_lib(): + # XXX this one seems to be a bit too platform-specific. Check + # how to test it on windows correctly (using so_prefix?) + # and what are alternatives to LD_LIBRARY_PATH + eci = ExternalCompilationInfo() + c_source = """ + int f(int a, int b) + { + return (a + b); + } + """ + tmpdir = udir.join('external_lib').ensure(dir=1) + c_file = tmpdir.join('libc_lib.c') + c_file.write(c_source) + l = platform.compile([c_file], eci, standalone=False) + eci = ExternalCompilationInfo( + libraries = ['c_lib'], + library_dirs = [str(tmpdir)] + ) + rffi_platform.verify_eci(eci) Modified: pypy/branch/parser-compiler/pypy/translator/backendopt/test/test_merge_if_blocks.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/backendopt/test/test_merge_if_blocks.py (original) +++ pypy/branch/parser-compiler/pypy/translator/backendopt/test/test_merge_if_blocks.py Fri Jul 17 21:55:59 2009 @@ -122,6 +122,16 @@ assert res == 6 +def test_merge_with_or(): + def merge(n): + if n == 5: + return 4 + elif n == 14 or n == 2: + return 16 + else: + return 7 + do_test_merge(merge, [5, 6, 14, 2, 3, 123]) + def test_dont_merge(): def merge(n, m): Modified: pypy/branch/parser-compiler/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/c/genc.py (original) +++ pypy/branch/parser-compiler/pypy/translator/c/genc.py Fri Jul 17 21:55:59 2009 @@ -105,7 +105,7 @@ def __init__(self, translator, entrypoint, config, gcpolicy=None): self.translator = translator self.entrypoint = entrypoint - self.entrypoint_name = self.entrypoint.func_name + self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) self.originalentrypoint = entrypoint self.config = config self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/ @@ -156,8 +156,13 @@ # build entrypoint and eventually other things to expose pf = self.getentrypointptr() - pfname = db.get(pf) - self.c_entrypoint_name = pfname + if isinstance(pf, list): + for one_pf in pf: + db.get(one_pf) + self.c_entrypoint_name = None + else: + pfname = db.get(pf) + self.c_entrypoint_name = pfname db.complete() self.collect_compilation_info(db) @@ -209,7 +214,6 @@ if db is None: db = self.build_database() pf = self.getentrypointptr() - pfname = db.get(pf) if self.modulename is None: self.modulename = uniquemodulename('testing') modulename = self.modulename @@ -229,6 +233,7 @@ self.eci, defines = defines) else: + pfname = db.get(pf) if self.config.translation.instrument: defines['INSTRUMENT'] = 1 if CBuilder.have___thread: Modified: pypy/branch/parser-compiler/pypy/translator/c/node.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/c/node.py (original) +++ pypy/branch/parser-compiler/pypy/translator/c/node.py Fri Jul 17 21:55:59 2009 @@ -13,6 +13,7 @@ from pypy.rlib.rarithmetic import isinf, isnan from pypy.translator.c import extfunc from pypy.translator.tool.cbuild import ExternalCompilationInfo +from py.builtin import BaseException def needs_gcheader(T): if not isinstance(T, ContainerType): @@ -691,7 +692,11 @@ self.db = db self.T = T self.obj = obj - if getattr(obj, 'external', None) == 'C' and not db.need_sandboxing(obj): + callable = getattr(obj, '_callable', None) + if (callable is not None and + getattr(callable, 'c_name', None) is not None): + self.name = forcename or obj._callable.c_name + elif getattr(obj, 'external', None) == 'C' and not db.need_sandboxing(obj): self.name = forcename or self.basename() else: self.name = (forcename or @@ -888,7 +893,7 @@ return 'Py_None' import types, py if isinstance(value, (type, types.ClassType)): - if (issubclass(value, Exception) and + if (issubclass(value, BaseException) and (value.__module__ == 'exceptions' or value is py.magic.AssertionError)): return 'PyExc_' + value.__name__ Modified: pypy/branch/parser-compiler/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/c/test/test_genc.py (original) +++ pypy/branch/parser-compiler/pypy/translator/c/test/test_genc.py Fri Jul 17 21:55:59 2009 @@ -12,7 +12,6 @@ from pypy.translator.gensupp import uniquemodulename from pypy.translator.backendopt.all import backend_optimizations from pypy.translator.interactive import Translation -from pypy import conftest def compile(fn, argtypes, view=False, gcpolicy="ref", backendopt=True, annotatorpolicy=None): @@ -24,7 +23,7 @@ # XXX fish t.driver.config.translation.countmallocs = True compiled_fn = t.compile_c() - if conftest.option.view: + if py.test.config.option.view: t.view() malloc_counters = t.driver.cbuilder.get_malloc_counters() def checking_fn(*args, **kwds): @@ -393,3 +392,16 @@ fn = compile(f, []) fn() + +def test_name(): + def f(): + return 3 + + f.c_name = 'pypy_xyz_f' + + t = Translation(f, [], backend="c") + t.annotate() + compiled_fn = t.compile_c() + if py.test.config.option.view: + t.view() + assert 'pypy_xyz_f' in t.driver.cbuilder.c_source_filename.read() Modified: pypy/branch/parser-compiler/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/parser-compiler/pypy/translator/c/test/test_newgc.py Fri Jul 17 21:55:59 2009 @@ -655,19 +655,6 @@ res = fn() assert res == 123 - def test_framework_malloc_gc(self): - py.test.skip('in-progress') - A = lltype.GcStruct('A', ('value', lltype.Signed)) - - def f(): - p = lltype.malloc(A, flavor='gc') - p.value = 123 - llop.gc__collect(lltype.Void) - return p.value - fn = self.getcompiled(f) - res = fn() - assert res == 123 - def test_framework_del_seeing_new_types(self): class B(object): pass Modified: pypy/branch/parser-compiler/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/parser-compiler/pypy/translator/c/test/test_standalone.py Fri Jul 17 21:55:59 2009 @@ -77,11 +77,11 @@ cbuilder.compile() counters_fname = udir.join("_counters_") - os.putenv('_INSTRUMENT_COUNTERS', str(counters_fname)) + os.environ['_INSTRUMENT_COUNTERS'] = str(counters_fname) try: data = cbuilder.cmdexec() finally: - os.unsetenv('_INSTRUMENT_COUNTERS') + del os.environ['_INSTRUMENT_COUNTERS'] f = counters_fname.open('rb') counters_data = f.read() Modified: pypy/branch/parser-compiler/pypy/translator/cli/src/ll_os.cs ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/cli/src/ll_os.cs (original) +++ pypy/branch/parser-compiler/pypy/translator/cli/src/ll_os.cs Fri Jul 17 21:55:59 2009 @@ -261,7 +261,8 @@ public static void ll_os_close(int fd) { FileStream stream = getfd(fd).GetStream(); - stream.Close(); + if (stream != null) // stdin/stdout/stderr files don't have a stream + stream.Close(); FileDescriptors.Remove(fd); } Modified: pypy/branch/parser-compiler/pypy/translator/driver.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/driver.py (original) +++ pypy/branch/parser-compiler/pypy/translator/driver.py Fri Jul 17 21:55:59 2009 @@ -304,6 +304,7 @@ assert self.libdef is not None for func, inputtypes in self.libdef.functions: annotator.build_types(func, inputtypes) + func.c_name = func.func_name self.sanity_check_annotation() annotator.simplify() # @@ -422,15 +423,20 @@ if translator.annotator is not None: translator.frozen = True - standalone = self.standalone - - if standalone: - from pypy.translator.c.genc import CStandaloneBuilder as CBuilder + if self.libdef is not None: + cbuilder = self.libdef.getcbuilder(self.translator, self.config) + self.standalone = False + standalone = False else: - from pypy.translator.c.genc import CExtModuleBuilder as CBuilder - cbuilder = CBuilder(self.translator, self.entry_point, - config=self.config) - cbuilder.stackless = self.config.translation.stackless + standalone = self.standalone + + if standalone: + from pypy.translator.c.genc import CStandaloneBuilder as CBuilder + else: + from pypy.translator.c.genc import CExtModuleBuilder as CBuilder + cbuilder = CBuilder(self.translator, self.entry_point, + config=self.config) + cbuilder.stackless = self.config.translation.stackless if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name database = cbuilder.build_database() @@ -477,7 +483,7 @@ def task_compile_c(self): # xxx messy cbuilder = self.cbuilder cbuilder.compile() - + if self.standalone: self.c_entryp = cbuilder.executable_name self.create_exe() Modified: pypy/branch/parser-compiler/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/parser-compiler/pypy/translator/goal/targetpypystandalone.py Fri Jul 17 21:55:59 2009 @@ -199,6 +199,16 @@ wrapstr = 'space.wrap(%r)' % (options) pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr + if config.translation.backend in ["cli", "jvm"] and sys.platform == "win32": + # HACK: The ftruncate implementation in streamio.py which is used for the Win32 platform + # is specific for the C backend and can't be generated on CLI or JVM. Because of that, + # we have to patch it out. + from pypy.rlib import streamio + def ftruncate_win32_dummy(fd, size): pass + def _setfd_binary_dummy(fd): pass + streamio.ftruncate_win32 = ftruncate_win32_dummy + streamio._setfd_binary = _setfd_binary_dummy + return self.get_entry_point(config) def jitpolicy(self, driver): Modified: pypy/branch/parser-compiler/pypy/translator/goal/translate.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/goal/translate.py (original) +++ pypy/branch/parser-compiler/pypy/translator/goal/translate.py Fri Jul 17 21:55:59 2009 @@ -54,6 +54,10 @@ "cProfile (to debug the speed of the translation process)", default=False, cmdline="--profile"), + BoolOption("pdb", + "Always run pdb even if the translation succeeds", + default=False, + cmdline="--pdb"), BoolOption("batch", "Don't run interactive helpers", default=False, cmdline="--batch", negation=False), IntOption("huge", "Threshold in the number of functions after which " @@ -279,7 +283,8 @@ debug(True) raise SystemExit(1) else: - debug(False) + if translateconfig.pdb: + debug(False) if __name__ == '__main__': Modified: pypy/branch/parser-compiler/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/jvm/genjvm.py (original) +++ pypy/branch/parser-compiler/pypy/translator/jvm/genjvm.py Fri Jul 17 21:55:59 2009 @@ -3,6 +3,7 @@ """ import sys +import os import py from py.compat import subprocess @@ -197,7 +198,7 @@ cmd = [getoption('java'), '-Xmx256M', # increase the heapsize so the microbenchmarks run '-cp', - str(self.javadir)+":"+str(self.jnajar), + str(self.javadir)+os.pathsep+str(self.jnajar), self.package+".Main"] + strargs print "Invoking java to run the code" stdout, stderr, retval = self._invoke(cmd, True) Modified: pypy/branch/parser-compiler/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/__init__.py Fri Jul 17 21:55:59 2009 @@ -52,6 +52,8 @@ name = "abstract platform" c_environ = None + so_prefixes = [''] + def __init__(self, cc): if self.__class__ is Platform: raise TypeError("You should not instantiate Platform class directly") @@ -70,7 +72,12 @@ ofiles.append(self._compile_c_file(self.cc, cfile, compile_args)) return ofiles - def execute(self, executable, args=None, env=None): + def execute(self, executable, args=None, env=None, compilation_info=None): + if env is None: + env = os.environ.copy() + if compilation_info is not None: + env['LD_LIBRARY_PATH'] = ':'.join( + [str(i) for i in compilation_info.library_dirs]) returncode, stdout, stderr = _run_subprocess(str(executable), args, env) return ExecutionResult(returncode, stdout, stderr) @@ -120,7 +127,7 @@ cflags = self.cflags + extra return (cflags + list(eci.compile_extra) + args) - def _link_args_from_eci(self, eci): + def _link_args_from_eci(self, eci, standalone): library_dirs = self._libdirs(eci.library_dirs) libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) @@ -137,8 +144,8 @@ exe_name += '.' + self.exe_ext else: exe_name += '.' + self.so_ext - return self._link(self.cc, ofiles, self._link_args_from_eci(eci), - standalone, exe_name) + largs = self._link_args_from_eci(eci, standalone) + return self._link(self.cc, ofiles, largs, standalone, exe_name) # below are some detailed informations for platforms Modified: pypy/branch/parser-compiler/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/darwin.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/darwin.py Fri Jul 17 21:55:59 2009 @@ -38,8 +38,8 @@ args.append(f) return args - def _link_args_from_eci(self, eci): - args = super(Darwin, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + args = super(Darwin, self)._link_args_from_eci(eci, standalone) frameworks = self._frameworks(eci.frameworks) include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) Modified: pypy/branch/parser-compiler/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/linux.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/linux.py Fri Jul 17 21:55:59 2009 @@ -13,6 +13,7 @@ standalone_only = [] shared_only = [] so_ext = 'so' + so_prefixes = ['lib', ''] def _args_for_shared(self, args): return ['-shared'] + args Modified: pypy/branch/parser-compiler/pypy/translator/platform/test/test_maemo.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/test/test_maemo.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/test/test_maemo.py Fri Jul 17 21:55:59 2009 @@ -31,3 +31,6 @@ executable = self.platform.compile([cfile], eci) res = self.platform.execute(executable) self.check_res(res) + + def test_environment_inheritance(self): + py.test.skip("FIXME") Modified: pypy/branch/parser-compiler/pypy/translator/platform/test/test_platform.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/test/test_platform.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/test/test_platform.py Fri Jul 17 21:55:59 2009 @@ -1,5 +1,5 @@ -import py, sys +import py, sys, ctypes, os from pypy.tool.udir import udir from pypy.translator.platform import CompilationError, Platform from pypy.translator.platform import host @@ -102,6 +102,18 @@ res = self.platform.execute(executable) assert res.out.startswith('4.0') + def test_environment_inheritance(self): + # make sure that environment is inherited + cmd = 'import os; print os.environ["_SOME_VARIABLE_%d"]' + res = self.platform.execute('python', ['-c', cmd % 1], + env={'_SOME_VARIABLE_1':'xyz'}) + assert 'xyz' in res.out + os.environ['_SOME_VARIABLE_2'] = 'zyz' + try: + res = self.platform.execute('python', ['-c', cmd % 2]) + assert 'zyz' in res.out + finally: + del os.environ['_SOME_VARIABLE_2'] def test_equality(): class X(Platform): Modified: pypy/branch/parser-compiler/pypy/translator/platform/windows.py ============================================================================== --- pypy/branch/parser-compiler/pypy/translator/platform/windows.py (original) +++ pypy/branch/parser-compiler/pypy/translator/platform/windows.py Fri Jul 17 21:55:59 2009 @@ -133,8 +133,9 @@ def _args_for_shared(self, args): return ['/dll'] + args - def _link_args_from_eci(self, eci): - args = super(MsvcPlatform, self)._link_args_from_eci(eci) + def _link_args_from_eci(self, eci, standalone): + # Windows needs to resolve all symbols even for DLLs + args = super(MsvcPlatform, self)._link_args_from_eci(eci, standalone=True) return args + ['/EXPORT:%s' % symbol for symbol in eci.export_symbols] def _compile_c_file(self, cc, cfile, compile_args): From benjamin at codespeak.net Fri Jul 17 22:02:19 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 22:02:19 +0200 (CEST) Subject: [pypy-svn] r66327 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090717200219.72419169EB3@codespeak.net> Author: benjamin Date: Fri Jul 17 22:02:18 2009 New Revision: 66327 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: more annotator assistance Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Fri Jul 17 22:02:18 2009 @@ -83,6 +83,7 @@ endDFA = automata.DFA([], []) # make the annotator happy line = '' + pos = 0 lines.append("") for line in lines: lnum = lnum + 1 From benjamin at codespeak.net Fri Jul 17 22:09:38 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 22:09:38 +0200 (CEST) Subject: [pypy-svn] r66328 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090717200938.26255169ED4@codespeak.net> Author: benjamin Date: Fri Jul 17 22:09:37 2009 New Revision: 66328 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Log: let the annotator see this variable Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Fri Jul 17 22:09:37 2009 @@ -97,6 +97,7 @@ def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) + sym_id = 0 # for the annotator while True: dfa, state_index, node = self.stack[-1] states, first = dfa From benjamin at codespeak.net Fri Jul 17 22:16:41 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 22:16:41 +0200 (CEST) Subject: [pypy-svn] r66329 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090717201641.39FB9169F2C@codespeak.net> Author: benjamin Date: Fri Jul 17 22:16:40 2009 New Revision: 66329 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: typo Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Fri Jul 17 22:16:40 2009 @@ -226,7 +226,7 @@ if start Author: benjamin Date: Fri Jul 17 22:28:39 2009 New Revision: 66330 Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Log: trick the annotator into making this resizable Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/pyparser.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Fri Jul 17 22:28:39 2009 @@ -16,7 +16,9 @@ def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column): if node.children is not None: - seq_w = [None]*(1 + len(node.children)) + seq_w = [None]*len(node.children) + # This is a hack to make the annotator think seq_w is resizable. + seq_w.append(None) seq_w[0] = space.wrap(node.type) for i in range(1, len(node.children) + 1): seq_w[i] = self._build_app_tree(space, node.children[i - 1], From benjamin at codespeak.net Fri Jul 17 22:48:12 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 22:48:12 +0200 (CEST) Subject: [pypy-svn] r66331 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090717204812.A364A169ED4@codespeak.net> Author: benjamin Date: Fri Jul 17 22:48:12 2009 New Revision: 66331 Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Log: don't use append at all here Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/pyparser.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Fri Jul 17 22:48:12 2009 @@ -16,20 +16,20 @@ def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column): if node.children is not None: - seq_w = [None]*len(node.children) - # This is a hack to make the annotator think seq_w is resizable. - seq_w.append(None) + seq_w = [None]*(len(node.children) + 1) seq_w[0] = space.wrap(node.type) for i in range(1, len(node.children) + 1): seq_w[i] = self._build_app_tree(space, node.children[i - 1], seq_maker, with_lineno, with_column) else: - seq_w = [space.wrap(node.type), space.wrap(node.value)] + seq_w = [None]*(2 + with_lineno + with_column) + seq_w[0] = space.wrap(node.type) + seq_w[1] = space.wrap(node.value) if with_lineno: - seq_w.append(space.wrap(node.lineno)) + seq_w[2] = space.wrap(node.lineno) if with_column: - seq_w.append(space.wrap(node.column)) + seq_w[3] = space.wrap(node.column) return seq_maker(seq_w) def descr_issuite(self, space): From benjamin at codespeak.net Fri Jul 17 23:15:47 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:15:47 +0200 (CEST) Subject: [pypy-svn] r66332 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090717211547.4EF09169E0D@codespeak.net> Author: benjamin Date: Fri Jul 17 23:15:46 2009 New Revision: 66332 Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Log: specialize the seq_maker argument Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/pyparser.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Fri Jul 17 23:15:46 2009 @@ -6,6 +6,7 @@ from pypy.interpreter.pyparser import pyparse, pygram, error from pypy.interpreter.astcompiler.astbuilder import ast_from_node from pypy.interpreter.astcompiler.codegen import compile_ast +from pypy.rlib.objectmodel import specialize class STType(Wrappable): @@ -14,6 +15,7 @@ self.tree = tree self.mode = mode + @specialize.arg(3) def _build_app_tree(self, space, node, seq_maker, with_lineno, with_column): if node.children is not None: seq_w = [None]*(len(node.children) + 1) From benjamin at codespeak.net Fri Jul 17 23:22:17 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:22:17 +0200 (CEST) Subject: [pypy-svn] r66333 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717212217.96706169EAB@codespeak.net> Author: benjamin Date: Fri Jul 17 23:22:17 2009 New Revision: 66333 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: 'prove' to the annotator that the slice indices will not be negative Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 23:22:17 2009 @@ -518,17 +518,17 @@ def _import_as(self, alias): source_name = alias.name dot = source_name.find(".") - if dot != -1: + if dot > 0: start = dot + 1 while True: dot = source_name.find(".", start) - if dot == -1: + if dot > 0: end = len(source_name) else: end = dot attr = source_name[start:end] self.emit_op_name(ops.LOAD_ATTR, self.names, attr) - if dot == -1: + if dot < 0: break start = dot + 1 self.name_op(alias.asname, ast.Store) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 17 23:22:17 2009 @@ -361,7 +361,7 @@ if store_name == "*": return True dot = store_name.find(".") - if dot != -1: + if dot > 0: store_name = store_name[:dot] self.note_symbol(store_name, SYM_ASSIGNED) return False From benjamin at codespeak.net Fri Jul 17 23:30:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:30:02 +0200 (CEST) Subject: [pypy-svn] r66334 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717213002.256E6169F80@codespeak.net> Author: benjamin Date: Fri Jul 17 23:30:01 2009 New Revision: 66334 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: fix annotation Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 23:30:01 2009 @@ -818,6 +818,7 @@ if call.kwargs: call.kwargs.walkabout(self) call_type |= 2 + op = 0 if call_type == 0: op = ops.CALL_FUNCTION elif call_type == 1: From benjamin at codespeak.net Fri Jul 17 23:38:49 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:38:49 +0200 (CEST) Subject: [pypy-svn] r66335 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717213849.10441169F75@codespeak.net> Author: benjamin Date: Fri Jul 17 23:38:49 2009 New Revision: 66335 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: help annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 23:38:49 2009 @@ -716,6 +716,7 @@ self.update_position(comp) comp.left.walkabout(self) ops_count = len(comp.ops) + cleanup = None if ops_count > 1: cleanup = self.new_block() comp.comparators[0].walkabout(self) From benjamin at codespeak.net Fri Jul 17 23:47:27 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:47:27 +0200 (CEST) Subject: [pypy-svn] r66336 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717214727.A540F169F7E@codespeak.net> Author: benjamin Date: Fri Jul 17 23:47:27 2009 New Revision: 66336 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: typo Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 23:47:27 2009 @@ -167,7 +167,7 @@ try: op = kind[ctx] except KeyError: - if kind is names_ops_deref and ctx == ast.Del: + if kind is name_ops_deref and ctx == ast.Del: raise SyntaxError("Can't delete variable used in " "nested scopes: %r" % (identifier,)) raise AssertionError("Unkown name operation") From benjamin at codespeak.net Fri Jul 17 23:54:27 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 17 Jul 2009 23:54:27 +0200 (CEST) Subject: [pypy-svn] r66337 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717215427.A27B2169F77@codespeak.net> Author: benjamin Date: Fri Jul 17 23:54:26 2009 New Revision: 66337 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: hint to annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 17 23:54:26 2009 @@ -601,6 +601,7 @@ body_block = self.new_block() cleanup = self.new_block() exit_storage = self.current_temporary_name() + temp_result = None if wih.optional_vars: temp_result = self.current_temporary_name() wih.context_expr.walkabout(self) From benjamin at codespeak.net Sat Jul 18 00:04:06 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:04:06 +0200 (CEST) Subject: [pypy-svn] r66338 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090717220406.BC810169F77@codespeak.net> Author: benjamin Date: Sat Jul 18 00:04:05 2009 New Revision: 66338 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: fix while loops with a constant false test Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 00:04:05 2009 @@ -436,7 +436,7 @@ test_constant = misc.expr_constant(self.space, wh.test) if test_constant == misc.CONST_FALSE: if wh.orelse: - self.visit_sequence(orelse) + self.visit_sequence(wh.orelse) else: end = self.new_block() if test_constant == misc.CONST_NOT_CONST: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Sat Jul 18 00:04:05 2009 @@ -434,6 +434,12 @@ comments[:0] = [comment] comment = '' """, 'comments', ['# foo', 42] + yield self.simple_test, """ + while 0: + pass + else: + x = 1 + """, "x", 1 def test_return_lineno(self): # the point of this test is to check that there is no code associated From benjamin at codespeak.net Sat Jul 18 00:12:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:12:03 +0200 (CEST) Subject: [pypy-svn] r66339 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717221203.5B539169F81@codespeak.net> Author: benjamin Date: Sat Jul 18 00:12:02 2009 New Revision: 66339 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: put hint on local variable Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 00:12:02 2009 @@ -417,15 +417,16 @@ ast.GenericASTVisitor.visit_With(self, wih) def visit_arguments(self, arguments): - assert isinstance(self.scope, FunctionScope) # Annotator hint. + scope = self.scope + assert isinstance(scope, FunctionScope) # Annotator hint. if arguments.args: self._handle_params(arguments.args, True) if arguments.vararg: self.note_symbol(arguments.vararg, SYM_PARAM) - self.scope.note_variable_arg(arguments.vararg) + scope.note_variable_arg(arguments.vararg) if arguments.kwarg: self.note_symbol(arguments.kwarg, SYM_PARAM) - self.scope.note_keywords_arg(arguments.kwarg) + scope.note_keywords_arg(arguments.kwarg) if arguments.args: self._handle_nested_params(arguments.args) From benjamin at codespeak.net Sat Jul 18 00:12:43 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:12:43 +0200 (CEST) Subject: [pypy-svn] r66340 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717221243.CC067169F81@codespeak.net> Author: benjamin Date: Sat Jul 18 00:12:43 2009 New Revision: 66340 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: help annotator find variable Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 00:12:43 2009 @@ -439,6 +439,7 @@ self.visit_sequence(wh.orelse) else: end = self.new_block() + anchor = None if test_constant == misc.CONST_NOT_CONST: anchor = self.new_block() self.emit_jump(ops.SETUP_LOOP, end) From benjamin at codespeak.net Sat Jul 18 00:13:34 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:13:34 +0200 (CEST) Subject: [pypy-svn] r66341 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717221334.2C2AD169F81@codespeak.net> Author: benjamin Date: Sat Jul 18 00:13:33 2009 New Revision: 66341 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: add dummy method Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 00:13:33 2009 @@ -122,6 +122,9 @@ self.done_with_future = False self._compile(tree) + def _compile(self, tree): + raise NotImplementedError + def current_temporary_name(self): name = "_[%i]" % (self.temporary_name_counter,) self.temporary_name_counter += 1 From benjamin at codespeak.net Sat Jul 18 00:20:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:20:25 +0200 (CEST) Subject: [pypy-svn] r66342 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717222025.11C49169F7F@codespeak.net> Author: benjamin Date: Sat Jul 18 00:20:25 2009 New Revision: 66342 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: this is technically not a visitor implementation Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 00:20:25 2009 @@ -350,10 +350,10 @@ def visit_ImportFrom(self, imp): for alias in imp.names: - if self.visit_alias(alias): + if self._visit_alias(alias): self.scope.note_import_star(imp) - def visit_alias(self, alias): + def _visit_alias(self, alias): if alias.asname: store_name = alias.asname else: From benjamin at codespeak.net Sat Jul 18 00:22:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:22:42 +0200 (CEST) Subject: [pypy-svn] r66343 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717222242.684D516849D@codespeak.net> Author: benjamin Date: Sat Jul 18 00:22:41 2009 New Revision: 66343 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: visit_Import needs visit_alias Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 00:22:41 2009 @@ -366,6 +366,9 @@ self.note_symbol(store_name, SYM_ASSIGNED) return False + def visit_alias(self, alias): + self._visit_alias(alias) + def visit_Exec(self, exc): self.scope.note_exec(exc) ast.GenericASTVisitor.visit_Exec(self, exc) From benjamin at codespeak.net Sat Jul 18 00:31:30 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 00:31:30 +0200 (CEST) Subject: [pypy-svn] r66344 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090717223130.261A0169F7F@codespeak.net> Author: benjamin Date: Sat Jul 18 00:31:29 2009 New Revision: 66344 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: tuples can't be None Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Sat Jul 18 00:31:29 2009 @@ -17,7 +17,7 @@ self.opcode = opcode self.arg = arg self.lineno = lineno - self.jump = None + self.has_jump = False def size(self): if self.opcode >= ops.HAVE_ARGUMENT: @@ -30,7 +30,7 @@ def jump_to(self, target, absolute=False): self.jump = (target, absolute) - + self.has_jump = True def __repr__(self): data = [ops.opname[self.opcode]] @@ -38,7 +38,7 @@ if self.opcode >= ops.HAVE_ARGUMENT: data.append(self.arg) template += " %i" - if self.jump: + if self.has_jump: data.append(self.jump[0]) template += " %s" template += ">" @@ -60,7 +60,7 @@ if self.next_block is not None: self.next_block._post_order(blocks) for instr in self.instructions: - if instr.jump: + if instr.has_jump: instr.jump[0]._post_order(blocks) blocks.append(self) self.marked = True @@ -203,7 +203,7 @@ offset = block.offset for instr in block.instructions: offset += instr.size() - if instr.jump: + if instr.has_jump: target, absolute = instr.jump if absolute: jump_arg = target.offset @@ -252,7 +252,7 @@ depth += _opcode_stack_effect(instr.opcode, instr.arg) if depth >= max_depth: max_depth = depth - if instr.jump: + if instr.has_jump: max_depth = self._recursive_stack_depth_walk(instr.jump[0], depth, max_depth) if instr.opcode == ops.JUMP_ABSOLUTE or \ From benjamin at codespeak.net Sat Jul 18 04:13:48 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:13:48 +0200 (CEST) Subject: [pypy-svn] r66345 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090718021348.A78E03180F5@codespeak.net> Author: benjamin Date: Sat Jul 18 04:13:47 2009 New Revision: 66345 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: add __slots__ to ast Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Sat Jul 18 04:13:47 2009 @@ -67,7 +67,9 @@ self.emit("") else: self.emit("class %s(AST):" % (base,)) - self.emit("pass", 1) + self.emit("") + slots = ", ".join(repr(attr.name.value) for attr in sum.attributes) + self.emit("__slots__ = (%s)" % (slots,), 1) self.emit("") for cons in sum.types: self.visit(cons, base, sum.attributes, simple) @@ -75,6 +77,8 @@ def visitProduct(self, product, name, simple): self.emit("class %s(AST):" % (name,)) + slots = ", ".join(repr(field.name.value) for field in product.fields) + self.emit("__slots__ = (%s)" % (slots,), 1) self.emit("") self.make_constructor(product.fields) self.emit("") @@ -95,7 +99,11 @@ def visitConstructor(self, cons, base, extra_attributes, simple): self.emit("class %s(%s):" % (cons.name, base)) self.emit("") - self.make_constructor(cons.fields + extra_attributes) + all_fields = cons.fields + extra_attributes + slots = ", ".join(repr(field.name.value) for field in all_fields) + self.emit("__slots__ = (%s)" % (slots,), 1) + self.emit("") + self.make_constructor(all_fields) self.emit("") self.emit("def walkabout(self, visitor):", 1) self.emit("visitor.visit_%s(self)" % (cons.name,), 2) From benjamin at codespeak.net Sat Jul 18 04:22:20 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:22:20 +0200 (CEST) Subject: [pypy-svn] r66346 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090718022220.C652F3180F7@codespeak.net> Author: benjamin Date: Sat Jul 18 04:22:19 2009 New Revision: 66346 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: add space Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Sat Jul 18 04:22:19 2009 @@ -77,6 +77,7 @@ def visitProduct(self, product, name, simple): self.emit("class %s(AST):" % (name,)) + self.emit("") slots = ", ".join(repr(field.name.value) for field in product.fields) self.emit("__slots__ = (%s)" % (slots,), 1) self.emit("") From benjamin at codespeak.net Sat Jul 18 04:23:28 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:23:28 +0200 (CEST) Subject: [pypy-svn] r66347 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090718022328.3A8C13180F7@codespeak.net> Author: benjamin Date: Sat Jul 18 04:23:27 2009 New Revision: 66347 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: give the base AST node __slots__ Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Sat Jul 18 04:23:27 2009 @@ -222,6 +222,8 @@ class AST(Wrappable): + __slots__ = () + __metaclass__ = extendabletype def walkabout(self, visitor): From benjamin at codespeak.net Sat Jul 18 04:34:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:34:59 +0200 (CEST) Subject: [pypy-svn] r66348 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718023459.858173180F8@codespeak.net> Author: benjamin Date: Sat Jul 18 04:34:59 2009 New Revision: 66348 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: use isinstance() Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 04:34:59 2009 @@ -112,49 +112,44 @@ def set_context(self, expr, ctx, node): error = None sequence = None - expr_type = expr.__class__ - if expr_type is ast.Attribute: + if isinstance(expr, ast.Attribute): if ctx == ast.Store: self.check_forbidden_name(expr.attr, node) expr.ctx = ctx - elif expr_type is ast.Subscript: + elif isinstance(expr, ast.Subscript): expr.ctx = ctx - elif expr_type is ast.Name: + elif isinstance(expr, ast.Name): if ctx == ast.Store: self.check_forbidden_name(expr.id, node) expr.ctx = ctx - elif expr_type is ast.List: + elif isinstance(expr, ast.List): expr.ctx = ctx sequence = expr.elts - elif expr_type is ast.Tuple: + elif isinstance(expr, ast.Tuple): if expr.elts: expr.ctx = ctx sequence = expr.elts else: error = "()" - elif expr_type is ast.Lambda: + elif isinstance(expr, ast.Lambda): error = "lambda" - elif expr_type is ast.Call: + elif isinstance(expr, ast.Call): error = "call" - elif expr_type is ast.BoolOp or \ - expr_type is ast.BinOp or \ - expr_type is ast.UnaryOp: + elif isinstance(expr, (ast.BoolOp, ast.BinOp, ast.UnaryOp)): error = "operator" - elif expr_type is ast.GeneratorExp: + elif isinstance(expr, ast.GeneratorExp): error = "generator expression" - elif expr_type is ast.Yield: + elif isinstance(expr, ast.Yield): error = "yield expression" - elif expr_type is ast.ListComp: + elif isinstance(expr, ast.ListComp): error = "list comprehension" - elif expr_type is ast.Dict or \ - expr_type is ast.Num or \ - expr_type is ast.Str: + elif isinstance(expr, (ast.Dict, ast.Num, ast.Str)): error = "literal" - elif expr_type is ast.Compare: + elif isinstance(expr, ast.Compare): error = "comparison" - elif expr_type is ast.IfExp: + elif isinstance(expr, ast.IfExp): error = "conditional expression" - elif expr_type is ast.Repr: + elif isinstance(expr, ast.Repr): error = "repr" else: raise AssertionError("unkown expression in set_context()") From benjamin at codespeak.net Sat Jul 18 04:46:32 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:46:32 +0200 (CEST) Subject: [pypy-svn] r66349 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718024632.2A6673180FC@codespeak.net> Author: benjamin Date: Sat Jul 18 04:46:31 2009 New Revision: 66349 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: *sigh* rpython doesn't support tuples as the second arg to isinstance Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 04:46:31 2009 @@ -135,7 +135,9 @@ error = "lambda" elif isinstance(expr, ast.Call): error = "call" - elif isinstance(expr, (ast.BoolOp, ast.BinOp, ast.UnaryOp)): + elif isinstance(expr, ast.BoolOp) or \ + isinstance(expr, ast.BinOp) or \ + isinstance(expr, ast.UnaryOp): error = "operator" elif isinstance(expr, ast.GeneratorExp): error = "generator expression" @@ -143,7 +145,9 @@ error = "yield expression" elif isinstance(expr, ast.ListComp): error = "list comprehension" - elif isinstance(expr, (ast.Dict, ast.Num, ast.Str)): + elif isinstance(expr, ast.Dict) or \ + isinstance(expr, ast.Num) or \ + isinstance(expr, ast.Str): error = "literal" elif isinstance(expr, ast.Compare): error = "comparison" From benjamin at codespeak.net Sat Jul 18 04:59:55 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 04:59:55 +0200 (CEST) Subject: [pypy-svn] r66350 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718025955.AB562498425@codespeak.net> Author: benjamin Date: Sat Jul 18 04:59:54 2009 New Revision: 66350 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: store the lineno and column directly on the scope Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 04:59:54 2009 @@ -26,8 +26,9 @@ can_be_optimized = False - def __init__(self, node, name): - self.node = node + def __init__(self, name, lineno=0, col_offset=0): + self.lineno = lineno + self.col_offset = col_offset self.parent = None self.name = name self.locals_fully_known = False @@ -60,7 +61,7 @@ if old_role & SYM_PARAM and role & SYM_PARAM: err = "duplicate argument '%s' in function definition" % \ (identifier,) - raise SyntaxError(err, self.node.lineno, self.node.col_offset) + raise SyntaxError(err, self.lineno, self.col_offset) new_role |= old_role self.roles[mangled] = new_role if role & SYM_PARAM: @@ -95,7 +96,7 @@ if flags & SYM_GLOBAL: if flags & SYM_PARAM: raise SyntaxError("name %r is both local and global" % (name,), - self.node.lineno, self.node.col_offset) + self.lineno, self.col_offset) self.symbols[name] = SCOPE_GLOBAL_EXPLICIT globs[name] = None if bound: @@ -178,8 +179,8 @@ class ModuleScope(Scope): - def __init__(self, module): - Scope.__init__(self, module, "top") + def __init__(self): + Scope.__init__(self, "top") class FunctionScope(Scope): @@ -187,7 +188,7 @@ can_be_optimized = True def __init__(self, func, name): - Scope.__init__(self, func, name) + Scope.__init__(self, name, func.lineno, func.col_offset) self.has_variable_arg = False self.has_keywords_arg = False self.is_generator = False @@ -269,7 +270,7 @@ _hide_bound_from_nested_scopes = True def __init__(self, clsdef): - Scope.__init__(self, clsdef, clsdef.name) + Scope.__init__(self, clsdef.name, clsdef.lineno, clsdef.col_offset) def mangle(self, name): return misc.mangle(name, self.name) @@ -284,9 +285,9 @@ self.scopes = {} self.scope = None self.stack = [] - top = ModuleScope(module) + top = ModuleScope() self.globs = top.roles - self.push_scope(top) + self.push_scope(top, module) try: module.walkabout(self) top.finalize(None, {}, {}) @@ -296,11 +297,11 @@ self.pop_scope() assert not self.stack - def push_scope(self, scope): + def push_scope(self, scope, node): if self.stack: self.stack[-1].add_child(scope) self.stack.append(scope) - self.scopes[scope.node] = scope + self.scopes[node] = scope # Convenience self.scope = scope @@ -331,7 +332,7 @@ self.visit_sequence(func.args.defaults) if func.decorators: self.visit_sequence(func.decorators) - self.push_scope(FunctionScope(func, func.name)) + self.push_scope(FunctionScope(func, func.name), func) func.args.walkabout(self) self.visit_sequence(func.body) self.pop_scope() @@ -344,7 +345,7 @@ self.note_symbol(clsdef.name, SYM_ASSIGNED) if clsdef.bases: self.visit_sequence(clsdef.bases) - self.push_scope(ClassScope(clsdef)) + self.push_scope(ClassScope(clsdef), clsdef) self.visit_sequence(clsdef.body) self.pop_scope() @@ -392,7 +393,7 @@ def visit_Lambda(self, lamb): if lamb.args.defaults: self.visit_sequence(lamb.args.defaults) - self.push_scope(FunctionScope(lamb, "lambda")) + self.push_scope(FunctionScope(lamb, "lambda"), lamb) lamb.args.walkabout(self) lamb.body.walkabout(self) self.pop_scope() @@ -400,7 +401,7 @@ def visit_GeneratorExp(self, genexp): outer = genexp.generators[0] outer.iter.walkabout(self) - self.push_scope(FunctionScope(genexp, "genexp")) + self.push_scope(FunctionScope(genexp, "genexp"), genexp) self.implicit_arg(0) outer.target.walkabout(self) if outer.ifs: From benjamin at codespeak.net Sat Jul 18 05:20:00 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 05:20:00 +0200 (CEST) Subject: [pypy-svn] r66351 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718032000.EA082169E11@codespeak.net> Author: benjamin Date: Sat Jul 18 05:20:00 2009 New Revision: 66351 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: column -> col_offset Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 05:20:00 2009 @@ -904,7 +904,7 @@ break tmp_atom_expr = self.handle_trailer(trailer, atom_expr) tmp_atom_expr.lineno = atom_expr.lineno - tmp_atom_expr.column = atom_expr.col_offset + tmp_atom_expr.col_offset = atom_expr.col_offset atom_expr = tmp_atom_expr if power_node.children[-1].type == syms.factor: right = self.handle_expr(power_node.children[-1]) From arigo at codespeak.net Sat Jul 18 11:31:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 11:31:55 +0200 (CEST) Subject: [pypy-svn] r66352 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090718093155.E64023180F4@codespeak.net> Author: arigo Date: Sat Jul 18 11:31:54 2009 New Revision: 66352 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Add an even simpler test, skipped. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Sat Jul 18 11:31:54 2009 @@ -393,6 +393,17 @@ assert boxp2.knownclsbox.value == self.node_vtable_adr assert boxp3.knownclsbox.value == self.node_vtable_adr2 + def test_find_nodes_new_aliasing_0(self): + py.test.skip("in-progress") + ops = """ + [p1, p2] + p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + jump(p3, p3) + """ + # both p1 and p2 must be NotSpecNodes; it's not possible to pass + # the same Virtual both in p1 and p2 (at least so far). + self.find_nodes(ops, 'Not, Not') + def test_find_nodes_new_aliasing_1(self): py.test.skip("infinite loop") ops = """ From benjamin at codespeak.net Sat Jul 18 14:43:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 14:43:42 +0200 (CEST) Subject: [pypy-svn] r66353 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718124342.C204D169F9B@codespeak.net> Author: benjamin Date: Sat Jul 18 14:43:41 2009 New Revision: 66353 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: pass lineno and column directly to FunctionScope initializer Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 14:43:41 2009 @@ -187,8 +187,8 @@ can_be_optimized = True - def __init__(self, func, name): - Scope.__init__(self, name, func.lineno, func.col_offset) + def __init__(self, name, lineno, col_offset): + Scope.__init__(self, name, lineno, col_offset) self.has_variable_arg = False self.has_keywords_arg = False self.is_generator = False @@ -332,7 +332,8 @@ self.visit_sequence(func.args.defaults) if func.decorators: self.visit_sequence(func.decorators) - self.push_scope(FunctionScope(func, func.name), func) + new_scope = FunctionScope(func.name, func.lineno, func.col_offset) + self.push_scope(new_scope, func) func.args.walkabout(self) self.visit_sequence(func.body) self.pop_scope() @@ -393,7 +394,8 @@ def visit_Lambda(self, lamb): if lamb.args.defaults: self.visit_sequence(lamb.args.defaults) - self.push_scope(FunctionScope(lamb, "lambda"), lamb) + new_scope = FunctionScope("lambda", lamb.lineno, lamb.col_offset) + self.push_scope(new_scope, lamb) lamb.args.walkabout(self) lamb.body.walkabout(self) self.pop_scope() @@ -401,7 +403,8 @@ def visit_GeneratorExp(self, genexp): outer = genexp.generators[0] outer.iter.walkabout(self) - self.push_scope(FunctionScope(genexp, "genexp"), genexp) + new_scope = FunctionScope("genexp", genexp.lineno, genexp.col_offset) + self.push_scope(new_scope, genexp) self.implicit_arg(0) outer.target.walkabout(self) if outer.ifs: From benjamin at codespeak.net Sat Jul 18 15:28:28 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 15:28:28 +0200 (CEST) Subject: [pypy-svn] r66354 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718132828.A9D28169F6B@codespeak.net> Author: benjamin Date: Sat Jul 18 15:28:27 2009 New Revision: 66354 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: pass lineno directly to update_position Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Sat Jul 18 15:28:27 2009 @@ -186,10 +186,10 @@ index = self.add_const(obj) self.emit_op_arg(ops.LOAD_CONST, index) - def update_position(self, node): - self.lineno = node.lineno + def update_position(self, lineno): + self.lineno = lineno if self.first_lineno == -1: - self.first_lineno = node.lineno + self.first_lineno = lineno def _resolve_block_targets(self, blocks): last_extended_arg_count = 0 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 15:28:27 2009 @@ -233,7 +233,7 @@ else: num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func) - self.update_position(func) + self.update_position(func.lineno) self._make_function(code, num_defaults) self.name_op(func.name, ast.Store) @@ -244,11 +244,11 @@ else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam) - self.update_position(lam) + self.update_position(lam.lineno) self._make_function(code, default_count) def visit_ClassDef(self, cls): - self.update_position(cls) + self.update_position(cls.lineno) self.load_const(self.space.wrap(cls.name)) if cls.bases: bases_count = len(cls.bases) @@ -257,7 +257,7 @@ bases_count = 0 self.emit_op_arg(ops.BUILD_TUPLE, bases_count) code = self.sub_scope(ClassCodeGenerator, cls.name, cls) - self.update_position(cls) + self.update_position(cls.lineno) self._make_function(code, 0) self.emit_op_arg(ops.CALL_FUNCTION, 0) self.emit_op(ops.BUILD_CLASS) @@ -272,7 +272,7 @@ return inplace_operations[op] def visit_AugAssign(self, assign): - self.update_position(assign) + self.update_position(assign.lineno) target = assign.target if isinstance(target, ast.Attribute): attr = ast.Attribute(target.value, target.attr, ast.AugLoad, @@ -299,7 +299,7 @@ raise AssertionError("unkown augassign") def visit_Assert(self, asrt): - self.update_position(asrt) + self.update_position(asrt.lineno) end = self.new_block() asrt.test.walkabout(self) self.emit_jump(ops.JUMP_IF_TRUE, end) @@ -322,13 +322,13 @@ return binary_operations[op] def visit_BinOp(self, binop): - self.update_position(binop) + self.update_position(binop.lineno) binop.left.walkabout(self) binop.right.walkabout(self) self.emit_op(self._binop(binop.op)) def visit_Return(self, ret): - self.update_position(ret) + self.update_position(ret.lineno) if ret.value: ret.value.walkabout(self) else: @@ -336,6 +336,7 @@ self.emit_op(ops.RETURN_VALUE) def visit_Print(self, pr): + self.update_position(pr.lineno) have_dest = bool(pr.dest) if have_dest: pr.dest.walkabout(self) @@ -358,11 +359,11 @@ self.emit_op(ops.POP_TOP) def visit_Delete(self, delete): - self.update_position(delete) + self.update_position(delete.lineno) self.visit_sequence(delete.targets) def visit_If(self, if_): - self.update_position(if_) + self.update_position(if_.lineno) end = self.new_block() test_constant = misc.expr_constant(self.space, if_.test) if test_constant == misc.CONST_FALSE: @@ -384,7 +385,7 @@ self.use_next_block(end) def visit_Break(self, br): - self.update_position(br) + self.update_position(br.lineno) for f_block in self.frame_blocks: if f_block[0] == F_BLOCK_LOOP: break @@ -414,7 +415,7 @@ self.error("'continue' not allowed in 'finally' clause", cont) def visit_For(self, fr): - self.update_position(fr) + self.update_position(fr.lineno) start = self.new_block() cleanup = self.new_block() end = self.new_block() @@ -435,7 +436,7 @@ self.use_next_block(end) def visit_While(self, wh): - self.update_position(wh) + self.update_position(wh.lineno) test_constant = misc.expr_constant(self.space, wh.test) if test_constant == misc.CONST_FALSE: if wh.orelse: @@ -465,7 +466,7 @@ self.use_next_block(end) def visit_TryExcept(self, te): - self.update_position(te) + self.update_position(te.lineno) exc = self.new_block() otherwise = self.new_block() end = self.new_block() @@ -478,7 +479,7 @@ self.emit_jump(ops.JUMP_FORWARD, otherwise) self.use_next_block(exc) for handler in te.handlers: - self.update_position(handler) + self.update_position(handler.lineno) next_except = self.new_block() if handler.type: self.emit_op(ops.DUP_TOP) @@ -504,7 +505,7 @@ self.use_next_block(end) def visit_TryFinally(self, tf): - self.update_position(tf) + self.update_position(tf.lineno) end = self.new_block() self.emit_jump(ops.SETUP_FINALLY, end) body = self.use_next_block() @@ -538,7 +539,7 @@ self.name_op(alias.asname, ast.Store) def visit_Import(self, imp): - self.update_position(imp) + self.update_position(imp.lineno) for alias in imp.names: if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: level = 0 @@ -558,7 +559,7 @@ self.name_op(store_name, ast.Store) def visit_ImportFrom(self, imp): - self.update_position(imp) + self.update_position(imp.lineno) space = self.space if imp.module == "__future__": if self.done_with_future: @@ -592,7 +593,7 @@ self.emit_op(ops.POP_TOP) def visit_Assign(self, assign): - self.update_position(assign) + self.update_position(assign.lineno) assign.value.walkabout(self) duplications = len(assign.targets) - 1 for i in range(len(assign.targets)): @@ -601,7 +602,7 @@ assign.targets[i].walkabout(self) def visit_With(self, wih): - self.update_position(wih) + self.update_position(wih.lineno) body_block = self.new_block() cleanup = self.new_block() exit_storage = self.current_temporary_name() @@ -638,7 +639,7 @@ self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup) def visit_Raise(self, rais): - self.update_position(rais) + self.update_position(rais.lineno) arg = 0 if rais.type: rais.type.walkabout(self) @@ -652,7 +653,7 @@ self.emit_op_arg(ops.RAISE_VARARGS, arg) def visit_Exec(self, exc): - self.update_position(exc) + self.update_position(exc.lineno) exc.body.walkabout(self) if exc.globals: exc.globals.walkabout(self) @@ -670,10 +671,10 @@ pass def visit_Pass(self, pas): - self.update_position(pas) + self.update_position(pas.lineno) def visit_Expr(self, expr): - self.update_position(expr) + self.update_position(expr.lineno) if self.interactive: expr.value.walkabout(self) self.emit_op(ops.PRINT_EXPR) @@ -683,7 +684,7 @@ self.emit_op(ops.POP_TOP) def visit_Yield(self, yie): - self.update_position(yie) + self.update_position(yie.lineno) if yie.value: yie.value.walkabout(self) else: @@ -691,20 +692,20 @@ self.emit_op(ops.YIELD_VALUE) def visit_Num(self, num): - self.update_position(num) + self.update_position(num.lineno) self.load_const(num.n) def visit_Str(self, string): - self.update_position(string) + self.update_position(string.lineno) self.load_const(string.s) def visit_UnaryOp(self, op): - self.update_position(op) + self.update_position(op.lineno) op.operand.walkabout(self) self.emit_op(unary_operations[op.op]) def visit_BoolOp(self, op): - self.update_position(op) + self.update_position(op.lineno) if op.op == ast.And: instr = ops.JUMP_IF_FALSE else: @@ -718,7 +719,7 @@ self.use_next_block(end) def visit_Compare(self, comp): - self.update_position(comp) + self.update_position(comp.lineno) comp.left.walkabout(self) ops_count = len(comp.ops) cleanup = None @@ -746,7 +747,7 @@ self.use_next_block(end) def visit_IfExp(self, ifexp): - self.update_position(ifexp) + self.update_position(ifexp.lineno) end = self.new_block() otherwise = self.new_block() ifexp.test.walkabout(self) @@ -760,7 +761,7 @@ self.use_next_block(end) def visit_Tuple(self, tup): - self.update_position(tup) + self.update_position(tup.lineno) if tup.elts: elt_count = len(tup.elts) else: @@ -773,7 +774,7 @@ self.emit_op_arg(ops.BUILD_TUPLE, elt_count) def visit_List(self, l): - self.update_position(l) + self.update_position(l.lineno) if l.elts: elt_count = len(l.elts) else: @@ -786,7 +787,7 @@ self.emit_op_arg(ops.BUILD_LIST, elt_count) def visit_Dict(self, d): - self.update_position(d) + self.update_position(d.lineno) self.emit_op_arg(ops.BUILD_MAP, 0) if d.values: for i in range(len(d.values)): @@ -797,7 +798,7 @@ self.emit_op(ops.STORE_SUBSCR) def visit_Name(self, name): - self.update_position(name) + self.update_position(name.lineno) self.name_op(name.id, name.ctx) def visit_keyword(self, keyword): @@ -805,7 +806,7 @@ keyword.value.walkabout(self) def visit_Call(self, call): - self.update_position(call) + self.update_position(call.lineno) if self._optimize_builtin_call(call) or \ self._optimize_method_call(call): return @@ -913,7 +914,7 @@ self.name_op(list_name, ast.Del) def visit_ListComp(self, lc): - self.update_position(lc) + self.update_position(lc.lineno) tmp_name = self.current_temporary_name() self.emit_op_arg(ops.BUILD_LIST, 0) self.emit_op(ops.DUP_TOP) @@ -969,14 +970,14 @@ def visit_GeneratorExp(self, genexp): code = self.sub_scope(GenExpCodeGenerator, "", genexp) - self.update_position(genexp) + self.update_position(genexp.lineno) self._make_function(code) genexp.generators[0].iter.walkabout(self) self.emit_op(ops.GET_ITER) self.emit_op_arg(ops.CALL_FUNCTION, 1) def visit_Attribute(self, attr): - self.update_position(attr) + self.update_position(attr.lineno) names = self.names if attr.ctx != ast.AugStore: attr.value.walkabout(self) @@ -1076,7 +1077,7 @@ self.emit_op(subscr_operations[ctx]) def visit_Subscript(self, sub): - self.update_position(sub) + self.update_position(sub.lineno) if sub.ctx != ast.AugStore: sub.value.walkabout(self) self._compile_slice(sub.slice, sub.ctx) @@ -1115,7 +1116,7 @@ for i in range(len(args)): arg = args[i] if isinstance(arg, ast.Tuple): - self.update_position(arg) + self.update_position(arg.lineno) self.name_op(".%i" % (i,), ast.Load) arg.walkabout(self) @@ -1157,7 +1158,7 @@ def _compile(self, genexp): assert isinstance(genexp, ast.GeneratorExp) - self.update_position(genexp) + self.update_position(genexp.lineno) self._genexp_generator(genexp.generators, 0, genexp.elt) def _get_code_flags(self): From arigo at codespeak.net Sat Jul 18 15:35:02 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 15:35:02 +0200 (CEST) Subject: [pypy-svn] r66355 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718133502.52BBF169F6E@codespeak.net> Author: arigo Date: Sat Jul 18 15:35:01 2009 New Revision: 66355 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Add another pass, find_unique_nodes(), before doing the intersection. Makes the three test pass. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Sat Jul 18 15:35:01 2009 @@ -25,6 +25,10 @@ # ____________________________________________________________ +UNIQUE_UNKNOWN = '\x00' +UNIQUE_YES = '\x01' +UNIQUE_NO = '\x02' + class InstanceNode(object): """For the first phase: InstanceNode is used to match the start and the end of the loop, so it contains both 'origfields' that represents @@ -35,6 +39,7 @@ curfields = None # optimization; equivalent to an empty dict dependencies = None knownclsbox = None + unique = UNIQUE_UNKNOWN # for find_unique_nodes() def __init__(self, escaped, fromstart=False): self.escaped = escaped @@ -151,6 +156,8 @@ """Build the list of specnodes based on the result computed by this PerfectSpecializationFinder. """ + for box in op.args: + self.find_unique_nodes(self.getnode(box)) specnodes = [] assert len(self.inputnodes) == len(op.args) for i in range(len(op.args)): @@ -159,29 +166,42 @@ specnodes.append(self.intersect(inputnode, exitnode)) self.specnodes = specnodes + def find_unique_nodes(self, exitnode): + if (exitnode.escaped or exitnode.fromstart + or exitnode.knownclsbox is None + or exitnode.unique != UNIQUE_UNKNOWN): + # the exitnode is not suitable for being a virtual, or we + # encounter it more than once when doing the recursion + exitnode.unique = UNIQUE_NO + else: + exitnode.unique = UNIQUE_YES + if exitnode.curfields is not None: + for subnode in exitnode.curfields.values(): + self.find_unique_nodes(subnode) + def intersect(self, inputnode, exitnode): assert inputnode.fromstart - if exitnode.knownclsbox is None: - return prebuiltNotSpecNode # no known class at exit - if (inputnode.knownclsbox is not None and - not inputnode.knownclsbox.equals(exitnode.knownclsbox)): - return prebuiltNotSpecNode # mismatched known class at exit - # - # for the sequel, we know that the class is known and matches - if inputnode.escaped or exitnode.escaped or exitnode.fromstart: - if inputnode.knownclsbox is None: - return prebuiltNotSpecNode # class not needed at input - return FixedClassSpecNode(exitnode.knownclsbox) + if exitnode.unique == UNIQUE_NO: + # give a NotSpecNode or a FixedClassSpecNode + if (inputnode.knownclsbox is not None and + exitnode.knownclsbox is not None and + inputnode.knownclsbox.equals(exitnode.knownclsbox)): + # the class is known at the end, needed at the input, + # and matches + return FixedClassSpecNode(inputnode.knownclsbox) + else: + return prebuiltNotSpecNode # + assert exitnode.unique == UNIQUE_YES fields = [] d = exitnode.curfields if d is not None: - if inputnode.origfields is None: - inputnode.origfields = av_newdict() lst = d.keys() sort_descrs(lst) for ofs in lst: try: + if inputnode.origfields is None: + raise KeyError node = inputnode.origfields[ofs] except KeyError: # field stored at exit, but not read at input. Must Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Sat Jul 18 15:35:01 2009 @@ -394,7 +394,6 @@ assert boxp3.knownclsbox.value == self.node_vtable_adr2 def test_find_nodes_new_aliasing_0(self): - py.test.skip("in-progress") ops = """ [p1, p2] p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) @@ -405,7 +404,6 @@ self.find_nodes(ops, 'Not, Not') def test_find_nodes_new_aliasing_1(self): - py.test.skip("infinite loop") ops = """ [sum, p1] guard_class(p1, ConstClass(node_vtable)) @@ -423,11 +421,10 @@ """ # the issue is the cycle "p2->p2", which cannot be represented # with SpecNodes so far - self.find_nodes(ops, 'Not, Not', + self.find_nodes(ops, 'Not, Fixed(node_vtable)', boxkinds={'sum': BoxInt, 'sum2': BoxInt}) def test_find_nodes_new_aliasing_2(self): - py.test.skip("in-progress") ops = """ [p1, p2] escape(p2) From benjamin at codespeak.net Sat Jul 18 15:36:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 15:36:02 +0200 (CEST) Subject: [pypy-svn] r66356 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718133602.2A08D169F6E@codespeak.net> Author: benjamin Date: Sat Jul 18 15:36:01 2009 New Revision: 66356 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: help annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 15:36:01 2009 @@ -186,6 +186,7 @@ if body: start = 0 if self.is_docstring(body[0]): + assert isinstance(body[0], ast.Expr) start = 1 body[0].value.walkabout(self) self.name_op("__doc__", ast.Store) @@ -1101,6 +1102,7 @@ def _compile(self, func): assert isinstance(func, ast.FunctionDef) if self.is_docstring(func.body[0]): + assert isinstance(func.body[0], ast.Expr) self.add_const(func.body[0].value.s) start = 1 else: From arigo at codespeak.net Sat Jul 18 15:44:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 15:44:17 +0200 (CEST) Subject: [pypy-svn] r66357 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718134417.0E232169F8A@codespeak.net> Author: arigo Date: Sat Jul 18 15:44:16 2009 New Revision: 66357 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Add tests. A small fix. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Sat Jul 18 15:44:16 2009 @@ -193,6 +193,11 @@ return prebuiltNotSpecNode # assert exitnode.unique == UNIQUE_YES + if (inputnode.knownclsbox is not None and + not inputnode.knownclsbox.equals(exitnode.knownclsbox)): + # unique match, but the class is known to be a mismatch + return prebuiltNotSpecNode + # fields = [] d = exitnode.curfields if d is not None: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Sat Jul 18 15:44:16 2009 @@ -435,6 +435,28 @@ # in p1 a Virtual and not in p2, as they both come from the same p3. self.find_nodes(ops, 'Not, Not') + def test_find_nodes_new_mismatch(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail() + p2 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + jump(p2) + """ + self.find_nodes(ops, 'Not') + + def test_find_nodes_new_aliasing_mismatch(self): + ops = """ + [p0, p1] + guard_class(p0, ConstClass(node_vtable)) + fail() + guard_class(p1, ConstClass(node_vtable2)) + fail() + p2 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + jump(p2, p2) + """ + self.find_nodes(ops, 'Not, Fixed(node_vtable2)') + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 15:47:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 15:47:42 +0200 (CEST) Subject: [pypy-svn] r66358 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718134742.92B9B169F96@codespeak.net> Author: benjamin Date: Sat Jul 18 15:47:41 2009 New Revision: 66358 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: annotator hint Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 15:47:41 2009 @@ -969,6 +969,7 @@ ext_slice = ast.ExtSlice(slices) return ast.Subscript(left_expr, ext_slice, ast.Load, middle.lineno, middle.column) + assert isinstance(slices[0], ast.Index) elts = [idx.value for idx in slices] tup = ast.Tuple(elts, ast.Load, middle.lineno, middle.column) return ast.Subscript(left_expr, ast.Index(tup), ast.Load, From arigo at codespeak.net Sat Jul 18 15:49:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 15:49:42 +0200 (CEST) Subject: [pypy-svn] r66359 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718134942.70CC2169F96@codespeak.net> Author: arigo Date: Sat Jul 18 15:49:42 2009 New Revision: 66359 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Another test and fix. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Sat Jul 18 15:49:42 2009 @@ -181,7 +181,7 @@ def intersect(self, inputnode, exitnode): assert inputnode.fromstart - if exitnode.unique == UNIQUE_NO: + if exitnode.unique == UNIQUE_NO or inputnode.escaped: # give a NotSpecNode or a FixedClassSpecNode if (inputnode.knownclsbox is not None and exitnode.knownclsbox is not None and Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Sat Jul 18 15:49:42 2009 @@ -457,6 +457,15 @@ """ self.find_nodes(ops, 'Not, Fixed(node_vtable2)') + def test_find_nodes_new_escapes(self): + ops = """ + [p0] + escape(p0) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + jump(p1) + """ + self.find_nodes(ops, 'Not') + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From arigo at codespeak.net Sat Jul 18 15:52:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 15:52:01 +0200 (CEST) Subject: [pypy-svn] r66360 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090718135201.E2A8A169F96@codespeak.net> Author: arigo Date: Sat Jul 18 15:52:01 2009 New Revision: 66360 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Another (passing) test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Sat Jul 18 15:52:01 2009 @@ -466,6 +466,21 @@ """ self.find_nodes(ops, 'Not') + def test_find_nodes_new_unused(self): + ops = """ + [p0] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, p3, descr=nextdescr) + jump(p1) + """ + self.find_nodes(ops, ''' + Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Virtual(node_vtable)))''') + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 15:55:05 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 15:55:05 +0200 (CEST) Subject: [pypy-svn] r66361 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718135505.0DEB8169F9C@codespeak.net> Author: benjamin Date: Sat Jul 18 15:55:05 2009 New Revision: 66361 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: rewrite slightly Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 15:55:05 2009 @@ -186,9 +186,10 @@ if body: start = 0 if self.is_docstring(body[0]): - assert isinstance(body[0], ast.Expr) + doc_expr = body[0] + assert isinstance(doc_expr, ast.Expr) start = 1 - body[0].value.walkabout(self) + doc_expr.value.walkabout(self) self.name_op("__doc__", ast.Store) for i in range(start, len(body)): body[i].walkabout(self) From arigo at codespeak.net Sat Jul 18 15:59:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 15:59:09 +0200 (CEST) Subject: [pypy-svn] r66362 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718135909.986E8169F9E@codespeak.net> Author: arigo Date: Sat Jul 18 15:59:08 2009 New Revision: 66362 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py - copied, changed from r66359, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py - copied, changed from r66360, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py Log: Rename optimize.py to optimizeloop.py. The plan is for it to contain only the PerfectSpecializationFinder, which works on loops only (not bridges). Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Sat Jul 18 15:59:08 2009 @@ -0,0 +1,44 @@ +from pypy.rlib.objectmodel import r_dict +from pypy.rlib.unroll import unrolling_iterable +from pypy.jit.metainterp import resoperation + +# ____________________________________________________________ +# Misc. utilities + +def av_eq(self, other): + return self.sort_key() == other.sort_key() + +def av_hash(self): + return self.sort_key() + +def av_newdict(): + return r_dict(av_eq, av_hash) + +def _findall(Class, name_prefix): + result = [] + for value, name in resoperation.opname.items(): + if hasattr(Class, name_prefix + name): + result.append((value, getattr(Class, name_prefix + name))) + return unrolling_iterable(result) + +def partition(array, left, right): + last_item = array[right] + pivot = last_item.sort_key() + storeindex = left + for i in range(left, right): + if array[i].sort_key() <= pivot: + array[i], array[storeindex] = array[storeindex], array[i] + storeindex += 1 + # Move pivot to its final place + array[storeindex], array[right] = last_item, array[storeindex] + return storeindex + +def quicksort(array, left, right): + # sort array[left:right+1] (i.e. bounds included) + if right > left: + pivotnewindex = partition(array, left, right) + quicksort(array, left, pivotnewindex - 1) + quicksort(array, pivotnewindex + 1, right) + +def sort_descrs(lst): + quicksort(lst, 0, len(lst)-1) Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py (from r66359, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py Sat Jul 18 15:59:08 2009 @@ -1,27 +1,8 @@ -from pypy.rlib.objectmodel import r_dict -from pypy.rlib.unroll import unrolling_iterable -from pypy.jit.metainterp import resoperation from pypy.jit.metainterp.specnode import prebuiltNotSpecNode from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.history import AbstractValue - - -def av_eq(self, other): - return self.sort_key() == other.sort_key() - -def av_hash(self): - return self.sort_key() - -def av_newdict(): - return r_dict(av_eq, av_hash) - -def _findall(Class, name_prefix): - result = [] - for value, name in resoperation.opname.items(): - if hasattr(Class, name_prefix + name): - result.append((value, getattr(Class, name_prefix + name))) - return unrolling_iterable(result) +from pypy.jit.metainterp.optimize import av_newdict, _findall, sort_descrs # ____________________________________________________________ @@ -218,27 +199,3 @@ return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') - -# ____________________________________________________________ - -def partition(array, left, right): - last_item = array[right] - pivot = last_item.sort_key() - storeindex = left - for i in range(left, right): - if array[i].sort_key() <= pivot: - array[i], array[storeindex] = array[storeindex], array[i] - storeindex += 1 - # Move pivot to its final place - array[storeindex], array[right] = last_item, array[storeindex] - return storeindex - -def quicksort(array, left, right): - # sort array[left:right+1] (i.e. bounds included) - if right > left: - pivotnewindex = partition(array, left, right) - quicksort(array, left, pivotnewindex - 1) - quicksort(array, pivotnewindex + 1, right) - -def sort_descrs(lst): - quicksort(lst, 0, len(lst)-1) Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py (from r66360, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py Sat Jul 18 15:59:08 2009 @@ -8,7 +8,7 @@ from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, ConstAddr, TreeLoop, BoxObj, ConstObj, AbstractDescr) -from pypy.jit.metainterp.optimize import PerfectSpecializationFinder +from pypy.jit.metainterp.optimizeloop import PerfectSpecializationFinder from pypy.jit.metainterp.optimize import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import FixedClassSpecNode From benjamin at codespeak.net Sat Jul 18 16:03:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 16:03:59 +0200 (CEST) Subject: [pypy-svn] r66363 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718140359.1E149169FA3@codespeak.net> Author: benjamin Date: Sat Jul 18 16:03:58 2009 New Revision: 66363 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: pass lineno directly Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 16:03:58 2009 @@ -131,8 +131,8 @@ assert self.scope.lookup(name) != symtable.SCOPE_UNKNOWN return name - def sub_scope(self, kind, name, node): - generator = kind(self.space, name, node, node.lineno, self.symbols, + def sub_scope(self, kind, name, node, lineno): + generator = kind(self.space, name, node, lineno, self.symbols, self.compile_info) return generator.assemble() @@ -234,7 +234,8 @@ num_defaults = len(func.args.defaults) else: num_defaults = 0 - code = self.sub_scope(FunctionCodeGenerator, func.name, func) + code = self.sub_scope(FunctionCodeGenerator, func.name, func, + func.lineno) self.update_position(func.lineno) self._make_function(code, num_defaults) self.name_op(func.name, ast.Store) @@ -245,7 +246,7 @@ default_count = len(lam.args.defaults) else: default_count = 0 - code = self.sub_scope(LambdaCodeGenerator, "", lam) + code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) self.update_position(lam.lineno) self._make_function(code, default_count) @@ -258,7 +259,7 @@ else: bases_count = 0 self.emit_op_arg(ops.BUILD_TUPLE, bases_count) - code = self.sub_scope(ClassCodeGenerator, cls.name, cls) + code = self.sub_scope(ClassCodeGenerator, cls.name, cls, cls.lineno) self.update_position(cls.lineno) self._make_function(code, 0) self.emit_op_arg(ops.CALL_FUNCTION, 0) @@ -971,7 +972,8 @@ self.use_next_block(end) def visit_GeneratorExp(self, genexp): - code = self.sub_scope(GenExpCodeGenerator, "", genexp) + code = self.sub_scope(GenExpCodeGenerator, "", genexp, + genexp.lineno) self.update_position(genexp.lineno) self._make_function(code) genexp.generators[0].iter.walkabout(self) From benjamin at codespeak.net Sat Jul 18 18:58:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 18:58:35 +0200 (CEST) Subject: [pypy-svn] r66364 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718165835.408DC16854B@codespeak.net> Author: benjamin Date: Sat Jul 18 18:58:34 2009 New Revision: 66364 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: assertion for the annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 18:58:34 2009 @@ -356,6 +356,7 @@ self.scope.note_import_star(imp) def _visit_alias(self, alias): + assert isinstance(alias, ast.alias) if alias.asname: store_name = alias.asname else: From arigo at codespeak.net Sat Jul 18 19:01:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 19:01:34 +0200 (CEST) Subject: [pypy-svn] r66365 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718170134.4895B169E11@codespeak.net> Author: arigo Date: Sat Jul 18 19:01:33 2009 New Revision: 66365 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py - copied, changed from r66363, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py - copied, changed from r66363, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Log: Reorganize things a bit yet another time. Start on the BridgeSpecializationFinder; only one test passing. Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (from r66363, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeloop.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sat Jul 18 19:01:33 2009 @@ -1,7 +1,9 @@ -from pypy.jit.metainterp.specnode import prebuiltNotSpecNode +from pypy.jit.metainterp.specnode import SpecNode +from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.history import AbstractValue +from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.optimize import av_newdict, _findall, sort_descrs # ____________________________________________________________ @@ -42,6 +44,18 @@ for box in deps: box.mark_escaped() + def set_unique_nodes(self): + if (self.escaped or self.fromstart or self.knownclsbox is None + or self.unique != UNIQUE_UNKNOWN): + # this node is not suitable for being a virtual, or we + # encounter it more than once when doing the recursion + self.unique = UNIQUE_NO + else: + self.unique = UNIQUE_YES + if self.curfields is not None: + for subnode in self.curfields.values(): + subnode.set_unique_nodes() + def __repr__(self): flags = '' if self.escaped: flags += 'e' @@ -49,8 +63,11 @@ if self.knownclsbox: flags += 'c' return "" % (flags,) +# ____________________________________________________________ +# General find_nodes_xxx() interface, for both loops and bridges -class PerfectSpecializationFinder(object): +class NodeFinder(object): + """Abstract base class.""" node_escaped = InstanceNode(escaped=True) def __init__(self): @@ -59,15 +76,8 @@ def getnode(self, box): return self.nodes.get(box, self.node_escaped) - def find_nodes(self, loop): - inputnodes = [] - for box in loop.inputargs: - instnode = InstanceNode(escaped=False, fromstart=True) - inputnodes.append(instnode) - self.nodes[box] = instnode - self.inputnodes = inputnodes - # - for op in loop.operations: + def find_nodes(self, operations): + for op in operations: opnum = op.opnum for value, func in find_nodes_ops: if opnum == value: @@ -134,11 +144,38 @@ instnode.knownclsbox = clsbox def find_nodes_JUMP(self, op): - """Build the list of specnodes based on the result - computed by this PerfectSpecializationFinder. - """ + # only set up the 'unique' field of the InstanceNodes; + # real handling comes later (build_result_specnodes() for loops). for box in op.args: - self.find_unique_nodes(self.getnode(box)) + self.getnode(box).set_unique_nodes() + + def find_nodes_FAIL(self, op): + xxx + +find_nodes_ops = _findall(NodeFinder, 'find_nodes_') + +# ____________________________________________________________ +# Perfect specialization -- for loops only + +class PerfectSpecializationFinder(NodeFinder): + + def find_nodes_loop(self, loop): + self.setup_input_nodes(loop.inputargs) + self.find_nodes(loop.operations) + self.build_result_specnodes(loop.operations[-1]) + + def setup_input_nodes(self, inputargs): + inputnodes = [] + for box in inputargs: + instnode = InstanceNode(escaped=False, fromstart=True) + inputnodes.append(instnode) + self.nodes[box] = instnode + self.inputnodes = inputnodes + + def build_result_specnodes(self, op): + # Build the list of specnodes based on the result + # computed by NodeFinder.find_nodes(). + assert op.opnum == rop.JUMP specnodes = [] assert len(self.inputnodes) == len(op.args) for i in range(len(op.args)): @@ -147,19 +184,6 @@ specnodes.append(self.intersect(inputnode, exitnode)) self.specnodes = specnodes - def find_unique_nodes(self, exitnode): - if (exitnode.escaped or exitnode.fromstart - or exitnode.knownclsbox is None - or exitnode.unique != UNIQUE_UNKNOWN): - # the exitnode is not suitable for being a virtual, or we - # encounter it more than once when doing the recursion - exitnode.unique = UNIQUE_NO - else: - exitnode.unique = UNIQUE_YES - if exitnode.curfields is not None: - for subnode in exitnode.curfields.values(): - self.find_unique_nodes(subnode) - def intersect(self, inputnode, exitnode): assert inputnode.fromstart if exitnode.unique == UNIQUE_NO or inputnode.escaped: @@ -198,4 +222,56 @@ fields.append((ofs, specnode)) return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) -find_nodes_ops = _findall(PerfectSpecializationFinder, 'find_nodes_') +# ____________________________________________________________ +# A subclass of NodeFinder for bridges only + +class __extend__(SpecNode): + def make_instance_node(self): + raise NotImplementedError + def matches_instance_node(self, exitnode): + raise NotImplementedError + +class __extend__(NotSpecNode): + def make_instance_node(self): + return NodeFinder.node_escaped + def matches_instance_node(self, exitnode): + return True + +class __extend__(FixedClassSpecNode): + def make_instance_node(self): + instnode = InstanceNode(escaped=True) + instnode.knownclsbox = self.known_class + return instnode + def matches_instance_node(self, exitnode): + xxx + +class __extend__(VirtualInstanceSpecNode): + def make_instance_node(self): + instnode = InstanceNode(escaped=False) + instnode.curfields = av_newdict() + for ofs, subspecnode in self.fields: + instnode.curfields[ofs] = subspecnode.make_instance_node() + return instnode + def matches_instance_node(self, exitnode): + if exitnode.unique == UNIQUE_NO: + return False + else: + xxx + +class BridgeSpecializationFinder(NodeFinder): + + def setup_bridge_input_nodes(self, specnodes, inputargs): + assert len(specnodes) == len(inputargs) + for i in range(len(inputargs)): + instnode = specnodes[i].make_instance_node() + box = inputargs[i] + self.nodes[box] = instnode + + def bridge_matches(self, jump_op, nextloop_specnodes): + assert jump_op.opnum == rop.JUMP + assert len(jump_op.args) == len(nextloop_specnodes) + for i in range(len(nextloop_specnodes)): + exitnode = self.getnode(jump_op.args[i]) + if not nextloop_specnodes[i].matches_instance_node(exitnode): + return False + return True Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Sat Jul 18 19:01:33 2009 @@ -1,6 +1,8 @@ +from pypy.tool.pairtype import extendabletype class SpecNode(object): + __metaclass__ = extendabletype # extended in optimizefindnode.py __slots__ = () def extract_runtime_data(self, cpu, valuebox, resultlist): Copied: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (from r66363, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py) ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeloop.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 19:01:33 2009 @@ -8,7 +8,8 @@ from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr, Const, ConstAddr, TreeLoop, BoxObj, ConstObj, AbstractDescr) -from pypy.jit.metainterp.optimizeloop import PerfectSpecializationFinder +from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder +from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder from pypy.jit.metainterp.optimize import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import FixedClassSpecNode @@ -135,7 +136,7 @@ equaloplists(optimized.operations, self.parse(expected).operations) - def assert_specnodes(self, specnodes, text): + def unpack_specnodes(self, text): # def constclass(cls_vtable): if self.type_system == 'lltype': @@ -156,6 +157,10 @@ 'Fixed': makeFixed, 'Virtual': makeVirtual} lst = eval('[' + text + ']', self.namespace, context) + return lst + + def check_specnodes(self, specnodes, text): + lst = self.unpack_specnodes(text) assert len(specnodes) == len(lst) for x, y in zip(specnodes, lst): assert x._equals(y) @@ -164,9 +169,8 @@ def find_nodes(self, ops, spectext, boxkinds=None): loop = self.parse(ops, boxkinds=boxkinds) perfect_specialization_finder = PerfectSpecializationFinder() - perfect_specialization_finder.find_nodes(loop) - assert self.assert_specnodes(perfect_specialization_finder.specnodes, - spectext) + perfect_specialization_finder.find_nodes_loop(loop) + self.check_specnodes(perfect_specialization_finder.specnodes, spectext) return (loop.getboxes(), perfect_specialization_finder.getnode) def test_find_nodes_simple(self): @@ -481,6 +485,35 @@ nextdescr=Virtual(node_vtable, nextdescr=Virtual(node_vtable)))''') + # ------------------------------ + # Bridge tests + + def find_bridge(self, ops, inputspectext, outputspectext, boxkinds=None, + mismatch=False): + inputspecnodes = self.unpack_specnodes(inputspectext) + outputspecnodes = self.unpack_specnodes(outputspectext) + bridge = self.parse(ops, boxkinds=boxkinds) + bridge_specialization_finder = BridgeSpecializationFinder() + bridge_specialization_finder.setup_bridge_input_nodes(inputspecnodes, + bridge.inputargs) + bridge_specialization_finder.find_nodes(bridge.operations) + matches = bridge_specialization_finder.bridge_matches( + bridge.operations[-1], + outputspecnodes) + if mismatch: + assert not matches + else: + assert matches + + def test_bridge_simple(self): + ops = """ + [i0] + i1 = int_add(i0, 1) + jump(i1) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', mismatch=True) + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 19:06:26 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:06:26 +0200 (CEST) Subject: [pypy-svn] r66366 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718170626.F14C6169FA5@codespeak.net> Author: benjamin Date: Sat Jul 18 19:06:26 2009 New Revision: 66366 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: rewrite to appease annotator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 19:06:26 2009 @@ -1105,8 +1105,9 @@ def _compile(self, func): assert isinstance(func, ast.FunctionDef) if self.is_docstring(func.body[0]): - assert isinstance(func.body[0], ast.Expr) - self.add_const(func.body[0].value.s) + doc_string = func.body[0] + assert isinstance(doc_string, ast.Expr) + self.add_const(doc_string.value.s) start = 1 else: self.add_const(self.space.w_None) From arigo at codespeak.net Sat Jul 18 19:07:21 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 19:07:21 +0200 (CEST) Subject: [pypy-svn] r66367 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718170721.2B7CF169F5E@codespeak.net> Author: arigo Date: Sat Jul 18 19:07:20 2009 New Revision: 66367 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: test_bridge_simple_known_class. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sat Jul 18 19:07:20 2009 @@ -243,7 +243,8 @@ instnode.knownclsbox = self.known_class return instnode def matches_instance_node(self, exitnode): - xxx + return (exitnode.knownclsbox is not None and + self.known_class.equals(exitnode.knownclsbox)) class __extend__(VirtualInstanceSpecNode): def make_instance_node(self): @@ -260,6 +261,11 @@ class BridgeSpecializationFinder(NodeFinder): + def find_nodes_bridge(self, bridge, specnodes): + if specnodes is not None: + self.setup_bridge_input_nodes(specnodes, bridge.inputargs) + self.find_nodes(bridge.operations) + def setup_bridge_input_nodes(self, specnodes, inputargs): assert len(specnodes) == len(inputargs) for i in range(len(inputargs)): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 19:07:20 2009 @@ -494,9 +494,7 @@ outputspecnodes = self.unpack_specnodes(outputspectext) bridge = self.parse(ops, boxkinds=boxkinds) bridge_specialization_finder = BridgeSpecializationFinder() - bridge_specialization_finder.setup_bridge_input_nodes(inputspecnodes, - bridge.inputargs) - bridge_specialization_finder.find_nodes(bridge.operations) + bridge_specialization_finder.find_nodes_bridge(bridge, inputspecnodes) matches = bridge_specialization_finder.bridge_matches( bridge.operations[-1], outputspecnodes) @@ -514,6 +512,20 @@ self.find_bridge(ops, 'Not', 'Not') self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', mismatch=True) + def test_bridge_simple_known_class(self): + ops = """ + [p0] + setfield_gc(p0, 123, descr=valuedescr) + jump(p0) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Fixed(node_vtable)', 'Not') + self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable)') + # + self.find_bridge(ops, 'Not', 'Fixed(node_vtable)', mismatch=True) + self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable2)', + mismatch=True) + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 19:14:20 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:14:20 +0200 (CEST) Subject: [pypy-svn] r66368 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718171420.6FF1816846E@codespeak.net> Author: benjamin Date: Sat Jul 18 19:14:19 2009 New Revision: 66368 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: rewrite so the annotator knows each element is more specific Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 19:14:19 2009 @@ -969,8 +969,10 @@ ext_slice = ast.ExtSlice(slices) return ast.Subscript(left_expr, ext_slice, ast.Load, middle.lineno, middle.column) - assert isinstance(slices[0], ast.Index) - elts = [idx.value for idx in slices] + elts = [] + for idx in slices: + assert isinstance(idx, ast.Index) + elts.append(idx.value) tup = ast.Tuple(elts, ast.Load, middle.lineno, middle.column) return ast.Subscript(left_expr, ast.Index(tup), ast.Load, middle.lineno, middle.column) From benjamin at codespeak.net Sat Jul 18 19:35:45 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:35:45 +0200 (CEST) Subject: [pypy-svn] r66369 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718173545.DA7F62A809D@codespeak.net> Author: benjamin Date: Sat Jul 18 19:35:44 2009 New Revision: 66369 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: use walkabout Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 19:35:44 2009 @@ -1107,7 +1107,7 @@ if self.is_docstring(func.body[0]): doc_string = func.body[0] assert isinstance(doc_string, ast.Expr) - self.add_const(doc_string.value.s) + doc_string.value.walkabout(self) start = 1 else: self.add_const(self.space.w_None) From benjamin at codespeak.net Sat Jul 18 19:39:00 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:39:00 +0200 (CEST) Subject: [pypy-svn] r66370 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718173900.12A422A809D@codespeak.net> Author: benjamin Date: Sat Jul 18 19:38:58 2009 New Revision: 66370 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: annotator assertion Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 19:38:58 2009 @@ -544,6 +544,7 @@ def visit_Import(self, imp): self.update_position(imp.lineno) for alias in imp.names: + assert isinstance(alias, ast.alias) if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: level = 0 else: From arigo at codespeak.net Sat Jul 18 19:46:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 19:46:42 +0200 (CEST) Subject: [pypy-svn] r66371 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718174642.9D0572A80A7@codespeak.net> Author: arigo Date: Sat Jul 18 19:46:41 2009 New Revision: 66371 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Finish bridge_matches() -- case of virtuals. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sat Jul 18 19:46:41 2009 @@ -249,15 +249,37 @@ class __extend__(VirtualInstanceSpecNode): def make_instance_node(self): instnode = InstanceNode(escaped=False) + instnode.knownclsbox = self.known_class instnode.curfields = av_newdict() for ofs, subspecnode in self.fields: instnode.curfields[ofs] = subspecnode.make_instance_node() return instnode + def matches_instance_node(self, exitnode): if exitnode.unique == UNIQUE_NO: return False - else: - xxx + # + assert exitnode.unique == UNIQUE_YES + if not self.known_class.equals(exitnode.knownclsbox): + # unique match, but the class is known to be a mismatch + return False + # + d = exitnode.curfields + seen = 0 + for ofs, subspecnode in self.fields: + try: + if d is None: + raise KeyError + instnode = d[ofs] + seen += 1 + except KeyError: + instnode = NodeFinder.node_escaped + if not subspecnode.matches_instance_node(instnode): + return False + if d is not None and len(d) > seen: + return False # some key is in d but not in self.fields + return True + class BridgeSpecializationFinder(NodeFinder): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 19:46:41 2009 @@ -526,6 +526,53 @@ self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable2)', mismatch=True) + def test_bridge_simple_virtual(self): + ops = """ + [p0] + setfield_gc(p0, 123, descr=valuedescr) + jump(p0) + """ + self.find_bridge(ops, 'Virtual(node_vtable)', 'Not') + self.find_bridge(ops, 'Virtual(node_vtable)', 'Fixed(node_vtable)') + self.find_bridge(ops, 'Virtual(node_vtable)', + 'Virtual(node_vtable, valuedescr=Not)') + self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', + 'Virtual(node_vtable, valuedescr=Not)') + self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Not)''') + self.find_bridge(ops, '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Fixed(node_vtable2))''', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Fixed(node_vtable2))''') + self.find_bridge(ops, '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Fixed(node_vtable2))''', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Not)''') + # + self.find_bridge(ops, 'Virtual(node_vtable)', 'Virtual(node_vtable)', + mismatch=True) # because of missing valuedescr + self.find_bridge(ops, 'Virtual(node_vtable)', 'Fixed(node_vtable2)', + mismatch=True) # bad class + self.find_bridge(ops, 'Virtual(node_vtable)', + 'Virtual(node_vtable2, valuedescr=Not)', + mismatch=True) # bad class + self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Fixed(node_vtable))''', + mismatch=True) # unexpected nextdescr + self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Virtual(node_vtable))''', + mismatch=True) # unexpected nextdescr + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From arigo at codespeak.net Sat Jul 18 19:50:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 19:50:56 +0200 (CEST) Subject: [pypy-svn] r66372 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090718175056.5C3BD2A80A5@codespeak.net> Author: arigo Date: Sat Jul 18 19:50:55 2009 New Revision: 66372 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Another test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 19:50:55 2009 @@ -526,7 +526,32 @@ self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable2)', mismatch=True) - def test_bridge_simple_virtual(self): + def test_bridge_simple_virtual_1(self): + ops = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p0, i0, descr=valuedescr) + jump(p0) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Not', 'Fixed(node_vtable)') + self.find_bridge(ops, 'Not', 'Virtual(node_vtable, valuedescr=Not)') + self.find_bridge(ops, 'Not', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Not)''') + # + self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', + mismatch=True) + self.find_bridge(ops, 'Not', 'Virtual(node_vtable, nextdescr=Not)', + mismatch=True) + self.find_bridge(ops, 'Not', + '''Virtual(node_vtable, + valuedescr=Not, + nextdescr=Fixed(node_vtable))''', + mismatch=True) + + def test_bridge_simple_virtual_2(self): ops = """ [p0] setfield_gc(p0, 123, descr=valuedescr) From benjamin at codespeak.net Sat Jul 18 19:54:47 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:54:47 +0200 (CEST) Subject: [pypy-svn] r66373 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718175447.22564169EB1@codespeak.net> Author: benjamin Date: Sat Jul 18 19:54:46 2009 New Revision: 66373 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: add assertion Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 19:54:46 2009 @@ -1177,6 +1177,7 @@ for_node = comp_for.children[1] for_targets = self.handle_exprlist(for_node, ast.Store) expr = handle_source_expression(comp_for.children[3]) + assert isinstance(expr, ast.expr) if len(for_node.children) == 1: comp = ast.comprehension(for_targets[0], expr, None) else: From benjamin at codespeak.net Sat Jul 18 19:55:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 19:55:44 +0200 (CEST) Subject: [pypy-svn] r66374 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718175544.F2BCE2A80A8@codespeak.net> Author: benjamin Date: Sat Jul 18 19:55:44 2009 New Revision: 66374 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: fix import statements Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 19:55:44 2009 @@ -527,10 +527,10 @@ source_name = alias.name dot = source_name.find(".") if dot > 0: - start = dot + 1 while True: + start = dot + 1 dot = source_name.find(".", start) - if dot > 0: + if dot < 0: end = len(source_name) else: end = dot @@ -538,7 +538,6 @@ self.emit_op_name(ops.LOAD_ATTR, self.names, attr) if dot < 0: break - start = dot + 1 self.name_op(alias.asname, ast.Store) def visit_Import(self, imp): From arigo at codespeak.net Sat Jul 18 19:56:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 19:56:55 +0200 (CEST) Subject: [pypy-svn] r66375 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090718175655.D4E282A80A5@codespeak.net> Author: arigo Date: Sat Jul 18 19:56:55 2009 New Revision: 66375 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Add a test (passing) about aliasing mismatch in bridges. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 19:56:55 2009 @@ -542,14 +542,16 @@ nextdescr=Not)''') # self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', - mismatch=True) + mismatch=True) # missing valuedescr self.find_bridge(ops, 'Not', 'Virtual(node_vtable, nextdescr=Not)', - mismatch=True) + mismatch=True) # missing valuedescr + self.find_bridge(ops, 'Not', 'Virtual(node_vtable2, valuedescr=Not)', + mismatch=True) # bad class self.find_bridge(ops, 'Not', '''Virtual(node_vtable, valuedescr=Not, nextdescr=Fixed(node_vtable))''', - mismatch=True) + mismatch=True) # nextdescr too precise def test_bridge_simple_virtual_2(self): ops = """ @@ -598,6 +600,23 @@ nextdescr=Virtual(node_vtable))''', mismatch=True) # unexpected nextdescr + def test_bridge_virtual_mismatch_1(self): + ops = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p0, i0, descr=valuedescr) + jump(p0, p0) + """ + self.find_bridge(ops, 'Not', 'Not, Not') + self.find_bridge(ops, 'Not', 'Not, Fixed(node_vtable)') + self.find_bridge(ops, 'Not', 'Fixed(node_vtable), Not') + self.find_bridge(ops, 'Not', 'Fixed(node_vtable), Fixed(node_vtable)') + # + self.find_bridge(ops, 'Not', + '''Virtual(node_vtable, valuedescr=Not), + Virtual(node_vtable, valuedescr=Not)''', + mismatch=True) # duplicate p0 + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 20:02:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:02:44 +0200 (CEST) Subject: [pypy-svn] r66376 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718180244.8C372169EB1@codespeak.net> Author: benjamin Date: Sat Jul 18 20:02:43 2009 New Revision: 66376 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: more assertions Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 20:02:43 2009 @@ -576,7 +576,10 @@ else: level = imp.level self.load_const(space.wrap(level)) - names_w = [space.wrap(alias.name) for alias in imp.names] + names_w = [] + for alias in imp.names: + assert isinstance(alias, ast.alias) + names_w.append(space.wrap(alias.name)) self.load_const(space.newtuple(names_w)) if imp.module: mod_name = imp.module @@ -587,6 +590,7 @@ self.emit_op(ops.IMPORT_STAR) else: for alias in imp.names: + assert isinstance(alias, ast.alias) self.emit_op_name(ops.IMPORT_FROM, self.names, alias.name) if alias.asname: store_name = alias.asname From benjamin at codespeak.net Sat Jul 18 20:11:22 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:11:22 +0200 (CEST) Subject: [pypy-svn] r66377 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718181122.4FD3A2A809D@codespeak.net> Author: benjamin Date: Sat Jul 18 20:11:21 2009 New Revision: 66377 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: tuple lists can't be resized Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 20:11:21 2009 @@ -576,10 +576,10 @@ else: level = imp.level self.load_const(space.wrap(level)) - names_w = [] - for alias in imp.names: + names_w = [None]*len(imp.names) + for i in range(len(imp.names)): assert isinstance(alias, ast.alias) - names_w.append(space.wrap(alias.name)) + names_w[i] = space.wrap(imp.names[i]) self.load_const(space.newtuple(names_w)) if imp.module: mod_name = imp.module From arigo at codespeak.net Sat Jul 18 20:13:59 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 18 Jul 2009 20:13:59 +0200 (CEST) Subject: [pypy-svn] r66378 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090718181359.C67072A809D@codespeak.net> Author: arigo Date: Sat Jul 18 20:13:59 2009 New Revision: 66378 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: More tests. One small fix. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sat Jul 18 20:13:59 2009 @@ -233,7 +233,7 @@ class __extend__(NotSpecNode): def make_instance_node(self): - return NodeFinder.node_escaped + return InstanceNode(escaped=True) def matches_instance_node(self, exitnode): return True Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sat Jul 18 20:13:59 2009 @@ -167,6 +167,7 @@ return True def find_nodes(self, ops, spectext, boxkinds=None): + assert boxkinds is None or isinstance(boxkinds, dict) loop = self.parse(ops, boxkinds=boxkinds) perfect_specialization_finder = PerfectSpecializationFinder() perfect_specialization_finder.find_nodes_loop(loop) @@ -490,6 +491,7 @@ def find_bridge(self, ops, inputspectext, outputspectext, boxkinds=None, mismatch=False): + assert boxkinds is None or isinstance(boxkinds, dict) inputspecnodes = self.unpack_specnodes(inputspectext) outputspecnodes = self.unpack_specnodes(outputspectext) bridge = self.parse(ops, boxkinds=boxkinds) @@ -617,6 +619,57 @@ Virtual(node_vtable, valuedescr=Not)''', mismatch=True) # duplicate p0 + def test_bridge_guard_class(self): + ops = """ + [p1] + p2 = getfield_gc(p1, descr=nextdescr) + guard_class(p2, ConstClass(node_vtable)) + fail() + jump(p2) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Not', 'Fixed(node_vtable)') + self.find_bridge(ops, 'Virtual(node_vtable2, nextdescr=Not)', + 'Fixed(node_vtable)') + self.find_bridge(ops, + '''Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Fixed(node_vtable2)))''', + '''Virtual(node_vtable, + nextdescr=Fixed(node_vtable2))''') + # + self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', + mismatch=True) + + def test_bridge_unused(self): + ops = """ + [] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, p3, descr=nextdescr) + jump(p1) + """ + self.find_bridge(ops, '', + '''Not''') + self.find_bridge(ops, '', + '''Virtual(node_vtable, + nextdescr=Not)''') + self.find_bridge(ops, '', + '''Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Not))''') + self.find_bridge(ops, '', + '''Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Virtual(node_vtable)))''') + self.find_bridge(ops, '', + '''Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Virtual(node_vtable, + nextdescr=Not)))''') + class TestLLtype(BaseTestOptimize, LLtypeMixin): pass From benjamin at codespeak.net Sat Jul 18 20:31:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:31:21 +0200 (CEST) Subject: [pypy-svn] r66379 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718183121.86998169F2C@codespeak.net> Author: benjamin Date: Sat Jul 18 20:31:20 2009 New Revision: 66379 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: typos Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 20:31:20 2009 @@ -578,8 +578,9 @@ self.load_const(space.wrap(level)) names_w = [None]*len(imp.names) for i in range(len(imp.names)): + alias = imp.names[i] assert isinstance(alias, ast.alias) - names_w[i] = space.wrap(imp.names[i]) + names_w[i] = space.wrap(alias.name) self.load_const(space.newtuple(names_w)) if imp.module: mod_name = imp.module From benjamin at codespeak.net Sat Jul 18 20:40:10 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:40:10 +0200 (CEST) Subject: [pypy-svn] r66380 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718184010.EAF012A8088@codespeak.net> Author: benjamin Date: Sat Jul 18 20:40:10 2009 New Revision: 66380 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: move assertion Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 20:40:10 2009 @@ -1177,7 +1177,6 @@ for_node = comp_for.children[1] for_targets = self.handle_exprlist(for_node, ast.Store) expr = handle_source_expression(comp_for.children[3]) - assert isinstance(expr, ast.expr) if len(for_node.children) == 1: comp = ast.comprehension(for_targets[0], expr, None) else: @@ -1198,6 +1197,7 @@ comp.ifs = ifs if comp_for.type == iter_type: comp_for = comp_for.children[0] + assert isinstance(comp, ast.comprehension) comps.append(comp) return elt, comps From benjamin at codespeak.net Sat Jul 18 20:46:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:46:31 +0200 (CEST) Subject: [pypy-svn] r66381 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718184631.71CB22A809D@codespeak.net> Author: benjamin Date: Sat Jul 18 20:46:30 2009 New Revision: 66381 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: - to _ Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 20:46:30 2009 @@ -1020,7 +1020,7 @@ elif stack_count == 1: self.emit_op_arg(ops.DUP_TOPX, 2) elif stack_count == 2: - self.emit_op-arg(ops.DUP_TOPX, 3) + self.emit_op_arg(ops.DUP_TOPX, 3) elif ctx == ast.AugStore: if stack_count == 0: self.emit_op(ops.ROT_TWO) From benjamin at codespeak.net Sat Jul 18 20:53:07 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 20:53:07 +0200 (CEST) Subject: [pypy-svn] r66382 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718185307.EE4012A8088@codespeak.net> Author: benjamin Date: Sat Jul 18 20:53:07 2009 New Revision: 66382 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: add hint Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 20:53:07 2009 @@ -1177,6 +1177,7 @@ for_node = comp_for.children[1] for_targets = self.handle_exprlist(for_node, ast.Store) expr = handle_source_expression(comp_for.children[3]) + assert isinstance(expr, ast.expr) if len(for_node.children) == 1: comp = ast.comprehension(for_targets[0], expr, None) else: From benjamin at codespeak.net Sat Jul 18 21:00:47 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:00:47 +0200 (CEST) Subject: [pypy-svn] r66383 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718190047.B6ADD3180FC@codespeak.net> Author: benjamin Date: Sat Jul 18 21:00:46 2009 New Revision: 66383 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: hint Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:00:46 2009 @@ -888,7 +888,9 @@ if_cleanup = self.new_block() anchor = self.new_block() gen = gens[gen_index] - gen.iter.walkabout(self) + it = gen.iter + assert isinstance(it, ast.expr) + it.walkabout(self) self.emit_op(ops.GET_ITER) self.use_next_block(start) self.emit_jump(ops.FOR_ITER, anchor) From benjamin at codespeak.net Sat Jul 18 21:07:49 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:07:49 +0200 (CEST) Subject: [pypy-svn] r66384 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718190749.A154F3180FC@codespeak.net> Author: benjamin Date: Sat Jul 18 21:07:47 2009 New Revision: 66384 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: a different assertion Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:07:47 2009 @@ -888,9 +888,8 @@ if_cleanup = self.new_block() anchor = self.new_block() gen = gens[gen_index] - it = gen.iter - assert isinstance(it, ast.expr) - it.walkabout(self) + assert isinstance(gen, ast.comprehension) + gen.iter.walkabout(self) self.emit_op(ops.GET_ITER) self.use_next_block(start) self.emit_jump(ops.FOR_ITER, anchor) From benjamin at codespeak.net Sat Jul 18 21:13:56 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:13:56 +0200 (CEST) Subject: [pypy-svn] r66385 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718191356.D1F9F169F65@codespeak.net> Author: benjamin Date: Sat Jul 18 21:13:54 2009 New Revision: 66385 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: slices have to be > 0 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:13:54 2009 @@ -555,7 +555,7 @@ self._import_as(alias) else: dot = alias.name.find(".") - if dot == -1: + if dot < 0: store_name = alias.name else: store_name = alias.name[:dot] From benjamin at codespeak.net Sat Jul 18 21:20:51 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:20:51 +0200 (CEST) Subject: [pypy-svn] r66386 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718192051.0DA6C169F6F@codespeak.net> Author: benjamin Date: Sat Jul 18 21:20:51 2009 New Revision: 66386 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: annotator hint Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 21:20:51 2009 @@ -403,6 +403,7 @@ def visit_GeneratorExp(self, genexp): outer = genexp.generators[0] + assert isinstance(outer, ast.comprehension) outer.iter.walkabout(self) new_scope = FunctionScope("genexp", genexp.lineno, genexp.col_offset) self.push_scope(new_scope, genexp) From benjamin at codespeak.net Sat Jul 18 21:28:29 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:28:29 +0200 (CEST) Subject: [pypy-svn] r66387 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718192829.F11DE169F8F@codespeak.net> Author: benjamin Date: Sat Jul 18 21:28:29 2009 New Revision: 66387 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: annotator help Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:28:29 2009 @@ -852,7 +852,9 @@ not self._call_has_simple_args(call) or \ not isinstance(call.func, ast.Name): return False - name_scope = self.scope.lookup(call.func.id) + func_name = call.func + assert isinstance(func_name, ast.Name) + name_scope = self.scope.lookup(func_name) if name_scope == symtable.SCOPE_GLOBAL_IMPLICIT or \ name_scope == symtable.SCOPE_UNKNOWN: builtin_index = BUILTIN_TO_INDEX.get(call.func.id, -1) From benjamin at codespeak.net Sat Jul 18 21:36:32 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:36:32 +0200 (CEST) Subject: [pypy-svn] r66388 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718193632.10FE7169F8A@codespeak.net> Author: benjamin Date: Sat Jul 18 21:36:31 2009 New Revision: 66388 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: help the annotator for alias Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:36:31 2009 @@ -587,7 +587,8 @@ else: mod_name = "" self.emit_op_name(ops.IMPORT_NAME, self.names, mod_name) - if len(imp.names) == 1 and imp.names[0].name == "*": + if len(imp.names) == 1 and isinstance(imp.names[0], ast.alias) and \ + imp.names[0].name == "*": self.emit_op(ops.IMPORT_STAR) else: for alias in imp.names: From benjamin at codespeak.net Sat Jul 18 21:43:40 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:43:40 +0200 (CEST) Subject: [pypy-svn] r66389 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718194340.2C750169F8A@codespeak.net> Author: benjamin Date: Sat Jul 18 21:43:39 2009 New Revision: 66389 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: hint Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:43:39 2009 @@ -940,6 +940,7 @@ anchor = self.new_block() end = self.new_block() gen = generators[gen_index] + assert isinstance(gen, ast.comprehension) self.emit_jump(ops.SETUP_LOOP, end) self.push_frame_block(F_BLOCK_LOOP, start) if gen_index == 0: From benjamin at codespeak.net Sat Jul 18 21:52:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 21:52:16 +0200 (CEST) Subject: [pypy-svn] r66390 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718195216.1BB26169F6B@codespeak.net> Author: benjamin Date: Sat Jul 18 21:52:15 2009 New Revision: 66390 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: fix annotation hints Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 21:52:15 2009 @@ -587,8 +587,9 @@ else: mod_name = "" self.emit_op_name(ops.IMPORT_NAME, self.names, mod_name) - if len(imp.names) == 1 and isinstance(imp.names[0], ast.alias) and \ - imp.names[0].name == "*": + first = imp.names[0] + assert isinstance(first, ast.alias) + if len(imp.names) == 1 and first.name == "*": self.emit_op(ops.IMPORT_STAR) else: for alias in imp.names: From benjamin at codespeak.net Sat Jul 18 22:00:29 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:00:29 +0200 (CEST) Subject: [pypy-svn] r66391 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718200029.38B88169F6B@codespeak.net> Author: benjamin Date: Sat Jul 18 22:00:27 2009 New Revision: 66391 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: fetch id attribute Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 22:00:27 2009 @@ -856,7 +856,7 @@ return False func_name = call.func assert isinstance(func_name, ast.Name) - name_scope = self.scope.lookup(func_name) + name_scope = self.scope.lookup(func_name.id) if name_scope == symtable.SCOPE_GLOBAL_IMPLICIT or \ name_scope == symtable.SCOPE_UNKNOWN: builtin_index = BUILTIN_TO_INDEX.get(call.func.id, -1) From benjamin at codespeak.net Sat Jul 18 22:07:00 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:07:00 +0200 (CEST) Subject: [pypy-svn] r66392 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718200700.14AFE169F89@codespeak.net> Author: benjamin Date: Sat Jul 18 22:06:59 2009 New Revision: 66392 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: help the annotator in optimize_method_call Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 22:06:59 2009 @@ -876,8 +876,10 @@ not self._call_has_simple_args(call) or \ not isinstance(call.func, ast.Attribute): return False - call.func.value.walkabout(self) - self.emit_op_name(ops.LOOKUP_METHOD, self.names, call.func.attr) + attr_lookup = call.func + assert isinstance(attr_lookup, ast.Attribute) + attr_lookup.value.walkabout(self) + self.emit_op_name(ops.LOOKUP_METHOD, self.names, attr_lookup.attr) if call.args: self.visit_sequence(call.args) arg_count = len(call.args) From benjamin at codespeak.net Sat Jul 18 22:13:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:13:16 +0200 (CEST) Subject: [pypy-svn] r66393 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718201316.87209169F89@codespeak.net> Author: benjamin Date: Sat Jul 18 22:13:16 2009 New Revision: 66393 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: use variable the annotator knows the type of Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 22:13:16 2009 @@ -859,7 +859,7 @@ name_scope = self.scope.lookup(func_name.id) if name_scope == symtable.SCOPE_GLOBAL_IMPLICIT or \ name_scope == symtable.SCOPE_UNKNOWN: - builtin_index = BUILTIN_TO_INDEX.get(call.func.id, -1) + builtin_index = BUILTIN_TO_INDEX.get(func_name.id, -1) if builtin_index != -1: if call.args: args_count = len(call.args) From benjamin at codespeak.net Sat Jul 18 22:19:46 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:19:46 +0200 (CEST) Subject: [pypy-svn] r66394 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718201946.EFA4B169FB0@codespeak.net> Author: benjamin Date: Sat Jul 18 22:19:46 2009 New Revision: 66394 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: annotate excepthandler right Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 22:19:46 2009 @@ -482,6 +482,7 @@ self.emit_jump(ops.JUMP_FORWARD, otherwise) self.use_next_block(exc) for handler in te.handlers: + assert isinstance(handler, ast.excepthandler) self.update_position(handler.lineno) next_except = self.new_block() if handler.type: From benjamin at codespeak.net Sat Jul 18 22:26:38 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:26:38 +0200 (CEST) Subject: [pypy-svn] r66395 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718202638.D8C31169FB5@codespeak.net> Author: benjamin Date: Sat Jul 18 22:26:38 2009 New Revision: 66395 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: fix name of unroller Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Sat Jul 18 22:26:38 2009 @@ -499,7 +499,7 @@ def _opcode_stack_effect(op, arg): if we_are_translated(): - for possible_op in ops.unrolling_op_descs: + for possible_op in ops.unrolling_opcode_descs: if op == possible_op.index: return _stack_effect_computers[op](arg) else: From benjamin at codespeak.net Sat Jul 18 22:32:57 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:32:57 +0200 (CEST) Subject: [pypy-svn] r66396 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718203257.404BF169FB8@codespeak.net> Author: benjamin Date: Sat Jul 18 22:32:56 2009 New Revision: 66396 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: prefix with '_' Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Sat Jul 18 22:32:56 2009 @@ -492,7 +492,7 @@ _stack_effect_computers[getattr(ops, name[9:])] = func for op, value in _static_opcode_stack_effects.iteritems(): def func(arg, _value=value): - return value + return _value _stack_effect_computers[op] = func del name, func, op, value From benjamin at codespeak.net Sat Jul 18 22:45:09 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 22:45:09 +0200 (CEST) Subject: [pypy-svn] r66397 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718204509.9D6EF169F63@codespeak.net> Author: benjamin Date: Sat Jul 18 22:45:09 2009 New Revision: 66397 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: it's silly to have to make a copy, but oh well.. Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Sat Jul 18 22:45:09 2009 @@ -298,7 +298,7 @@ stack_depth, flags, bytecode, - consts_w, + list(consts_w), names, var_names, self.compile_info.filename, From benjamin at codespeak.net Sat Jul 18 23:00:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 23:00:42 +0200 (CEST) Subject: [pypy-svn] r66398 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718210042.B5754169FB0@codespeak.net> Author: benjamin Date: Sat Jul 18 23:00:41 2009 New Revision: 66398 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: tell the annotator the type Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 23:00:41 2009 @@ -990,7 +990,9 @@ genexp.lineno) self.update_position(genexp.lineno) self._make_function(code) - genexp.generators[0].iter.walkabout(self) + first_comp = genexp.generators[0] + assert isinstance(first_comp, ast.comprehension) + first_comp.iter.walkabout(self) self.emit_op(ops.GET_ITER) self.emit_op_arg(ops.CALL_FUNCTION, 1) From benjamin at codespeak.net Sat Jul 18 23:10:58 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 23:10:58 +0200 (CEST) Subject: [pypy-svn] r66399 - in pypy/branch/parser-compiler/pypy: config doc/config interpreter module/dyngram Message-ID: <20090718211058.08377169FB3@codespeak.net> Author: benjamin Date: Sat Jul 18 23:10:58 2009 New Revision: 66399 Removed: pypy/branch/parser-compiler/pypy/doc/config/objspace.usemodules.dyngram.txt pypy/branch/parser-compiler/pypy/module/dyngram/ Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Log: remove dynamic grammar code Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/parser-compiler/pypy/config/pypyoption.py (original) +++ pypy/branch/parser-compiler/pypy/config/pypyoption.py Sat Jul 18 23:10:58 2009 @@ -26,7 +26,7 @@ working_modules.update(dict.fromkeys( ["_socket", "unicodedata", "mmap", "fcntl", "rctime" , "select", "zipimport", "_lsprof", - "crypt", "signal", "dyngram", "_rawffi", "termios", "zlib", + "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl"] )) Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Sat Jul 18 23:10:58 2009 @@ -252,45 +252,3 @@ assert isinstance(code, PyCode) space.timer.stop("PythonAST compile") return code - - # interface for pypy.module.recparser - def get_parser(self): - return self.parser - - def source2ast(self, source, mode='exec'): - from pypy.interpreter.pyparser.astbuilder import AstBuilder - builder = AstBuilder(self.parser, self.grammar_version, - space=self.space) - self.parser.parse_source(source, mode, builder) - return builder.rule_stack[-1] - - -def install_compiler_hook(space, w_callable): -# if not space.get( w_callable ): -# raise OperationError( space.w_TypeError( space.wrap( "must have a callable" ) ) - space.default_compiler.w_compile_hook = w_callable - -def insert_grammar_rule(space, w_rule, w_buildfuncs): - """inserts new grammar rules to the default compiler""" - from pypy.interpreter import function - rule = space.str_w(w_rule) - #buildfuncs_w = w_buildfuncs.content - buildfuncs = {} - #for w_name, w_func in buildfuncs_w.iteritems(): - # buildfuncs[space.str_w(w_name)] = space.unwrap(w_func) - w_iter = space.iter(w_buildfuncs) - while 1: - try: - w_key = space.next(w_iter) - w_func = space.getitem(w_buildfuncs, w_key) - buildfuncs[space.str_w(w_key)] = space.interp_w(function.Function, w_func) - except OperationError, e: - if not e.match(space, space.w_StopIteration): - raise - break - space.default_compiler.additional_rules = buildfuncs - space.default_compiler.parser.insert_rule(rule) - -# XXX cyclic import -#from pypy.interpreter.baseobjspace import ObjSpace -#insert_grammar_rule.unwrap_spec = [ObjSpace, str, dict] From benjamin at codespeak.net Sat Jul 18 23:23:30 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 23:23:30 +0200 (CEST) Subject: [pypy-svn] r66400 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718212330.28276169FB9@codespeak.net> Author: benjamin Date: Sat Jul 18 23:23:29 2009 New Revision: 66400 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: specialize comprehension helper Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sat Jul 18 23:23:29 2009 @@ -3,6 +3,7 @@ from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser import parsestring +from pypy.rlib.objectmodel import specialize def ast_from_node(space, n, compile_info): @@ -1167,6 +1168,7 @@ return count iter_node = first_child.children[2] + @specialize.arg(5) def comprehension_helper(self, comp_node, for_type, if_type, iter_type, handle_source_expression): elt = self.handle_expr(comp_node.children[0]) From benjamin at codespeak.net Sat Jul 18 23:40:55 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 23:40:55 +0200 (CEST) Subject: [pypy-svn] r66401 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718214055.B569416800E@codespeak.net> Author: benjamin Date: Sat Jul 18 23:40:54 2009 New Revision: 66401 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: use '%d' which is supported by rpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sat Jul 18 23:40:54 2009 @@ -126,7 +126,7 @@ raise NotImplementedError def current_temporary_name(self): - name = "_[%i]" % (self.temporary_name_counter,) + name = "_[%d]" % (self.temporary_name_counter,) self.temporary_name_counter += 1 assert self.scope.lookup(name) != symtable.SCOPE_UNKNOWN return name Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sat Jul 18 23:40:54 2009 @@ -50,7 +50,7 @@ return self.roles.get(self.mangle(name), SYM_BLANK) def new_temporary_name(self): - self.note_symbol("_[%i]" % (self.temp_name_counter,), SYM_ASSIGNED) + self.note_symbol("_[%d]" % (self.temp_name_counter,), SYM_ASSIGNED) self.temp_name_counter += 1 def note_symbol(self, identifier, role): From benjamin at codespeak.net Sat Jul 18 23:55:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 18 Jul 2009 23:55:42 +0200 (CEST) Subject: [pypy-svn] r66402 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090718215542.A3B35169FAE@codespeak.net> Author: benjamin Date: Sat Jul 18 23:55:41 2009 New Revision: 66402 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: add a test for deleting cells Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Sat Jul 18 23:55:41 2009 @@ -413,6 +413,14 @@ decl = str(decl) + "\n" yield self.st, decl, 'x', (1, 2, 3, 4) + source = """def f(a): + del a + def x(): + a +""" + exc = py.test.raises(SyntaxError, self.run, source).value + assert exc.msg == "Can't delete variable used in nested scopes: 'a'" + def test_try_except_finally(self): yield self.simple_test, """ try: From benjamin at codespeak.net Sun Jul 19 00:02:55 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:02:55 +0200 (CEST) Subject: [pypy-svn] r66403 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718220255.93E0B169F84@codespeak.net> Author: benjamin Date: Sun Jul 19 00:02:55 2009 New Revision: 66403 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: replace lstrip Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Sun Jul 19 00:02:55 2009 @@ -1052,10 +1052,15 @@ base = 16 elif len(raw) > 1: base = 8 - raw = raw.lstrip("0xX") - if not raw: - raw = "0" - elif not raw[0].isdigit(): + # strip leading characters + i = 0 + limit = len(raw) - 1 + while i < limit: + if raw[i] not in "0xX": + break + i += 1 + raw = raw[i:] + if not raw[0].isdigit(): raw = "0" + raw w_num_str = self.space.wrap(raw) w_index = None From benjamin at codespeak.net Sun Jul 19 00:12:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:12:35 +0200 (CEST) Subject: [pypy-svn] r66404 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090718221235.307B1169F84@codespeak.net> Author: benjamin Date: Sun Jul 19 00:12:35 2009 New Revision: 66404 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Log: assign a dummy value in __init__ Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Sun Jul 19 00:12:35 2009 @@ -86,6 +86,7 @@ def __init__(self, grammar): self.grammar = grammar self.root = None + self.stack = None def prepare(self, start=-1): if start == -1: From benjamin at codespeak.net Sun Jul 19 00:12:59 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:12:59 +0200 (CEST) Subject: [pypy-svn] r66405 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718221259.15995169F84@codespeak.net> Author: benjamin Date: Sun Jul 19 00:12:58 2009 New Revision: 66405 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: more %i to %d Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sun Jul 19 00:12:58 2009 @@ -1139,7 +1139,7 @@ arg = args[i] if isinstance(arg, ast.Tuple): self.update_position(arg.lineno) - self.name_op(".%i" % (i,), ast.Load) + self.name_op(".%d" % (i,), ast.Load) arg.walkabout(self) def _get_code_flags(self): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sun Jul 19 00:12:58 2009 @@ -316,7 +316,7 @@ return self.scopes[scope_node] def implicit_arg(self, pos): - name = ".%i" % (pos,) + name = ".%d" % (pos,) self.note_symbol(name, SYM_PARAM) def note_symbol(self, identifier, role): From benjamin at codespeak.net Sun Jul 19 00:27:11 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:27:11 +0200 (CEST) Subject: [pypy-svn] r66406 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718222711.5AF9D169FAF@codespeak.net> Author: benjamin Date: Sun Jul 19 00:27:10 2009 New Revision: 66406 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: do string formatting on constants Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sun Jul 19 00:27:10 2009 @@ -243,25 +243,26 @@ def _check_optimization(self): if (self.has_free or self.child_has_free) and not self.optimized: err = None + if self.child_has_free: + trailer = "contains a nested function with free variables" + else: + trailer = "is a nested function" + name = self.name if self.import_star: node = self.import_star if self.bare_exec: err = "function %r uses import * and bare exec, " \ - "which are illegal because it %s" + "which are illegal because it %s" % (name, trailer) else: - err = "import * is not allowed in function %r because it %s" + err = "import * is not allowed in function %r because " \ + "it %s" % (name, trailer) elif self.bare_exec: node = self.bare_exec err = "unqualified exec is not allowed in function %r " \ - "because it %s" + "because it %s" % (name, trailer) else: raise AssertionError("unkown reason for unoptimization") - if self.child_has_free: - trailer = "contains a nested function with free variables" - else: - trailer = "is a nested function" - raise SyntaxError(err % (self.name, trailer), node.lineno, - node.col_offset) + raise SyntaxError(err, node.lineno, node.col_offset) self.locals_fully_known = self.optimized and not self.has_exec @@ -385,9 +386,11 @@ old_role = self.scope.lookup_role(name) if old_role & (SYM_USED | SYM_ASSIGNED): if old_role & SYM_ASSIGNED: - msg = "name '%s' is assigned to before global declaration" + msg = "name '%s' is assigned to before global declaration" \ + % (name,) else: - msg = "name '%s' is used prior to global declaration" + msg = "name '%s' is used prior to global declaration" % \ + (name,) misc.syntax_warning(self.space, msg, self.compile_info.filename, glob.lineno, glob.col_offset) self.note_symbol(name, SYM_GLOBAL) From benjamin at codespeak.net Sun Jul 19 00:36:57 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:36:57 +0200 (CEST) Subject: [pypy-svn] r66407 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718223657.EF331169FB9@codespeak.net> Author: benjamin Date: Sun Jul 19 00:36:56 2009 New Revision: 66407 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: '%r' is not rpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Sun Jul 19 00:36:56 2009 @@ -95,8 +95,8 @@ def _finalize_name(self, name, flags, local, bound, free, globs): if flags & SYM_GLOBAL: if flags & SYM_PARAM: - raise SyntaxError("name %r is both local and global" % (name,), - self.lineno, self.col_offset) + err = "name '%s' is both local and global" % (name,) + raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_GLOBAL_EXPLICIT globs[name] = None if bound: @@ -251,14 +251,14 @@ if self.import_star: node = self.import_star if self.bare_exec: - err = "function %r uses import * and bare exec, " \ + err = "function '%s' uses import * and bare exec, " \ "which are illegal because it %s" % (name, trailer) else: - err = "import * is not allowed in function %r because " \ + err = "import * is not allowed in function '%s' because " \ "it %s" % (name, trailer) elif self.bare_exec: node = self.bare_exec - err = "unqualified exec is not allowed in function %r " \ + err = "unqualified exec is not allowed in function '%s' " \ "because it %s" % (name, trailer) else: raise AssertionError("unkown reason for unoptimization") From benjamin at codespeak.net Sun Jul 19 00:46:14 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 00:46:14 +0200 (CEST) Subject: [pypy-svn] r66408 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090718224614.49894169FB6@codespeak.net> Author: benjamin Date: Sun Jul 19 00:46:13 2009 New Revision: 66408 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: change %r to %s Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sun Jul 19 00:46:13 2009 @@ -172,7 +172,7 @@ except KeyError: if kind is name_ops_deref and ctx == ast.Del: raise SyntaxError("Can't delete variable used in " - "nested scopes: %r" % (identifier,)) + "nested scopes: '%s'" % (identifier,)) raise AssertionError("Unkown name operation") self.emit_op_arg(op, self.add_name(container, identifier)) From benjamin at codespeak.net Sun Jul 19 05:39:45 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 05:39:45 +0200 (CEST) Subject: [pypy-svn] r66409 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090719033945.80187169F84@codespeak.net> Author: benjamin Date: Sun Jul 19 05:39:43 2009 New Revision: 66409 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: implement backquote repr, for which there was not a single test Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sun Jul 19 05:39:43 2009 @@ -996,6 +996,11 @@ self.emit_op(ops.GET_ITER) self.emit_op_arg(ops.CALL_FUNCTION, 1) + def visit_Repr(self, rep): + self.update_position(rep.lineno) + rep.value.walkabout(self) + self.emit_op(ops.UNARY_CONVERT) + def visit_Attribute(self, attr): self.update_position(attr.lineno) names = self.names Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Sun Jul 19 05:39:43 2009 @@ -712,6 +712,9 @@ def test_lambda(self): yield self.st, "y = lambda x: x", "y(4)", 4 + def test_backquote_repr(self): + yield self.st, "x = None; y = `x`", "y", "None" + class AppTestPrint: From arigo at codespeak.net Sun Jul 19 11:49:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 11:49:34 +0200 (CEST) Subject: [pypy-svn] r66410 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719094934.EFBA5498440@codespeak.net> Author: arigo Date: Sun Jul 19 11:49:33 2009 New Revision: 66410 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py - copied unchanged from r66409, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Rename optimize to optimizeutil, as it only contains general utilities. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sun Jul 19 11:49:33 2009 @@ -4,7 +4,7 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.history import AbstractValue from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.optimize import av_newdict, _findall, sort_descrs +from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs # ____________________________________________________________ Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Jul 19 11:49:33 2009 @@ -10,7 +10,7 @@ ConstObj, AbstractDescr) from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder -from pypy.jit.metainterp.optimize import sort_descrs +from pypy.jit.metainterp.optimizeutil import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode From arigo at codespeak.net Sun Jul 19 12:18:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 12:18:03 +0200 (CEST) Subject: [pypy-svn] r66411 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090719101803.110B2169FA1@codespeak.net> Author: arigo Date: Sun Jul 19 12:18:03 2009 New Revision: 66411 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Two tests showing bugs. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Jul 19 12:18:03 2009 @@ -315,6 +315,34 @@ assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox.value == self.node_vtable_adr2 + def test_find_nodes_guard_class_bug(self): + py.test.skip("oups") + ops = """ + [p1] + p2 = escape() + setfield_gc(p1, p2, descr=nextdescr) + p3 = getfield_gc(p1, descr=nextdescr) + guard_class(p3, ConstClass(node_vtable)) + fail() + jump(p1) + """ + self.find_nodes(ops, 'Not') + + def test_find_nodes_external_change(self): + py.test.skip("oups") + ops = """ + [p0, p3] + guard_class(p3, ConstClass(node_vtable)) + fail() + p1 = getfield_gc(p0, descr=nextdescr) + guard_class(p1, ConstClass(node_vtable)) + fail() + escape() # might change the field! + p2 = getfield_gc(p0, descr=nextdescr) + jump(p0, p2) + """ + self.find_nodes(ops, 'Not, Not') + def test_find_nodes_new_1(self): ops = """ [p1] From arigo at codespeak.net Sun Jul 19 17:59:28 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 17:59:28 +0200 (CEST) Subject: [pypy-svn] r66412 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719155928.53D3B169FAB@codespeak.net> Author: arigo Date: Sun Jul 19 17:59:26 2009 New Revision: 66412 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Simplify optimizefindnode by not tracking FixedClassSpecNodes any more. Solves the bugs basically by removing them. I think that we might need another approach to track FixedClassSpecNodes; for now I will just focus on Virtuals. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sun Jul 19 17:59:26 2009 @@ -1,6 +1,5 @@ from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode -from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.history import AbstractValue from pypy.jit.metainterp.resoperation import rop @@ -18,14 +17,16 @@ the field's status at the start and 'curfields' that represents it at the current point (== the end when the first phase is complete). """ + escaped = False # if True, then all the rest of the info is pointless + unique = UNIQUE_UNKNOWN # for find_unique_nodes() + + # fields used to store the shape of the potential Virtual + knownclsbox = None origfields = None # optimization; equivalent to an empty dict curfields = None # optimization; equivalent to an empty dict dependencies = None - knownclsbox = None - unique = UNIQUE_UNKNOWN # for find_unique_nodes() - def __init__(self, escaped, fromstart=False): - self.escaped = escaped + def __init__(self, fromstart=False): self.fromstart = fromstart def add_escape_dependency(self, other): @@ -68,7 +69,8 @@ class NodeFinder(object): """Abstract base class.""" - node_escaped = InstanceNode(escaped=True) + node_escaped = InstanceNode() + node_escaped.escaped = True def __init__(self): self.nodes = {} # Box -> InstanceNode @@ -93,7 +95,7 @@ self.getnode(box).mark_escaped() def find_nodes_NEW_WITH_VTABLE(self, op): - instnode = InstanceNode(escaped=False) + instnode = InstanceNode() instnode.knownclsbox = op.args[0] self.nodes[op.result] = instnode @@ -119,36 +121,28 @@ fieldnode = instnode.curfields[field] elif instnode.origfields is not None and field in instnode.origfields: fieldnode = instnode.origfields[field] - else: - fieldnode = InstanceNode(escaped=False, - fromstart=instnode.fromstart) + elif instnode.fromstart: + fieldnode = InstanceNode(fromstart=True) instnode.add_escape_dependency(fieldnode) if instnode.origfields is None: instnode.origfields = av_newdict() instnode.origfields[field] = fieldnode + else: + return # nothing to be gained from tracking the field self.nodes[op.result] = fieldnode def find_nodes_GETFIELD_GC_PURE(self, op): self.find_nodes_GETFIELD_GC(op) - def find_nodes_GUARD_CLASS(self, op): - instbox = op.args[0] - clsbox = op.args[1] - try: - instnode = self.nodes[instbox] - except KeyError: - instnode = self.nodes[instbox] = InstanceNode(escaped=True) - assert instnode is not self.node_escaped - assert (instnode.knownclsbox is None or - instnode.knownclsbox.equals(clsbox)) - instnode.knownclsbox = clsbox - def find_nodes_JUMP(self, op): # only set up the 'unique' field of the InstanceNodes; # real handling comes later (build_result_specnodes() for loops). for box in op.args: self.getnode(box).set_unique_nodes() + def find_nodes_GUARD_CLASS(self, op): + pass # prevent default handling + def find_nodes_FAIL(self, op): xxx @@ -158,6 +152,7 @@ # Perfect specialization -- for loops only class PerfectSpecializationFinder(NodeFinder): + node_fromstart = InstanceNode(fromstart=True) def find_nodes_loop(self, loop): self.setup_input_nodes(loop.inputargs) @@ -167,7 +162,7 @@ def setup_input_nodes(self, inputargs): inputnodes = [] for box in inputargs: - instnode = InstanceNode(escaped=False, fromstart=True) + instnode = InstanceNode(fromstart=True) inputnodes.append(instnode) self.nodes[box] = instnode self.inputnodes = inputnodes @@ -187,15 +182,8 @@ def intersect(self, inputnode, exitnode): assert inputnode.fromstart if exitnode.unique == UNIQUE_NO or inputnode.escaped: - # give a NotSpecNode or a FixedClassSpecNode - if (inputnode.knownclsbox is not None and - exitnode.knownclsbox is not None and - inputnode.knownclsbox.equals(exitnode.knownclsbox)): - # the class is known at the end, needed at the input, - # and matches - return FixedClassSpecNode(inputnode.knownclsbox) - else: - return prebuiltNotSpecNode + # give a NotSpecNode + return prebuiltNotSpecNode # assert exitnode.unique == UNIQUE_YES if (inputnode.knownclsbox is not None and @@ -217,7 +205,7 @@ # field stored at exit, but not read at input. Must # still be allocated, otherwise it will be incorrectly # uninitialized after a guard failure. - node = InstanceNode(escaped=False, fromstart=True) + node = self.node_fromstart specnode = self.intersect(node, d[ofs]) fields.append((ofs, specnode)) return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) @@ -233,22 +221,13 @@ class __extend__(NotSpecNode): def make_instance_node(self): - return InstanceNode(escaped=True) + return NodeFinder.node_escaped def matches_instance_node(self, exitnode): return True -class __extend__(FixedClassSpecNode): - def make_instance_node(self): - instnode = InstanceNode(escaped=True) - instnode.knownclsbox = self.known_class - return instnode - def matches_instance_node(self, exitnode): - return (exitnode.knownclsbox is not None and - self.known_class.equals(exitnode.knownclsbox)) - class __extend__(VirtualInstanceSpecNode): def make_instance_node(self): - instnode = InstanceNode(escaped=False) + instnode = InstanceNode() instnode.knownclsbox = self.known_class instnode.curfields = av_newdict() for ofs, subspecnode in self.fields: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Sun Jul 19 17:59:26 2009 @@ -19,18 +19,9 @@ prebuiltNotSpecNode = NotSpecNode() -class FixedClassSpecNode(SpecNode): - def __init__(self, known_class): - self.known_class = known_class - - def _equals(self, other): # for tests only - return (type(other) is FixedClassSpecNode and - self.known_class.equals(other.known_class)) - - -class VirtualInstanceSpecNode(FixedClassSpecNode): +class VirtualInstanceSpecNode(SpecNode): def __init__(self, known_class, fields): - FixedClassSpecNode.__init__(self, known_class) + self.known_class = known_class self.fields = fields def _equals(self, other): # for tests only Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Jul 19 17:59:26 2009 @@ -12,7 +12,6 @@ from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder from pypy.jit.metainterp.optimizeutil import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode -from pypy.jit.metainterp.specnode import FixedClassSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.test.oparser import parse @@ -144,8 +143,6 @@ self.cpu) else: return ConstObj(ootype.cast_to_object(cls_vtable)) - def makeFixed(cls_vtable): - return FixedClassSpecNode(constclass(cls_vtable)) def makeVirtual(cls_vtable, **kwds_fields): fields = [] for key, value in kwds_fields.items(): @@ -154,7 +151,6 @@ return VirtualInstanceSpecNode(constclass(cls_vtable), fields) # context = {'Not': prebuiltNotSpecNode, - 'Fixed': makeFixed, 'Virtual': makeVirtual} lst = eval('[' + text + ']', self.namespace, context) return lst @@ -230,119 +226,6 @@ assert getnode(boxes.p3).fromstart assert not getnode(boxes.p4).fromstart - def test_find_nodes_guard_class_1(self): - ops = """ - [p1] - guard_class(p1, ConstClass(node_vtable)) - fail() - jump(p1) - """ - boxes, getnode = self.find_nodes(ops, 'Fixed(node_vtable)') - boxp1 = getnode(boxes.p1) - assert boxp1.knownclsbox.value == self.node_vtable_adr - - def test_find_nodes_guard_class_2(self): - ops = """ - [p1] - p2 = getfield_gc(p1, descr=nextdescr) - guard_class(p2, ConstClass(node_vtable)) - fail() - jump(p1) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - boxp1 = getnode(boxes.p1) - boxp2 = getnode(boxes.p2) - assert boxp1.knownclsbox is None - assert boxp2.knownclsbox.value == self.node_vtable_adr - - def test_find_nodes_guard_class_outonly(self): - ops = """ - [p1] - p2 = escape() - guard_class(p2, ConstClass(node_vtable)) - fail() - jump(p2) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - boxp1 = getnode(boxes.p1) - boxp2 = getnode(boxes.p2) - assert boxp1.knownclsbox is None - assert boxp2.knownclsbox.value == self.node_vtable_adr - - def test_find_nodes_guard_class_inonly(self): - ops = """ - [p1] - guard_class(p1, ConstClass(node_vtable)) - fail() - p2 = escape() - jump(p2) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - boxp1 = getnode(boxes.p1) - boxp2 = getnode(boxes.p2) - assert boxp1.knownclsbox.value == self.node_vtable_adr - assert boxp2.knownclsbox is None - - def test_find_nodes_guard_class_inout(self): - ops = """ - [p1] - guard_class(p1, ConstClass(node_vtable)) - fail() - p2 = escape() - guard_class(p2, ConstClass(node_vtable)) - fail() - jump(p2) - """ - boxes, getnode = self.find_nodes(ops, 'Fixed(node_vtable)') - boxp1 = getnode(boxes.p1) - boxp2 = getnode(boxes.p2) - assert boxp1.knownclsbox.value == self.node_vtable_adr - assert boxp2.knownclsbox.value == self.node_vtable_adr - - def test_find_nodes_guard_class_mismatch(self): - ops = """ - [p1] - guard_class(p1, ConstClass(node_vtable)) - fail() - p2 = escape() - guard_class(p2, ConstClass(node_vtable2)) - fail() - jump(p2) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - boxp1 = getnode(boxes.p1) - boxp2 = getnode(boxes.p2) - assert boxp1.knownclsbox.value == self.node_vtable_adr - assert boxp2.knownclsbox.value == self.node_vtable_adr2 - - def test_find_nodes_guard_class_bug(self): - py.test.skip("oups") - ops = """ - [p1] - p2 = escape() - setfield_gc(p1, p2, descr=nextdescr) - p3 = getfield_gc(p1, descr=nextdescr) - guard_class(p3, ConstClass(node_vtable)) - fail() - jump(p1) - """ - self.find_nodes(ops, 'Not') - - def test_find_nodes_external_change(self): - py.test.skip("oups") - ops = """ - [p0, p3] - guard_class(p3, ConstClass(node_vtable)) - fail() - p1 = getfield_gc(p0, descr=nextdescr) - guard_class(p1, ConstClass(node_vtable)) - fail() - escape() # might change the field! - p2 = getfield_gc(p0, descr=nextdescr) - jump(p0, p2) - """ - self.find_nodes(ops, 'Not, Not') - def test_find_nodes_new_1(self): ops = """ [p1] @@ -422,7 +305,6 @@ assert not boxp2.fromstart assert not boxp3.fromstart - assert boxp1.knownclsbox.value == self.node_vtable_adr assert boxp2.knownclsbox.value == self.node_vtable_adr assert boxp3.knownclsbox.value == self.node_vtable_adr2 @@ -454,7 +336,7 @@ """ # the issue is the cycle "p2->p2", which cannot be represented # with SpecNodes so far - self.find_nodes(ops, 'Not, Fixed(node_vtable)', + self.find_nodes(ops, 'Not, Not', boxkinds={'sum': BoxInt, 'sum2': BoxInt}) def test_find_nodes_new_aliasing_2(self): @@ -469,6 +351,7 @@ self.find_nodes(ops, 'Not, Not') def test_find_nodes_new_mismatch(self): + py.test.skip("gives a Virtual instead of Not -- not really wrong") ops = """ [p1] guard_class(p1, ConstClass(node_vtable)) @@ -488,7 +371,7 @@ p2 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) jump(p2, p2) """ - self.find_nodes(ops, 'Not, Fixed(node_vtable2)') + self.find_nodes(ops, 'Not, Not') def test_find_nodes_new_escapes(self): ops = """ @@ -549,12 +432,6 @@ jump(p0) """ self.find_bridge(ops, 'Not', 'Not') - self.find_bridge(ops, 'Fixed(node_vtable)', 'Not') - self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable)') - # - self.find_bridge(ops, 'Not', 'Fixed(node_vtable)', mismatch=True) - self.find_bridge(ops, 'Fixed(node_vtable)', 'Fixed(node_vtable2)', - mismatch=True) def test_bridge_simple_virtual_1(self): ops = """ @@ -564,7 +441,6 @@ jump(p0) """ self.find_bridge(ops, 'Not', 'Not') - self.find_bridge(ops, 'Not', 'Fixed(node_vtable)') self.find_bridge(ops, 'Not', 'Virtual(node_vtable, valuedescr=Not)') self.find_bridge(ops, 'Not', '''Virtual(node_vtable, @@ -577,11 +453,6 @@ mismatch=True) # missing valuedescr self.find_bridge(ops, 'Not', 'Virtual(node_vtable2, valuedescr=Not)', mismatch=True) # bad class - self.find_bridge(ops, 'Not', - '''Virtual(node_vtable, - valuedescr=Not, - nextdescr=Fixed(node_vtable))''', - mismatch=True) # nextdescr too precise def test_bridge_simple_virtual_2(self): ops = """ @@ -590,7 +461,6 @@ jump(p0) """ self.find_bridge(ops, 'Virtual(node_vtable)', 'Not') - self.find_bridge(ops, 'Virtual(node_vtable)', 'Fixed(node_vtable)') self.find_bridge(ops, 'Virtual(node_vtable)', 'Virtual(node_vtable, valuedescr=Not)') self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', @@ -601,34 +471,16 @@ nextdescr=Not)''') self.find_bridge(ops, '''Virtual(node_vtable, valuedescr=Not, - nextdescr=Fixed(node_vtable2))''', - '''Virtual(node_vtable, - valuedescr=Not, - nextdescr=Fixed(node_vtable2))''') - self.find_bridge(ops, '''Virtual(node_vtable, - valuedescr=Not, - nextdescr=Fixed(node_vtable2))''', + nextdescr=Not)''', '''Virtual(node_vtable, valuedescr=Not, nextdescr=Not)''') # self.find_bridge(ops, 'Virtual(node_vtable)', 'Virtual(node_vtable)', mismatch=True) # because of missing valuedescr - self.find_bridge(ops, 'Virtual(node_vtable)', 'Fixed(node_vtable2)', - mismatch=True) # bad class self.find_bridge(ops, 'Virtual(node_vtable)', 'Virtual(node_vtable2, valuedescr=Not)', mismatch=True) # bad class - self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', - '''Virtual(node_vtable, - valuedescr=Not, - nextdescr=Fixed(node_vtable))''', - mismatch=True) # unexpected nextdescr - self.find_bridge(ops, 'Virtual(node_vtable, valuedescr=Not)', - '''Virtual(node_vtable, - valuedescr=Not, - nextdescr=Virtual(node_vtable))''', - mismatch=True) # unexpected nextdescr def test_bridge_virtual_mismatch_1(self): ops = """ @@ -638,9 +490,6 @@ jump(p0, p0) """ self.find_bridge(ops, 'Not', 'Not, Not') - self.find_bridge(ops, 'Not', 'Not, Fixed(node_vtable)') - self.find_bridge(ops, 'Not', 'Fixed(node_vtable), Not') - self.find_bridge(ops, 'Not', 'Fixed(node_vtable), Fixed(node_vtable)') # self.find_bridge(ops, 'Not', '''Virtual(node_vtable, valuedescr=Not), @@ -656,15 +505,13 @@ jump(p2) """ self.find_bridge(ops, 'Not', 'Not') - self.find_bridge(ops, 'Not', 'Fixed(node_vtable)') - self.find_bridge(ops, 'Virtual(node_vtable2, nextdescr=Not)', - 'Fixed(node_vtable)') + self.find_bridge(ops, 'Virtual(node_vtable2, nextdescr=Not)', 'Not') self.find_bridge(ops, '''Virtual(node_vtable, nextdescr=Virtual(node_vtable, - nextdescr=Fixed(node_vtable2)))''', + nextdescr=Not))''', '''Virtual(node_vtable, - nextdescr=Fixed(node_vtable2))''') + nextdescr=Not)''') # self.find_bridge(ops, 'Not', 'Virtual(node_vtable)', mismatch=True) From arigo at codespeak.net Sun Jul 19 18:12:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 18:12:01 +0200 (CEST) Subject: [pypy-svn] r66413 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090719161201.82DB4169F98@codespeak.net> Author: arigo Date: Sun Jul 19 18:12:00 2009 New Revision: 66413 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Log: Comment. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Sun Jul 19 18:12:00 2009 @@ -27,7 +27,7 @@ dependencies = None def __init__(self, fromstart=False): - self.fromstart = fromstart + self.fromstart = fromstart # for loops only: present since the start def add_escape_dependency(self, other): assert not self.escaped From arigo at codespeak.net Sun Jul 19 18:25:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 18:25:47 +0200 (CEST) Subject: [pypy-svn] r66414 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719162547.0CF6A169FC4@codespeak.net> Author: arigo Date: Sun Jul 19 18:25:46 2009 New Revision: 66414 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (contents, props changed) pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (contents, props changed) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Add optimizeopt.py, the module in which to put the actual optimization of the loop's list of operations. Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Sun Jul 19 18:25:46 2009 @@ -0,0 +1,3 @@ + +def optimize(loop): + pass Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Sun Jul 19 18:25:46 2009 @@ -15,44 +15,6 @@ from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.test.oparser import parse -# ____________________________________________________________ - -def equaloplists(oplist1, oplist2): - print '-'*20, 'Comparing lists', '-'*20 - for op1, op2 in zip(oplist1, oplist2): - txt1 = str(op1) - txt2 = str(op2) - while txt1 or txt2: - print '%-39s| %s' % (txt1[:39], txt2[:39]) - txt1 = txt1[39:] - txt2 = txt2[39:] - assert op1.opnum == op2.opnum - assert len(op1.args) == len(op2.args) - for x, y in zip(op1.args, op2.args): - assert x == y - assert op1.result == op2.result - assert op1.descr == op2.descr - if op1.suboperations: - assert equaloplists(op1.suboperations, op2.suboperations) - assert len(oplist1) == len(oplist2) - print '-'*57 - return True - -def test_equaloplists(): - ops = """ - [i0] - i1 = int_add(i0, 1) - guard_true(i1) - i2 = int_add(i1, 1) - fail(i2) - jump(i1) - """ - loop1 = parse(ops) - loop2 = parse(ops) - loop3 = parse(ops.replace("i2 = int_add", "i2 = int_sub")) - assert equaloplists(loop1.operations, loop2.operations) - py.test.raises(AssertionError, - "equaloplists(loop1.operations, loop3.operations)") def test_sort_descrs(): class PseudoDescr(AbstractDescr): @@ -60,11 +22,12 @@ self.n = n def sort_key(self): return self.n - lst = [PseudoDescr(2), PseudoDescr(3), PseudoDescr(6)] - lst2 = lst[:] - random.shuffle(lst2) - sort_descrs(lst2) - assert lst2 == lst + for i in range(17): + lst = [PseudoDescr(j) for j in range(i)] + lst2 = lst[:] + random.shuffle(lst2) + sort_descrs(lst2) + assert lst2 == lst # ____________________________________________________________ @@ -122,19 +85,13 @@ node_vtable_adr2: cpu.typedescrof(NODE2)} namespace = locals() -# ____________________________________________________________ - -class BaseTestOptimize(object): +class BaseTest(object): def parse(self, s, boxkinds=None): return parse(s, self.cpu, self.namespace, type_system=self.type_system, boxkinds=boxkinds) - def assert_equal(self, optimized, expected): - equaloplists(optimized.operations, - self.parse(expected).operations) - def unpack_specnodes(self, text): # def constclass(cls_vtable): @@ -162,6 +119,10 @@ assert x._equals(y) return True +# ____________________________________________________________ + +class BaseTestOptimizeFindNode(BaseTest): + def find_nodes(self, ops, spectext, boxkinds=None): assert boxkinds is None or isinstance(boxkinds, dict) loop = self.parse(ops, boxkinds=boxkinds) @@ -546,8 +507,8 @@ nextdescr=Not)))''') -class TestLLtype(BaseTestOptimize, LLtypeMixin): +class TestLLtype(BaseTestOptimizeFindNode, LLtypeMixin): pass -class TestOOtype(BaseTestOptimize, OOtypeMixin): +class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): pass Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Sun Jul 19 18:25:46 2009 @@ -0,0 +1,78 @@ +import py +from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, + OOtypeMixin, + BaseTest) +from pypy.jit.metainterp.optimizeopt import optimize +from pypy.jit.metainterp.test.oparser import parse + +# ____________________________________________________________ + +def equaloplists(oplist1, oplist2): + print '-'*20, 'Comparing lists', '-'*20 + for op1, op2 in zip(oplist1, oplist2): + txt1 = str(op1) + txt2 = str(op2) + while txt1 or txt2: + print '%-39s| %s' % (txt1[:39], txt2[:39]) + txt1 = txt1[39:] + txt2 = txt2[39:] + assert op1.opnum == op2.opnum + assert len(op1.args) == len(op2.args) + for x, y in zip(op1.args, op2.args): + assert x == y + assert op1.result == op2.result + assert op1.descr == op2.descr + if op1.suboperations: + assert equaloplists(op1.suboperations, op2.suboperations) + assert len(oplist1) == len(oplist2) + print '-'*57 + return True + +def test_equaloplists(): + ops = """ + [i0] + i1 = int_add(i0, 1) + guard_true(i1) + i2 = int_add(i1, 1) + fail(i2) + jump(i1) + """ + loop1 = parse(ops) + loop2 = parse(ops) + loop3 = parse(ops.replace("i2 = int_add", "i2 = int_sub")) + assert equaloplists(loop1.operations, loop2.operations) + py.test.raises(AssertionError, + "equaloplists(loop1.operations, loop3.operations)") + +# ____________________________________________________________ + +class BaseTestOptimizeOpt(BaseTest): + + def assert_equal(self, optimized, expected): + assert optimized.inputargs == expected.inputargs + assert equaloplists(optimized.operations, + expected.operations) + + def optimize(self, ops, spectext, optops, boxkinds=None): + loop = self.parse(ops, boxkinds=boxkinds) + loop.specnodes = self.unpack_specnodes(spectext) + optimize(loop) + expected = self.parse(optops, boxkinds=boxkinds) + self.assert_equal(loop, expected) + + def test_simple(self): + ops = """ + [i] + i0 = int_sub(i, 1) + guard_value(i0, 0) + fail(i0) + jump(i0) + """ + self.optimize(ops, 'Not', ops) + + +class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): + pass + +class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): + pass From arigo at codespeak.net Sun Jul 19 18:57:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 18:57:10 +0200 (CEST) Subject: [pypy-svn] r66415 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719165710.4EA67169F8F@codespeak.net> Author: arigo Date: Sun Jul 19 18:57:09 2009 New Revision: 66415 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Basic constant propagation. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Sun Jul 19 18:57:09 2009 @@ -1,3 +1,85 @@ +from pypy.jit.metainterp.history import Const, Box +from pypy.jit.metainterp.optimizeutil import av_newdict, _findall +from pypy.rlib.objectmodel import we_are_translated + def optimize(loop): - pass + """Optimize loop.operations to make it match the input of loop.specnodes + and to remove internal overheadish operations. Note that loop.specnodes + must be applicable to the loop; you will probably get an AssertionError + if not. + """ + Optimizer().optimize(loop) + +# ____________________________________________________________ + + +class Optimizer(object): + + def __init__(self): + # set of Boxes; the value is not stored in the dict, but can + # be accessed as the Box.getint() or get_(). + self._consts = {} + + def is_constant(self, box): + return isinstance(box, Const) or box in self._consts + + def make_constant(self, box): + assert isinstance(box, Box) + self._consts[box] = box.constbox() + + def rename_argument(self, box): + return self._consts.get(box, box) + + # ---------- + + def optimize(self, loop): + self.newoperations = [] + for op in loop.operations: + opnum = op.opnum + for value, func in optimize_ops: + if opnum == value: + func(self, op) + break + else: + self.optimize_default(op) + loop.operations = self.newoperations + + def emit_operation(self, op): + op2 = op.clone() + op2.args = [self.rename_argument(box) for box in op.args] + self.newoperations.append(op2) + + def optimize_default(self, op): + if op.is_always_pure(): + for box in op.args: + if not self.is_constant(box): + break + else: + # all constant arguments: constant-fold away + self.make_constant(op.result) + return + # otherwise, the operation remains + self.emit_operation(op) + + def optimize_GUARD_VALUE(self, op): + if self.is_constant(op.args[0]): + assert isinstance(op.args[1], Const) + assert self.rename_argument(op.args[0]).get_() == op.args[1].get_() + else: + self.emit_operation(op) + + def optimize_GUARD_TRUE(self, op): + if self.is_constant(op.args[0]): + assert self.rename_argument(op.args[0]).getint() + else: + self.emit_operation(op) + + def optimize_GUARD_FALSE(self, op): + if self.is_constant(op.args[0]): + assert not self.rename_argument(op.args[0]).getint() + else: + self.emit_operation(op) + + +optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Sun Jul 19 18:57:09 2009 @@ -22,7 +22,7 @@ class VirtualInstanceSpecNode(SpecNode): def __init__(self, known_class, fields): self.known_class = known_class - self.fields = fields + self.fields = fields # list: [(fieldofs, subspecnode)] def _equals(self, other): # for tests only ok = (type(other) is VirtualInstanceSpecNode and Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Sun Jul 19 18:57:09 2009 @@ -53,8 +53,9 @@ assert equaloplists(optimized.operations, expected.operations) - def optimize(self, ops, spectext, optops, boxkinds=None): + def optimize(self, ops, spectext, optops, boxkinds=None, **values): loop = self.parse(ops, boxkinds=boxkinds) + loop.setvalues(**values) loop.specnodes = self.unpack_specnodes(spectext) optimize(loop) expected = self.parse(optops, boxkinds=boxkinds) @@ -70,6 +71,26 @@ """ self.optimize(ops, 'Not', ops) + def test_constant_propagate(self): + ops = """ + [] + i0 = int_add(2, 3) + i1 = int_is_true(i0) + guard_true(i1) + fail() + i2 = bool_not(i1) + guard_false(i2) + fail() + guard_value(i0, 5) + fail() + jump() + """ + expected = """ + [] + jump() + """ + self.optimize(ops, '', expected, i0=5, i1=1, i2=0) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From benjamin at codespeak.net Sun Jul 19 19:02:32 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sun, 19 Jul 2009 19:02:32 +0200 (CEST) Subject: [pypy-svn] r66416 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler test Message-ID: <20090719170232.9919F169F98@codespeak.net> Author: benjamin Date: Sun Jul 19 19:02:31 2009 New Revision: 66416 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Log: handle decorators Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Sun Jul 19 19:02:31 2009 @@ -238,6 +238,9 @@ func.lineno) self.update_position(func.lineno) self._make_function(code, num_defaults) + if func.decorators: + for i in range(len(func.decorators)): + self.emit_op_arg(ops.CALL_FUNCTION, 1) self.name_op(func.name, ast.Store) def visit_Lambda(self, lam): Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Sun Jul 19 19:02:31 2009 @@ -284,6 +284,37 @@ exec s + +class AppTestDecorators: + + def test_function_decorators(self): + def other(): + return 4 + def dec(f): + return other + ns = {} + ns["dec"] = dec + exec """@dec +def g(): + pass""" in ns + assert ns["g"] is other + assert ns["g"]() == 4 + + def test_application_order(self): + def dec1(f): + record.append(1) + return f + def dec2(f): + record.append(2) + return f + record = [] + ns = {"dec1" : dec1, "dec2" : dec2} + exec """@dec1 + at dec2 +def g(): pass""" in ns + assert record == [2, 1] + + class AppTestWith: def test_with_simple(self): From arigo at codespeak.net Sun Jul 19 19:30:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 19:30:51 +0200 (CEST) Subject: [pypy-svn] r66417 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719173051.14449169F65@codespeak.net> Author: arigo Date: Sun Jul 19 19:30:51 2009 New Revision: 66417 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Progress. Implement GUARD_CLASS and replace the _consts dict with an _equals mapping. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Sun Jul 19 19:30:51 2009 @@ -17,43 +17,57 @@ class Optimizer(object): def __init__(self): - # set of Boxes; the value is not stored in the dict, but can - # be accessed as the Box.getint() or get_(). - self._consts = {} + self._equals = {} # mapping Box -> Box-or-Const + self._known_classes = {} # mapping Box -> ConstClass - def is_constant(self, box): - return isinstance(box, Const) or box in self._consts + def deref(self, box): + while box in self._equals: + # follow the chain: box -> box2 -> box3 -> ... + box2 = self._equals[box] + if box2 not in self._equals: + return box2 + # compress the mapping one step (rare case) + box3 = self._equals[box2] + self._equals[box] = box3 + box = box3 + return box def make_constant(self, box): assert isinstance(box, Box) - self._consts[box] = box.constbox() + assert box not in self._equals + self._equals[box] = box.constbox() - def rename_argument(self, box): - return self._consts.get(box, box) + def has_constant_class(self, box): + return isinstance(box, Const) or box in self._known_classes + + def make_constant_class(self, box, clsbox): + assert isinstance(box, Box) + assert isinstance(clsbox, Const) + self._known_classes[box] = clsbox # ---------- def optimize(self, loop): self.newoperations = [] for op in loop.operations: - opnum = op.opnum + op2 = op.clone() + op2.args = [self.deref(box) for box in op.args] + opnum = op2.opnum for value, func in optimize_ops: if opnum == value: - func(self, op) + func(self, op2) break else: - self.optimize_default(op) + self.optimize_default(op2) loop.operations = self.newoperations def emit_operation(self, op): - op2 = op.clone() - op2.args = [self.rename_argument(box) for box in op.args] - self.newoperations.append(op2) + self.newoperations.append(op) def optimize_default(self, op): if op.is_always_pure(): for box in op.args: - if not self.is_constant(box): + if not isinstance(box, Const): break else: # all constant arguments: constant-fold away @@ -63,23 +77,31 @@ self.emit_operation(op) def optimize_GUARD_VALUE(self, op): - if self.is_constant(op.args[0]): - assert isinstance(op.args[1], Const) - assert self.rename_argument(op.args[0]).get_() == op.args[1].get_() - else: + assert isinstance(op.args[1], Const) + assert op.args[0].get_() == op.args[1].get_() + if not isinstance(op.args[0], Const): self.emit_operation(op) + self.make_constant(op.args[0]) def optimize_GUARD_TRUE(self, op): - if self.is_constant(op.args[0]): - assert self.rename_argument(op.args[0]).getint() - else: + assert op.args[0].getint() + if not isinstance(op.args[0], Const): self.emit_operation(op) + self.make_constant(op.args[0]) def optimize_GUARD_FALSE(self, op): - if self.is_constant(op.args[0]): - assert not self.rename_argument(op.args[0]).getint() - else: + assert not op.args[0].getint() + if not isinstance(op.args[0], Const): + self.emit_operation(op) + self.make_constant(op.args[0]) + + def optimize_GUARD_CLASS(self, op): + instbox = op.args[0] + clsbox = op.args[1] + # XXX should probably assert that the class is right + if not self.has_constant_class(instbox): self.emit_operation(op) + self.make_constant_class(instbox, clsbox) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Sun Jul 19 19:30:51 2009 @@ -3,6 +3,7 @@ OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize +from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse # ____________________________________________________________ @@ -67,7 +68,7 @@ i0 = int_sub(i, 1) guard_value(i0, 0) fail(i0) - jump(i0) + jump(i) """ self.optimize(ops, 'Not', ops) @@ -91,6 +92,61 @@ """ self.optimize(ops, '', expected, i0=5, i1=1, i2=0) + def test_constfold_all(self): + for op in range(rop.INT_ADD, rop.BOOL_NOT+1): + try: + op = opname[op] + except KeyError: + continue + ops = """ + [] + i1 = %s(3, 2) + jump() + """ % op.lower() + expected = """ + [] + jump() + """ + self.optimize(ops, '', expected) + + # ---------- + + def test_remove_guard_class(self): + ops = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + guard_class(p0, ConstClass(node_vtable)) + fail() + jump(p0) + """ + expected = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + jump(p0) + """ + self.optimize(ops, 'Not', expected) + + def test_remove_consecutive_guard_value_constfold(self): + ops = """ + [i0] + guard_value(i0, 0) + fail() + i1 = int_add(i0, 1) + guard_value(i1, 1) + fail() + i2 = int_add(i1, 2) + jump(i2) + """ + expected = """ + [i0] + guard_value(i0, 0) + fail() + jump(3) + """ + self.optimize(ops, 'Not', expected, i0=0, i1=1, i2=3) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Sun Jul 19 19:44:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 19:44:13 +0200 (CEST) Subject: [pypy-svn] r66418 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719174413.2FDC6169F8E@codespeak.net> Author: arigo Date: Sun Jul 19 19:44:12 2009 New Revision: 66418 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Optimize oononnull and ooisnull. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Sun Jul 19 19:44:12 2009 @@ -103,5 +103,19 @@ self.emit_operation(op) self.make_constant_class(instbox, clsbox) + def optimize_OONONNULL(self, op): + if self.has_constant_class(op.args[0]): + assert op.result.getint() == 1 + self.make_constant(op.result) + else: + self.optimize_default(op) + + def optimize_OOISNULL(self, op): + if self.has_constant_class(op.args[0]): + assert op.result.getint() == 0 + self.make_constant(op.result) + else: + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Sun Jul 19 19:44:12 2009 @@ -70,7 +70,7 @@ fail(i0) jump(i) """ - self.optimize(ops, 'Not', ops) + self.optimize(ops, 'Not', ops, i0=0) def test_constant_propagate(self): ops = """ @@ -147,6 +147,48 @@ """ self.optimize(ops, 'Not', expected, i0=0, i1=1, i2=3) + def test_ooisnull_oononnull_1(self): + ops = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + i0 = oononnull(p0) + guard_true(i0) + fail() + i1 = ooisnull(p0) + guard_false(i1) + fail() + jump(p0) + """ + expected = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + jump(p0) + """ + self.optimize(ops, 'Not', expected, i0=1, i1=0) + + def test_ooisnull_oononnull_2(self): + py.test.skip("less important") + ops = """ + [p0] + i0 = oononnull(p0) # p0 != NULL + guard_true(i0) + fail() + i1 = ooisnull(p0) + guard_false(i1) + fail() + jump(p0) + """ + expected = """ + [p0] + i0 = oononnull(p0) + guard_true(i0) + fail() + jump(p0) + """ + self.optimize(ops, 'Not', expected, i0=1, i1=0) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Sun Jul 19 19:58:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 19 Jul 2009 19:58:13 +0200 (CEST) Subject: [pypy-svn] r66419 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090719175813.5544516855E@codespeak.net> Author: arigo Date: Sun Jul 19 19:58:09 2009 New Revision: 66419 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Treat oois(_, NULL) and ooisnot(_, NULL) as ooisnull(_) and oononnull(_). Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py Sun Jul 19 19:58:09 2009 @@ -89,6 +89,9 @@ def get_(self): raise NotImplementedError + def nonnull(self): + raise NotImplementedError + def clonebox(self): raise NotImplementedError @@ -216,6 +219,9 @@ def get_(self): return self.value + def nonnull(self): + return self.value != 0 + def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -261,6 +267,9 @@ def get_(self): return llmemory.cast_adr_to_int(self.value) + def nonnull(self): + return self.value != llmemory.NULL + def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.getint()) @@ -296,6 +305,9 @@ def getaddr(self, cpu): return llmemory.cast_ptr_to_adr(self.value) + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_ptr(j, self.value) @@ -330,6 +342,9 @@ else: return 0 + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_obj(j, self.value) @@ -417,6 +432,9 @@ def get_(self): return self.value + def nonnull(self): + return self.value != 0 + def set_future_value(self, cpu, j): cpu.set_future_value_int(j, self.value) @@ -451,6 +469,9 @@ def get_(self): return lltype.cast_ptr_to_int(self.value) + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_ptr(j, self.value) @@ -486,6 +507,9 @@ else: return 0 + def nonnull(self): + return bool(self.value) + def set_future_value(self, cpu, j): cpu.set_future_value_obj(j, self.value) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Sun Jul 19 19:58:09 2009 @@ -1,4 +1,5 @@ from pypy.jit.metainterp.history import Const, Box +from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.optimizeutil import av_newdict, _findall from pypy.rlib.objectmodel import we_are_translated @@ -117,5 +118,29 @@ else: self.optimize_default(op) + def optimize_OOISNOT(self, op): + if isinstance(op.args[1], Const) and not op.args[1].nonnull(): + op.opnum = rop.OONONNULL + del op.args[1] + self.optimize_OONONNULL(op) + elif isinstance(op.args[0], Const) and not op.args[0].nonnull(): + op.opnum = rop.OONONNULL + del op.args[0] + self.optimize_OONONNULL(op) + else: + self.optimize_default(op) + + def optimize_OOIS(self, op): + if isinstance(op.args[1], Const) and not op.args[1].nonnull(): + op.opnum = rop.OOISNULL + del op.args[1] + self.optimize_OOISNULL(op) + elif isinstance(op.args[0], Const) and not op.args[0].nonnull(): + op.opnum = rop.OOISNULL + del op.args[0] + self.optimize_OOISNULL(op) + else: + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Sun Jul 19 19:58:09 2009 @@ -101,6 +101,11 @@ return ConstObj(ootype.cast_to_object(self.consts[name])) elif arg == 'None': return None + elif arg == 'NULL': + if self.type_system == 'lltype': + return ConstPtr(ConstPtr.value) + else: + return ConstObj(ConstObj.value) return self.vars[arg] def parse_op(self, line): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Sun Jul 19 19:58:09 2009 @@ -189,6 +189,33 @@ """ self.optimize(ops, 'Not', expected, i0=1, i1=0) + def test_oois_1(self): + ops = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + i0 = ooisnot(p0, NULL) + guard_true(i0) + fail() + i1 = oois(p0, NULL) + guard_false(i1) + fail() + i2 = ooisnot(NULL, p0) + guard_true(i0) + fail() + i3 = oois(NULL, p0) + guard_false(i1) + fail() + jump(p0) + """ + expected = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + jump(p0) + """ + self.optimize(ops, 'Not', expected, i0=1, i1=0, i2=1, i3=0) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From benjamin at codespeak.net Mon Jul 20 00:25:10 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 00:25:10 +0200 (CEST) Subject: [pypy-svn] r66420 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090719222510.D91573180FC@codespeak.net> Author: benjamin Date: Mon Jul 20 00:25:09 2009 New Revision: 66420 Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py Log: add ParserError exception which isn't actually used for anything Modified: pypy/branch/parser-compiler/pypy/module/parser/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/__init__.py Mon Jul 20 00:25:09 2009 @@ -6,7 +6,9 @@ applevel_name = 'parser' - appleveldefs = {} + appleveldefs = { + 'ParserError' : 'app_helpers.ParserError' + } interpleveldefs = { '__name__' : '(space.wrap("parser"))', From benjamin at codespeak.net Mon Jul 20 00:26:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 00:26:25 +0200 (CEST) Subject: [pypy-svn] r66421 - pypy/branch/parser-compiler/pypy/module/parser Message-ID: <20090719222625.4ED693180FF@codespeak.net> Author: benjamin Date: Mon Jul 20 00:26:24 2009 New Revision: 66421 Added: pypy/branch/parser-compiler/pypy/module/parser/app_helpers.py (contents, props changed) Log: add app level exception Added: pypy/branch/parser-compiler/pypy/module/parser/app_helpers.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/parser/app_helpers.py Mon Jul 20 00:26:24 2009 @@ -0,0 +1,2 @@ +class ParserError(Exception): + pass From benjamin at codespeak.net Mon Jul 20 01:25:53 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 01:25:53 +0200 (CEST) Subject: [pypy-svn] r66422 - in pypy/branch/parser-compiler/pypy: interpreter/astcompiler module/parser Message-ID: <20090719232553.A7595169FA1@codespeak.net> Author: benjamin Date: Mon Jul 20 01:25:52 2009 New Revision: 66422 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Log: make keyword args consistent with cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 01:25:52 2009 @@ -14,38 +14,38 @@ return TopLevelCodeGenerator(space, module, symbols, info).assemble() -name_ops_default = { +name_ops_default = misc.dict_to_switch({ ast.Load : ops.LOAD_NAME, ast.Store : ops.STORE_NAME, ast.Del : ops.DELETE_NAME -} +}) -name_ops_fast = { +name_ops_fast = misc.dict_to_switch({ ast.Load : ops.LOAD_FAST, ast.Store : ops.STORE_FAST, ast.Del : ops.DELETE_FAST -} +}) -name_ops_deref = { +name_ops_deref = misc.dict_to_switch({ ast.Load : ops.LOAD_DEREF, ast.Store : ops.STORE_DEREF, -} +}) -name_ops_global = { +name_ops_global = misc.dict_to_switch({ ast.Load : ops.LOAD_GLOBAL, ast.Store : ops.STORE_GLOBAL, ast.Del : ops.DELETE_GLOBAL -} +}) -unary_operations = { +unary_operations = misc.dict_to_switch({ ast.Invert : ops.UNARY_INVERT, ast.Not : ops.UNARY_NOT, ast.UAdd : ops.UNARY_POSITIVE, ast.USub : ops.UNARY_NEGATIVE -} +}) -binary_operations = { +binary_operations = misc.dict_to_switch({ ast.Add : ops.BINARY_ADD, ast.Sub : ops.BINARY_SUBTRACT, ast.Mult : ops.BINARY_MULTIPLY, @@ -57,9 +57,9 @@ ast.BitAnd : ops.BINARY_AND, ast.BitXor : ops.BINARY_XOR, ast.FloorDiv : ops.BINARY_FLOOR_DIVIDE -} +}) -inplace_operations = { +inplace_operations = misc.dict_to_switch({ ast.Add : ops.INPLACE_ADD, ast.Sub : ops.INPLACE_SUBTRACT, ast.Mult : ops.INPLACE_MULTIPLY, @@ -71,9 +71,9 @@ ast.BitAnd : ops.INPLACE_AND, ast.BitXor : ops.INPLACE_XOR, ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE -} +}) -compare_operations = { +compare_operations = misc.dict_to_switch({ ast.Eq : 2, ast.NotEq : 3, ast.Lt : 0, @@ -84,23 +84,23 @@ ast.NotIn : 7, ast.Is : 8, ast.IsNot : 9 -} +}) -subscr_operations = { +subscr_operations = misc.dict_to_switch({ ast.AugLoad : ops.BINARY_SUBSCR, ast.Load : ops.BINARY_SUBSCR, ast.AugStore : ops.STORE_SUBSCR, ast.Store : ops.STORE_SUBSCR, ast.Del : ops.DELETE_SUBSCR -} +}) -slice_operations = { +slice_operations = misc.dict_to_switch({ ast.AugLoad : ops.SLICE, ast.Load : ops.SLICE, ast.AugStore : ops.STORE_SLICE, ast.Store : ops.STORE_SLICE, ast.Del : ops.DELETE_SLICE -} +}) F_BLOCK_LOOP = 0 @@ -168,7 +168,7 @@ elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: kind = name_ops_global try: - op = kind[ctx] + op = kind(ctx) except KeyError: if kind is name_ops_deref and ctx == ast.Del: raise SyntaxError("Can't delete variable used in " @@ -275,7 +275,7 @@ return ops.INPLACE_TRUE_DIVIDE else: return ops.INPLACE_DIVIDE - return inplace_operations[op] + return inplace_operations(op) def visit_AugAssign(self, assign): self.update_position(assign.lineno) @@ -325,7 +325,7 @@ return ops.BINARY_TRUE_DIVIDE else: return ops.BINARY_DIVIDE - return binary_operations[op] + return binary_operations(op) def visit_BinOp(self, binop): self.update_position(binop.lineno) @@ -716,7 +716,7 @@ def visit_UnaryOp(self, op): self.update_position(op.lineno) op.operand.walkabout(self) - self.emit_op(unary_operations[op.op]) + self.emit_op(unary_operations(op.op)) def visit_BoolOp(self, op): self.update_position(op.lineno) @@ -743,14 +743,14 @@ for i in range(1, ops_count): self.emit_op(ops.DUP_TOP) self.emit_op(ops.ROT_THREE) - op_kind = compare_operations[comp.ops[i - 1]] + op_kind = compare_operations(comp.ops[i - 1]) self.emit_op_arg(ops.COMPARE_OP, op_kind) self.emit_jump(ops.JUMP_IF_FALSE, cleanup) self.emit_op(ops.POP_TOP) if i < (ops_count - 1): comp.comparators[i].walkabout(self) comp.comparators[-1].walkabout(self) - last_kind = compare_operations[comp.ops[-1]] + last_kind = compare_operations(comp.ops[-1]) self.emit_op_arg(ops.COMPARE_OP, last_kind) if ops_count > 1: end = self.new_block() @@ -1047,7 +1047,7 @@ self.emit_op(ops.ROT_THREE) elif stack_count == 2: self.emit_op(ops.ROT_FOUR) - self.emit_op(slice_operations[ctx] + slice_offset) + self.emit_op(slice_operations(ctx) + slice_offset) def _complex_slice(self, slc, ctx): if slc.lower: @@ -1102,7 +1102,7 @@ self.emit_op_arg(ops.DUP_TOPX, 2) elif ctx == ast.AugStore: self.emit_op(ops.ROT_THREE) - self.emit_op(subscr_operations[ctx]) + self.emit_op(subscr_operations(ctx)) def visit_Subscript(self, sub): self.update_position(sub.lineno) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Mon Jul 20 01:25:52 2009 @@ -1,5 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.astcompiler import ast2 as ast +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.unroll import unrolling_iterable app = gateway.applevel(""" @@ -38,6 +40,21 @@ return False +def dict_to_switch(d): + def lookup(query): + if we_are_translated(): + for key, value in unrolling_iteritems: + if key == query: + return value + else: + raise KeyError + else: + return d[query] + lookup._always_inline_ = True + unrolling_iteritems = unrolling_iterable(d.iteritems()) + return lookup + + def flatten(tup): elts = [] for elt in tup: Modified: pypy/branch/parser-compiler/pypy/module/parser/pyparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/parser/pyparser.py (original) +++ pypy/branch/parser-compiler/pypy/module/parser/pyparser.py Mon Jul 20 01:25:52 2009 @@ -42,14 +42,14 @@ return space.wrap(self.tree.type == pygram.syms.eval_input) descr_isexpr.unwrap_spec = ["self", ObjSpace] - def descr_totuple(self, space, with_lineno=False, with_offset=False): + def descr_totuple(self, space, line_info=False, col_info=False): return self._build_app_tree(space, self.tree, space.newtuple, - with_lineno, with_offset) + line_info, col_info) descr_totuple.unwrap_spec = ["self", ObjSpace, bool, bool] - def descr_tolist(self, space, with_lineno=False, with_offset=False): + def descr_tolist(self, space, line_info=False, col_info=False): return self._build_app_tree(space, self.tree, space.newlist, - with_lineno, with_offset) + line_info, col_info) descr_tolist.unwrap_spec = ["self", ObjSpace, bool, bool] def descr_compile(self, space, filename=""): From benjamin at codespeak.net Mon Jul 20 01:26:53 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 01:26:53 +0200 (CEST) Subject: [pypy-svn] r66423 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090719232653.87A75169FA6@codespeak.net> Author: benjamin Date: Mon Jul 20 01:26:53 2009 New Revision: 66423 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: revert unintended changes Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 01:26:53 2009 @@ -14,38 +14,38 @@ return TopLevelCodeGenerator(space, module, symbols, info).assemble() -name_ops_default = misc.dict_to_switch({ +name_ops_default = { ast.Load : ops.LOAD_NAME, ast.Store : ops.STORE_NAME, ast.Del : ops.DELETE_NAME -}) +} -name_ops_fast = misc.dict_to_switch({ +name_ops_fast = { ast.Load : ops.LOAD_FAST, ast.Store : ops.STORE_FAST, ast.Del : ops.DELETE_FAST -}) +} -name_ops_deref = misc.dict_to_switch({ +name_ops_deref = { ast.Load : ops.LOAD_DEREF, ast.Store : ops.STORE_DEREF, -}) +} -name_ops_global = misc.dict_to_switch({ +name_ops_global = { ast.Load : ops.LOAD_GLOBAL, ast.Store : ops.STORE_GLOBAL, ast.Del : ops.DELETE_GLOBAL -}) +} -unary_operations = misc.dict_to_switch({ +unary_operations = { ast.Invert : ops.UNARY_INVERT, ast.Not : ops.UNARY_NOT, ast.UAdd : ops.UNARY_POSITIVE, ast.USub : ops.UNARY_NEGATIVE -}) +} -binary_operations = misc.dict_to_switch({ +binary_operations = { ast.Add : ops.BINARY_ADD, ast.Sub : ops.BINARY_SUBTRACT, ast.Mult : ops.BINARY_MULTIPLY, @@ -57,9 +57,9 @@ ast.BitAnd : ops.BINARY_AND, ast.BitXor : ops.BINARY_XOR, ast.FloorDiv : ops.BINARY_FLOOR_DIVIDE -}) +} -inplace_operations = misc.dict_to_switch({ +inplace_operations = { ast.Add : ops.INPLACE_ADD, ast.Sub : ops.INPLACE_SUBTRACT, ast.Mult : ops.INPLACE_MULTIPLY, @@ -71,9 +71,9 @@ ast.BitAnd : ops.INPLACE_AND, ast.BitXor : ops.INPLACE_XOR, ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE -}) +} -compare_operations = misc.dict_to_switch({ +compare_operations = { ast.Eq : 2, ast.NotEq : 3, ast.Lt : 0, @@ -84,23 +84,23 @@ ast.NotIn : 7, ast.Is : 8, ast.IsNot : 9 -}) +} -subscr_operations = misc.dict_to_switch({ +subscr_operations = { ast.AugLoad : ops.BINARY_SUBSCR, ast.Load : ops.BINARY_SUBSCR, ast.AugStore : ops.STORE_SUBSCR, ast.Store : ops.STORE_SUBSCR, ast.Del : ops.DELETE_SUBSCR -}) +} -slice_operations = misc.dict_to_switch({ +slice_operations = { ast.AugLoad : ops.SLICE, ast.Load : ops.SLICE, ast.AugStore : ops.STORE_SLICE, ast.Store : ops.STORE_SLICE, ast.Del : ops.DELETE_SLICE -}) +} F_BLOCK_LOOP = 0 @@ -168,7 +168,7 @@ elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: kind = name_ops_global try: - op = kind(ctx) + op = kind[ctx] except KeyError: if kind is name_ops_deref and ctx == ast.Del: raise SyntaxError("Can't delete variable used in " @@ -275,7 +275,7 @@ return ops.INPLACE_TRUE_DIVIDE else: return ops.INPLACE_DIVIDE - return inplace_operations(op) + return inplace_operations[op] def visit_AugAssign(self, assign): self.update_position(assign.lineno) @@ -325,7 +325,7 @@ return ops.BINARY_TRUE_DIVIDE else: return ops.BINARY_DIVIDE - return binary_operations(op) + return binary_operations[op] def visit_BinOp(self, binop): self.update_position(binop.lineno) @@ -716,7 +716,7 @@ def visit_UnaryOp(self, op): self.update_position(op.lineno) op.operand.walkabout(self) - self.emit_op(unary_operations(op.op)) + self.emit_op(unary_operations[op.op]) def visit_BoolOp(self, op): self.update_position(op.lineno) @@ -743,14 +743,14 @@ for i in range(1, ops_count): self.emit_op(ops.DUP_TOP) self.emit_op(ops.ROT_THREE) - op_kind = compare_operations(comp.ops[i - 1]) + op_kind = compare_operations[comp.ops[i - 1]] self.emit_op_arg(ops.COMPARE_OP, op_kind) self.emit_jump(ops.JUMP_IF_FALSE, cleanup) self.emit_op(ops.POP_TOP) if i < (ops_count - 1): comp.comparators[i].walkabout(self) comp.comparators[-1].walkabout(self) - last_kind = compare_operations(comp.ops[-1]) + last_kind = compare_operations[comp.ops[-1]] self.emit_op_arg(ops.COMPARE_OP, last_kind) if ops_count > 1: end = self.new_block() @@ -1047,7 +1047,7 @@ self.emit_op(ops.ROT_THREE) elif stack_count == 2: self.emit_op(ops.ROT_FOUR) - self.emit_op(slice_operations(ctx) + slice_offset) + self.emit_op(slice_operations[ctx] + slice_offset) def _complex_slice(self, slc, ctx): if slc.lower: @@ -1102,7 +1102,7 @@ self.emit_op_arg(ops.DUP_TOPX, 2) elif ctx == ast.AugStore: self.emit_op(ops.ROT_THREE) - self.emit_op(subscr_operations(ctx)) + self.emit_op(subscr_operations[ctx]) def visit_Subscript(self, sub): self.update_position(sub.lineno) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Mon Jul 20 01:26:53 2009 @@ -1,7 +1,5 @@ from pypy.interpreter import gateway from pypy.interpreter.astcompiler import ast2 as ast -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.unroll import unrolling_iterable app = gateway.applevel(""" @@ -40,21 +38,6 @@ return False -def dict_to_switch(d): - def lookup(query): - if we_are_translated(): - for key, value in unrolling_iteritems: - if key == query: - return value - else: - raise KeyError - else: - return d[query] - lookup._always_inline_ = True - unrolling_iteritems = unrolling_iterable(d.iteritems()) - return lookup - - def flatten(tup): elts = [] for elt in tup: From benjamin at codespeak.net Mon Jul 20 01:29:38 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 01:29:38 +0200 (CEST) Subject: [pypy-svn] r66424 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090719232938.8BD7F169FA6@codespeak.net> Author: benjamin Date: Mon Jul 20 01:29:37 2009 New Revision: 66424 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Log: turn static dict lookups into switches Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 01:29:37 2009 @@ -489,10 +489,12 @@ _stack_effect_computers = {} for name, func in globals().items(): if name.startswith("_compute_"): + func._always_inline_ = True _stack_effect_computers[getattr(ops, name[9:])] = func for op, value in _static_opcode_stack_effects.iteritems(): def func(arg, _value=value): return _value + func._always_inline_ = True _stack_effect_computers[op] = func del name, func, op, value Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 01:29:37 2009 @@ -14,38 +14,38 @@ return TopLevelCodeGenerator(space, module, symbols, info).assemble() -name_ops_default = { +name_ops_default = misc.dict_to_switch({ ast.Load : ops.LOAD_NAME, ast.Store : ops.STORE_NAME, ast.Del : ops.DELETE_NAME -} +}) -name_ops_fast = { +name_ops_fast = misc.dict_to_switch({ ast.Load : ops.LOAD_FAST, ast.Store : ops.STORE_FAST, ast.Del : ops.DELETE_FAST -} +}) -name_ops_deref = { +name_ops_deref = misc.dict_to_switch({ ast.Load : ops.LOAD_DEREF, ast.Store : ops.STORE_DEREF, -} +}) -name_ops_global = { +name_ops_global = misc.dict_to_switch({ ast.Load : ops.LOAD_GLOBAL, ast.Store : ops.STORE_GLOBAL, ast.Del : ops.DELETE_GLOBAL -} +}) -unary_operations = { +unary_operations = misc.dict_to_switch({ ast.Invert : ops.UNARY_INVERT, ast.Not : ops.UNARY_NOT, ast.UAdd : ops.UNARY_POSITIVE, ast.USub : ops.UNARY_NEGATIVE -} +}) -binary_operations = { +binary_operations = misc.dict_to_switch({ ast.Add : ops.BINARY_ADD, ast.Sub : ops.BINARY_SUBTRACT, ast.Mult : ops.BINARY_MULTIPLY, @@ -57,9 +57,9 @@ ast.BitAnd : ops.BINARY_AND, ast.BitXor : ops.BINARY_XOR, ast.FloorDiv : ops.BINARY_FLOOR_DIVIDE -} +}) -inplace_operations = { +inplace_operations = misc.dict_to_switch({ ast.Add : ops.INPLACE_ADD, ast.Sub : ops.INPLACE_SUBTRACT, ast.Mult : ops.INPLACE_MULTIPLY, @@ -71,9 +71,9 @@ ast.BitAnd : ops.INPLACE_AND, ast.BitXor : ops.INPLACE_XOR, ast.FloorDiv : ops.INPLACE_FLOOR_DIVIDE -} +}) -compare_operations = { +compare_operations = misc.dict_to_switch({ ast.Eq : 2, ast.NotEq : 3, ast.Lt : 0, @@ -84,23 +84,23 @@ ast.NotIn : 7, ast.Is : 8, ast.IsNot : 9 -} +}) -subscr_operations = { +subscr_operations = misc.dict_to_switch({ ast.AugLoad : ops.BINARY_SUBSCR, ast.Load : ops.BINARY_SUBSCR, ast.AugStore : ops.STORE_SUBSCR, ast.Store : ops.STORE_SUBSCR, ast.Del : ops.DELETE_SUBSCR -} +}) -slice_operations = { +slice_operations = misc.dict_to_switch({ ast.AugLoad : ops.SLICE, ast.Load : ops.SLICE, ast.AugStore : ops.STORE_SLICE, ast.Store : ops.STORE_SLICE, ast.Del : ops.DELETE_SLICE -} +}) F_BLOCK_LOOP = 0 @@ -168,7 +168,7 @@ elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: kind = name_ops_global try: - op = kind[ctx] + op = kind(ctx) except KeyError: if kind is name_ops_deref and ctx == ast.Del: raise SyntaxError("Can't delete variable used in " @@ -275,7 +275,7 @@ return ops.INPLACE_TRUE_DIVIDE else: return ops.INPLACE_DIVIDE - return inplace_operations[op] + return inplace_operations(op) def visit_AugAssign(self, assign): self.update_position(assign.lineno) @@ -325,7 +325,7 @@ return ops.BINARY_TRUE_DIVIDE else: return ops.BINARY_DIVIDE - return binary_operations[op] + return binary_operations(op) def visit_BinOp(self, binop): self.update_position(binop.lineno) @@ -716,7 +716,7 @@ def visit_UnaryOp(self, op): self.update_position(op.lineno) op.operand.walkabout(self) - self.emit_op(unary_operations[op.op]) + self.emit_op(unary_operations(op.op)) def visit_BoolOp(self, op): self.update_position(op.lineno) @@ -743,14 +743,14 @@ for i in range(1, ops_count): self.emit_op(ops.DUP_TOP) self.emit_op(ops.ROT_THREE) - op_kind = compare_operations[comp.ops[i - 1]] + op_kind = compare_operations(comp.ops[i - 1]) self.emit_op_arg(ops.COMPARE_OP, op_kind) self.emit_jump(ops.JUMP_IF_FALSE, cleanup) self.emit_op(ops.POP_TOP) if i < (ops_count - 1): comp.comparators[i].walkabout(self) comp.comparators[-1].walkabout(self) - last_kind = compare_operations[comp.ops[-1]] + last_kind = compare_operations(comp.ops[-1]) self.emit_op_arg(ops.COMPARE_OP, last_kind) if ops_count > 1: end = self.new_block() @@ -1047,7 +1047,7 @@ self.emit_op(ops.ROT_THREE) elif stack_count == 2: self.emit_op(ops.ROT_FOUR) - self.emit_op(slice_operations[ctx] + slice_offset) + self.emit_op(slice_operations(ctx) + slice_offset) def _complex_slice(self, slc, ctx): if slc.lower: @@ -1102,7 +1102,7 @@ self.emit_op_arg(ops.DUP_TOPX, 2) elif ctx == ast.AugStore: self.emit_op(ops.ROT_THREE) - self.emit_op(subscr_operations[ctx]) + self.emit_op(subscr_operations(ctx)) def visit_Subscript(self, sub): self.update_position(sub.lineno) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Mon Jul 20 01:29:37 2009 @@ -1,5 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.astcompiler import ast2 as ast +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.unroll import unrolling_iterable app = gateway.applevel(""" @@ -38,6 +40,21 @@ return False +def dict_to_switch(d): + def lookup(query): + if we_are_translated(): + for key, value in unrolling_iteritems: + if key == query: + return value + else: + raise KeyError + else: + return d[query] + lookup._always_inline_ = True + unrolling_iteritems = unrolling_iterable(d.iteritems()) + return lookup + + def flatten(tup): elts = [] for elt in tup: From benjamin at codespeak.net Mon Jul 20 01:55:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 01:55:16 +0200 (CEST) Subject: [pypy-svn] r66425 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090719235516.0E064169FA6@codespeak.net> Author: benjamin Date: Mon Jul 20 01:55:15 2009 New Revision: 66425 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: refactor name_op to help new switches Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 01:55:15 2009 @@ -150,30 +150,30 @@ def name_op(self, identifier, ctx): scope = self.scope.lookup(identifier) - kind = name_ops_default + op = ops.NOP container = self.names if scope == symtable.SCOPE_LOCAL: if self.scope.can_be_optimized: container = self.var_names - kind = name_ops_fast + op = name_ops_fast(ctx) elif scope == symtable.SCOPE_FREE: - kind = name_ops_deref + op = name_ops_deref(ctx) container = self.free_vars elif scope == symtable.SCOPE_CELL: - kind = name_ops_deref + try: + op = name_ops_deref(ctx) + except KeyError: + assert ctx == ast.Del + raise SyntaxError("Can't delete variable used in " + "nested scopes: '%s'" % (identifier,)) container = self.cell_vars elif scope == symtable.SCOPE_GLOBAL_IMPLICIT: if self.scope.locals_fully_known: - kind = name_ops_global + op = name_ops_global(ctx) elif scope == symtable.SCOPE_GLOBAL_EXPLICIT: - kind = name_ops_global - try: - op = kind(ctx) - except KeyError: - if kind is name_ops_deref and ctx == ast.Del: - raise SyntaxError("Can't delete variable used in " - "nested scopes: '%s'" % (identifier,)) - raise AssertionError("Unkown name operation") + op = name_ops_global(ctx) + if op == ops.NOP: + op = name_ops_default(ctx) self.emit_op_arg(op, self.add_name(container, identifier)) def is_docstring(self, node): From benjamin at codespeak.net Mon Jul 20 04:16:52 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 04:16:52 +0200 (CEST) Subject: [pypy-svn] r66426 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720021652.1A6F949842A@codespeak.net> Author: benjamin Date: Mon Jul 20 04:16:51 2009 New Revision: 66426 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: allow call to be constant folded Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 04:16:51 2009 @@ -503,7 +503,7 @@ if we_are_translated(): for possible_op in ops.unrolling_opcode_descs: if op == possible_op.index: - return _stack_effect_computers[op](arg) + return _stack_effect_computers[possible_op](arg) else: raise AssertionError("unkown opcode: %s" % (op,)) else: From benjamin at codespeak.net Mon Jul 20 04:23:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 04:23:42 +0200 (CEST) Subject: [pypy-svn] r66427 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720022342.6376149842A@codespeak.net> Author: benjamin Date: Mon Jul 20 04:23:41 2009 New Revision: 66427 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: typo Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 04:23:41 2009 @@ -503,7 +503,7 @@ if we_are_translated(): for possible_op in ops.unrolling_opcode_descs: if op == possible_op.index: - return _stack_effect_computers[possible_op](arg) + return _stack_effect_computers[possible_op.index](arg) else: raise AssertionError("unkown opcode: %s" % (op,)) else: From benjamin at codespeak.net Mon Jul 20 04:35:09 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 04:35:09 +0200 (CEST) Subject: [pypy-svn] r66428 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090720023509.9FEEE169E9B@codespeak.net> Author: benjamin Date: Mon Jul 20 04:35:09 2009 New Revision: 66428 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: don't declare slots already in the base class Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Mon Jul 20 04:35:09 2009 @@ -101,7 +101,7 @@ self.emit("class %s(%s):" % (cons.name, base)) self.emit("") all_fields = cons.fields + extra_attributes - slots = ", ".join(repr(field.name.value) for field in all_fields) + slots = ", ".join(repr(field.name.value) for field in cons.fields) self.emit("__slots__ = (%s)" % (slots,), 1) self.emit("") self.make_constructor(all_fields) From fijal at codespeak.net Mon Jul 20 11:00:48 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 20 Jul 2009 11:00:48 +0200 (CEST) Subject: [pypy-svn] r66429 - in pypy/branch/pyjitpl5/pypy/module/micronumpy: . test Message-ID: <20090720090048.0E86E2A8088@codespeak.net> Author: fijal Date: Mon Jul 20 11:00:47 2009 New Revision: 66429 Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Log: Implement some basics of multi-dim arrays Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/numarray.py Mon Jul 20 11:00:47 2009 @@ -5,13 +5,15 @@ from pypy.interpreter.gateway import interp2app, NoneNotWrapped from pypy.rlib.debug import make_sure_not_resized -class NumArray(Wrappable): +class BaseNumArray(Wrappable): + pass + +class NumArray(BaseNumArray): def __init__(self, space, dim, dtype): self.dim = dim self.space = space # ignore dtype for now - assert len(dim) == 1 - self.storage = [0] * dim[0] + self.storage = [0] * dim make_sure_not_resized(self.storage) def descr_getitem(self, index): @@ -44,11 +46,70 @@ __len__ = interp2app(NumArray.descr_len), ) +def compute_pos(space, indexes, dim): + current = 1 + pos = 0 + for i in range(len(indexes)): + index = indexes[i] + d = dim[i] + if index >= d or index <= -d - 1: + raise OperationError(space.w_IndexError, + space.wrap("invalid index")) + if index < 0: + index = d + index + pos += index * current + current *= d + return pos + +class MultiDimArray(BaseNumArray): + def __init__(self, space, dim, dtype): + self.dim = dim + self.space = space + # ignore dtype for now + size = 1 + for el in dim: + size *= el + self.storage = [0] * size + make_sure_not_resized(self.storage) + + def _unpack_indexes(self, space, w_index): + indexes = [space.int_w(w_i) for w_i in space.viewiterable(w_index)] + if len(indexes) != len(self.dim): + raise OperationError(space.w_IndexError, space.wrap( + 'Wrong index')) + return indexes + + def descr_getitem(self, w_index): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.dim) + return space.wrap(self.storage[pos]) + descr_getitem.unwrap_spec = ['self', W_Root] + + def descr_setitem(self, w_index, value): + space = self.space + indexes = self._unpack_indexes(space, w_index) + pos = compute_pos(space, indexes, self.dim) + self.storage[pos] = value + return space.w_None + descr_setitem.unwrap_spec = ['self', W_Root, int] + + def descr_len(self): + return self.space.wrap(self.dim[0]) + descr_len.unwrap_spec = ['self'] + +MultiDimArray.typedef = TypeDef( + 'NumArray', + __getitem__ = interp2app(MultiDimArray.descr_getitem), + __setitem__ = interp2app(MultiDimArray.descr_setitem), + __len__ = interp2app(MultiDimArray.descr_len), +) + def unpack_dim(space, w_dim): if space.is_true(space.isinstance(w_dim, space.w_int)): return [space.int_w(w_dim)] - else: - raise NotImplementedError + dim_w = space.viewiterable(w_dim) + return [space.int_w(w_i) for w_i in dim_w] def unpack_dtype(space, w_dtype): if space.is_w(w_dtype, space.w_int): @@ -59,5 +120,8 @@ def zeros(space, w_dim, w_dtype): dim = unpack_dim(space, w_dim) dtype = unpack_dtype(space, w_dtype) - return space.wrap(NumArray(space, dim, dtype)) + if len(dim) == 1: + return space.wrap(NumArray(space, dim[0], dtype)) + else: + return space.wrap(MultiDimArray(space, dim, dtype)) zeros.unwrap_spec = [ObjSpace, W_Root, W_Root] Modified: pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py (original) +++ pypy/branch/pyjitpl5/pypy/module/micronumpy/test/test_numpy.py Mon Jul 20 11:00:47 2009 @@ -39,3 +39,27 @@ assert x[4] == 0 assert len(x) == 5 raises(ValueError, minimum, ar, zeros(3, dtype=int)) + +class AppTestMultiDim(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=('micronumpy',)) + + def test_multidim(self): + from numpy import zeros + ar = zeros((3, 3), dtype=int) + assert ar[0, 2] == 0 + raises(IndexError, ar.__getitem__, (3, 0)) + assert ar[-2, 1] == 0 + + def test_multidim_getset(self): + from numpy import zeros + ar = zeros((3, 3, 3), dtype=int) + ar[1, 2, 1] = 3 + assert ar[1, 2, 1] == 3 + assert ar[-2, 2, 1] == 3 + assert ar[2, 2, 1] == 0 + assert ar[-2, 2, -2] == 3 + + def test_len(self): + from numpy import zeros + assert len(zeros((3, 2, 1), dtype=int)) == 3 From fijal at codespeak.net Mon Jul 20 11:52:48 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 20 Jul 2009 11:52:48 +0200 (CEST) Subject: [pypy-svn] r66430 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090720095248.91EFA169E97@codespeak.net> Author: fijal Date: Mon Jul 20 11:52:48 2009 New Revision: 66430 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize2.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py Log: skip deprecated tests. they should probably go away, but I don't feel like stomping on armin's feet Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize2.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize2.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize2.py Mon Jul 20 11:52:48 2009 @@ -1,4 +1,5 @@ import py +py.test.skip("deprecated") from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py Mon Jul 20 11:52:48 2009 @@ -1,4 +1,5 @@ import py +py.test.skip("deprecated") from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py Mon Jul 20 11:52:48 2009 @@ -1,4 +1,5 @@ import py +py.test.skip("deprecated") from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE From benjamin at codespeak.net Mon Jul 20 14:33:36 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 14:33:36 +0200 (CEST) Subject: [pypy-svn] r66431 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720123336.7FAB9169FA8@codespeak.net> Author: benjamin Date: Mon Jul 20 14:33:35 2009 New Revision: 66431 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: add NOP to the list Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 14:33:35 2009 @@ -317,6 +317,8 @@ _static_opcode_stack_effects = { + ops.NOP, + ops.POP_TOP : -1, ops.ROT_TWO : 0, ops.ROT_THREE : 0, From benjamin at codespeak.net Mon Jul 20 14:39:04 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 14:39:04 +0200 (CEST) Subject: [pypy-svn] r66432 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720123904.E079C169E76@codespeak.net> Author: benjamin Date: Mon Jul 20 14:39:04 2009 New Revision: 66432 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: fix syntax Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 14:39:04 2009 @@ -317,7 +317,7 @@ _static_opcode_stack_effects = { - ops.NOP, + ops.NOP : 0, ops.POP_TOP : -1, ops.ROT_TWO : 0, From benjamin at codespeak.net Mon Jul 20 15:07:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 15:07:50 +0200 (CEST) Subject: [pypy-svn] r66433 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720130750.DD182168073@codespeak.net> Author: benjamin Date: Mon Jul 20 15:07:49 2009 New Revision: 66433 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: another unused opcode Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 15:07:49 2009 @@ -318,6 +318,7 @@ _static_opcode_stack_effects = { ops.NOP : 0, + ops.STOP_CODE : 0, ops.POP_TOP : -1, ops.ROT_TWO : 0, From benjamin at codespeak.net Mon Jul 20 15:17:24 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 15:17:24 +0200 (CEST) Subject: [pypy-svn] r66434 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720131724.1BCDF169FAE@codespeak.net> Author: benjamin Date: Mon Jul 20 15:17:23 2009 New Revision: 66434 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: pass over EXTENDED_ARG Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 15:17:23 2009 @@ -505,6 +505,8 @@ def _opcode_stack_effect(op, arg): if we_are_translated(): for possible_op in ops.unrolling_opcode_descs: + if possible_op == ops.EXTENDED_ARG: + continue if op == possible_op.index: return _stack_effect_computers[possible_op.index](arg) else: From benjamin at codespeak.net Mon Jul 20 15:25:33 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 15:25:33 +0200 (CEST) Subject: [pypy-svn] r66435 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720132533.2A152169E74@codespeak.net> Author: benjamin Date: Mon Jul 20 15:25:32 2009 New Revision: 66435 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: compare with .index attribute Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 15:25:32 2009 @@ -505,7 +505,7 @@ def _opcode_stack_effect(op, arg): if we_are_translated(): for possible_op in ops.unrolling_opcode_descs: - if possible_op == ops.EXTENDED_ARG: + if possible_op.index == ops.EXTENDED_ARG: continue if op == possible_op.index: return _stack_effect_computers[possible_op.index](arg) From benjamin at codespeak.net Mon Jul 20 16:26:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 16:26:03 +0200 (CEST) Subject: [pypy-svn] r66436 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler/test pyparser pyparser/test Message-ID: <20090720142603.ABC3F169F41@codespeak.net> Author: benjamin Date: Mon Jul 20 16:26:02 2009 New Revision: 66436 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: use cpython's error message Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Mon Jul 20 16:26:02 2009 @@ -672,7 +672,7 @@ try: self.simple_test(source, None, None) except IndentationError, e: - assert e.msg == 'expected indented block' + assert e.msg == 'expected an indented block' else: raise Exception("DID NOT RAISE") Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Mon Jul 20 16:26:02 2009 @@ -130,7 +130,7 @@ if tp == pygram.tokens.INDENT: msg = "unexpected indent" elif e.expected == pygram.tokens.INDENT: - msg = "expected indented block" + msg = "expected an indented block" else: new_err = error.SyntaxError msg = "invalid syntax" Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Mon Jul 20 16:26:02 2009 @@ -81,7 +81,7 @@ def f(): pass""" exc = py.test.raises(IndentationError, parse, input).value - assert exc.msg == "expected indented block" + assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") assert exc.offset == 4 From fijal at codespeak.net Mon Jul 20 16:32:35 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 20 Jul 2009 16:32:35 +0200 (CEST) Subject: [pypy-svn] r66437 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090720143235.E7997169E6C@codespeak.net> Author: fijal Date: Mon Jul 20 16:32:33 2009 New Revision: 66437 Modified: pypy/extradoc/talk/euroscipy2009/abstract.txt Log: Abstract as it went Modified: pypy/extradoc/talk/euroscipy2009/abstract.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/abstract.txt (original) +++ pypy/extradoc/talk/euroscipy2009/abstract.txt Mon Jul 20 16:32:33 2009 @@ -19,6 +19,8 @@ JIT but won't go into too many architectural details. We will also present how JIT can make the life easier for simple -postprocessing of data generated by EULAG model. XXX extend. +postprocessing of data generated by nonhydrostatic model of geophysical +flows - EULAG for the case of boundary layer clouds observed during +various experimental campaign. .. _`March 2009`: http://morepypy.blogspot.com/2009/03/good-news-everyone.html From fijal at codespeak.net Mon Jul 20 16:32:54 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 20 Jul 2009 16:32:54 +0200 (CEST) Subject: [pypy-svn] r66438 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090720143254.77660169F49@codespeak.net> Author: fijal Date: Mon Jul 20 16:32:53 2009 New Revision: 66438 Added: pypy/extradoc/talk/euroscipy2009/talk.txt (contents, props changed) Log: First draft of a talk - very outlinish Added: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Mon Jul 20 16:32:53 2009 @@ -0,0 +1,87 @@ + +============================================= +PyPy's JIT for scientific python computations +============================================= + +Contents +======== + +* Introduction to PyPy + +* Introduction to JIT + +* A bit about EULAG model + +* JIT & numpy integration + +* Future + +PyPy - what's that? +=================== + +xxx + +Motivation behind the project +============================= + +xxx + +Status of PyPy today +==================== + +xxx + +Python's performance +==================== + +* Python as a language (not CPython) + +* Highly dynamic + +* Impossible to efficiently optimize statically + +* Dynamic compilation to the rescue + +JIT - motivation +================ + +* Psyco hard to maintain and extend + +xxx + +JIT - status +============ + +* JIT compiler generator, not JIT compiler + + +XXXX +EULAG slides +XXXX + +Data postprocessing +=================== + +* Short programs (~200 python LOC) + +* Relatively simple logic + +* Numpy for operations, matplotlib for output + +* Massive datasets + +* Sometimes require walking array elements + +* Sometimes slow ..... + +Data postprocessing - solution +============================== + +* Implement minimal version of numeric for PyPy + +* Reuse the JIT for speedups + +* ~10x speedups over CPython version + +* Still about 4x slower than C or matrix operations + From benjamin at codespeak.net Mon Jul 20 16:38:29 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 16:38:29 +0200 (CEST) Subject: [pypy-svn] r66439 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090720143829.E253B169F48@codespeak.net> Author: benjamin Date: Mon Jul 20 16:38:29 2009 New Revision: 66439 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Log: implement attribute deletion Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 16:38:29 2009 @@ -1019,6 +1019,10 @@ self.emit_op_name(ops.STORE_ATTR, names, attr.attr) elif attr.ctx == ast.Store: self.emit_op_name(ops.STORE_ATTR, names, attr.attr) + elif attr.ctx == ast.Del: + self.emit_op_name(ops.DELETE_ATTR, names, attr.attr) + else: + raise AssertionError("unknown context") def _simple_slice(self, slc, ctx): slice_offset = 0 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Mon Jul 20 16:38:29 2009 @@ -715,6 +715,18 @@ def test_backquote_repr(self): yield self.st, "x = None; y = `x`", "y", "None" + def test_deleting_attributes(self): + test = """class X(): + x = 3 +del X.x +try: + X.x +except AttributeError: + pass +else: + raise AssertionError("attribute not removed")""" + yield self.st, test, "X.__name__", "X" + class AppTestPrint: From arigo at codespeak.net Mon Jul 20 17:08:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 17:08:10 +0200 (CEST) Subject: [pypy-svn] r66440 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720150810.A0586169F9C@codespeak.net> Author: arigo Date: Mon Jul 20 17:08:09 2009 New Revision: 66440 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Fix optimizefindnode: a CALL_PURE operation has no side-effect, but it must still force its arguments to be non-virtual. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Mon Jul 20 17:08:09 2009 @@ -89,7 +89,7 @@ self.find_nodes_default(op) def find_nodes_default(self, op): - if not op.has_no_side_effect(): + if not op.has_no_side_effect_ptr(): # default case: mark the arguments as escaping for box in op.args: self.getnode(box).mark_escaped() Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Mon Jul 20 17:08:09 2009 @@ -81,6 +81,10 @@ def has_no_side_effect(self): return rop._NOSIDEEFFECT_FIRST <= self.opnum <= rop._NOSIDEEFFECT_LAST + def has_no_side_effect_ptr(self): + return (rop._NOSIDEEFFECT_PTR_FIRST <= self.opnum <= + rop._NOSIDEEFFECT_PTR_LAST) + def can_raise(self): return rop._CANRAISE_FIRST <= self.opnum <= rop._CANRAISE_LAST @@ -151,6 +155,7 @@ INT_INVERT = 62 BOOL_NOT = 63 # + _NOSIDEEFFECT_PTR_FIRST = 70 # -- start of no_side_effect_ptr operations -- OONONNULL = 70 OOISNULL = 71 OOIS = 72 @@ -176,6 +181,7 @@ GETARRAYITEM_GC = 120 GETFIELD_GC = 121 GETFIELD_RAW = 122 + _NOSIDEEFFECT_PTR_LAST = 129 # -- end of no_side_effect_ptr operations -- _NOSIDEEFFECT_LAST = 129 # ----- end of no_side_effect operations ----- NEW = 130 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Jul 20 17:08:09 2009 @@ -358,6 +358,62 @@ nextdescr=Virtual(node_vtable, nextdescr=Virtual(node_vtable)))''') + def test_find_nodes_oois(self): + ops = """ + [p3, p4, p2] + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + i1 = oononnull(p0) + guard_true(i1) + fail() + i2 = ooisnull(p0) + guard_false(i2) + fail() + i3 = ooisnot(p0, NULL) + guard_true(i3) + fail() + i4 = oois(p0, NULL) + guard_false(i4) + fail() + i5 = ooisnot(NULL, p0) + guard_true(i5) + fail() + i6 = oois(NULL, p0) + guard_false(i6) + fail() + i7 = ooisnot(p0, p1) + guard_true(i7) + fail() + i8 = oois(p0, p1) + guard_false(i8) + fail() + i9 = ooisnot(p0, p2) + guard_true(i9) + fail() + i10 = oois(p0, p2) + guard_false(i10) + fail() + i11 = ooisnot(p2, p1) + guard_true(i11) + fail() + i12 = oois(p2, p1) + guard_false(i12) + fail() + jump(p0, p1, p2) + """ + self.find_nodes(ops, '''Virtual(node_vtable), + Virtual(node_vtable), + Not''') + + def test_find_nodes_call(self): + ops = """ + [i0, p2] + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + i1 = call_pure(i0, p0) # forces p0 to not be virtual + jump(i1, p0) + """ + self.find_nodes(ops, 'Not, Not') + # ------------------------------ # Bridge tests From arigo at codespeak.net Mon Jul 20 17:11:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 17:11:30 +0200 (CEST) Subject: [pypy-svn] r66441 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090720151130.EEB7D169F9E@codespeak.net> Author: arigo Date: Mon Jul 20 17:11:30 2009 New Revision: 66441 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Log: Move OOSEND_PURE out of the _NOSIDEEFFECT_PTR group, towards CALL_PURE. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Mon Jul 20 17:11:30 2009 @@ -117,10 +117,11 @@ _GUARD_FOLDABLE_LAST = 11 GUARD_NO_EXCEPTION = 13 GUARD_EXCEPTION = 14 - _GUARD_LAST = 19 # ----- end of guard operations ----- + _GUARD_LAST = 14 # ----- end of guard operations ----- - _NOSIDEEFFECT_FIRST = 20 # ----- start of no_side_effect operations ----- - _ALWAYS_PURE_FIRST = 20 # ----- start of always_pure operations ----- + _NOSIDEEFFECT_FIRST = 19 # ----- start of no_side_effect operations ----- + _ALWAYS_PURE_FIRST = 19 # ----- start of always_pure operations ----- + OOSEND_PURE = 19 # ootype operation CALL_PURE = 20 # CAST_INT_TO_PTR = 21 @@ -174,9 +175,8 @@ OOIDENTITYHASH = 85 INSTANCEOF = 86 SUBCLASSOF = 87 - OOSEND_PURE = 88 # - _ALWAYS_PURE_LAST = 88 # ----- end of always_pure operations ----- + _ALWAYS_PURE_LAST = 87 # ----- end of always_pure operations ----- GETARRAYITEM_GC = 120 GETFIELD_GC = 121 From benjamin at codespeak.net Mon Jul 20 17:25:43 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 17:25:43 +0200 (CEST) Subject: [pypy-svn] r66442 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090720152543.A3FDD169F62@codespeak.net> Author: benjamin Date: Mon Jul 20 17:25:42 2009 New Revision: 66442 Removed: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_ast.py Log: remove outdated test From arigo at codespeak.net Mon Jul 20 17:41:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 17:41:39 +0200 (CEST) Subject: [pypy-svn] r66443 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090720154139.66A45169E97@codespeak.net> Author: arigo Date: Mon Jul 20 17:41:38 2009 New Revision: 66443 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Log: After discussion with fijal on irc, we found out that the NEW_* operations don't have side-effects on their own. (We disregard running finalizers; see discussion in the irc logs.) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Mon Jul 20 17:41:38 2009 @@ -181,12 +181,12 @@ GETARRAYITEM_GC = 120 GETFIELD_GC = 121 GETFIELD_RAW = 122 + NEW = 123 + NEW_WITH_VTABLE = 124 + NEW_ARRAY = 125 _NOSIDEEFFECT_PTR_LAST = 129 # -- end of no_side_effect_ptr operations -- _NOSIDEEFFECT_LAST = 129 # ----- end of no_side_effect operations ----- - NEW = 130 - NEW_WITH_VTABLE = 131 - NEW_ARRAY = 132 SETARRAYITEM_GC = 133 SETFIELD_GC = 134 SETFIELD_RAW = 135 From benjamin at codespeak.net Mon Jul 20 17:49:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 17:49:50 +0200 (CEST) Subject: [pypy-svn] r66444 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720154950.78EDC169FA6@codespeak.net> Author: benjamin Date: Mon Jul 20 17:49:49 2009 New Revision: 66444 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: fix spelling Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Mon Jul 20 17:49:49 2009 @@ -510,7 +510,7 @@ if op == possible_op.index: return _stack_effect_computers[possible_op.index](arg) else: - raise AssertionError("unkown opcode: %s" % (op,)) + raise AssertionError("unknown opcode: %s" % (op,)) else: try: return _static_opcode_stack_effects[op] Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Mon Jul 20 17:49:49 2009 @@ -85,7 +85,7 @@ stmts.append(self.handle_stmt(stmt)) return ast.Interactive(stmts) else: - raise AssertionError("unkown root node") + raise AssertionError("unknown root node") def number_of_statements(self, n): stmt_type = n.type @@ -157,7 +157,7 @@ elif isinstance(expr, ast.Repr): error = "repr" else: - raise AssertionError("unkown expression in set_context()") + raise AssertionError("unknown expression in set_context()") if error is not None: if ctx == ast.Store: action = "assign to" @@ -219,7 +219,7 @@ return ast.Raise(exc, value, traceback, flow_node.lineno, flow_node.column) else: - raise AssertionError("unkown flow statement") + raise AssertionError("unknown flow statement") def alias_for_import_name(self, import_name, store=True): while True: @@ -259,7 +259,7 @@ elif import_name_type == tokens.STAR: return ast.alias("*", None) else: - raise AssertionError("unkown import name") + raise AssertionError("unknown import name") def handle_import_stmt(self, import_node): import_node = import_node.children[0] @@ -298,7 +298,7 @@ self.error("trailing comma is only allowed with " "surronding parenthesis", names_node) else: - raise AssertionError("unkown import node") + raise AssertionError("unknown import node") if star_import: aliases = [self.alias_for_import_name(names_node)] else: @@ -309,7 +309,7 @@ return ast.ImportFrom(modname, aliases, dot_count, import_node.lineno, import_node.column) else: - raise AssertionError("unkown import node") + raise AssertionError("unknown import node") def handle_global_stmt(self, global_node): names = [global_node.children[i].value @@ -405,7 +405,7 @@ body = self.handle_suite(if_node.children[3]) return ast.If(expr, body, otherwise, if_node.lineno, if_node.column) else: - raise AssertionError("unkown if statement configuration") + raise AssertionError("unknown if statement configuration") def handle_while_stmt(self, while_node): loop_test = self.handle_expr(while_node.children[1]) @@ -610,7 +610,7 @@ self.check_forbidden_name(keywords_arg, name_node) i += 3 else: - raise AssertionError("unkown node in argument list") + raise AssertionError("unknown node in argument list") if not defaults: defaults = None if not args: @@ -689,7 +689,7 @@ else: raise AssertionError("unhandled compound statement") else: - raise AssertionError("unkown statment type") + raise AssertionError("unknown statment type") def handle_expr_stmt(self, stmt): if len(stmt.children) == 1: @@ -804,7 +804,7 @@ elif expr_node_type == syms.power: return self.handle_power(expr_node) else: - raise AssertionError("unkown expr") + raise AssertionError("unknown expr") def handle_lambdef(self, lambdef_node): expr = self.handle_expr(lambdef_node.children[-1]) @@ -1132,7 +1132,7 @@ expr = self.handle_testlist(atom_node.children[1]) return ast.Repr(expr, atom_node.lineno, atom_node.column) else: - raise AssertionError("unkown atom") + raise AssertionError("unknown atom") def handle_testlist_gexp(self, gexp_node): if len(gexp_node.children) > 1 and \ Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 17:49:49 2009 @@ -302,7 +302,7 @@ self.emit_op(self._op_for_augassign(assign.op)) self.name_op(target.id, ast.Store) else: - raise AssertionError("unkown augassign") + raise AssertionError("unknown augassign") def visit_Assert(self, asrt): self.update_position(asrt.lineno) @@ -1076,7 +1076,7 @@ elif isinstance(slc, ast.Index): slc.value.walkabout(self) else: - raise AssertionError("unkown nested slice type") + raise AssertionError("unknown nested slice type") def _compile_slice(self, slc, ctx): if isinstance(slc, ast.Index): @@ -1101,7 +1101,7 @@ self._nested_slice(dim, ctx) self.emit_op_arg(ops.BUILD_TUPLE, len(slc.dims)) else: - raise AssertionError("unkown slice type") + raise AssertionError("unknown slice type") if ctx == ast.AugLoad: self.emit_op_arg(ops.DUP_TOPX, 2) elif ctx == ast.AugStore: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Mon Jul 20 17:49:49 2009 @@ -261,7 +261,7 @@ err = "unqualified exec is not allowed in function '%s' " \ "because it %s" % (name, trailer) else: - raise AssertionError("unkown reason for unoptimization") + raise AssertionError("unknown reason for unoptimization") raise SyntaxError(err, node.lineno, node.col_offset) self.locals_fully_known = self.optimized and not self.has_exec @@ -451,7 +451,7 @@ if is_toplevel: self.implicit_arg(i) else: - raise AssertionError("unkown parameter type") + raise AssertionError("unknown parameter type") if not is_toplevel: self._handle_nested_params(params) From benjamin at codespeak.net Mon Jul 20 18:13:03 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 18:13:03 +0200 (CEST) Subject: [pypy-svn] r66445 - in pypy/branch/parser-compiler: lib-python/modified-2.5.2 pypy/interpreter/astcompiler Message-ID: <20090720161303.685C7169E74@codespeak.net> Author: benjamin Date: Mon Jul 20 18:13:02 2009 New Revision: 66445 Added: pypy/branch/parser-compiler/lib-python/modified-2.5.2/symbol.py (contents, props changed) pypy/branch/parser-compiler/lib-python/modified-2.5.2/token.py (contents, props changed) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: generate symbol and token from our parser's data Added: pypy/branch/parser-compiler/lib-python/modified-2.5.2/symbol.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/symbol.py Mon Jul 20 18:13:02 2009 @@ -0,0 +1,114 @@ +#! /usr/bin/env python + +"""Non-terminal symbols of Python grammar (from "graminit.h").""" + +# This file is automatically generated; please don't muck it up! +# +# To update the symbols in this file, 'cd' to the top directory of +# the python source tree after building the interpreter and run: +# +# python Lib/symbol.py + +#--start constants-- +dotted_as_names = 276 +import_as_name = 297 +try_stmt = 332 +eval_input = 279 +small_stmt = 320 +augassign = 264 +argument = 260 +or_test = 310 +fplist = 290 +import_as_names = 298 +return_stmt = 316 +testlist_safe = 330 +not_test = 307 +listmaker = 306 +except_clause = 280 +list_if = 304 +old_test = 309 +arglist = 259 +import_from = 299 +gen_iter = 294 +break_stmt = 265 +dictmaker = 274 +comp_op = 267 +import_stmt = 301 +with_var = 336 +parameters = 311 +continue_stmt = 270 +fpdef = 289 +shift_expr = 317 +dotted_as_name = 275 +testlist_gexp = 329 +list_iter = 305 +exec_stmt = 281 +factor = 285 +list_for = 303 +global_stmt = 295 +subscript = 322 +decorators = 272 +compound_stmt = 269 +and_expr = 257 +yield_stmt = 339 +dotted_name = 277 +yield_expr = 338 +power = 313 +print_stmt = 314 +gen_for = 292 +subscriptlist = 323 +testlist = 327 +classdef = 266 +and_test = 258 +encoding_decl = 278 +assert_stmt = 262 +test = 326 +for_stmt = 288 +stmt = 321 +lambdef = 302 +atom = 263 +funcdef = 291 +expr_stmt = 283 +old_lambdef = 308 +exprlist = 284 +decorator = 271 +pass_stmt = 312 +sliceop = 319 +comparison = 268 +term = 325 +if_stmt = 296 +arith_expr = 261 +expr = 282 +raise_stmt = 315 +import_name = 300 +gen_if = 293 +del_stmt = 273 +while_stmt = 334 +varargslist = 333 +testlist1 = 328 +suite = 324 +single_input = 256 +simple_stmt = 318 +with_stmt = 335 +xor_expr = 337 +flow_stmt = 287 +trailer = 331 +file_input = 286 +#--end constants-- + +sym_name = {} +for _name, _value in globals().items(): + if type(_value) is type(0): + sym_name[_value] = _name + + +if __name__ == "__main__": + import os + import sys + import token + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../../")) + try: + from pypy.interpreter.pyparser import pygram + token.main(pygram.python_grammar.symbol_ids, __file__) + finally: + sys.path.pop() Added: pypy/branch/parser-compiler/lib-python/modified-2.5.2/token.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/token.py Mon Jul 20 18:13:02 2009 @@ -0,0 +1,127 @@ +#! /usr/bin/env python + +"""Token constants (from "token.h").""" + +# This file is automatically generated; please don't muck it up! +# +# To update the symbols in this file, 'cd' to the top directory of +# the python source tree after building the interpreter and run: +# +# python Lib/token.py + +#--start constants-- +DEDENT = 6 +LPAR = 7 +STAR = 16 +AMPER = 19 +LESS = 20 +SLASHEQUAL = 40 +NUMBER = 2 +RPAR = 8 +CIRCUMFLEX = 33 +NOTEQUAL = 29 +VBAR = 18 +BACKQUOTE = 25 +DOUBLESTAR = 36 +MINUS = 15 +DOT = 23 +STRING = 3 +STAREQUAL = 39 +GREATEREQUAL = 31 +MINEQUAL = 38 +LEFTSHIFTEQUAL = 45 +SEMI = 13 +CIRCUMFLEXEQUAL = 44 +NEWLINE = 4 +DOUBLESLASHEQUAL = 49 +COLON = 11 +PERCENTEQUAL = 41 +TILDE = 32 +PLUS = 14 +ERRORTOKEN = 52 +RSQB = 10 +EQEQUAL = 28 +COMMENT = 53 +AMPEREQUAL = 42 +RIGHTSHIFT = 35 +RBRACE = 27 +NT_OFFSET = 256 +PERCENT = 24 +DOUBLESLASH = 48 +DOUBLESTAREQUAL = 47 +EQUAL = 22 +PLUSEQUAL = 37 +AT = 50 +SLASH = 17 +LESSEQUAL = 30 +NL = 54 +LSQB = 9 +N_TOKENS = 55 +RIGHTSHIFTEQUAL = 46 +GREATER = 21 +LBRACE = 26 +INDENT = 5 +NAME = 1 +VBAREQUAL = 43 +LEFTSHIFT = 34 +COMMA = 12 +ENDMARKER = 0 +OP = 51 +#--end constants-- + +tok_name = {} +for _name, _value in globals().items(): + if type(_value) is type(0): + tok_name[_value] = _name + + +def ISTERMINAL(x): + return x < NT_OFFSET + +def ISNONTERMINAL(x): + return x >= NT_OFFSET + +def ISEOF(x): + return x == ENDMARKER + + +def main(data, outFileName): + # load the output skeleton from the target: + try: + fp = open(outFileName) + except IOError, err: + sys.stderr.write("I/O error: %s\n" % str(err)) + sys.exit(2) + format = fp.read().split("\n") + fp.close() + try: + start = format.index("#--start constants--") + 1 + end = format.index("#--end constants--") + except ValueError: + sys.stderr.write("target does not contain format markers") + sys.exit(3) + lines = [] + for key, val in data.iteritems(): + lines.append("%s = %d" % (key, val)) + format[start:end] = lines + try: + fp = open(outFileName, 'w') + except IOError, err: + sys.stderr.write("I/O error: %s\n" % str(err)) + sys.exit(4) + fp.write("\n".join(format)) + fp.close() + + +if __name__ == "__main__": + import os + import sys + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../../")) + try: + from pypy.interpreter.pyparser import pytoken + data = pytoken.python_tokens.copy() + data["N_TOKENS"] = len(data) + data["NT_OFFSET"] = 256 + main(data, __file__) + finally: + sys.path.pop() Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 18:13:02 2009 @@ -1007,19 +1007,20 @@ def visit_Attribute(self, attr): self.update_position(attr.lineno) names = self.names - if attr.ctx != ast.AugStore: + ctx = attr.ctx + if ctx != ast.AugStore: attr.value.walkabout(self) - if attr.ctx == ast.AugLoad: + if ctx == ast.AugLoad: self.emit_op(ops.DUP_TOP) self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif attr.ctx == ast.Load: + elif ctx == ast.Load: self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif attr.ctx == ast.AugStore: + elif ctx == ast.AugStore: self.emit_op(ops.ROT_TWO) self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif attr.ctx == ast.Store: + elif ctx == ast.Store: self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif attr.ctx == ast.Del: + elif ctx == ast.Del: self.emit_op_name(ops.DELETE_ATTR, names, attr.attr) else: raise AssertionError("unknown context") From benjamin at codespeak.net Mon Jul 20 18:14:39 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 18:14:39 +0200 (CEST) Subject: [pypy-svn] r66446 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720161439.6556C169E74@codespeak.net> Author: benjamin Date: Mon Jul 20 18:14:38 2009 New Revision: 66446 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: revert unintended change Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 18:14:38 2009 @@ -1007,20 +1007,19 @@ def visit_Attribute(self, attr): self.update_position(attr.lineno) names = self.names - ctx = attr.ctx - if ctx != ast.AugStore: + if attr.ctx != ast.AugStore: attr.value.walkabout(self) - if ctx == ast.AugLoad: + if attr.ctx == ast.AugLoad: self.emit_op(ops.DUP_TOP) self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif ctx == ast.Load: + elif attr.ctx == ast.Load: self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif ctx == ast.AugStore: + elif attr.ctx == ast.AugStore: self.emit_op(ops.ROT_TWO) self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif ctx == ast.Store: + elif attr.ctx == ast.Store: self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif ctx == ast.Del: + elif attr.ctx == ast.Del: self.emit_op_name(ops.DELETE_ATTR, names, attr.attr) else: raise AssertionError("unknown context") From benjamin at codespeak.net Mon Jul 20 18:15:52 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 18:15:52 +0200 (CEST) Subject: [pypy-svn] r66447 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090720161552.A2C58169EAB@codespeak.net> Author: benjamin Date: Mon Jul 20 18:15:51 2009 New Revision: 66447 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: allow this to be folded into a switch Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Mon Jul 20 18:15:51 2009 @@ -1007,19 +1007,20 @@ def visit_Attribute(self, attr): self.update_position(attr.lineno) names = self.names - if attr.ctx != ast.AugStore: + ctx = attr.ctx + if ctx != ast.AugStore: attr.value.walkabout(self) - if attr.ctx == ast.AugLoad: + if ctx == ast.AugLoad: self.emit_op(ops.DUP_TOP) self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif attr.ctx == ast.Load: + elif ctx == ast.Load: self.emit_op_name(ops.LOAD_ATTR, names, attr.attr) - elif attr.ctx == ast.AugStore: + elif ctx == ast.AugStore: self.emit_op(ops.ROT_TWO) self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif attr.ctx == ast.Store: + elif ctx == ast.Store: self.emit_op_name(ops.STORE_ATTR, names, attr.attr) - elif attr.ctx == ast.Del: + elif ctx == ast.Del: self.emit_op_name(ops.DELETE_ATTR, names, attr.attr) else: raise AssertionError("unknown context") From arigo at codespeak.net Mon Jul 20 18:24:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 18:24:08 +0200 (CEST) Subject: [pypy-svn] r66448 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090720162408.9448E169E74@codespeak.net> Author: arigo Date: Mon Jul 20 18:24:08 2009 New Revision: 66448 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Log: When parsing a variable called e.g. 'i5', make sure that automatically-numbered variables created afterwards don't also get the name 'i5'. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Mon Jul 20 18:24:08 2009 @@ -64,9 +64,11 @@ if elem.startswith('i'): # integer box = BoxInt() + _box_counter_more_than(elem[1:]) elif elem.startswith('p'): # pointer box = BoxPtr() + _box_counter_more_than(elem[1:]) else: for prefix, boxclass in self.boxkinds.iteritems(): if elem.startswith(prefix): @@ -213,3 +215,7 @@ def parse(descr, cpu=None, namespace={}, type_system='lltype', boxkinds=None): return OpParser(descr, cpu, namespace, type_system, boxkinds).parse() + +def _box_counter_more_than(s): + if s.isdigit(): + Box._counter = max(Box._counter, int(s)+1) From arigo at codespeak.net Mon Jul 20 18:24:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 18:24:23 +0200 (CEST) Subject: [pypy-svn] r66449 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720162423.E3ACE169E74@codespeak.net> Author: arigo Date: Mon Jul 20 18:24:23 2009 New Revision: 66449 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Passes the first virtual tests. Yay! Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 18:24:23 2009 @@ -1,27 +1,69 @@ -from pypy.jit.metainterp.history import Const, Box +from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxObj +from pypy.jit.metainterp.history import AbstractValue from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.specnode import SpecNode +from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.optimizeutil import av_newdict, _findall from pypy.rlib.objectmodel import we_are_translated -def optimize(loop): +def optimize(cpu, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes must be applicable to the loop; you will probably get an AssertionError if not. """ - Optimizer().optimize(loop) + optimizer = Optimizer(cpu, loop) + optimizer.setup_virtuals() + optimizer.propagate_forward() # ____________________________________________________________ +class VirtualBox(AbstractValue): + def __init__(self): + self.fields = av_newdict() # ofs -> Box + + def nonnull(self): + return True + + +class __extend__(SpecNode): + def setup_virtual_node(self, optimizer, box, newinputargs): + newinputargs.append(box) + def teardown_virtual_node(self, box, newexitargs): + newexitargs.append(box) + +class __extend__(VirtualInstanceSpecNode): + def setup_virtual_node(self, optimizer, box, newinputargs): + vbox = optimizer.make_virtual(box, self.known_class) + for ofs, subspecnode in self.fields: + subbox = optimizer.make_box(ofs) + vbox.fields[ofs] = subbox + subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) + def teardown_virtual_node(self, box, newexitargs): + assert isinstance(box, VirtualBox) + for ofs, subspecnode in self.fields: + # XXX if ofs not in box.fields:... + subspecnode.teardown_virtual_node(box.fields[ofs], newexitargs) + class Optimizer(object): - def __init__(self): - self._equals = {} # mapping Box -> Box-or-Const - self._known_classes = {} # mapping Box -> ConstClass + def __init__(self, cpu, loop): + self.cpu = cpu + self.loop = loop + # Boxes used a keys to _equals have been proven to be equal to + # something else: another Box, a Const, or a VirtualBox. Boxes + # and VirtualBoxes can also be listed in _known_classes if we + # know their class (or just know that they are non-null, in which + # case we use None). Boxes *must not* be keys in both dicts. + self._equals = {} # mapping Box -> Box/Const/VirtualBox + self._known_classes = {} # mapping Box -> ConstClass/None def deref(self, box): + """Maps a Box/Const to the corresponding Box/Const/VirtualBox + by following the dict _equals. + """ while box in self._equals: # follow the chain: box -> box2 -> box3 -> ... box2 = self._equals[box] @@ -33,24 +75,67 @@ box = box3 return box - def make_constant(self, box): + def _make_equal(self, box, box2): assert isinstance(box, Box) assert box not in self._equals - self._equals[box] = box.constbox() + assert box not in self._known_classes + self._equals[box] = box2 + + def make_constant(self, box): + """Mark the given Box as actually representing a Const value.""" + self._make_equal(box, box.constbox()) + + def make_virtual(self, box, clsbox): + """Mark the given Box as actually representing a VirtualBox value.""" + vbox = VirtualBox() + self._make_equal(box, vbox) + self.make_constant_class(vbox, clsbox) + return vbox def has_constant_class(self, box): - return isinstance(box, Const) or box in self._known_classes + return (isinstance(box, Const) or + self._known_classes.get(box, None) is not None) def make_constant_class(self, box, clsbox): - assert isinstance(box, Box) assert isinstance(clsbox, Const) + assert box not in self._equals self._known_classes[box] = clsbox + def make_nonnull(self, box): + assert box not in self._equals + self._known_classes.setdefault(box, None) + + def make_box(self, fieldofs): + if fieldofs.is_pointer_field(): + if not self.cpu.is_oo: + return BoxPtr() + else: + return BoxObj() + else: + return BoxInt() + + def known_nonnull(self, box): + if isinstance(box, Box): + return box in self._known_classes + else: + return box.nonnull() # Consts or VirtualBoxes + # ---------- - def optimize(self, loop): + def setup_virtuals(self): + inputargs = self.loop.inputargs + specnodes = self.loop.specnodes + assert len(inputargs) == len(specnodes) + newinputargs = [] + for i in range(len(inputargs)): + specnodes[i].setup_virtual_node(self, inputargs[i], newinputargs) + self.loop.inputargs = newinputargs + + # ---------- + + def propagate_forward(self): self.newoperations = [] - for op in loop.operations: + for op in self.loop.operations: op2 = op.clone() op2.args = [self.deref(box) for box in op.args] opnum = op2.opnum @@ -60,9 +145,11 @@ break else: self.optimize_default(op2) - loop.operations = self.newoperations + self.loop.operations = self.newoperations def emit_operation(self, op): + for x in op.args: + assert not isinstance(x, VirtualBox) self.newoperations.append(op) def optimize_default(self, op): @@ -77,6 +164,16 @@ # otherwise, the operation remains self.emit_operation(op) + def optimize_JUMP(self, op): + orgop = self.loop.operations[-1] + exitargs = [] + specnodes = orgop.jump_target.specnodes + assert len(op.args) == len(specnodes) + for i in range(len(specnodes)): + specnodes[i].teardown_virtual_node(op.args[i], exitargs) + op.args = exitargs + self.emit_operation(op) + def optimize_GUARD_VALUE(self, op): assert isinstance(op.args[1], Const) assert op.args[0].get_() == op.args[1].get_() @@ -119,11 +216,14 @@ self.optimize_default(op) def optimize_OOISNOT(self, op): - if isinstance(op.args[1], Const) and not op.args[1].nonnull(): + if (isinstance(op.args[0], VirtualBox) or + isinstance(op.args[1], VirtualBox)): + self.make_constant(op.result) + elif self.known_nonnull(op.args[1]): op.opnum = rop.OONONNULL del op.args[1] self.optimize_OONONNULL(op) - elif isinstance(op.args[0], Const) and not op.args[0].nonnull(): + elif self.known_nonnull(op.args[0]): op.opnum = rop.OONONNULL del op.args[0] self.optimize_OONONNULL(op) @@ -131,16 +231,35 @@ self.optimize_default(op) def optimize_OOIS(self, op): - if isinstance(op.args[1], Const) and not op.args[1].nonnull(): + if (isinstance(op.args[0], VirtualBox) or + isinstance(op.args[1], VirtualBox)): + self.make_constant(op.result) + elif self.known_nonnull(op.args[1]): op.opnum = rop.OOISNULL del op.args[1] self.optimize_OOISNULL(op) - elif isinstance(op.args[0], Const) and not op.args[0].nonnull(): + elif self.known_nonnull(op.args[0]): op.opnum = rop.OOISNULL del op.args[0] self.optimize_OOISNULL(op) else: self.optimize_default(op) + def optimize_GETFIELD_GC(self, op): + instbox = op.args[0] + if isinstance(instbox, VirtualBox): + self._make_equal(op.result, instbox.fields[op.descr]) + else: + self.optimize_default(op) + + optimize_GETFIELD_PURE_GC = optimize_GETFIELD_GC + + def optimize_SETFIELD_GC(self, op): + instbox = op.args[0] + if isinstance(instbox, VirtualBox): + instbox.fields[op.descr] = op.args[1] + else: + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 18:24:23 2009 @@ -8,7 +8,7 @@ # ____________________________________________________________ -def equaloplists(oplist1, oplist2): +def equaloplists(oplist1, oplist2, remap={}): print '-'*20, 'Comparing lists', '-'*20 for op1, op2 in zip(oplist1, oplist2): txt1 = str(op1) @@ -20,11 +20,11 @@ assert op1.opnum == op2.opnum assert len(op1.args) == len(op2.args) for x, y in zip(op1.args, op2.args): - assert x == y - assert op1.result == op2.result + assert x == remap.get(y, y) + assert op1.result == remap.get(op2.result, op2.result) assert op1.descr == op2.descr if op1.suboperations: - assert equaloplists(op1.suboperations, op2.suboperations) + assert equaloplists(op1.suboperations, op2.suboperations, remap) assert len(oplist1) == len(oplist2) print '-'*57 return True @@ -45,20 +45,57 @@ py.test.raises(AssertionError, "equaloplists(loop1.operations, loop3.operations)") +def test_equaloplists_remap(): + ops1 = """ + [i0] + i1 = int_add(i0, 1) + guard_true(i1) + i2 = int_add(i1, 1) + fail(i2) + jump(i1) + """ + ops2 = """ + [i3] + i1 = int_add(i3, 1) + guard_true(i1) + i5 = int_add(i1, 1) + fail(i5) + jump(i1) + """ + loop1 = parse(ops1) + loop2 = parse(ops2) + py.test.raises(AssertionError, + "equaloplists(loop1.operations, loop2.operations)") + i0 = loop1.inputargs[0] + i3 = loop2.inputargs[0] + i2 = loop1.operations[1].suboperations[0].result + i5 = loop2.operations[1].suboperations[0].result + assert equaloplists(loop1.operations, loop2.operations, + {i3: i0, i5: i2}) + # ____________________________________________________________ class BaseTestOptimizeOpt(BaseTest): def assert_equal(self, optimized, expected): - assert optimized.inputargs == expected.inputargs + assert len(optimized.inputargs) == len(expected.inputargs) + remap = {} + for box1, box2 in zip(optimized.inputargs, expected.inputargs): + assert box1.__class__ == box2.__class__ + remap[box2] = box1 assert equaloplists(optimized.operations, - expected.operations) + expected.operations, + remap) - def optimize(self, ops, spectext, optops, boxkinds=None, **values): + def optimize_loop(self, ops, spectext, optops, boxkinds=None, **values): loop = self.parse(ops, boxkinds=boxkinds) loop.setvalues(**values) loop.specnodes = self.unpack_specnodes(spectext) - optimize(loop) + assert loop.operations[-1].opnum == rop.JUMP + loop.operations[-1].jump_target = loop + # + optimize(self.cpu, loop) + # expected = self.parse(optops, boxkinds=boxkinds) self.assert_equal(loop, expected) @@ -70,7 +107,7 @@ fail(i0) jump(i) """ - self.optimize(ops, 'Not', ops, i0=0) + self.optimize_loop(ops, 'Not', ops, i0=0) def test_constant_propagate(self): ops = """ @@ -90,7 +127,7 @@ [] jump() """ - self.optimize(ops, '', expected, i0=5, i1=1, i2=0) + self.optimize_loop(ops, '', expected, i0=5, i1=1, i2=0) def test_constfold_all(self): for op in range(rop.INT_ADD, rop.BOOL_NOT+1): @@ -107,7 +144,7 @@ [] jump() """ - self.optimize(ops, '', expected) + self.optimize_loop(ops, '', expected) # ---------- @@ -126,7 +163,7 @@ fail() jump(p0) """ - self.optimize(ops, 'Not', expected) + self.optimize_loop(ops, 'Not', expected) def test_remove_consecutive_guard_value_constfold(self): ops = """ @@ -145,7 +182,7 @@ fail() jump(3) """ - self.optimize(ops, 'Not', expected, i0=0, i1=1, i2=3) + self.optimize_loop(ops, 'Not', expected, i0=0, i1=1, i2=3) def test_ooisnull_oononnull_1(self): ops = """ @@ -166,7 +203,7 @@ fail() jump(p0) """ - self.optimize(ops, 'Not', expected, i0=1, i1=0) + self.optimize_loop(ops, 'Not', expected, i0=1, i1=0) def test_ooisnull_oononnull_2(self): py.test.skip("less important") @@ -187,7 +224,7 @@ fail() jump(p0) """ - self.optimize(ops, 'Not', expected, i0=1, i1=0) + self.optimize_loop(ops, 'Not', expected, i0=1, i1=0) def test_oois_1(self): ops = """ @@ -214,7 +251,84 @@ fail() jump(p0) """ - self.optimize(ops, 'Not', expected, i0=1, i1=0, i2=1, i3=0) + self.optimize_loop(ops, 'Not', expected, i0=1, i1=0, i2=1, i3=0) + + # ---------- + + def test_virtual_1(self): + ops = """ + [i, p0] + i0 = getfield_gc(p0, descr=valuedescr) + i1 = int_add(i0, i) + setfield_gc(p0, i1, descr=valuedescr) + jump(i, p0) + """ + expected = """ + [i, i2] + i1 = int_add(i2, i) + jump(i, i1) + """ + self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', + expected) + + def test_virtual_oois(self): + ops = """ + [p0, p1, p2] + i1 = oononnull(p0) + guard_true(i1) + fail() + i2 = ooisnull(p0) + guard_false(i2) + fail() + i3 = ooisnot(p0, NULL) + guard_true(i3) + fail() + i4 = oois(p0, NULL) + guard_false(i4) + fail() + i5 = ooisnot(NULL, p0) + guard_true(i5) + fail() + i6 = oois(NULL, p0) + guard_false(i6) + fail() + i7 = ooisnot(p0, p1) + guard_true(i7) + fail() + i8 = oois(p0, p1) + guard_false(i8) + fail() + i9 = ooisnot(p0, p2) + guard_true(i9) + fail() + i10 = oois(p0, p2) + guard_false(i10) + fail() + i11 = ooisnot(p2, p1) + guard_true(i11) + fail() + i12 = oois(p2, p1) + guard_false(i12) + fail() + jump(p0, p1, p2) + """ + expected = """ + [p2] + # all constant-folded :-) + jump(p2) + """ + self.optimize_loop(ops, '''Virtual(node_vtable), + Virtual(node_vtable), + Not''', + expected, + i1=1, i2=0, i3=1, i4=0, i5=1, i6=0, + i7=1, i8=0, i9=1, i10=0, i11=1, i12=0) + # + # to be complete, we also check the no-opt case where most comparisons + # are not removed. The exact set of comparisons removed depends on + # the details of the algorithm... + expected2 = ops + self.optimize_loop(ops, 'Not, Not, Not', expected2) class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): From arigo at codespeak.net Mon Jul 20 18:33:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 18:33:04 +0200 (CEST) Subject: [pypy-svn] r66450 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720163304.3256D169E11@codespeak.net> Author: arigo Date: Mon Jul 20 18:33:03 2009 New Revision: 66450 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Actually use known_nonnull(). Simplify deref(). Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 18:33:03 2009 @@ -64,15 +64,8 @@ """Maps a Box/Const to the corresponding Box/Const/VirtualBox by following the dict _equals. """ - while box in self._equals: - # follow the chain: box -> box2 -> box3 -> ... - box2 = self._equals[box] - if box2 not in self._equals: - return box2 - # compress the mapping one step (rare case) - box3 = self._equals[box2] - self._equals[box] = box3 - box = box3 + box = self._equals.get(box, box) + assert box not in self._equals return box def _make_equal(self, box, box2): @@ -202,14 +195,14 @@ self.make_constant_class(instbox, clsbox) def optimize_OONONNULL(self, op): - if self.has_constant_class(op.args[0]): + if self.known_nonnull(op.args[0]): assert op.result.getint() == 1 self.make_constant(op.result) else: self.optimize_default(op) def optimize_OOISNULL(self, op): - if self.has_constant_class(op.args[0]): + if self.known_nonnull(op.args[0]): assert op.result.getint() == 0 self.make_constant(op.result) else: @@ -250,6 +243,7 @@ if isinstance(instbox, VirtualBox): self._make_equal(op.result, instbox.fields[op.descr]) else: + self.make_nonnull(instbox) self.optimize_default(op) optimize_GETFIELD_PURE_GC = optimize_GETFIELD_GC @@ -259,6 +253,7 @@ if isinstance(instbox, VirtualBox): instbox.fields[op.descr] = op.args[1] else: + self.make_nonnull(instbox) self.optimize_default(op) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 18:33:03 2009 @@ -253,6 +253,38 @@ """ self.optimize_loop(ops, 'Not', expected, i0=1, i1=0, i2=1, i3=0) + def test_nonnull_1(self): + ops = """ + [p0] + setfield_gc(p0, 5, descr=valuedescr) # forces p0 != NULL + i0 = ooisnot(p0, NULL) + guard_true(i0) + fail() + i1 = oois(p0, NULL) + guard_false(i1) + fail() + i2 = ooisnot(NULL, p0) + guard_true(i0) + fail() + i3 = oois(NULL, p0) + guard_false(i1) + fail() + i4 = oononnull(p0) + guard_true(i4) + fail() + i5 = ooisnull(p0) + guard_false(i5) + fail() + jump(p0) + """ + expected = """ + [p0] + setfield_gc(p0, 5, descr=valuedescr) + jump(p0) + """ + self.optimize_loop(ops, 'Not', expected, + i0=1, i1=0, i2=1, i3=0, i4=1, i5=0) + # ---------- def test_virtual_1(self): From arigo at codespeak.net Mon Jul 20 18:49:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 18:49:57 +0200 (CEST) Subject: [pypy-svn] r66451 - pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph Message-ID: <20090720164957.406D216849D@codespeak.net> Author: arigo Date: Mon Jul 20 18:49:56 2009 New Revision: 66451 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py Log: Forgot to check this in. A small extension needed. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py Mon Jul 20 18:49:56 2009 @@ -43,6 +43,9 @@ def sort_key(self): return self.ofs + def is_pointer_field(self): + return self.typeinfo == 'p' + def equals(self, other): if not isinstance(other, Descr): return False @@ -679,10 +682,14 @@ self.getfield = getfield self.setfield = setfield + self._is_pointer_field = (history.getkind(T) == 'obj') def sort_key(self): return self._keys.getkey((self.TYPE, self.fieldname)) + def is_pointer_field(self): + return self._is_pointer_field + def equals(self, other): return self.TYPE == other.TYPE and \ self.fieldname == other.fieldname From benjamin at codespeak.net Mon Jul 20 18:56:17 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Mon, 20 Jul 2009 18:56:17 +0200 (CEST) Subject: [pypy-svn] r66452 - in pypy/branch/parser-compiler: lib-python/modified-2.5.2 pypy/config pypy/module/symbol pypy/module/token pypy/module/token/test Message-ID: <20090720165617.77CB7169E22@codespeak.net> Author: benjamin Date: Mon Jul 20 18:56:13 2009 New Revision: 66452 Added: pypy/branch/parser-compiler/pypy/module/token/ pypy/branch/parser-compiler/pypy/module/token/__init__.py (contents, props changed) pypy/branch/parser-compiler/pypy/module/token/test/ pypy/branch/parser-compiler/pypy/module/token/test/test_token.py (contents, props changed) Removed: pypy/branch/parser-compiler/lib-python/modified-2.5.2/symbol.py pypy/branch/parser-compiler/lib-python/modified-2.5.2/token.py Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py pypy/branch/parser-compiler/pypy/module/symbol/__init__.py Log: implement token and symbol as builtin modules Modified: pypy/branch/parser-compiler/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/parser-compiler/pypy/config/pypyoption.py (original) +++ pypy/branch/parser-compiler/pypy/config/pypyoption.py Mon Jul 20 18:56:13 2009 @@ -18,7 +18,7 @@ default_modules.update(dict.fromkeys( ["_codecs", "gc", "_weakref", "marshal", "errno", "math", "_sre", "_pickle_support", "operator", - "parser", "symbol", "_random", "__pypy__"])) + "parser", "symbol", "token", "_random", "__pypy__"])) # --allworkingmodules Modified: pypy/branch/parser-compiler/pypy/module/symbol/__init__.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/symbol/__init__.py (original) +++ pypy/branch/parser-compiler/pypy/module/symbol/__init__.py Mon Jul 20 18:56:13 2009 @@ -5,10 +5,7 @@ """ from pypy.interpreter.mixedmodule import MixedModule - -# Forward imports so they run at startup time -import pypy.interpreter.pyparser.pythonlexer -import pypy.interpreter.pyparser.pythonparse +from pypy.interpreter.pyparser import pygram class Module(MixedModule): @@ -17,23 +14,11 @@ interpleveldefs = {} # see below -# Export the values from our custom symbol module. -# Skip negative values (the corresponding symbols are not visible in -# pure Python). -sym_name = {} - -def _init_symbols(grammar_version): - global sym_name - +def _init_symbols(): sym_name = {} - from pypy.interpreter.pyparser.pythonparse import make_pyparser - parser = make_pyparser(grammar_version) - - for name, val in parser.symbols.items(): - if val >= 0: - Module.interpleveldefs[name] = 'space.wrap(%d)' % val - sym_name[val] = name + for name, val in pygram.python_grammar.symbol_ids.iteritems(): + Module.interpleveldefs[name] = 'space.wrap(%d)' % val + sym_name[val] = name Module.interpleveldefs['sym_name'] = 'space.wrap(%r)' % (sym_name,) -# This is very evil -_init_symbols('2.5') +_init_symbols() Added: pypy/branch/parser-compiler/pypy/module/token/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/token/__init__.py Mon Jul 20 18:56:13 2009 @@ -0,0 +1,38 @@ +from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.pyparser import pytoken, pygram + + +class Module(MixedModule): + + appleveldefs = {} + interpleveldefs = { + "NT_OFFSET" : "space.wrap(256)", + "ISTERMINAL" : "__init__.isterminal", + "ISNONTERMINAL" : "__init__.isnonterminal", + "ISEOF" : "__init__.iseof" + } + + +def _init_tokens(): + tok_name = {} + for tok, id in pytoken.python_tokens.iteritems(): + Module.interpleveldefs[tok] = "space.wrap(%d)" % (id,) + tok_name[id] = tok + Module.interpleveldefs["tok_name"] = "space.wrap(%r)" % (tok_name,) + Module.interpleveldefs["N_TOKENS"] = "space.wrap(%d)" % len(tok_name) + +_init_tokens() + + +def isterminal(space, tok): + return space.wrap(tok < 256) +isterminal.unwrap_spec = [ObjSpace, int] + +def isnonterminal(space, tok): + return space.wrap(tok >= 256) +isnonterminal.unwrap_spec = [ObjSpace, int] + +def iseof(space, tok): + return space.wrap(tok == pygram.tokens.ENDMARKER) +iseof.unwrap_spec = [ObjSpace, int] Added: pypy/branch/parser-compiler/pypy/module/token/test/test_token.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/module/token/test/test_token.py Mon Jul 20 18:56:13 2009 @@ -0,0 +1,18 @@ +class AppTestToken: + + def setup_class(cls): + cls.w_token = cls.space.appexec([], """(): + import token + return token""") + + def test_isterminal(self): + assert self.token.ISTERMINAL(self.token.ENDMARKER) + assert not self.token.ISTERMINAL(300) + + def test_isnonterminal(self): + assert self.token.ISNONTERMINAL(300) + assert not self.token.ISNONTERMINAL(self.token.NAME) + + def test_iseof(self): + assert self.token.ISEOF(self.token.ENDMARKER) + assert not self.token.ISEOF(self.token.NAME) From arigo at codespeak.net Mon Jul 20 19:06:16 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 19:06:16 +0200 (CEST) Subject: [pypy-svn] r66453 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090720170616.2D016169E74@codespeak.net> Author: arigo Date: Mon Jul 20 19:06:14 2009 New Revision: 66453 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Another test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Jul 20 19:06:14 2009 @@ -414,6 +414,20 @@ """ self.find_nodes(ops, 'Not, Not') + def test_find_nodes_default_field(self): + ops = """ + [p0] + i0 = getfield_gc(p0, descr=valuedescr) + guard_value(i0, 0) + fail() + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + # the field 'value' has its default value of 0 + jump(p1) + """ + # Unclear if the answer must contain or not the 'value' field. + # For now it does not. + self.find_nodes(ops, 'Virtual(node_vtable)') + # ------------------------------ # Bridge tests From arigo at codespeak.net Mon Jul 20 20:04:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 20:04:24 +0200 (CEST) Subject: [pypy-svn] r66454 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090720180424.09277169E0A@codespeak.net> Author: arigo Date: Mon Jul 20 20:04:23 2009 New Revision: 66454 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py Log: Not used. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py Mon Jul 20 20:04:23 2009 @@ -231,8 +231,6 @@ def _getrepr_(self): return self.value - sort_key = getint - def repr_rpython(self): return repr_rpython(self, 'ci') From arigo at codespeak.net Mon Jul 20 20:05:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 20:05:15 +0200 (CEST) Subject: [pypy-svn] r66455 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720180515.98054169E28@codespeak.net> Author: arigo Date: Mon Jul 20 20:05:14 2009 New Revision: 66455 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Fix test_find_nodes_default_field, with a comment about why the previous answer was wrong after all. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Mon Jul 20 20:05:14 2009 @@ -193,6 +193,13 @@ # fields = [] d = exitnode.curfields + if inputnode.origfields is not None: + if d is not None: + d = d.copy() + else: + d = av_newdict() + for ofs in inputnode.origfields: + d.setdefault(ofs, self.node_escaped) if d is not None: lst = d.keys() sort_descrs(lst) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Jul 20 20:05:14 2009 @@ -424,9 +424,9 @@ # the field 'value' has its default value of 0 jump(p1) """ - # Unclear if the answer must contain or not the 'value' field. - # For now it does not. - self.find_nodes(ops, 'Virtual(node_vtable)') + # The answer must contain the 'value' field, because otherwise + # we might get incorrect results: when tracing, maybe i0 was not 0. + self.find_nodes(ops, 'Virtual(node_vtable, valuedescr=Not)') # ------------------------------ # Bridge tests From arigo at codespeak.net Mon Jul 20 20:13:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 20:13:55 +0200 (CEST) Subject: [pypy-svn] r66456 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720181355.55844169DF6@codespeak.net> Author: arigo Date: Mon Jul 20 20:13:54 2009 New Revision: 66456 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: A first simple test about NEW_WITH_VTABLE. Skipped the more complicated test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 20:13:54 2009 @@ -241,6 +241,7 @@ def optimize_GETFIELD_GC(self, op): instbox = op.args[0] if isinstance(instbox, VirtualBox): + # optimizefindnode should ensure that 'op.descr in instbox.fields' self._make_equal(op.result, instbox.fields[op.descr]) else: self.make_nonnull(instbox) @@ -256,5 +257,8 @@ self.make_nonnull(instbox) self.optimize_default(op) + def optimize_NEW_WITH_VTABLE(self, op): + self.make_virtual(op.result, op.args[0]) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 20:13:54 2009 @@ -303,6 +303,23 @@ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected) + def test_virtual_2(self): + ops = """ + [i, p0] + i0 = getfield_gc(p0, descr=valuedescr) + i1 = int_add(i0, i) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i1, descr=valuedescr) + jump(i, p1) + """ + expected = """ + [i, i2] + i1 = int_add(i2, i) + jump(i, i1) + """ + self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', + expected) + def test_virtual_oois(self): ops = """ [p0, p1, p2] @@ -362,6 +379,28 @@ expected2 = ops self.optimize_loop(ops, 'Not, Not, Not', expected2) + def test_virtual_default_field(self): + py.test.skip("in-progress") + ops = """ + [p0] + i0 = getfield_gc(p0, descr=valuedescr) + guard_value(i0, 0) + fail() + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + # the field 'value' has its default value of 0 + jump(p1) + """ + expected = """ + [i] + guard_value(i, 0) + fail() + jump(0) + """ + # the 'expected' is sub-optimal, but it should be done by another later + # optimization step. See test_find_nodes_default_field() for why. + self.optimize_loop(ops, 'Virtual(node_vtable, valuedescr=Not)', + expected) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Mon Jul 20 20:18:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 20:18:45 +0200 (CEST) Subject: [pypy-svn] r66457 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720181845.EBFFD169DF6@codespeak.net> Author: arigo Date: Mon Jul 20 20:18:45 2009 New Revision: 66457 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Pass the next test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py Mon Jul 20 20:18:45 2009 @@ -317,6 +317,8 @@ def repr_rpython(self): return repr_rpython(self, 'cp') +CONST_NULL = ConstPtr(ConstPtr.value) + class ConstObj(Const): type = OBJ value = ootype.NULL @@ -360,6 +362,8 @@ def repr_rpython(self): return repr_rpython(self, 'co') +CONST_NULL_OBJ = ConstObj(ConstObj.value) + class Box(AbstractValue): __slots__ = () _extended_display = True Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 20:18:45 2009 @@ -1,5 +1,5 @@ from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxObj -from pypy.jit.metainterp.history import AbstractValue +from pypy.jit.metainterp import history from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode @@ -19,7 +19,7 @@ # ____________________________________________________________ -class VirtualBox(AbstractValue): +class VirtualBox(history.AbstractValue): def __init__(self): self.fields = av_newdict() # ofs -> Box @@ -30,7 +30,7 @@ class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): newinputargs.append(box) - def teardown_virtual_node(self, box, newexitargs): + def teardown_virtual_node(self, optimizer, box, newexitargs): newexitargs.append(box) class __extend__(VirtualInstanceSpecNode): @@ -40,11 +40,14 @@ subbox = optimizer.make_box(ofs) vbox.fields[ofs] = subbox subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) - def teardown_virtual_node(self, box, newexitargs): + def teardown_virtual_node(self, optimizer, box, newexitargs): assert isinstance(box, VirtualBox) for ofs, subspecnode in self.fields: - # XXX if ofs not in box.fields:... - subspecnode.teardown_virtual_node(box.fields[ofs], newexitargs) + try: + subbox = box.fields[ofs] + except KeyError: + subbox = optimizer.make_const(ofs) + subspecnode.teardown_virtual_node(optimizer, subbox, newexitargs) class Optimizer(object): @@ -107,6 +110,15 @@ else: return BoxInt() + def make_const(self, fieldofs): + if fieldofs.is_pointer_field(): + if not self.cpu.is_oo: + return history.CONST_NULL + else: + return history.CONST_NULL_OBJ + else: + return history.CONST_FALSE + def known_nonnull(self, box): if isinstance(box, Box): return box in self._known_classes @@ -163,7 +175,7 @@ specnodes = orgop.jump_target.specnodes assert len(op.args) == len(specnodes) for i in range(len(specnodes)): - specnodes[i].teardown_virtual_node(op.args[i], exitargs) + specnodes[i].teardown_virtual_node(self, op.args[i], exitargs) op.args = exitargs self.emit_operation(op) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 20:18:45 2009 @@ -380,7 +380,6 @@ self.optimize_loop(ops, 'Not, Not, Not', expected2) def test_virtual_default_field(self): - py.test.skip("in-progress") ops = """ [p0] i0 = getfield_gc(p0, descr=valuedescr) From arigo at codespeak.net Mon Jul 20 22:57:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 22:57:22 +0200 (CEST) Subject: [pypy-svn] r66459 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720205722.DC3EB169DFB@codespeak.net> Author: arigo Date: Mon Jul 20 22:57:21 2009 New Revision: 66459 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: In-progress: refactoring the data model. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 22:57:21 2009 @@ -1,6 +1,6 @@ from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxObj from pypy.jit.metainterp import history -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.optimizeutil import av_newdict, _findall @@ -19,12 +19,84 @@ # ____________________________________________________________ -class VirtualBox(history.AbstractValue): - def __init__(self): - self.fields = av_newdict() # ofs -> Box +LEVEL_UNKNOWN = 0 +LEVEL_NONNULL = 1 +LEVEL_KNOWNCLASS = 2 +LEVEL_CONSTANT = 3 + + +class InstanceValue(object): + level = LEVEL_UNKNOWN + + def __init__(self, box): + self.box = box + if isinstance(box, Const): + self.level = LEVEL_CONSTANT + + def force_box(self): + return self.box + + def is_constant(self): + return self.level == LEVEL_CONSTANT + + def is_null(self): + return self.is_constant() and not self.box.nonnull() + + def make_constant(self): + """Mark 'self' as actually representing a Const value.""" + self.box = self.force_box().constbox() + self.level = LEVEL_CONSTANT + + def has_constant_class(self): + return self.level >= LEVEL_KNOWNCLASS + + def make_constant_class(self): + if self.level < LEVEL_KNOWNCLASS: + self.level = LEVEL_KNOWNCLASS + + def is_nonnull(self): + level = self.level + if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS: + return True + elif level == LEVEL_CONSTANT: + return self.box.nonnull() + else: + return False + + def make_nonnull(self): + if self.level < LEVEL_NONNULL: + self.level = LEVEL_NONNULL + + def is_virtual(self): + return self.box is None + + +class ConstantValue(InstanceValue): + level = LEVEL_CONSTANT + + def __init__(self, box): + self.box = box + + +class VirtualValue(InstanceValue): + box = None + level = LEVEL_KNOWNCLASS + + def __init__(self, optimizer): + self.optimizer = optimizer + self._fields = av_newdict() - def nonnull(self): - return True + def getfield(self, ofs): + return self._fields[ofs] + + def setfield(self, ofs, fieldvalue): + self._fields[ofs] = fieldvalue + + def force_box(self): + if self.box is None: + optimizer = self.optimizer + xxxx + return self.box class __extend__(SpecNode): @@ -35,18 +107,19 @@ class __extend__(VirtualInstanceSpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): + import py; py.test.skip("in-progress") vbox = optimizer.make_virtual(box, self.known_class) for ofs, subspecnode in self.fields: - subbox = optimizer.make_box(ofs) + subbox = optimizer.new_box(ofs) vbox.fields[ofs] = subbox subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) def teardown_virtual_node(self, optimizer, box, newexitargs): - assert isinstance(box, VirtualBox) + assert isinstance(box, VirtualBox) and box.escaped_to is None for ofs, subspecnode in self.fields: try: subbox = box.fields[ofs] except KeyError: - subbox = optimizer.make_const(ofs) + subbox = optimizer.new_const(ofs) subspecnode.teardown_virtual_node(optimizer, subbox, newexitargs) @@ -55,53 +128,34 @@ def __init__(self, cpu, loop): self.cpu = cpu self.loop = loop - # Boxes used a keys to _equals have been proven to be equal to - # something else: another Box, a Const, or a VirtualBox. Boxes - # and VirtualBoxes can also be listed in _known_classes if we - # know their class (or just know that they are non-null, in which - # case we use None). Boxes *must not* be keys in both dicts. - self._equals = {} # mapping Box -> Box/Const/VirtualBox - self._known_classes = {} # mapping Box -> ConstClass/None - - def deref(self, box): - """Maps a Box/Const to the corresponding Box/Const/VirtualBox - by following the dict _equals. - """ - box = self._equals.get(box, box) - assert box not in self._equals - return box - - def _make_equal(self, box, box2): - assert isinstance(box, Box) - assert box not in self._equals - assert box not in self._known_classes - self._equals[box] = box2 + self.values = {} + + def getvalue(self, box): + try: + value = self.values[box] + except KeyError: + value = self.values[box] = InstanceValue(box) + return value + + def is_constant(self, box): + if isinstance(box, Const): + return True + try: + return self.values[box].is_constant() + except KeyError: + return False + + def make_equal_to(self, box, value): + assert box not in self.values + self.values[box] = value def make_constant(self, box): - """Mark the given Box as actually representing a Const value.""" - self._make_equal(box, box.constbox()) + self.make_equal_to(box, ConstantValue(box.constbox())) - def make_virtual(self, box, clsbox): - """Mark the given Box as actually representing a VirtualBox value.""" - vbox = VirtualBox() - self._make_equal(box, vbox) - self.make_constant_class(vbox, clsbox) - return vbox - - def has_constant_class(self, box): - return (isinstance(box, Const) or - self._known_classes.get(box, None) is not None) - - def make_constant_class(self, box, clsbox): - assert isinstance(clsbox, Const) - assert box not in self._equals - self._known_classes[box] = clsbox - - def make_nonnull(self, box): - assert box not in self._equals - self._known_classes.setdefault(box, None) + def known_nonnull(self, box): + return self.getvalue(box).is_nonnull() - def make_box(self, fieldofs): + def new_box(self, fieldofs): if fieldofs.is_pointer_field(): if not self.cpu.is_oo: return BoxPtr() @@ -110,7 +164,7 @@ else: return BoxInt() - def make_const(self, fieldofs): + def new_const(self, fieldofs): if fieldofs.is_pointer_field(): if not self.cpu.is_oo: return history.CONST_NULL @@ -119,12 +173,6 @@ else: return history.CONST_FALSE - def known_nonnull(self, box): - if isinstance(box, Box): - return box in self._known_classes - else: - return box.nonnull() # Consts or VirtualBoxes - # ---------- def setup_virtuals(self): @@ -141,26 +189,32 @@ def propagate_forward(self): self.newoperations = [] for op in self.loop.operations: - op2 = op.clone() - op2.args = [self.deref(box) for box in op.args] - opnum = op2.opnum + opnum = op.opnum for value, func in optimize_ops: if opnum == value: - func(self, op2) + func(self, op) break else: - self.optimize_default(op2) + self.optimize_default(op) self.loop.operations = self.newoperations def emit_operation(self, op): - for x in op.args: - assert not isinstance(x, VirtualBox) + must_clone = True + for i in range(len(op.args)): + arg = op.args[i] + if arg in self.values: + box = self.values[arg].force_box() + if box is not arg: + if must_clone: + op = op.clone() + must_clone = False + op.args[i] = box self.newoperations.append(op) def optimize_default(self, op): if op.is_always_pure(): - for box in op.args: - if not isinstance(box, Const): + for arg in op.args: + if not self.is_constant(arg): break else: # all constant arguments: constant-fold away @@ -169,7 +223,7 @@ # otherwise, the operation remains self.emit_operation(op) - def optimize_JUMP(self, op): + def XXX_optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] specnodes = orgop.jump_target.specnodes @@ -177,34 +231,34 @@ for i in range(len(specnodes)): specnodes[i].teardown_virtual_node(self, op.args[i], exitargs) op.args = exitargs + #XXX return op + + def optimize_guard(self, op): + value = self.getvalue(op.args[0]) + if value.is_constant(): + return self.emit_operation(op) + value.make_constant() def optimize_GUARD_VALUE(self, op): assert isinstance(op.args[1], Const) - assert op.args[0].get_() == op.args[1].get_() - if not isinstance(op.args[0], Const): - self.emit_operation(op) - self.make_constant(op.args[0]) + self.optimize_guard(op) def optimize_GUARD_TRUE(self, op): assert op.args[0].getint() - if not isinstance(op.args[0], Const): - self.emit_operation(op) - self.make_constant(op.args[0]) + self.optimize_guard(op) def optimize_GUARD_FALSE(self, op): assert not op.args[0].getint() - if not isinstance(op.args[0], Const): - self.emit_operation(op) - self.make_constant(op.args[0]) + self.optimize_guard(op) def optimize_GUARD_CLASS(self, op): - instbox = op.args[0] - clsbox = op.args[1] # XXX should probably assert that the class is right - if not self.has_constant_class(instbox): - self.emit_operation(op) - self.make_constant_class(instbox, clsbox) + value = self.getvalue(op.args[0]) + if value.has_constant_class(): + return + self.emit_operation(op) + value.make_constant_class() def optimize_OONONNULL(self, op): if self.known_nonnull(op.args[0]): @@ -221,56 +275,56 @@ self.optimize_default(op) def optimize_OOISNOT(self, op): - if (isinstance(op.args[0], VirtualBox) or - isinstance(op.args[1], VirtualBox)): + value0 = self.getvalue(op.args[0]) + value1 = self.getvalue(op.args[1]) + if value0.is_virtual() or value1.is_virtual(): self.make_constant(op.result) - elif self.known_nonnull(op.args[1]): - op.opnum = rop.OONONNULL - del op.args[1] + elif value1.is_null(): + op = ResOperation(rop.OONONNULL, [op.args[0]], op.result) self.optimize_OONONNULL(op) - elif self.known_nonnull(op.args[0]): - op.opnum = rop.OONONNULL - del op.args[0] + elif value0.is_null(): + op = ResOperation(rop.OONONNULL, [op.args[1]], op.result) self.optimize_OONONNULL(op) else: self.optimize_default(op) def optimize_OOIS(self, op): - if (isinstance(op.args[0], VirtualBox) or - isinstance(op.args[1], VirtualBox)): + value0 = self.getvalue(op.args[0]) + value1 = self.getvalue(op.args[1]) + if value0.is_virtual() or value1.is_virtual(): self.make_constant(op.result) - elif self.known_nonnull(op.args[1]): - op.opnum = rop.OOISNULL - del op.args[1] + elif value1.is_null(): + op = ResOperation(rop.OOISNULL, [op.args[0]], op.result) self.optimize_OOISNULL(op) - elif self.known_nonnull(op.args[0]): - op.opnum = rop.OOISNULL - del op.args[0] + elif value0.is_null(): + op = ResOperation(rop.OOISNULL, [op.args[1]], op.result) self.optimize_OOISNULL(op) else: self.optimize_default(op) def optimize_GETFIELD_GC(self, op): - instbox = op.args[0] - if isinstance(instbox, VirtualBox): - # optimizefindnode should ensure that 'op.descr in instbox.fields' - self._make_equal(op.result, instbox.fields[op.descr]) + value = self.getvalue(op.args[0]) + if value.is_virtual(): + # optimizefindnode should ensure that we don't get a KeyError + fieldvalue = value.getfield(op.descr) + self.make_equal_to(op.result, fieldvalue) else: - self.make_nonnull(instbox) + value.make_nonnull() self.optimize_default(op) - optimize_GETFIELD_PURE_GC = optimize_GETFIELD_GC + def optimize_GETFIELD_PURE_GC(self, op): + xxx # optimize_GETFIELD_GC def optimize_SETFIELD_GC(self, op): - instbox = op.args[0] - if isinstance(instbox, VirtualBox): - instbox.fields[op.descr] = op.args[1] + value = self.getvalue(op.args[0]) + if value.is_virtual(): + value.setfield(op.descr, self.getvalue(op.args[1])) else: - self.make_nonnull(instbox) + value.make_nonnull() self.optimize_default(op) def optimize_NEW_WITH_VTABLE(self, op): - self.make_virtual(op.result, op.args[0]) + import py; py.test.skip("in-progress") optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 22:57:21 2009 @@ -400,6 +400,57 @@ self.optimize_loop(ops, 'Virtual(node_vtable, valuedescr=Not)', expected) + def test_virtual_3(self): + ops = """ + [i] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i, descr=valuedescr) + i0 = getfield_gc(p1, descr=valuedescr) + i1 = int_add(i0, 1) + jump(i1) + """ + expected = """ + [i] + i1 = int_add(i, 1) + jump(i1) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_nonvirtual_1(self): + ops = """ + [i] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i, descr=valuedescr) + i0 = getfield_gc(p1, descr=valuedescr) + i1 = int_add(i0, 1) + escape(p1) + escape(p1) + jump(i1) + """ + expected = """ + [i] + i1 = int_add(i, 1) + p9 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p9, i, descr=valuedescr) + escape(p9) + escape(p9) + jump(i1) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_nonvirtual_2(self): + ops = """ + [i, p0] + i0 = getfield_gc(p0, descr=valuedescr) + escape(p0) + i1 = int_add(i0, i) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i1, descr=valuedescr) + jump(i, p1) + """ + expected = ops + self.optimize_loop(ops, 'Not, Not', expected) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Mon Jul 20 23:09:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 23:09:13 +0200 (CEST) Subject: [pypy-svn] r66460 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720210913.CFE66169E03@codespeak.net> Author: arigo Date: Mon Jul 20 23:09:12 2009 New Revision: 66460 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: In-progress. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 23:09:12 2009 @@ -89,6 +89,9 @@ def getfield(self, ofs): return self._fields[ofs] + def getfield_default(self, ofs, default): + return self._fields.get(ofs, default) + def setfield(self, ofs, fieldvalue): self._fields[ofs] = fieldvalue @@ -102,25 +105,21 @@ class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): newinputargs.append(box) - def teardown_virtual_node(self, optimizer, box, newexitargs): - newexitargs.append(box) + def teardown_virtual_node(self, optimizer, value, newexitargs): + newexitargs.append(value.force_box()) class __extend__(VirtualInstanceSpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): - import py; py.test.skip("in-progress") - vbox = optimizer.make_virtual(box, self.known_class) + vvalue = optimizer.make_virtual(box) for ofs, subspecnode in self.fields: subbox = optimizer.new_box(ofs) - vbox.fields[ofs] = subbox + vvalue.setfield(ofs, optimizer.getvalue(subbox)) subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) - def teardown_virtual_node(self, optimizer, box, newexitargs): - assert isinstance(box, VirtualBox) and box.escaped_to is None + def teardown_virtual_node(self, optimizer, value, newexitargs): + assert value.is_virtual() for ofs, subspecnode in self.fields: - try: - subbox = box.fields[ofs] - except KeyError: - subbox = optimizer.new_const(ofs) - subspecnode.teardown_virtual_node(optimizer, subbox, newexitargs) + subvalue = value.getfield_default(ofs, optimizer.new_const(ofs)) + subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) class Optimizer(object): @@ -155,6 +154,11 @@ def known_nonnull(self, box): return self.getvalue(box).is_nonnull() + def make_virtual(self, box): + vvalue = VirtualValue(self) + self.make_equal_to(box, vvalue) + return vvalue + def new_box(self, fieldofs): if fieldofs.is_pointer_field(): if not self.cpu.is_oo: @@ -198,8 +202,7 @@ self.optimize_default(op) self.loop.operations = self.newoperations - def emit_operation(self, op): - must_clone = True + def emit_operation(self, op, must_clone=True): for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -223,15 +226,17 @@ # otherwise, the operation remains self.emit_operation(op) - def XXX_optimize_JUMP(self, op): + def optimize_JUMP(self, op): orgop = self.loop.operations[-1] exitargs = [] specnodes = orgop.jump_target.specnodes assert len(op.args) == len(specnodes) for i in range(len(specnodes)): - specnodes[i].teardown_virtual_node(self, op.args[i], exitargs) + value = self.getvalue(op.args[i]) + specnodes[i].teardown_virtual_node(self, value, exitargs) + op = op.clone() op.args = exitargs - #XXX return op + self.emit_operation(op, must_clone=False) def optimize_guard(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 23:09:12 2009 @@ -376,7 +376,46 @@ # to be complete, we also check the no-opt case where most comparisons # are not removed. The exact set of comparisons removed depends on # the details of the algorithm... - expected2 = ops + expected2 = """ + [p0, p1, p2] + i1 = oononnull(p0) + guard_true(i1) + fail() + i2 = ooisnull(p0) + guard_false(i2) + fail() + i3 = oononnull(p0) + guard_true(i3) + fail() + i4 = ooisnull(p0) + guard_false(i4) + fail() + i5 = oononnull(p0) + guard_true(i5) + fail() + i6 = ooisnull(p0) + guard_false(i6) + fail() + i7 = ooisnot(p0, p1) + guard_true(i7) + fail() + i8 = oois(p0, p1) + guard_false(i8) + fail() + i9 = ooisnot(p0, p2) + guard_true(i9) + fail() + i10 = oois(p0, p2) + guard_false(i10) + fail() + i11 = ooisnot(p2, p1) + guard_true(i11) + fail() + i12 = oois(p2, p1) + guard_false(i12) + fail() + jump(p0, p1, p2) + """ self.optimize_loop(ops, 'Not, Not, Not', expected2) def test_virtual_default_field(self): From arigo at codespeak.net Mon Jul 20 23:13:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 23:13:51 +0200 (CEST) Subject: [pypy-svn] r66461 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090720211351.7B176169E0B@codespeak.net> Author: arigo Date: Mon Jul 20 23:13:50 2009 New Revision: 66461 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Log: Implement NEW_WITH_VTABLE again. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py Mon Jul 20 23:13:50 2009 @@ -317,8 +317,6 @@ def repr_rpython(self): return repr_rpython(self, 'cp') -CONST_NULL = ConstPtr(ConstPtr.value) - class ConstObj(Const): type = OBJ value = ootype.NULL @@ -362,8 +360,6 @@ def repr_rpython(self): return repr_rpython(self, 'co') -CONST_NULL_OBJ = ConstObj(ConstObj.value) - class Box(AbstractValue): __slots__ = () _extended_display = True Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 23:13:50 2009 @@ -1,5 +1,5 @@ -from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxObj -from pypy.jit.metainterp import history +from pypy.jit.metainterp.history import Box, BoxInt, BoxPtr, BoxObj +from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode @@ -77,6 +77,10 @@ def __init__(self, box): self.box = box +CVAL_ZERO = ConstantValue(ConstInt(0)) +CVAL_NULLPTR = ConstantValue(ConstPtr(ConstPtr.value)) +CVAL_NULLOBJ = ConstantValue(ConstObj(ConstObj.value)) + class VirtualValue(InstanceValue): box = None @@ -98,7 +102,7 @@ def force_box(self): if self.box is None: optimizer = self.optimizer - xxxx + import py; py.test.skip("in-progress") return self.box @@ -171,11 +175,11 @@ def new_const(self, fieldofs): if fieldofs.is_pointer_field(): if not self.cpu.is_oo: - return history.CONST_NULL + return CVAL_NULLPTR else: - return history.CONST_NULL_OBJ + return CVAL_NULLOBJ else: - return history.CONST_FALSE + return CVAL_ZERO # ---------- @@ -329,7 +333,7 @@ self.optimize_default(op) def optimize_NEW_WITH_VTABLE(self, op): - import py; py.test.skip("in-progress") + self.make_virtual(op.result) optimize_ops = _findall(Optimizer, 'optimize_') From arigo at codespeak.net Mon Jul 20 23:23:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 20 Jul 2009 23:23:30 +0200 (CEST) Subject: [pypy-svn] r66462 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090720212330.55F7B169E23@codespeak.net> Author: arigo Date: Mon Jul 20 23:23:28 2009 New Revision: 66462 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Pass the remaining tests. Yay! Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Mon Jul 20 23:23:28 2009 @@ -86,8 +86,9 @@ box = None level = LEVEL_KNOWNCLASS - def __init__(self, optimizer): + def __init__(self, optimizer, source_op=None): self.optimizer = optimizer + self.source_op = source_op # the NEW_WITH_VTABLE operation building it self._fields = av_newdict() def getfield(self, ofs): @@ -101,8 +102,16 @@ def force_box(self): if self.box is None: - optimizer = self.optimizer - import py; py.test.skip("in-progress") + assert self.source_op is not None # otherwise, we are trying + # to force a Virtual from a specnode computed by optimizefindnode. + newoperations = self.optimizer.newoperations + newoperations.append(self.source_op) + self.box = box = self.source_op.result + for ofs in self._fields: + subbox = self._fields[ofs].force_box() + op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, + descr=ofs) + newoperations.append(op) return self.box @@ -158,8 +167,8 @@ def known_nonnull(self, box): return self.getvalue(box).is_nonnull() - def make_virtual(self, box): - vvalue = VirtualValue(self) + def make_virtual(self, box, source_op=None): + vvalue = VirtualValue(self, source_op) self.make_equal_to(box, vvalue) return vvalue @@ -333,7 +342,7 @@ self.optimize_default(op) def optimize_NEW_WITH_VTABLE(self, op): - self.make_virtual(op.result) + self.make_virtual(op.result, op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 20 23:23:28 2009 @@ -469,10 +469,10 @@ expected = """ [i] i1 = int_add(i, 1) - p9 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p9, i, descr=valuedescr) - escape(p9) - escape(p9) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i, descr=valuedescr) + escape(p1) + escape(p1) jump(i1) """ self.optimize_loop(ops, 'Not', expected) From benjamin at codespeak.net Tue Jul 21 00:06:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 00:06:44 +0200 (CEST) Subject: [pypy-svn] r66463 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090720220644.1F633169E18@codespeak.net> Author: benjamin Date: Tue Jul 21 00:06:43 2009 New Revision: 66463 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: non-extended slices have a step of None Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 21 00:06:43 2009 @@ -935,7 +935,11 @@ if third_child.type == syms.test: upper = self.handle_expr(third_child) last_child = slice_node.children[-1] - if last_child.type == syms.sliceop and len(last_child.children) == 2: + if last_child.type == syms.sliceop: + if len(last_child.children) == 1: + step = ast.Name("None", ast.Load, last_child.lineno, + last_child.column) + else: step_child = last_child.children[1] if step_child.type == syms.test: step = self.handle_expr(step_child) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 21 00:06:43 2009 @@ -965,26 +965,40 @@ assert sub.ctx == ast.Load assert isinstance(sub.slice, ast.Index) assert isinstance(sub.slice.value, ast.Name) - for input in (":", "::"): - slc = self.get_first_expr("x[%s]" % (input,)).slice - assert slc.upper is None - assert slc.lower is None - assert slc.step is None - for input in ("1:", "1::"): - slc = self.get_first_expr("x[%s]" % (input,)).slice - assert isinstance(slc.lower, ast.Num) - assert slc.upper is None - assert slc.step is None - for input in (":2", ":2:"): - slc = self.get_first_expr("x[%s]" % (input,)).slice - assert slc.lower is None - assert isinstance(slc.upper, ast.Num) - assert slc.step is None - for input in ("2:2:", "2:2"): - slc = self.get_first_expr("x[%s]" % (input,)).slice - assert isinstance(slc.lower, ast.Num) - assert isinstance(slc.upper, ast.Num) - assert slc.step is None + slc = self.get_first_expr("x[:]").slice + assert slc.upper is None + assert slc.lower is None + assert slc.step is None + slc = self.get_first_expr("x[::]").slice + assert slc.upper is None + assert slc.lower is None + assert isinstance(slc.step, ast.Name) + assert slc.step.id == "None" + assert slc.step.ctx == ast.Load + slc = self.get_first_expr("x[1:]").slice + assert isinstance(slc.lower, ast.Num) + assert slc.upper is None + assert slc.step is None + slc = self.get_first_expr("x[1::]").slice + assert isinstance(slc.lower, ast.Num) + assert slc.upper is None + assert isinstance(slc.step, ast.Name) + slc = self.get_first_expr("x[:2]").slice + assert slc.lower is None + assert isinstance(slc.upper, ast.Num) + assert slc.step is None + slc = self.get_first_expr("x[:2:]").slice + assert slc.lower is None + assert isinstance(slc.upper, ast.Num) + assert isinstance(slc.step, ast.Name) + slc = self.get_first_expr("x[2:2]").slice + assert isinstance(slc.lower, ast.Num) + assert isinstance(slc.upper, ast.Num) + assert slc.step is None + slc = self.get_first_expr("x[2:2:]").slice + assert isinstance(slc.lower, ast.Num) + assert isinstance(slc.upper, ast.Num) + assert isinstance(slc.step, ast.Name) slc = self.get_first_expr("x[::2]").slice assert slc.lower is None assert slc.upper is None From benjamin at codespeak.net Tue Jul 21 01:37:26 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 01:37:26 +0200 (CEST) Subject: [pypy-svn] r66464 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090720233726.B26DF169E0D@codespeak.net> Author: benjamin Date: Tue Jul 21 01:37:25 2009 New Revision: 66464 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: fix parsing of negative hex and octal literals Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 21 01:37:25 2009 @@ -1051,6 +1051,11 @@ def parse_number(self, raw): base = 10 + if raw.startswith("-"): + negative = True + raw = raw.lstrip("-") + else: + negative = False if raw.startswith("0"): if len(raw) > 2 and raw[1] in "Xx": base = 16 @@ -1066,6 +1071,8 @@ raw = raw[i:] if not raw[0].isdigit(): raw = "0" + raw + if negative: + raw = "-" + raw w_num_str = self.space.wrap(raw) w_index = None w_base = self.space.wrap(base) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 21 01:37:25 2009 @@ -1078,6 +1078,7 @@ assert space.eq_w(get_num("00000"), space.wrap(0)) assert space.eq_w(get_num("-3"), space.wrap(-3)) assert space.eq_w(get_num("-0"), space.wrap(0)) + assert space.eq_w(get_num("-0xAAAAAAL"), space.wrap(-0xAAAAAAL)) n = get_num(str(-sys.maxint - 1)) assert space.is_true(space.isinstance(n, space.w_int)) From benjamin at codespeak.net Tue Jul 21 02:03:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 02:03:31 +0200 (CEST) Subject: [pypy-svn] r66465 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090721000331.74733169E0C@codespeak.net> Author: benjamin Date: Tue Jul 21 02:03:30 2009 New Revision: 66465 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: make error message consistent with cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Tue Jul 21 02:03:30 2009 @@ -135,7 +135,7 @@ elif isinstance(expr, ast.Lambda): error = "lambda" elif isinstance(expr, ast.Call): - error = "call" + error = "function call" elif isinstance(expr, ast.BoolOp) or \ isinstance(expr, ast.BinOp) or \ isinstance(expr, ast.UnaryOp): @@ -581,8 +581,8 @@ i += 2 have_default = True elif have_default: - self.error("non-default argument after default one", - arguments_node) + msg = "non-default argument follows default argument" + self.error(msg, arguments_node) if len(argument.children) == 3: sub_arg = argument.children[1] if len(sub_arg.children) != 1: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 21 02:03:30 2009 @@ -547,7 +547,7 @@ assert args.kwarg == "f" input = "def f(a=b, c): pass" exc = py.test.raises(SyntaxError, self.get_ast, input).value - assert exc.msg == "non-default argument after default one" + assert exc.msg == "non-default argument follows default argument"" def test_decorator(self): func = self.get_first_stmt("@dec\ndef f(): pass") @@ -690,7 +690,7 @@ assert all(elt.ctx == ast.Store for elt in seq.elts) invalid_stores = ( ("(lambda x: x)", "lambda"), - ("f()", "call"), + ("f()", "function call"), ("~x", "operator"), ("+x", "operator"), ("-x", "operator"), From benjamin at codespeak.net Tue Jul 21 02:04:38 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 02:04:38 +0200 (CEST) Subject: [pypy-svn] r66466 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090721000438.099CF169E0C@codespeak.net> Author: benjamin Date: Tue Jul 21 02:04:38 2009 New Revision: 66466 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Log: fix syntax Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Tue Jul 21 02:04:38 2009 @@ -547,7 +547,7 @@ assert args.kwarg == "f" input = "def f(a=b, c): pass" exc = py.test.raises(SyntaxError, self.get_ast, input).value - assert exc.msg == "non-default argument follows default argument"" + assert exc.msg == "non-default argument follows default argument" def test_decorator(self): func = self.get_first_stmt("@dec\ndef f(): pass") From benjamin at codespeak.net Tue Jul 21 04:25:23 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 04:25:23 +0200 (CEST) Subject: [pypy-svn] r66467 - in pypy/branch/parser-compiler/pypy: interpreter module/__builtin__/test Message-ID: <20090721022523.4B6ED169E27@codespeak.net> Author: benjamin Date: Tue Jul 21 04:25:22 2009 New Revision: 66467 Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_builtin.py Log: reject invalid flags to compile() Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Tue Jul 21 04:25:22 2009 @@ -231,8 +231,15 @@ from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError from pypy.interpreter.astcompiler.astbuilder import ast_from_node from pypy.interpreter.astcompiler.codegen import compile_ast + from pypy.interpreter.astcompiler import consts space = self.space + + if flags & ~(consts.PyCF_SOURCE_IS_UTF8 | consts.PyCF_DONT_IMPLY_DEDENT + | self.futureFlags.allowed_flags): + raise OperationError(space.w_ValueError, + space.wrap("invalid compile flags")) + space.timer.start("PythonAST compile") try: flags |= getFutures(self.futureFlags, source) Modified: pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_builtin.py ============================================================================== --- pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_builtin.py (original) +++ pypy/branch/parser-compiler/pypy/module/__builtin__/test/test_builtin.py Tue Jul 21 04:25:22 2009 @@ -400,6 +400,7 @@ raises(SyntaxError, compile, '-', '?', 'eval') raises(ValueError, compile, '"\\xt"', '?', 'eval') raises(ValueError, compile, '1+2', '?', 'maybenot') + raises(ValueError, compile, "\n", "", "exec", 0xff) raises(TypeError, compile, '1+2', 12, 34) def test_unicode_compile(self): From benjamin at codespeak.net Tue Jul 21 04:40:09 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 04:40:09 +0200 (CEST) Subject: [pypy-svn] r66468 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721024009.DE6C5169E44@codespeak.net> Author: benjamin Date: Tue Jul 21 04:40:09 2009 New Revision: 66468 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: make error message consistent with cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Tue Jul 21 04:40:09 2009 @@ -413,12 +413,13 @@ self.emit_jump(ops.CONTINUE_LOOP, block, True) break if self.frame_blocks[i][0] == F_BLOCK_FINALLY_END: - self.error("'continue' not allowed in 'finally' clause", + self.error("'continue' not supported inside 'finally' " \ + "clause", cont) else: self.error("'continue' outside loop", cont) elif current_block == F_BLOCK_FINALLY_END: - self.error("'continue' not allowed in 'finally' clause", cont) + self.error("'continue' not supported inside 'finally' clause", cont) def visit_For(self, fr): self.update_position(fr.lineno) From benjamin at codespeak.net Tue Jul 21 04:45:00 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 04:45:00 +0200 (CEST) Subject: [pypy-svn] r66469 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721024500.1BEE116857B@codespeak.net> Author: benjamin Date: Tue Jul 21 04:44:59 2009 New Revision: 66469 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: pass filename argument correctly Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Tue Jul 21 04:44:59 2009 @@ -146,7 +146,7 @@ def error(self, msg, node): raise SyntaxError(msg, node.lineno, node.col_offset, - self.compile_info.filename) + filename=self.compile_info.filename) def name_op(self, identifier, ctx): scope = self.scope.lookup(identifier) From benjamin at codespeak.net Tue Jul 21 04:57:28 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 04:57:28 +0200 (CEST) Subject: [pypy-svn] r66470 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090721025728.690A9169E18@codespeak.net> Author: benjamin Date: Tue Jul 21 04:57:26 2009 New Revision: 66470 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_syntax.py Log: these are clearly implementation details Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_syntax.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_syntax.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_syntax.py Tue Jul 21 04:57:26 2009 @@ -164,13 +164,13 @@ >>> (x for x in x) += 1 # doctest: +ELLIPSIS Traceback (most recent call last): -SyntaxError: augmented assign...generator expression not possible (, line 1) +SyntaxError: can't assign to generator expression (, line 1) >>> None += 1 # doctest: +ELLIPSIS Traceback (most recent call last): SyntaxError: assignment to None... (, line 1) >>> f() += 1 Traceback (most recent call last): -SyntaxError: illegal expression for augmented assignment (, line 1) +SyntaxError: can't assign to function call (, line 1) Test continue in finally in weird combinations. From benjamin at codespeak.net Tue Jul 21 05:19:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 05:19:21 +0200 (CEST) Subject: [pypy-svn] r66471 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721031921.206D7169E1B@codespeak.net> Author: benjamin Date: Tue Jul 21 05:19:19 2009 New Revision: 66471 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: fix some subtle lineno tests Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Tue Jul 21 05:19:19 2009 @@ -541,7 +541,7 @@ # after the loading of "b". This works with the C Python # compiler because it only generates a SET_LINENO instruction # for the assignment. - if line >= 0: + if line > 0: addr = offset - self.current_off if not addr and not line: return From benjamin at codespeak.net Tue Jul 21 14:56:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 14:56:21 +0200 (CEST) Subject: [pypy-svn] r66481 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721125621.ED00A169E29@codespeak.net> Author: benjamin Date: Tue Jul 21 14:56:20 2009 New Revision: 66481 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: switch to cpython error message Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Tue Jul 21 14:56:20 2009 @@ -401,7 +401,7 @@ def visit_Continue(self, cont): if not self.frame_blocks: - self.error("'continue' outside look", cont) + self.error("'continue' not properly in loop", cont) current_block, block = self.frame_blocks[-1] if current_block == F_BLOCK_LOOP: self.emit_jump(ops.JUMP_ABSOLUTE, block, True) @@ -417,7 +417,7 @@ "clause", cont) else: - self.error("'continue' outside loop", cont) + self.error("'continue' not properly in loop", cont) elif current_block == F_BLOCK_FINALLY_END: self.error("'continue' not supported inside 'finally' clause", cont) From benjamin at codespeak.net Tue Jul 21 15:58:07 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 15:58:07 +0200 (CEST) Subject: [pypy-svn] r66483 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721135807.004E3168560@codespeak.net> Author: benjamin Date: Tue Jul 21 15:58:06 2009 New Revision: 66483 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: try to generate lnotab EXACTLY like cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Tue Jul 21 15:58:06 2009 @@ -12,11 +12,10 @@ class Instruction(object): - def __init__(self, opcode, lineno, arg=0): - assert lineno != -1 + def __init__(self, opcode, arg=0): self.opcode = opcode self.arg = arg - self.lineno = lineno + self.lineno = 0 self.has_jump = False def size(self): @@ -122,7 +121,6 @@ self.name = name self.first_lineno = first_lineno self.compile_info = compile_info - self.lineno = -1 self.first_block = self.new_block() self.use_block(self.first_block) self.names = {} @@ -132,6 +130,8 @@ self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars)) self.w_consts = space.newdict() self.argcount = 0 + self.lineno_set = False + self.lineno = 0 self.add_none_to_final_return = True def new_block(self): @@ -149,14 +149,21 @@ return block def emit_op(self, op): - instr = Instruction(op, self.lineno) + instr = Instruction(op) + if not self.lineno_set: + instr.lineno = self.lineno + self.lineno_set = True self.instrs.append(instr) if op == ops.RETURN_VALUE: self.current_block.have_return = True return instr def emit_op_arg(self, op, arg): - self.instrs.append(Instruction(op, self.lineno, arg)) + instr = Instruction(op, arg) + if not self.lineno_set: + instr.lineno = self.lineno + self.lineno_set = True + self.instrs.append(instr) def emit_op_name(self, op, container, name): self.emit_op_arg(op, self.add_name(container, name)) @@ -188,8 +195,7 @@ def update_position(self, lineno): self.lineno = lineno - if self.first_lineno == -1: - self.first_lineno = lineno + self.lineno_set = False def _resolve_block_targets(self, blocks): last_extended_arg_count = 0 @@ -269,18 +275,22 @@ for block in blocks: offset = block.offset for instr in block.instructions: - lineno_table_builder.note_lineno_here(offset, instr.lineno) + if instr.lineno: + lineno_table_builder.note_lineno_here(offset, instr.lineno) offset += instr.size() return lineno_table_builder.get_table() def assemble(self): - if self.lineno == -1: - self.lineno = self.first_lineno if not self.current_block.have_return: self.use_next_block() if self.add_none_to_final_return: self.load_const(self.space.w_None) self.emit_op(ops.RETURN_VALUE) + if self.first_lineno == -1: + if self.first_block.instructions: + self.first_lineno = self.first_block.instructions[0].lineno + else: + self.first_lineno = 1 blocks = self.first_block.post_order() self._resolve_block_targets(blocks) lnotab = self._build_lnotab(blocks) @@ -531,6 +541,7 @@ def note_lineno_here(self, offset, lineno): # compute deltas line = lineno - self.current_line + addr = offset - self.current_off # Python assumes that lineno always increases with # increasing bytecode address (lnotab is unsigned char). # Depending on when SET_LINENO instructions are emitted @@ -541,10 +552,7 @@ # after the loading of "b". This works with the C Python # compiler because it only generates a SET_LINENO instruction # for the assignment. - if line > 0: - addr = offset - self.current_off - if not addr and not line: - return + if line or addr: push = self.table.append while addr > 255: push(chr(255)) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Tue Jul 21 15:58:06 2009 @@ -227,6 +227,7 @@ self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults) def visit_FunctionDef(self, func): + self.update_position(func.lineno) if func.decorators: self.visit_sequence(func.decorators) if func.args.defaults: @@ -236,7 +237,6 @@ num_defaults = 0 code = self.sub_scope(FunctionCodeGenerator, func.name, func, func.lineno) - self.update_position(func.lineno) self._make_function(code, num_defaults) if func.decorators: for i in range(len(func.decorators)): @@ -244,13 +244,13 @@ self.name_op(func.name, ast.Store) def visit_Lambda(self, lam): + self.update_position(lam.lineno) if lam.args.defaults: self.visit_sequence(lam.args.defaults) default_count = len(lam.args.defaults) else: default_count = 0 code = self.sub_scope(LambdaCodeGenerator, "", lam, lam.lineno) - self.update_position(lam.lineno) self._make_function(code, default_count) def visit_ClassDef(self, cls): @@ -263,7 +263,6 @@ bases_count = 0 self.emit_op_arg(ops.BUILD_TUPLE, bases_count) code = self.sub_scope(ClassCodeGenerator, cls.name, cls, cls.lineno) - self.update_position(cls.lineno) self._make_function(code, 0) self.emit_op_arg(ops.CALL_FUNCTION, 0) self.emit_op(ops.BUILD_CLASS) From arigo at codespeak.net Tue Jul 21 19:34:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 21 Jul 2009 19:34:40 +0200 (CEST) Subject: [pypy-svn] r66484 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090721173440.38F97168561@codespeak.net> Author: arigo Date: Tue Jul 21 19:34:38 2009 New Revision: 66484 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Another test copied from test_optimize2.py. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 21 19:34:38 2009 @@ -285,6 +285,20 @@ self.optimize_loop(ops, 'Not', expected, i0=1, i1=0, i2=1, i3=0, i4=1, i5=0) + def test_const_guard_value(self): + ops = """ + [] + i = int_add(5, 3) + guard_value(i, 8) + fail() + jump() + """ + expected = """ + [] + jump() + """ + self.optimize_loop(ops, '', expected, i=8) + # ---------- def test_virtual_1(self): From cfbolz at codespeak.net Tue Jul 21 19:34:52 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 21 Jul 2009 19:34:52 +0200 (CEST) Subject: [pypy-svn] r66485 - in pypy/branch/io-lang/pypy: module/_stackless module/_stackless/test rlib rlib/test Message-ID: <20090721173452.E0887169E16@codespeak.net> Author: cfbolz Date: Tue Jul 21 19:34:51 2009 New Revision: 66485 Added: pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py (contents, props changed) - copied, changed from r66473, pypy/branch/io-lang/pypy/module/_stackless/test/test_interp_coroutine.py Removed: pypy/branch/io-lang/pypy/module/_stackless/test/test_interp_coroutine.py Modified: pypy/branch/io-lang/pypy/module/_stackless/interp_clonable.py pypy/branch/io-lang/pypy/module/_stackless/interp_coroutine.py pypy/branch/io-lang/pypy/module/_stackless/interp_greenlet.py pypy/branch/io-lang/pypy/rlib/rcoroutine.py Log: Make rlib.rcoroutine actually usable from anywhere other than the Python interpreter. he changes there are mostly a reindent. Modified: pypy/branch/io-lang/pypy/module/_stackless/interp_clonable.py ============================================================================== --- pypy/branch/io-lang/pypy/module/_stackless/interp_clonable.py (original) +++ pypy/branch/io-lang/pypy/module/_stackless/interp_clonable.py Tue Jul 21 19:34:51 2009 @@ -3,7 +3,7 @@ from pypy.interpreter.gateway import interp2app, ObjSpace, W_Root from pypy.module._stackless.interp_coroutine import AppCoroutine, AppCoState from pypy.module._stackless.interp_coroutine import makeStaticMethod -from pypy.rlib.rcoroutine import AbstractThunk +from pypy.module._stackless.rcoroutine import AbstractThunk from pypy.module._stackless.rclonable import InterpClonableMixin Modified: pypy/branch/io-lang/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/io-lang/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/io-lang/pypy/module/_stackless/interp_coroutine.py Tue Jul 21 19:34:51 2009 @@ -24,7 +24,7 @@ from pypy.interpreter.function import StaticMethod from pypy.module._stackless.stackless_flags import StacklessFlags -from pypy.rlib.rcoroutine import Coroutine, BaseCoState, AbstractThunk +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk from pypy.rlib import rstack # for resume points from pypy.tool import stdlib_opcode as pythonopcode Modified: pypy/branch/io-lang/pypy/module/_stackless/interp_greenlet.py ============================================================================== --- pypy/branch/io-lang/pypy/module/_stackless/interp_greenlet.py (original) +++ pypy/branch/io-lang/pypy/module/_stackless/interp_greenlet.py Tue Jul 21 19:34:51 2009 @@ -4,8 +4,8 @@ from pypy.interpreter.gateway import NoneNotWrapped from pypy.interpreter.error import OperationError -from pypy.rlib.rcoroutine import Coroutine, BaseCoState -from pypy.rlib.rcoroutine import AbstractThunk, syncstate +from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState +from pypy.module._stackless.rcoroutine import AbstractThunk, syncstate from pypy.module._stackless.interp_coroutine import makeStaticMethod Modified: pypy/branch/io-lang/pypy/rlib/rcoroutine.py ============================================================================== --- pypy/branch/io-lang/pypy/rlib/rcoroutine.py (original) +++ pypy/branch/io-lang/pypy/rlib/rcoroutine.py Tue Jul 21 19:34:51 2009 @@ -29,7 +29,6 @@ The type of a switch is determined by the target's costate. """ -from pypy.interpreter.baseobjspace import Wrappable from pypy.rlib.rstack import yield_current_frame_to_caller, resume_point from pypy.rlib.objectmodel import we_are_translated @@ -55,270 +54,272 @@ import sys, os +def make_coroutine_classes(baseclass): + class BaseCoState(object): + def __init__(self): + self.current = self.main = None + + def __repr__(self): + "NOT_RPYTHON" + # for debugging only + return '<%s current=%r>' % (self.__class__.__name__, self.current) + + def update(self, new): + syncstate.leaving = self.current + syncstate.entering = new + self.current = new + frame, new.frame = new.frame, None + return frame + + + class CoState(BaseCoState): + def __init__(self): + BaseCoState.__init__(self) + self.current = self.main = Coroutine(self) -class BaseCoState(object): - def __init__(self): - self.current = self.main = None - - def __repr__(self): - "NOT_RPYTHON" - # for debugging only - return '<%s current=%r>' % (self.__class__.__name__, self.current) - - def update(self, new): - syncstate.leaving = self.current - syncstate.entering = new - self.current = new - frame, new.frame = new.frame, None - return frame - - -class CoState(BaseCoState): - def __init__(self): - BaseCoState.__init__(self) - self.current = self.main = Coroutine(self) - -class CoroutineDamage(SystemError): - pass - - -class SyncState(object): - def __init__(self): - self.reset() - - def reset(self): - self.default_costate = None - self.leaving = None - self.entering = None - self.things_to_do = False - self.temp_exc = None - self.to_delete = [] - - def switched(self, incoming_frame): - left = syncstate.leaving - entered = syncstate.entering - syncstate.leaving = syncstate.entering = None - if left is not None: # mostly to work around an annotation problem; - # should not really be None - left.frame = incoming_frame - left.goodbye() - if entered is not None: - entered.hello() - if self.things_to_do: - self._do_things_to_do() - - def push_exception(self, exc): - self.things_to_do = True - self.temp_exc = exc - - def check_for_zombie(self, obj): - return co in self.to_delete - - def postpone_deletion(self, obj): - self.to_delete.append(obj) - self.things_to_do = True - - def _do_things_to_do(self): - if self.temp_exc is not None: - # somebody left an unhandled exception and switched to us. - # this both provides default exception handling and the - # way to inject an exception, like CoroutineExit. - e, self.temp_exc = self.temp_exc, None - self.things_to_do = bool(self.to_delete) - raise e - while self.to_delete: - delete, self.to_delete = self.to_delete, [] - for obj in delete: - obj.parent = obj.costate.current - obj._kill_finally() - else: - self.things_to_do = False - -syncstate = SyncState() + class CoroutineDamage(SystemError): + pass -class CoroutineExit(SystemExit): - # XXX SystemExit's __init__ creates problems in bookkeeper. - def __init__(self): - pass + class SyncState(object): + def __init__(self): + self.reset() + + def reset(self): + self.default_costate = None + self.leaving = None + self.entering = None + self.things_to_do = False + self.temp_exc = None + self.to_delete = [] -class AbstractThunk(object): - def call(self): - raise NotImplementedError("abstract base class") - - -class Coroutine(Wrappable): - def __init__(self, state=None): - self.frame = None - if state is None: - state = self._get_default_costate() - self.costate = state - self.parent = None - self.thunk = None - - def __repr__(self): - 'NOT_RPYTHON' - # just for debugging - if hasattr(self, '__name__'): - return '' % (self.__name__, self.frame, self.thunk is not None) - else: - return '' % (self.frame, self.thunk is not None) + def switched(self, incoming_frame): + left = syncstate.leaving + entered = syncstate.entering + syncstate.leaving = syncstate.entering = None + if left is not None: # mostly to work around an annotation problem; + # should not really be None + left.frame = incoming_frame + left.goodbye() + if entered is not None: + entered.hello() + if self.things_to_do: + self._do_things_to_do() + + def push_exception(self, exc): + self.things_to_do = True + self.temp_exc = exc + + def check_for_zombie(self, obj): + return co in self.to_delete + + def postpone_deletion(self, obj): + self.to_delete.append(obj) + self.things_to_do = True + + def _do_things_to_do(self): + if self.temp_exc is not None: + # somebody left an unhandled exception and switched to us. + # this both provides default exception handling and the + # way to inject an exception, like CoroutineExit. + e, self.temp_exc = self.temp_exc, None + self.things_to_do = bool(self.to_delete) + raise e + while self.to_delete: + delete, self.to_delete = self.to_delete, [] + for obj in delete: + obj.parent = obj.costate.current + obj._kill_finally() + else: + self.things_to_do = False + + syncstate = SyncState() + + + class CoroutineExit(SystemExit): + # XXX SystemExit's __init__ creates problems in bookkeeper. + def __init__(self): + pass - def _get_default_costate(): - state = syncstate.default_costate - if state is None: - state = syncstate.default_costate = CoState() - return state - _get_default_costate = staticmethod(_get_default_costate) - - def _get_default_parent(self): - return self.costate.current - - def bind(self, thunk): - assert isinstance(thunk, AbstractThunk) - if self.frame is not None: - raise CoroutineDamage - if self.parent is None: - self.parent = self._get_default_parent() - assert self.parent is not None - self.thunk = thunk - if we_are_translated(): - self.frame = self._bind() - else: - self.frame = self._greenlet_bind() + class AbstractThunk(object): + def call(self): + raise NotImplementedError("abstract base class") + + + class Coroutine(baseclass): + def __init__(self, state=None): + self.frame = None + if state is None: + state = self._get_default_costate() + self.costate = state + self.parent = None + self.thunk = None + + def __repr__(self): + 'NOT_RPYTHON' + # just for debugging + if hasattr(self, '__name__'): + return '' % (self.__name__, self.frame, self.thunk is not None) + else: + return '' % (self.frame, self.thunk is not None) + + def _get_default_costate(): + state = syncstate.default_costate + if state is None: + state = syncstate.default_costate = CoState() + return state + _get_default_costate = staticmethod(_get_default_costate) + + def _get_default_parent(self): + return self.costate.current + + def bind(self, thunk): + assert isinstance(thunk, AbstractThunk) + if self.frame is not None: + raise CoroutineDamage + if self.parent is None: + self.parent = self._get_default_parent() + assert self.parent is not None + self.thunk = thunk + if we_are_translated(): + self.frame = self._bind() + else: + self.frame = self._greenlet_bind() + + def _greenlet_bind(self): + weak = [self] + def _greenlet_execute(incoming_frame): + try: + chain2go2next = weak[0]._execute(incoming_frame) + except: + # no exception is supposed to get out of _execute() + # better report it directly into the main greenlet then, + # and hidden to prevent catching + main_greenlet.throw(AssertionError( + "unexpected exception out of Coroutine._execute()", + *sys.exc_info())) + assert 0 + del weak[0] + greenlet.getcurrent().parent = chain2go2next.greenlet + return None # as the result of the FrameChain.switch() + chain = FrameChain(_greenlet_execute) + return chain + + def _bind(self): + state = self.costate + incoming_frame = yield_current_frame_to_caller() + self = state.current + return self._execute(incoming_frame) - def _greenlet_bind(self): - weak = [self] - def _greenlet_execute(incoming_frame): - try: - chain2go2next = weak[0]._execute(incoming_frame) - except: - # no exception is supposed to get out of _execute() - # better report it directly into the main greenlet then, - # and hidden to prevent catching - main_greenlet.throw(AssertionError( - "unexpected exception out of Coroutine._execute()", - *sys.exc_info())) - assert 0 - del weak[0] - greenlet.getcurrent().parent = chain2go2next.greenlet - return None # as the result of the FrameChain.switch() - chain = FrameChain(_greenlet_execute) - return chain - - def _bind(self): - state = self.costate - incoming_frame = yield_current_frame_to_caller() - self = state.current - return self._execute(incoming_frame) - - def _execute(self, incoming_frame): - state = self.costate - try: + def _execute(self, incoming_frame): + state = self.costate try: try: - exc = None - thunk = self.thunk - self.thunk = None - syncstate.switched(incoming_frame) - thunk.call() - resume_point("coroutine__bind", state) - except Exception, e: - exc = e - raise - finally: - # warning! we must reload the 'self' from the costate, - # because after a clone() the 'self' of both copies - # point to the original! - self = state.current - self.finish(exc) - except CoroutineExit: - # ignore a shutdown exception + try: + exc = None + thunk = self.thunk + self.thunk = None + syncstate.switched(incoming_frame) + thunk.call() + resume_point("coroutine__bind", state) + except Exception, e: + exc = e + raise + finally: + # warning! we must reload the 'self' from the costate, + # because after a clone() the 'self' of both copies + # point to the original! + self = state.current + self.finish(exc) + except CoroutineExit: + # ignore a shutdown exception + pass + except Exception, e: + # redirect all unhandled exceptions to the parent + syncstate.push_exception(e) + while self.parent is not None and self.parent.frame is None: + # greenlet behavior is fine + self.parent = self.parent.parent + return state.update(self.parent) + + def switch(self): + if self.frame is None: + # considered a programming error. + # greenlets and tasklets have different ideas about this. + raise CoroutineDamage + state = self.costate + incoming_frame = state.update(self).switch() + resume_point("coroutine_switch", state, returns=incoming_frame) + syncstate.switched(incoming_frame) + + def kill(self): + if self.frame is None: + return + state = self.costate + syncstate.push_exception(CoroutineExit()) + # careful here - if setting self.parent to state.current would + # create a loop, break it. The assumption is that 'self' + # will die, so that state.current's chain of parents can be + # modified to skip 'self' without too many people noticing. + p = state.current + if p is self or self.parent is None: + pass # killing the current of the main - don't change any parent + else: + while p.parent is not None: + if p.parent is self: + p.parent = self.parent + break + p = p.parent + self.parent = state.current + self.switch() + + def _kill_finally(self): + try: + self._userdel() + except Exception: + pass # maybe print a warning? + self.kill() + + def __del__(self): + # provide the necessary clean-up if this coro is left + # with a frame. + # note that AppCoroutine has to take care about this + # as well, including a check for user-supplied __del__. + # Additionally note that in the context of __del__, we are + # not in the position to issue a switch. + # we defer it completely. + if self.frame is not None and syncstate is not None: + syncstate.postpone_deletion(self) + + def _userdel(self): + # override this for exposed coros pass - except Exception, e: - # redirect all unhandled exceptions to the parent - syncstate.push_exception(e) - while self.parent is not None and self.parent.frame is None: - # greenlet behavior is fine - self.parent = self.parent.parent - return state.update(self.parent) - def switch(self): - if self.frame is None: - # considered a programming error. - # greenlets and tasklets have different ideas about this. - raise CoroutineDamage - state = self.costate - incoming_frame = state.update(self).switch() - resume_point("coroutine_switch", state, returns=incoming_frame) - syncstate.switched(incoming_frame) - - def kill(self): - if self.frame is None: - return - state = self.costate - syncstate.push_exception(CoroutineExit()) - # careful here - if setting self.parent to state.current would - # create a loop, break it. The assumption is that 'self' - # will die, so that state.current's chain of parents can be - # modified to skip 'self' without too many people noticing. - p = state.current - if p is self or self.parent is None: - pass # killing the current of the main - don't change any parent - else: - while p.parent is not None: - if p.parent is self: - p.parent = self.parent - break - p = p.parent - self.parent = state.current - self.switch() - - def _kill_finally(self): - try: - self._userdel() - except Exception: - pass # maybe print a warning? - self.kill() - - def __del__(self): - # provide the necessary clean-up if this coro is left - # with a frame. - # note that AppCoroutine has to take care about this - # as well, including a check for user-supplied __del__. - # Additionally note that in the context of __del__, we are - # not in the position to issue a switch. - # we defer it completely. - if self.frame is not None and syncstate is not None: - syncstate.postpone_deletion(self) + def is_alive(self): + return self.frame is not None or self is self.costate.current - def _userdel(self): - # override this for exposed coros - pass + def is_zombie(self): + return self.frame is not None and syncstate.check_for_zombie(self) - def is_alive(self): - return self.frame is not None or self is self.costate.current + def getcurrent(): + costate = Coroutine._get_default_costate() + return costate.current + getcurrent = staticmethod(getcurrent) - def is_zombie(self): - return self.frame is not None and syncstate.check_for_zombie(self) + def getmain(): + costate = Coroutine._get_default_costate() + return costate.main + getmain = staticmethod(getmain) - def getcurrent(): - costate = Coroutine._get_default_costate() - return costate.current - getcurrent = staticmethod(getcurrent) - - def getmain(): - costate = Coroutine._get_default_costate() - return costate.main - getmain = staticmethod(getmain) + def hello(self): + "Called when execution is transferred into this coroutine." - def hello(self): - "Called when execution is transferred into this coroutine." + def goodbye(self): + "Called just after execution is transferred away from this coroutine." - def goodbye(self): - "Called just after execution is transferred away from this coroutine." + def finish(self, exc=None): + "stephan forgot me" - def finish(self, exc=None): - "stephan forgot me" + return locals() # _________________________________________________ Copied: pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py (from r66473, pypy/branch/io-lang/pypy/module/_stackless/test/test_interp_coroutine.py) ============================================================================== --- pypy/branch/io-lang/pypy/module/_stackless/test/test_interp_coroutine.py (original) +++ pypy/branch/io-lang/pypy/rlib/test/test_rcoroutine.py Tue Jul 21 19:34:51 2009 @@ -4,10 +4,15 @@ import os from pypy import conftest; conftest.translation_test_so_skip_if_appdirect() -from pypy.rlib.rcoroutine import syncstate, Coroutine, AbstractThunk +from pypy.rlib.rcoroutine import make_coroutine_classes from pypy.translator.c.test.test_stackless import StacklessTest from pypy.translator.c import gc +d = make_coroutine_classes(object) +syncstate = d['syncstate'] +Coroutine = d['Coroutine'] +AbstractThunk = d['AbstractThunk'] + def output(stuff): os.write(2, stuff + '\n') From cfbolz at codespeak.net Tue Jul 21 19:39:06 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 21 Jul 2009 19:39:06 +0200 (CEST) Subject: [pypy-svn] r66486 - pypy/branch/io-lang/pypy/module/_stackless Message-ID: <20090721173906.37D4F169E72@codespeak.net> Author: cfbolz Date: Tue Jul 21 19:39:05 2009 New Revision: 66486 Added: pypy/branch/io-lang/pypy/module/_stackless/rcoroutine.py (contents, props changed) Log: forgot to add a file Added: pypy/branch/io-lang/pypy/module/_stackless/rcoroutine.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/module/_stackless/rcoroutine.py Tue Jul 21 19:39:05 2009 @@ -0,0 +1,9 @@ +from pypy.rlib.rcoroutine import make_coroutine_classes +from pypy.interpreter.baseobjspace import Wrappable + +d = make_coroutine_classes(Wrappable) + +Coroutine = d['Coroutine'] +BaseCoState = d['BaseCoState'] +AbstractThunk = d['AbstractThunk'] +syncstate = d['syncstate'] From arigo at codespeak.net Tue Jul 21 19:44:52 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 21 Jul 2009 19:44:52 +0200 (CEST) Subject: [pypy-svn] r66487 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090721174452.B31CB168473@codespeak.net> Author: arigo Date: Tue Jul 21 19:44:52 2009 New Revision: 66487 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Remove unneeded guard_no_exceptions. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Tue Jul 21 19:44:52 2009 @@ -204,6 +204,7 @@ # ---------- def propagate_forward(self): + self.exception_might_have_happened = False self.newoperations = [] for op in self.loop.operations: opnum = op.opnum @@ -226,6 +227,8 @@ must_clone = False op.args[i] = box self.newoperations.append(op) + if op.can_raise(): + self.exception_might_have_happened = True def optimize_default(self, op): if op.is_always_pure(): @@ -278,6 +281,12 @@ self.emit_operation(op) value.make_constant_class() + def optimize_GUARD_NO_EXCEPTION(self, op): + if not self.exception_might_have_happened: + return + self.emit_operation(op) + self.exception_might_have_happened = False + def optimize_OONONNULL(self, op): if self.known_nonnull(op.args[0]): assert op.result.getint() == 1 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 21 19:44:52 2009 @@ -301,6 +301,35 @@ # ---------- + def test_fold_guard_no_exception(self): + ops = """ + [i] + guard_no_exception() + fail() + i1 = int_add(i, 3) + guard_no_exception() + fail() + i2 = call(i1) + guard_no_exception() + fail(i1, i2) + guard_no_exception() + fail() + i3 = call(i2) + jump(i1) # the exception is considered lost when we loop back + """ + expected = """ + [i] + i1 = int_add(i, 3) + i2 = call(i1) + guard_no_exception() + fail(i1, i2) + i3 = call(i2) + jump(i1) + """ + self.optimize_loop(ops, 'Not', expected) + + # ---------- + def test_virtual_1(self): ops = """ [i, p0] From arigo at codespeak.net Tue Jul 21 19:53:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 21 Jul 2009 19:53:10 +0200 (CEST) Subject: [pypy-svn] r66488 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090721175310.31FF0168547@codespeak.net> Author: arigo Date: Tue Jul 21 19:53:09 2009 New Revision: 66488 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Handle the case of loops ending in a FAIL. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Tue Jul 21 19:53:09 2009 @@ -99,6 +99,9 @@ instnode.knownclsbox = op.args[0] self.nodes[op.result] = instnode + def find_nodes_GUARD_CLASS(self, op): + pass # prevent default handling + def find_nodes_SETFIELD_GC(self, op): instnode = self.getnode(op.args[0]) if instnode.escaped: @@ -140,11 +143,11 @@ for box in op.args: self.getnode(box).set_unique_nodes() - def find_nodes_GUARD_CLASS(self, op): - pass # prevent default handling - def find_nodes_FAIL(self, op): - xxx + # only for bridges, and only for the ones that end in a 'return' + # or 'raise'; all other cases end with a JUMP. + for box in op.args: + self.getnode(box).unique = UNIQUE_NO find_nodes_ops = _findall(NodeFinder, 'find_nodes_') @@ -282,7 +285,6 @@ self.nodes[box] = instnode def bridge_matches(self, jump_op, nextloop_specnodes): - assert jump_op.opnum == rop.JUMP assert len(jump_op.args) == len(nextloop_specnodes) for i in range(len(nextloop_specnodes)): exitnode = self.getnode(jump_op.args[i]) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Tue Jul 21 19:53:09 2009 @@ -576,6 +576,23 @@ nextdescr=Virtual(node_vtable, nextdescr=Not)))''') + def test_bridge_to_fail(self): + ops = """ + [i1] + i2 = int_add(i1, 5) + fail(i2) + """ + self.find_bridge(ops, 'Not', 'Not') + + def test_bridge_virtual_to_fail(self): + ops = """ + [i1] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i1, descr=valuedescr) + fail(p1) + """ + self.find_bridge(ops, 'Not', 'Not') + class TestLLtype(BaseTestOptimizeFindNode, LLtypeMixin): pass From arigo at codespeak.net Tue Jul 21 20:08:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 21 Jul 2009 20:08:29 +0200 (CEST) Subject: [pypy-svn] r66489 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090721180829.0E68716857E@codespeak.net> Author: arigo Date: Tue Jul 21 20:08:28 2009 New Revision: 66489 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Test GETFIELD_GC_PURE. Implemented just by calling GETFIELD_GC (see comment). Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Tue Jul 21 20:08:28 2009 @@ -339,8 +339,9 @@ value.make_nonnull() self.optimize_default(op) - def optimize_GETFIELD_PURE_GC(self, op): - xxx # optimize_GETFIELD_GC + # note: the following line does not mean that the two operations are + # completely equivalent, because GETFIELD_GC_PURE is_always_pure(). + optimize_GETFIELD_GC_PURE = optimize_GETFIELD_GC def optimize_SETFIELD_GC(self, op): value = self.getvalue(op.args[0]) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Tue Jul 21 20:08:28 2009 @@ -108,6 +108,12 @@ return ConstPtr(ConstPtr.value) else: return ConstObj(ConstObj.value) + elif arg.startswith('ConstPtr('): + name = arg[len('ConstPtr('):-1] + if self.type_system == 'lltype': + return ConstPtr(self.boxkinds[name]) + else: + return ConstObj(self.boxkinds[name]) return self.vars[arg] def parse_op(self, line): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Tue Jul 21 20:08:28 2009 @@ -533,6 +533,43 @@ expected = ops self.optimize_loop(ops, 'Not, Not', expected) + def test_getfield_gc_pure_1(self): + ops = """ + [i] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i, descr=valuedescr) + i1 = getfield_gc_pure(p1, descr=valuedescr) + jump(i1) + """ + expected = """ + [i] + jump(i) + """ + self.optimize_loop(ops, 'Not', expected) + + def test_getfield_gc_pure_2(self): + ops = """ + [i] + i1 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr) + jump(i1) + """ + expected = """ + [i] + jump(5) + """ + self.optimize_loop(ops, 'Not', expected, i1=5, + boxkinds={'myptr': self.nodebox.value}) + + def test_getfield_gc_nonpure_2(self): + ops = """ + [i] + i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr) + jump(i1) + """ + expected = ops + self.optimize_loop(ops, 'Not', expected, i1=5, + boxkinds={'myptr': self.nodebox.value}) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From benjamin at codespeak.net Tue Jul 21 21:08:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 21:08:31 +0200 (CEST) Subject: [pypy-svn] r66490 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721190831.99CA4168076@codespeak.net> Author: benjamin Date: Tue Jul 21 21:08:26 2009 New Revision: 66490 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: only update lineno if it greater than the last; also make lntob building more like cpython Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Tue Jul 21 21:08:26 2009 @@ -194,8 +194,9 @@ self.emit_op_arg(ops.LOAD_CONST, index) def update_position(self, lineno): - self.lineno = lineno - self.lineno_set = False + if lineno > self.lineno: + self.lineno = lineno + self.lineno_set = False def _resolve_block_targets(self, blocks): last_extended_arg_count = 0 @@ -541,6 +542,7 @@ def note_lineno_here(self, offset, lineno): # compute deltas line = lineno - self.current_line + assert line >= 0 addr = offset - self.current_off # Python assumes that lineno always increases with # increasing bytecode address (lnotab is unsigned char). @@ -559,12 +561,11 @@ push(chr(0)) addr -= 255 while line > 255: - push(chr(addr)) + push(chr(0)) push(chr(255)) line -= 255 addr = 0 - if addr > 0 or line > 0: - push(chr(addr)) - push(chr(line)) + push(chr(addr)) + push(chr(line)) self.current_line = lineno self.current_off = offset From benjamin at codespeak.net Tue Jul 21 21:52:47 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 21:52:47 +0200 (CEST) Subject: [pypy-svn] r66491 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090721195247.C98B0168556@codespeak.net> Author: benjamin Date: Tue Jul 21 21:52:45 2009 New Revision: 66491 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_grammar.py Log: we now use cpython's naming scheme for implicit tuple unpacking Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_grammar.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_grammar.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_grammar.py Tue Jul 21 21:52:45 2009 @@ -165,12 +165,7 @@ 'list',)) vereq(f5.func_code.co_varnames, ('(compound, first)', 'two', 'compound', 'first')) -elif check_impl_detail(pypy=True): - vereq(f4.func_code.co_varnames, - ('two', '.2', 'compound', 'argument', 'list')) - vereq(f5.func_code.co_varnames, - ('.0', 'two', 'compound', 'first')) -elif check_impl_detail(cpython=True): +elif check_impl_detail(cpython=True, pypy=True): vereq(f4.func_code.co_varnames, ('two', '.1', 'compound', 'argument', 'list')) vereq(f5.func_code.co_varnames, @@ -185,9 +180,7 @@ # thus, the names nested inside tuples must appear after these names. if check_impl_detail(jython=True): verify(v3.func_code.co_varnames == ('a', '(b, c)', 'rest', 'b', 'c')) -elif check_impl_detail(pypy=True): - vereq(v3.func_code.co_varnames, ('a', '.2', 'rest', 'b', 'c')) -elif check_impl_detail(cpython=True): +elif check_impl_detail(cpython=True, pypy=True): vereq(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c')) verify(v3(1, (2, 3), 4) == (1, 2, 3, (4,))) def d01(a=1): pass From benjamin at codespeak.net Tue Jul 21 21:58:18 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Tue, 21 Jul 2009 21:58:18 +0200 (CEST) Subject: [pypy-svn] r66492 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090721195818.382DF1684C3@codespeak.net> Author: benjamin Date: Tue Jul 21 21:58:18 2009 New Revision: 66492 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: move function's _compile out of AbstractFunctionGenerator Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Tue Jul 21 21:58:18 2009 @@ -1131,22 +1131,6 @@ class AbstractFunctionCodeGenerator(PythonCodeGenerator): - def _compile(self, func): - assert isinstance(func, ast.FunctionDef) - if self.is_docstring(func.body[0]): - doc_string = func.body[0] - assert isinstance(doc_string, ast.Expr) - doc_string.value.walkabout(self) - start = 1 - else: - self.add_const(self.space.w_None) - start = 0 - if func.args.args: - self._handle_nested_args(func.args.args) - self.argcount = len(func.args.args) - for i in range(start, len(func.body)): - func.body[i].walkabout(self) - def _handle_nested_args(self, args): for i in range(len(args)): arg = args[i] @@ -1175,7 +1159,22 @@ class FunctionCodeGenerator(AbstractFunctionCodeGenerator): - pass + + def _compile(self, func): + assert isinstance(func, ast.FunctionDef) + if self.is_docstring(func.body[0]): + doc_string = func.body[0] + assert isinstance(doc_string, ast.Expr) + doc_string.value.walkabout(self) + start = 1 + else: + self.add_const(self.space.w_None) + start = 0 + if func.args.args: + self._handle_nested_args(func.args.args) + self.argcount = len(func.args.args) + for i in range(start, len(func.body)): + func.body[i].walkabout(self) class LambdaCodeGenerator(AbstractFunctionCodeGenerator): From benjamin at codespeak.net Wed Jul 22 02:34:54 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 02:34:54 +0200 (CEST) Subject: [pypy-svn] r66493 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722003454.98CC3168539@codespeak.net> Author: benjamin Date: Wed Jul 22 02:34:53 2009 New Revision: 66493 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: push the last part of the address while writing out the line Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Wed Jul 22 02:34:53 2009 @@ -561,7 +561,7 @@ push(chr(0)) addr -= 255 while line > 255: - push(chr(0)) + push(chr(addr)) push(chr(255)) line -= 255 addr = 0 From benjamin at codespeak.net Wed Jul 22 03:35:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 03:35:44 +0200 (CEST) Subject: [pypy-svn] r66494 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722013544.094441684A4@codespeak.net> Author: benjamin Date: Wed Jul 22 03:35:43 2009 New Revision: 66494 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: force a new lineno to be set for every statement Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Wed Jul 22 03:35:43 2009 @@ -193,8 +193,10 @@ index = self.add_const(obj) self.emit_op_arg(ops.LOAD_CONST, index) - def update_position(self, lineno): - if lineno > self.lineno: + def update_position(self, lineno, force=False): + if force or lineno > self.lineno: + if force and lineno < self.lineno: + raise AssertionError self.lineno = lineno self.lineno_set = False Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Wed Jul 22 03:35:43 2009 @@ -227,7 +227,7 @@ self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults) def visit_FunctionDef(self, func): - self.update_position(func.lineno) + self.update_position(func.lineno, True) if func.decorators: self.visit_sequence(func.decorators) if func.args.defaults: @@ -254,7 +254,7 @@ self._make_function(code, default_count) def visit_ClassDef(self, cls): - self.update_position(cls.lineno) + self.update_position(cls.lineno, True) self.load_const(self.space.wrap(cls.name)) if cls.bases: bases_count = len(cls.bases) @@ -277,7 +277,7 @@ return inplace_operations(op) def visit_AugAssign(self, assign): - self.update_position(assign.lineno) + self.update_position(assign.lineno, True) target = assign.target if isinstance(target, ast.Attribute): attr = ast.Attribute(target.value, target.attr, ast.AugLoad, @@ -333,7 +333,7 @@ self.emit_op(self._binop(binop.op)) def visit_Return(self, ret): - self.update_position(ret.lineno) + self.update_position(ret.lineno, True) if ret.value: ret.value.walkabout(self) else: @@ -341,7 +341,7 @@ self.emit_op(ops.RETURN_VALUE) def visit_Print(self, pr): - self.update_position(pr.lineno) + self.update_position(pr.lineno, True) have_dest = bool(pr.dest) if have_dest: pr.dest.walkabout(self) @@ -364,11 +364,11 @@ self.emit_op(ops.POP_TOP) def visit_Delete(self, delete): - self.update_position(delete.lineno) + self.update_position(delete.lineno, True) self.visit_sequence(delete.targets) def visit_If(self, if_): - self.update_position(if_.lineno) + self.update_position(if_.lineno, True) end = self.new_block() test_constant = misc.expr_constant(self.space, if_.test) if test_constant == misc.CONST_FALSE: @@ -390,7 +390,7 @@ self.use_next_block(end) def visit_Break(self, br): - self.update_position(br.lineno) + self.update_position(br.lineno, True) for f_block in self.frame_blocks: if f_block[0] == F_BLOCK_LOOP: break @@ -421,7 +421,7 @@ self.error("'continue' not supported inside 'finally' clause", cont) def visit_For(self, fr): - self.update_position(fr.lineno) + self.update_position(fr.lineno, True) start = self.new_block() cleanup = self.new_block() end = self.new_block() @@ -442,7 +442,7 @@ self.use_next_block(end) def visit_While(self, wh): - self.update_position(wh.lineno) + self.update_position(wh.lineno, True) test_constant = misc.expr_constant(self.space, wh.test) if test_constant == misc.CONST_FALSE: if wh.orelse: @@ -472,7 +472,7 @@ self.use_next_block(end) def visit_TryExcept(self, te): - self.update_position(te.lineno) + self.update_position(te.lineno, True) exc = self.new_block() otherwise = self.new_block() end = self.new_block() @@ -486,7 +486,7 @@ self.use_next_block(exc) for handler in te.handlers: assert isinstance(handler, ast.excepthandler) - self.update_position(handler.lineno) + self.update_position(handler.lineno, True) next_except = self.new_block() if handler.type: self.emit_op(ops.DUP_TOP) @@ -512,7 +512,7 @@ self.use_next_block(end) def visit_TryFinally(self, tf): - self.update_position(tf.lineno) + self.update_position(tf.lineno, True) end = self.new_block() self.emit_jump(ops.SETUP_FINALLY, end) body = self.use_next_block() @@ -545,7 +545,7 @@ self.name_op(alias.asname, ast.Store) def visit_Import(self, imp): - self.update_position(imp.lineno) + self.update_position(imp.lineno, True) for alias in imp.names: assert isinstance(alias, ast.alias) if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: @@ -566,7 +566,7 @@ self.name_op(store_name, ast.Store) def visit_ImportFrom(self, imp): - self.update_position(imp.lineno) + self.update_position(imp.lineno, True) space = self.space if imp.module == "__future__": if self.done_with_future: @@ -607,7 +607,7 @@ self.emit_op(ops.POP_TOP) def visit_Assign(self, assign): - self.update_position(assign.lineno) + self.update_position(assign.lineno, True) assign.value.walkabout(self) duplications = len(assign.targets) - 1 for i in range(len(assign.targets)): @@ -616,7 +616,7 @@ assign.targets[i].walkabout(self) def visit_With(self, wih): - self.update_position(wih.lineno) + self.update_position(wih.lineno, True) body_block = self.new_block() cleanup = self.new_block() exit_storage = self.current_temporary_name() @@ -653,7 +653,7 @@ self.pop_frame_block(F_BLOCK_FINALLY_END, cleanup) def visit_Raise(self, rais): - self.update_position(rais.lineno) + self.update_position(rais.lineno, True) arg = 0 if rais.type: rais.type.walkabout(self) @@ -667,7 +667,7 @@ self.emit_op_arg(ops.RAISE_VARARGS, arg) def visit_Exec(self, exc): - self.update_position(exc.lineno) + self.update_position(exc.lineno, True) exc.body.walkabout(self) if exc.globals: exc.globals.walkabout(self) @@ -685,10 +685,10 @@ pass def visit_Pass(self, pas): - self.update_position(pas.lineno) + self.update_position(pas.lineno, True) def visit_Expr(self, expr): - self.update_position(expr.lineno) + self.update_position(expr.lineno, True) if self.interactive: expr.value.walkabout(self) self.emit_op(ops.PRINT_EXPR) From benjamin at codespeak.net Wed Jul 22 03:36:25 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 03:36:25 +0200 (CEST) Subject: [pypy-svn] r66495 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722013625.D6FA81684C0@codespeak.net> Author: benjamin Date: Wed Jul 22 03:36:24 2009 New Revision: 66495 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: note lineno for continue Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Wed Jul 22 03:36:24 2009 @@ -399,6 +399,7 @@ self.emit_op(ops.BREAK_LOOP) def visit_Continue(self, cont): + self.update_position(cont.lineno, True) if not self.frame_blocks: self.error("'continue' not properly in loop", cont) current_block, block = self.frame_blocks[-1] From benjamin at codespeak.net Wed Jul 22 03:37:45 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 03:37:45 +0200 (CEST) Subject: [pypy-svn] r66496 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722013745.509B21684C0@codespeak.net> Author: benjamin Date: Wed Jul 22 03:37:44 2009 New Revision: 66496 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: set lineno again for iter in for and while loops Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Wed Jul 22 03:37:44 2009 @@ -431,6 +431,8 @@ fr.iter.walkabout(self) self.emit_op(ops.GET_ITER) self.use_next_block(start) + # This adds another line, so each for iteration can be traced. + self.lineno_set = False self.emit_jump(ops.FOR_ITER, cleanup) fr.target.walkabout(self) self.visit_sequence(fr.body) @@ -458,6 +460,8 @@ self.push_frame_block(F_BLOCK_LOOP, loop) self.use_next_block(loop) if test_constant == misc.CONST_NOT_CONST: + # Force another lineno to be set for tracing purposes. + self.lineno_set = False wh.test.walkabout(self) self.emit_jump(ops.JUMP_IF_FALSE, anchor) self.emit_op(ops.POP_TOP) From arigo at codespeak.net Wed Jul 22 09:36:58 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 22 Jul 2009 09:36:58 +0200 (CEST) Subject: [pypy-svn] r66498 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090722073658.7D3BE169E72@codespeak.net> Author: arigo Date: Wed Jul 22 09:36:56 2009 New Revision: 66498 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Start handling the fail() operations attached to guards. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Wed Jul 22 09:36:56 2009 @@ -3,7 +3,7 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode -from pypy.jit.metainterp.optimizeutil import av_newdict, _findall +from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs from pypy.rlib.objectmodel import we_are_translated @@ -36,6 +36,10 @@ def force_box(self): return self.box + def get_args_for_fail(self, list): + if not self.is_constant(): + list.append(self.box) + def is_constant(self): return self.level == LEVEL_CONSTANT @@ -114,6 +118,15 @@ newoperations.append(op) return self.box + def get_args_for_fail(self, list): + if self.box is not None: + InstanceValue.get_args_for_fail(self, list) + else: + lst = self._fields.keys() + sort_descrs(lst) + for ofs in lst: + self._fields[ofs].get_args_for_fail(list) + class __extend__(SpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): @@ -217,6 +230,7 @@ self.loop.operations = self.newoperations def emit_operation(self, op, must_clone=True): + op1 = op for i in range(len(op.args)): arg = op.args[i] if arg in self.values: @@ -226,9 +240,32 @@ op = op.clone() must_clone = False op.args[i] = box - self.newoperations.append(op) - if op.can_raise(): + if op.is_guard(): + self.clone_guard(op, op1) + elif op.can_raise(): self.exception_might_have_happened = True + self.newoperations.append(op) + + def clone_guard(self, op2, op1): + assert len(op1.suboperations) == 1 + op_fail = op1.suboperations[0] + assert op_fail.opnum == rop.FAIL + # + newboxes = [] + for box in op_fail.args: + try: + value = self.values[box] + except KeyError: + newboxes.append(box) + else: + value.get_args_for_fail(newboxes) + # NB. we mutate op_fail in-place above. That's bad. Hopefully + # it does not really matter because no-one is going to look again + # at its unoptimized version. We cannot really clone it because of + # how the rest works (e.g. it is returned by cpu.execute_operations()). + op_fail.args = newboxes + op2.suboperations = op1.suboperations + op1.optimized = op2 def optimize_default(self, op): if op.is_always_pure(): @@ -250,9 +287,10 @@ for i in range(len(specnodes)): value = self.getvalue(op.args[i]) specnodes[i].teardown_virtual_node(self, value, exitargs) - op = op.clone() - op.args = exitargs - self.emit_operation(op, must_clone=False) + op2 = op.clone() + op2.args = exitargs + op2.jump_target = op.jump_target + self.emit_operation(op2, must_clone=False) def optimize_guard(self, op): value = self.getvalue(op.args[0]) @@ -263,14 +301,15 @@ def optimize_GUARD_VALUE(self, op): assert isinstance(op.args[1], Const) + assert op.args[0].getint() == op.args[1].getint() self.optimize_guard(op) def optimize_GUARD_TRUE(self, op): - assert op.args[0].getint() + assert op.args[0].getint() == 1 self.optimize_guard(op) def optimize_GUARD_FALSE(self, op): - assert not op.args[0].getint() + assert op.args[0].getint() == 0 self.optimize_guard(op) def optimize_GUARD_CLASS(self, op): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py Wed Jul 22 09:36:56 2009 @@ -34,9 +34,7 @@ self.descr = descr def clone(self): - res = ResOperation(self.opnum, self.args, self.result, self.descr) - res.jump_target = self.jump_target - return res + return ResOperation(self.opnum, self.args, self.result, self.descr) def __repr__(self): return self.repr() Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 22 09:36:56 2009 @@ -480,7 +480,7 @@ # the 'expected' is sub-optimal, but it should be done by another later # optimization step. See test_find_nodes_default_field() for why. self.optimize_loop(ops, 'Virtual(node_vtable, valuedescr=Not)', - expected) + expected, i0=0) def test_virtual_3(self): ops = """ @@ -570,6 +570,27 @@ self.optimize_loop(ops, 'Not', expected, i1=5, boxkinds={'myptr': self.nodebox.value}) + def test_expand_fail_arguments(self): + ops = """ + [i1, i2, i3, p3] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, 1, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p3, descr=nextdescr) + guard_true(i1) + fail(p1, i3) + jump(i2, i1, i3, p3) + """ + expected = """ + [i1, i2, i3, p3] + guard_true(i1) + fail(i2, p3, i3) + jump(i2, 1, i3, p3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Wed Jul 22 09:44:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 22 Jul 2009 09:44:29 +0200 (CEST) Subject: [pypy-svn] r66499 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090722074429.0BB66169E72@codespeak.net> Author: arigo Date: Wed Jul 22 09:44:29 2009 New Revision: 66499 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Protect against loops and duplicate arguments. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Wed Jul 22 09:44:29 2009 @@ -36,9 +36,11 @@ def force_box(self): return self.box - def get_args_for_fail(self, list): + def get_args_for_fail(self, list, memo): if not self.is_constant(): - list.append(self.box) + if self.box not in memo: + list.append(self.box) + memo[self.box] = None def is_constant(self): return self.level == LEVEL_CONSTANT @@ -118,14 +120,19 @@ newoperations.append(op) return self.box - def get_args_for_fail(self, list): + def get_args_for_fail(self, list, memo): if self.box is not None: - InstanceValue.get_args_for_fail(self, list) + InstanceValue.get_args_for_fail(self, list, memo) else: + if self.source_op is not None: #otherwise, no loop detection needed + keybox = self.source_op.result + if keybox in memo: + return + memo[keybox] = None lst = self._fields.keys() sort_descrs(lst) for ofs in lst: - self._fields[ofs].get_args_for_fail(list) + self._fields[ofs].get_args_for_fail(list, memo) class __extend__(SpecNode): @@ -251,14 +258,17 @@ op_fail = op1.suboperations[0] assert op_fail.opnum == rop.FAIL # + memo = {} newboxes = [] for box in op_fail.args: try: value = self.values[box] except KeyError: - newboxes.append(box) + if box not in memo: + newboxes.append(box) + memo[box] = None else: - value.get_args_for_fail(newboxes) + value.get_args_for_fail(newboxes, memo) # NB. we mutate op_fail in-place above. That's bad. Hopefully # it does not really matter because no-one is going to look again # at its unoptimized version. We cannot really clone it because of Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 22 09:44:29 2009 @@ -591,6 +591,24 @@ """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + def test_expand_fail_loop(self): + ops = """ + [i1, i2] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, p1, descr=nextdescr) + guard_true(i1) + fail(p1) + jump(i1, i2) + """ + expected = """ + [i1, i2] + guard_true(i1) + fail(i2) + jump(1, i2) + """ + self.optimize_loop(ops, 'Not, Not', expected, i1=1) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Wed Jul 22 09:47:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 22 Jul 2009 09:47:20 +0200 (CEST) Subject: [pypy-svn] r66500 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090722074720.BE811169E72@codespeak.net> Author: arigo Date: Wed Jul 22 09:47:20 2009 New Revision: 66500 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: A test for the "duplicate" part of the previous checkin. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 22 09:47:20 2009 @@ -609,6 +609,33 @@ """ self.optimize_loop(ops, 'Not, Not', expected, i1=1) + def test_expand_fail_duplicate(self): + ops = """ + [i1, i2] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + guard_true(i1) + fail(%s) + jump(i1, i2) + """ + expected = """ + [i1, i2] + guard_true(i1) + fail(i2) + jump(1, i2) + """ + self.optimize_loop(ops % 'p1', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p1, i2', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p1, i2, i2', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'i2, p1, i2', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'i2, i2, p1', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p1, p1', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p1, p2', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p2, p1', 'Not, Not', expected, i1=1) + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Wed Jul 22 11:33:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 22 Jul 2009 11:33:33 +0200 (CEST) Subject: [pypy-svn] r66502 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090722093333.A8800169E90@codespeak.net> Author: arigo Date: Wed Jul 22 11:33:31 2009 New Revision: 66502 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (contents, props changed) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/compile.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Log: Move encoding and decoding of the frames and boxes at a FAIL in its own file. Will then add tests and direct support for virtuals. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/compile.py Wed Jul 22 11:33:31 2009 @@ -242,15 +242,13 @@ class ResumeGuardDescr(AbstractDescr): - def __init__(self, resume_info, vable_nums, consts, - history, history_guard_index): - self.resume_info = resume_info - self.vable_nums = vable_nums # None if no virtualizable - self.counter = 0 + counter = 0 + + def __init__(self, history, history_guard_index): self.history = history assert history_guard_index >= 0 self.history_guard_index = history_guard_index - self.consts = consts + # this class also gets attributes stored by ResumeDataBuilder.finish() def handle_fail_op(self, metainterp_sd, fail_op): from pypy.jit.metainterp.pyjitpl import MetaInterp Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Wed Jul 22 11:33:31 2009 @@ -4,7 +4,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_print -from pypy.jit.metainterp import history, compile +from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.heaptracker import populate_type_cache @@ -832,13 +832,12 @@ if not we_are_translated(): self.metainterp._debug_history[-1][-1] = argboxes - def setup_resume_at_op(self, pc, nums, consts, liveboxes, - exception_target): + def setup_resume_at_op(self, pc, exception_target, env): if not we_are_translated(): - check_args(*liveboxes) + check_args(*env) self.pc = pc self.exception_target = exception_target - self.env = _consume_nums(nums, liveboxes, consts) + self.env = env if DEBUG >= 2: values = ' '.join([box.repr_rpython() for box in self.env]) log('setup_resume_at_op %s:%d [%s] %d' % (self.jitcode.name, @@ -869,28 +868,21 @@ return saved_pc = self.pc self.pc = pc - # XXX 'resume_info' should be shared, either partially or - # if possible totally - resume_info = [] - liveboxes = [] - consts = [] - memo = {} - if metainterp.staticdata.virtualizable_info is None: - vable_nums = None - else: - vable_nums = _generate_nums(metainterp.virtualizable_boxes, - memo, liveboxes, consts) + resumebuilder = resume.ResumeDataBuilder() + if metainterp.staticdata.virtualizable_info is not None: + resumebuilder.generate_boxes(metainterp.virtualizable_boxes) for frame in metainterp.framestack: - nums = _generate_nums(frame.env, memo, liveboxes, consts) - resume_info.append((frame.jitcode, frame.pc, nums, - frame.exception_target)) + resumebuilder.generate_frame_info(frame.jitcode, frame.pc, + frame.exception_target) + resumebuilder.generate_boxes(frame.env) if box is not None: moreargs = [box] + extraargs else: moreargs = list(extraargs) guard_op = metainterp.history.record(opnum, moreargs, None) - resumedescr = compile.ResumeGuardDescr(resume_info, vable_nums, consts, + resumedescr = compile.ResumeGuardDescr( metainterp.history, len(metainterp.history.operations)-1) + liveboxes = resumebuilder.finish(resumedescr) op = history.ResOperation(rop.FAIL, liveboxes, None, descr=resumedescr) guard_op.suboperations = [op] self.pc = saved_pc @@ -924,35 +916,6 @@ # ____________________________________________________________ -def _generate_nums(frameboxes, memo, liveboxes, consts): - nums = [] - for framebox in frameboxes: - assert framebox is not None - if isinstance(framebox, Box): - try: - num = memo[framebox] - except KeyError: - num = len(liveboxes) - memo[framebox] = num - liveboxes.append(framebox) - else: - num = ~len(consts) - consts.append(framebox) - nums.append(num) - return nums - -def _consume_nums(nums, liveboxes, consts): - env = [] - for num in nums: - if num >= 0: - box = liveboxes[num] - else: - box = consts[~num] - env.append(box) - return env - -# ____________________________________________________________ - class MetaInterpStaticData(object): virtualizable_info = None @@ -1476,10 +1439,7 @@ self.history = history.BlackHole(self.cpu) # the BlackHole is invalid because it doesn't start with # guard_failure.key.guard_op.suboperations, but that's fine - self.rebuild_state_after_failure(resumedescr.resume_info, - resumedescr.vable_nums, - resumedescr.consts, - guard_failure.args) + self.rebuild_state_after_failure(resumedescr, guard_failure.args) def initialize_virtualizable(self, original_boxes): vinfo = self.staticdata.virtualizable_info @@ -1548,14 +1508,13 @@ frame.generate_guard(frame.pc, rop.GUARD_NO_EXCEPTION, None, []) return False - def rebuild_state_after_failure(self, resume_info, vable_nums, consts, - newboxes): + def rebuild_state_after_failure(self, resumedescr, newboxes): if not we_are_translated(): self._debug_history.append(['guard_failure', None, None]) vinfo = self.staticdata.virtualizable_info + resumereader = resume.ResumeDataReader(resumedescr, newboxes) if vinfo is not None: - self.virtualizable_boxes = _consume_nums(vable_nums, - newboxes, consts) + self.virtualizable_boxes = resumereader.consume_boxes() # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_rti # is and stays NULL. @@ -1565,10 +1524,11 @@ self.synchronize_virtualizable() # self.framestack = [] - for jitcode, pc, nums, exception_target in resume_info: + while resumereader.has_more_frame_infos(): + jitcode, pc, exception_target = resumereader.consume_frame_info() + env = resumereader.consume_boxes() f = self.newframe(jitcode) - f.setup_resume_at_op(pc, nums, consts, newboxes, - exception_target) + f.setup_resume_at_op(pc, exception_target, env) def check_synchronized_virtualizable(self): if not we_are_translated(): Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Wed Jul 22 11:33:31 2009 @@ -0,0 +1,77 @@ +from pypy.jit.metainterp.history import Box + +# Logic to encode the chain of frames and the state of the boxes at a +# FAIL operation, and to decode it again. This is a bit advanced, +# because it needs to support optimize.py which encodes virtuals with +# arbitrary cycles. + +# XXX I guess that building the data so that it is compact as possible +# would be a big win. + + +class ResumeDataBuilder(object): + + def __init__(self): + self.memo = {} + self.liveboxes = [] + self.consts = [] + self.nums = [] + self.frame_infos = [] + + def generate_boxes(self, boxes): + for box in boxes: + assert box is not None + if isinstance(box, Box): + try: + num = self.memo[box] + except KeyError: + num = len(self.liveboxes) + self.liveboxes.append(box) + self.memo[box] = num + else: + num = -2 - len(self.consts) + self.consts.append(box) + self.nums.append(num) + self.nums.append(-1) + + def generate_frame_info(self, *frame_info): + self.frame_infos.append(frame_info) + + def finish(self, storage): + storage.rd_frame_infos = self.frame_infos[:] + storage.rd_nums = self.nums[:] + storage.rd_consts = self.consts[:] + return self.liveboxes + + +class ResumeDataReader(object): + i_frame_infos = 0 + i_boxes = 0 + + def __init__(self, storage, liveboxes): + self.frame_infos = storage.rd_frame_infos + self.nums = storage.rd_nums + self.consts = storage.rd_consts + self.liveboxes = liveboxes + + def consume_boxes(self): + boxes = [] + while True: + num = self.nums[self.i_boxes] + self.i_boxes += 1 + if num >= 0: + box = self.liveboxes[num] + elif num != -1: + box = self.consts[-2 - num] + else: + break + boxes.append(box) + return boxes + + def has_more_frame_infos(self): + return self.i_frame_infos < len(self.frame_infos) + + def consume_frame_info(self): + frame_info = self.frame_infos[self.i_frame_infos] + self.i_frame_infos += 1 + return frame_info From benjamin at codespeak.net Wed Jul 22 18:50:04 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 18:50:04 +0200 (CEST) Subject: [pypy-svn] r66505 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090722165004.DD9E0169E81@codespeak.net> Author: benjamin Date: Wed Jul 22 18:50:03 2009 New Revision: 66505 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_trace.py Log: we now follow cpython in noting the 'pass' as a line Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_trace.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_trace.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_trace.py Wed Jul 22 18:50:03 2009 @@ -336,10 +336,11 @@ for_example, [(0, 'call'), (1, 'line'), - (1, 'line'), + (2, 'line'), (1, 'line'), (2, 'line'), - (2, 'return')]) + (1, 'line'), + (1, 'return')]) def while_example(): # While expression should be traced on every loop From benjamin at codespeak.net Wed Jul 22 20:25:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:25:44 +0200 (CEST) Subject: [pypy-svn] r66507 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler pyparser Message-ID: <20090722182544.4BB3F169E7D@codespeak.net> Author: benjamin Date: Wed Jul 22 20:25:43 2009 New Revision: 66507 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: lots of checking for __future__ syntax errors Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Wed Jul 22 20:25:43 2009 @@ -6,6 +6,7 @@ consts, misc) from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops +from pypy.interpreter.pyparser import future from pypy.module.__builtin__.__init__ import BUILTIN_TO_INDEX @@ -118,8 +119,8 @@ self.symbols = symbols self.frame_blocks = [] self.interactive = False - self.temporary_name_counter = 1 self.done_with_future = False + self.temporary_name_counter = 1 self._compile(tree) def _compile(self, tree): @@ -551,6 +552,7 @@ def visit_Import(self, imp): self.update_position(imp.lineno, True) + self.done_with_future = True for alias in imp.names: assert isinstance(alias, ast.alias) if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: @@ -573,10 +575,21 @@ def visit_ImportFrom(self, imp): self.update_position(imp.lineno, True) space = self.space + first = imp.names[0] + assert isinstance(first, ast.alias) + star_import = len(imp.names) == 1 and first.name == "*" if imp.module == "__future__": - if self.done_with_future: - self.error("__future__ statements must appear before other " \ - "imports", imp) + if self.done_with_future or \ + imp.lineno > self.compile_info.last_future_import: + self.error("__future__ statements must appear at beginning " \ + "of file", imp) + if star_import: + self.error("* not valid in __future__ imports", imp) + for alias in imp.names: + assert isinstance(alias, ast.alias) + if alias.name not in future.futureFlags_2_5.compiler_features: + self.error("future feature %s is not defined" % + (alias.name,), imp) else: self.done_with_future = True if imp.level == 0 and \ @@ -596,9 +609,7 @@ else: mod_name = "" self.emit_op_name(ops.IMPORT_NAME, self.names, mod_name) - first = imp.names[0] - assert isinstance(first, ast.alias) - if len(imp.names) == 1 and first.name == "*": + if star_import: self.emit_op(ops.IMPORT_STAR) else: for alias in imp.names: Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Wed Jul 22 20:25:43 2009 @@ -242,8 +242,9 @@ space.timer.start("PythonAST compile") try: - flags |= getFutures(self.futureFlags, source) - info = CompileInfo(filename, mode, flags) + f_flags, future_lineno = getFutures(self.futureFlags, source) + flags |= f_flags + info = CompileInfo(filename, mode, flags, future_lineno) parse_tree = self.parser.parse_source(source, info) module = ast_from_node(space, parse_tree, info) code = compile_ast(space, module, info) Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Wed Jul 22 20:25:43 2009 @@ -33,7 +33,7 @@ futures.start() except DoneException, e: pass - return futures.flags + return futures.flags, futures.lineno class DoneException(Exception): pass @@ -67,6 +67,7 @@ self.futureFlags = futureFlags self.s = string self.pos = 0 + self.lineno = 1 self.docstringConsumed = False self.flags = 0 @@ -98,15 +99,23 @@ while 1: # Deal with a triple quoted docstring if self.getc() == '\\': self.pos += 2 - elif self.getc() != endchar: - self.pos += 1 else: - self.pos += 1 - if (self.getc() == endchar and - self.getc(+1) == endchar): - self.pos += 2 - self.consumeEmptyLine() - break + c = self.getc() + if c != endchar: + self.pos += 1 + if c == '\n': + self.lineno += 1 + elif c == '\r': + if self.getc() == '\n': + self.pos += 1 + self.lineno += 1 + else: + self.pos += 1 + if (self.getc() == endchar and + self.getc(+1) == endchar): + self.pos += 2 + self.consumeEmptyLine() + break else: # Deal with a single quoted docstring self.pos += 1 @@ -142,11 +151,16 @@ self.consumeWhitespace() self.start() elif self.getc() in '\r\n': + c = self.getc() self.pos += 1 - if self.getc() == '\n': - self.pos += 1 + if c == '\r': + if self.getc() == '\n': + self.pos += 1 + self.lineno += 1 + else: + self.lineno += 1 self.start() - + def consumeComment(self): self.pos += 1 while self.getc() not in '\r\n': @@ -196,11 +210,13 @@ c = self.getc() if c == '\n': self.pos += 1 + self.lineno += 1 continue elif c == '\r': self.pos += 1 if self.getc() == '\n': self.pos += 1 + self.lineno += 1 else: raise DoneException else: Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 22 20:25:43 2009 @@ -58,11 +58,12 @@ class CompileInfo(object): - def __init__(self, filename, mode="exec", flags=0): + def __init__(self, filename, mode="exec", flags=0, future_lineno=0): self.filename = filename self.mode = mode self.encoding = None self.flags = flags + self.last_future_import = future_lineno _targets = { From benjamin at codespeak.net Wed Jul 22 20:34:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:34:42 +0200 (CEST) Subject: [pypy-svn] r66508 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090722183442.00B131684A1@codespeak.net> Author: benjamin Date: Wed Jul 22 20:34:42 2009 New Revision: 66508 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: cpython error message Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 20:34:42 2009 @@ -199,15 +199,15 @@ def note_yield(self, yield_node): if self.return_with_value: - raise SyntaxError("return with value in generator", + raise SyntaxError("'return' with argument in generator", yield_node.lineno, yield_node.col_offset) self.is_generator = True def note_return(self, ret): if ret.value: if self.is_generator: - raise SyntaxError("return with value in generator", ret.lineno, - ret.col_offset) + raise SyntaxError("'return' with argument in generator", + ret.lineno, ret.col_offset) self.return_with_value = True def note_exec(self, exc): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 22 20:34:42 2009 @@ -313,7 +313,7 @@ for input in ("yield\n return x", "return x\n yield"): input = "def f():\n " + input exc = py.test.raises(SyntaxError, self.func_scope, input).value - assert exc.msg == "return with value in generator" + assert exc.msg == "'return' with argument in generator" scp = self.func_scope("def f():\n return\n yield x") def test_return(self): From benjamin at codespeak.net Wed Jul 22 20:44:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:44:16 +0200 (CEST) Subject: [pypy-svn] r66509 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090722184416.ACD9C168041@codespeak.net> Author: benjamin Date: Wed Jul 22 20:44:16 2009 New Revision: 66509 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: in -> inside Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 20:44:16 2009 @@ -199,7 +199,7 @@ def note_yield(self, yield_node): if self.return_with_value: - raise SyntaxError("'return' with argument in generator", + raise SyntaxError("'return' with argument inside generator", yield_node.lineno, yield_node.col_offset) self.is_generator = True Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 22 20:44:16 2009 @@ -313,7 +313,7 @@ for input in ("yield\n return x", "return x\n yield"): input = "def f():\n " + input exc = py.test.raises(SyntaxError, self.func_scope, input).value - assert exc.msg == "'return' with argument in generator" + assert exc.msg == "'return' with argument inside generator" scp = self.func_scope("def f():\n return\n yield x") def test_return(self): From benjamin at codespeak.net Wed Jul 22 20:52:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:52:15 +0200 (CEST) Subject: [pypy-svn] r66510 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722185215.8010A169E88@codespeak.net> Author: benjamin Date: Wed Jul 22 20:52:14 2009 New Revision: 66510 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: always error out on the yield node Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 20:52:14 2009 @@ -201,13 +201,15 @@ if self.return_with_value: raise SyntaxError("'return' with argument inside generator", yield_node.lineno, yield_node.col_offset) + self.yie_node = yield_node self.is_generator = True def note_return(self, ret): if ret.value: if self.is_generator: raise SyntaxError("'return' with argument in generator", - ret.lineno, ret.col_offset) + self.yie_node.lineno, + self.yie_node.col_offset) self.return_with_value = True def note_exec(self, exc): From benjamin at codespeak.net Wed Jul 22 20:52:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:52:50 +0200 (CEST) Subject: [pypy-svn] r66511 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090722185250.657D3169E88@codespeak.net> Author: benjamin Date: Wed Jul 22 20:52:50 2009 New Revision: 66511 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py Log: these error message do the same job Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py Wed Jul 22 20:52:50 2009 @@ -137,12 +137,12 @@ >>> (y for y in (1,2)) = 10 #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: assign to generator expression not possible... + SyntaxError: can't assign to generator expression >>> (y for y in (1,2)) += 10 #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: augmented assign to tuple literal or generator expression not possible... + SyntaxError: can't assign to generator expression From benjamin at codespeak.net Wed Jul 22 20:58:32 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 20:58:32 +0200 (CEST) Subject: [pypy-svn] r66512 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722185832.E5E1F1684BC@codespeak.net> Author: benjamin Date: Wed Jul 22 20:58:32 2009 New Revision: 66512 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: give the return statement as the error line Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 20:58:32 2009 @@ -200,17 +200,16 @@ def note_yield(self, yield_node): if self.return_with_value: raise SyntaxError("'return' with argument inside generator", - yield_node.lineno, yield_node.col_offset) - self.yie_node = yield_node + self.ret.lineno, self.ret.col_offset) self.is_generator = True def note_return(self, ret): if ret.value: if self.is_generator: raise SyntaxError("'return' with argument in generator", - self.yie_node.lineno, - self.yie_node.col_offset) + ret.lineno, ret.col_offset) self.return_with_value = True + self.ret = ret def note_exec(self, exc): Scope.note_exec(self, exc) From benjamin at codespeak.net Wed Jul 22 21:04:56 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 21:04:56 +0200 (CEST) Subject: [pypy-svn] r66513 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722190456.57DA9318101@codespeak.net> Author: benjamin Date: Wed Jul 22 21:04:53 2009 New Revision: 66513 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Log: another error message tweak Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 21:04:53 2009 @@ -206,7 +206,7 @@ def note_return(self, ret): if ret.value: if self.is_generator: - raise SyntaxError("'return' with argument in generator", + raise SyntaxError("'return' with argument inside generator", ret.lineno, ret.col_offset) self.return_with_value = True self.ret = ret From benjamin at codespeak.net Wed Jul 22 21:18:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 21:18:21 +0200 (CEST) Subject: [pypy-svn] r66514 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test Message-ID: <20090722191821.B3A00168060@codespeak.net> Author: benjamin Date: Wed Jul 22 21:18:13 2009 New Revision: 66514 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: add quotes Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Wed Jul 22 21:18:13 2009 @@ -69,7 +69,7 @@ return mangled def note_yield(self, yield_node): - raise SyntaxError("yield outside function", yield_node.lineno, + raise SyntaxError("'yield' outside function", yield_node.lineno, yield_node.col_offset) def note_return(self, ret): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Wed Jul 22 21:18:13 2009 @@ -309,7 +309,7 @@ assert scp.is_generator for input in ("yield x", "class y: yield x"): exc = py.test.raises(SyntaxError, self.mod_scope, "yield x").value - assert exc.msg == "yield outside function" + assert exc.msg == "'yield' outside function" for input in ("yield\n return x", "return x\n yield"): input = "def f():\n " + input exc = py.test.raises(SyntaxError, self.func_scope, input).value From benjamin at codespeak.net Wed Jul 22 21:20:42 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 21:20:42 +0200 (CEST) Subject: [pypy-svn] r66515 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090722192042.7B2841683D3@codespeak.net> Author: benjamin Date: Wed Jul 22 21:20:41 2009 New Revision: 66515 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py Log: these error message are imp details Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py Wed Jul 22 21:20:41 2009 @@ -1535,17 +1535,17 @@ >>> def f(): x = yield = y Traceback (most recent call last): ... -SyntaxError: assignment to yield expression not possible (, line 1) +SyntaxError: can't assign to yield expr (, line 1) >>> def f(): (yield bar) = y Traceback (most recent call last): ... -SyntaxError: assignment to yield expression not possible (, line 1) +SyntaxError: can't assign to yield expr (, line 1) >>> def f(): (yield bar) += y Traceback (most recent call last): ... -SyntaxError: augmented assignment to yield expression not possible (, line 1) +SyntaxError: can't assign to yield expr (, line 1) Now check some throw() conditions: From benjamin at codespeak.net Wed Jul 22 22:37:09 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 22:37:09 +0200 (CEST) Subject: [pypy-svn] r66517 - in pypy/branch/parser-compiler: lib-python/modified-2.5.2/test pypy/interpreter/astcompiler Message-ID: <20090722203709.001F5168550@codespeak.net> Author: benjamin Date: Wed Jul 22 22:37:08 2009 New Revision: 66517 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: spelling out expression is just fine Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py Wed Jul 22 22:37:08 2009 @@ -1535,17 +1535,17 @@ >>> def f(): x = yield = y Traceback (most recent call last): ... -SyntaxError: can't assign to yield expr (, line 1) +SyntaxError: ccan't assign to yield expression (, line 1) >>> def f(): (yield bar) = y Traceback (most recent call last): ... -SyntaxError: can't assign to yield expr (, line 1) +SyntaxError: can't assign to yield expression (, line 1) >>> def f(): (yield bar) += y Traceback (most recent call last): ... -SyntaxError: can't assign to yield expr (, line 1) +SyntaxError: can't assign to yield expression (, line 1) Now check some throw() conditions: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Wed Jul 22 22:37:08 2009 @@ -715,7 +715,7 @@ for i in range(0, len(stmt.children) - 2, 2): target_node = stmt.children[i] if target_node.type == syms.yield_expr: - self.error("can't assign to yield expr", target_node) + self.error("can't assign to yield expression", target_node) target_expr = self.handle_testlist(target_node) self.set_context(target_expr, ast.Store, target_node) targets.append(target_expr) From benjamin at codespeak.net Wed Jul 22 22:55:18 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Wed, 22 Jul 2009 22:55:18 +0200 (CEST) Subject: [pypy-svn] r66518 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090722205518.3879516849C@codespeak.net> Author: benjamin Date: Wed Jul 22 22:55:17 2009 New Revision: 66518 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py Log: fix typo Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_generators.py Wed Jul 22 22:55:17 2009 @@ -1535,7 +1535,7 @@ >>> def f(): x = yield = y Traceback (most recent call last): ... -SyntaxError: ccan't assign to yield expression (, line 1) +SyntaxError: can't assign to yield expression (, line 1) >>> def f(): (yield bar) = y Traceback (most recent call last): From benjamin at codespeak.net Thu Jul 23 00:03:18 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:03:18 +0200 (CEST) Subject: [pypy-svn] r66519 - in pypy/branch/parser-compiler/pypy/interpreter: pyparser test Message-ID: <20090722220318.89CB6168569@codespeak.net> Author: benjamin Date: Thu Jul 23 00:03:17 2009 New Revision: 66519 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Log: make the column be the start of the token Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Thu Jul 23 00:03:17 2009 @@ -96,14 +96,16 @@ endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch - tok = (tokens.STRING, contstr + line[:end], lnum, pos, line) + tok = (tokens.STRING, contstr + line[:end], strstart[0], + strstart[1], line) token_list.append(tok) last_comment = '' contstr, needcont = '', 0 contline = None elif (needcont and not line.endswith('\\\n') and not line.endswith('\\\r\n')): - tok = (tokens.ERRORTOKEN, contstr + line, lnum, pos, line) + tok = (tokens.ERRORTOKEN, contstr + line, strstart[0], + strstart[1], line) token_list.append(tok) last_comment = '' contstr = '' @@ -131,7 +133,7 @@ if column > indents[-1]: # count indents or dedents indents.append(column) - token_list.append((tokens.INDENT, line[:pos], lnum, pos, line)) + token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: indents = indents[:-1] @@ -164,11 +166,11 @@ token, initial = line[start:end], line[start] if initial in numchars or \ (initial == '.' and token != '.'): # ordinary number - token_list.append((tokens.NUMBER, token, lnum, pos, line)) + token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': if parenlev <= 0: - tok = (tokens.NEWLINE, last_comment, lnum, pos, line) + tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' elif initial == '#': @@ -180,10 +182,11 @@ if endmatch >= 0: # all on one line pos = endmatch token = line[start:pos] - tok = (tokens.STRING, token, lnum, pos, line) + tok = (tokens.STRING, token, lnum, start, line) token_list.append(tok) last_comment = '' else: + strstart = (lnum, start) contstr = line[start:] contline = line break @@ -191,17 +194,19 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string + strstart = (lnum, start) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string - tok = (tokens.STRING, token, lnum, pos, line) + tok = (tokens.STRING, token, lnum, start, line) token_list.append(tok) last_comment = '' elif initial in namechars: # ordinary name - token_list.append((tokens.NAME, token, lnum, pos, line)) + print token, start + token_list.append((tokens.NAME, token, lnum, start, line)) last_comment = '' elif initial == '\\': # continued stmt continued = 1 @@ -217,7 +222,7 @@ punct = python_opmap[token] else: punct = tokens.OP - token_list.append((punct, token, lnum, pos, line)) + token_list.append((punct, token, lnum, start, line)) last_comment = '' else: start = whiteSpaceDFA.recognize(line, pos) Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_syntax.py Thu Jul 23 00:03:17 2009 @@ -640,7 +640,7 @@ except SyntaxError, e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + 1 + assert e.offset == e.text.index('b') else: raise Exception("no SyntaxError??") From benjamin at codespeak.net Thu Jul 23 00:05:48 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:05:48 +0200 (CEST) Subject: [pypy-svn] r66520 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler pyparser Message-ID: <20090722220548.B96D4168482@codespeak.net> Author: benjamin Date: Thu Jul 23 00:05:47 2009 New Revision: 66520 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Log: check for invalid __future__ statements with the column offset Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 00:05:47 2009 @@ -119,7 +119,6 @@ self.symbols = symbols self.frame_blocks = [] self.interactive = False - self.done_with_future = False self.temporary_name_counter = 1 self._compile(tree) @@ -552,7 +551,6 @@ def visit_Import(self, imp): self.update_position(imp.lineno, True) - self.done_with_future = True for alias in imp.names: assert isinstance(alias, ast.alias) if self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: @@ -579,8 +577,10 @@ assert isinstance(first, ast.alias) star_import = len(imp.names) == 1 and first.name == "*" if imp.module == "__future__": - if self.done_with_future or \ - imp.lineno > self.compile_info.last_future_import: + last_line, last_offset = self.compile_info.last_future_import + if imp.lineno > last_line or \ + imp.lineno == last_line and imp.col_offset > last_offset: + print last_offset, imp.lineno, imp.col_offset self.error("__future__ statements must appear at beginning " \ "of file", imp) if star_import: @@ -590,8 +590,6 @@ if alias.name not in future.futureFlags_2_5.compiler_features: self.error("future feature %s is not defined" % (alias.name,), imp) - else: - self.done_with_future = True if imp.level == 0 and \ not self.compile_info.flags & consts.CO_FUTURE_ABSOLUTE_IMPORT: level = -1 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Thu Jul 23 00:05:47 2009 @@ -33,7 +33,7 @@ futures.start() except DoneException, e: pass - return futures.flags, futures.lineno + return futures.flags, (futures.lineno, futures.col_offset) class DoneException(Exception): pass @@ -68,6 +68,8 @@ self.s = string self.pos = 0 self.lineno = 1 + self.line_start_pos = 0 + self.col_offset = 0 self.docstringConsumed = False self.flags = 0 @@ -90,6 +92,10 @@ else: return + def atbol(self): + self.lineno += 1 + self.line_start_pos = self.pos + def consumeDocstring(self): self.docstringConsumed = True endchar = self.getc() @@ -104,11 +110,11 @@ if c != endchar: self.pos += 1 if c == '\n': - self.lineno += 1 + self.atbol() elif c == '\r': if self.getc() == '\n': self.pos += 1 - self.lineno += 1 + self.atbol() else: self.pos += 1 if (self.getc() == endchar and @@ -156,9 +162,9 @@ if c == '\r': if self.getc() == '\n': self.pos += 1 - self.lineno += 1 + self.atbol() else: - self.lineno += 1 + self.atbol() self.start() def consumeComment(self): @@ -168,6 +174,7 @@ self.consumeEmptyLine() def consumeFrom(self): + col_offset = self.pos - self.line_start_pos self.pos += 1 if self.getc() == 'r' and self.getc(+1) == 'o' and self.getc(+2) == 'm': self.docstringConsumed = True @@ -190,6 +197,7 @@ else: self.setFlag(self.getName()) self.getMore() + self.col_offset = col_offset self.consumeEmptyLine() else: return @@ -210,13 +218,13 @@ c = self.getc() if c == '\n': self.pos += 1 - self.lineno += 1 + self.atbol() continue elif c == '\r': self.pos += 1 if self.getc() == '\n': self.pos += 1 - self.lineno += 1 + self.atbol() else: raise DoneException else: From benjamin at codespeak.net Thu Jul 23 00:40:39 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:40:39 +0200 (CEST) Subject: [pypy-svn] r66521 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test Message-ID: <20090722224039.AAACD168547@codespeak.net> Author: benjamin Date: Thu Jul 23 00:40:39 2009 New Revision: 66521 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_futureautomaton.py Log: fix line detection for future imports Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Thu Jul 23 00:40:39 2009 @@ -67,7 +67,8 @@ self.futureFlags = futureFlags self.s = string self.pos = 0 - self.lineno = 1 + self.current_lineno = 1 + self.lineno = -1 self.line_start_pos = 0 self.col_offset = 0 self.docstringConsumed = False @@ -93,7 +94,7 @@ return def atbol(self): - self.lineno += 1 + self.current_lineno += 1 self.line_start_pos = self.pos def consumeDocstring(self): @@ -175,6 +176,7 @@ def consumeFrom(self): col_offset = self.pos - self.line_start_pos + line = self.current_lineno self.pos += 1 if self.getc() == 'r' and self.getc(+1) == 'o' and self.getc(+2) == 'm': self.docstringConsumed = True @@ -198,6 +200,7 @@ self.setFlag(self.getName()) self.getMore() self.col_offset = col_offset + self.lineno = line self.consumeEmptyLine() else: return Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_futureautomaton.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_futureautomaton.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_futureautomaton.py Thu Jul 23 00:40:39 2009 @@ -124,15 +124,17 @@ def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - flags = future.getFutures(future.futureFlags_2_5, s) + flags, pos = future.getFutures(future.futureFlags_2_5, s) assert flags == (fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED | fut.CO_FUTURE_WITH_STATEMENT) + assert pos == (3, 55) def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' - flags = future.getFutures(future.futureFlags_2_5, s) + flags, pos = future.getFutures(future.futureFlags_2_5, s) assert flags & fut.CO_FUTURE_WITH_STATEMENT == 0 + assert pos == (1, 0) def test_nonexisting(): From benjamin at codespeak.net Thu Jul 23 00:42:44 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:42:44 +0200 (CEST) Subject: [pypy-svn] r66522 - pypy/branch/parser-compiler/pypy/interpreter/pyparser/test Message-ID: <20090722224244.AAAF3168569@codespeak.net> Author: benjamin Date: Thu Jul 23 00:42:43 2009 New Revision: 66522 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Log: offsets are now at the beginning Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Thu Jul 23 00:42:43 2009 @@ -62,7 +62,7 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 12 + assert exc.offset == 5 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "\"blah").value assert exc.msg == "EOL while scanning single-quoted string" @@ -84,7 +84,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 4 + assert exc.offset == 0 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" From benjamin at codespeak.net Thu Jul 23 00:46:56 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:46:56 +0200 (CEST) Subject: [pypy-svn] r66523 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090722224656.437C7168569@codespeak.net> Author: benjamin Date: Thu Jul 23 00:46:55 2009 New Revision: 66523 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: remove print Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 00:46:55 2009 @@ -580,7 +580,6 @@ last_line, last_offset = self.compile_info.last_future_import if imp.lineno > last_line or \ imp.lineno == last_line and imp.col_offset > last_offset: - print last_offset, imp.lineno, imp.col_offset self.error("__future__ statements must appear at beginning " \ "of file", imp) if star_import: From benjamin at codespeak.net Thu Jul 23 00:47:15 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:47:15 +0200 (CEST) Subject: [pypy-svn] r66524 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090722224715.9BD3D168547@codespeak.net> Author: benjamin Date: Thu Jul 23 00:47:15 2009 New Revision: 66524 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: remove print Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Thu Jul 23 00:47:15 2009 @@ -205,7 +205,6 @@ token_list.append(tok) last_comment = '' elif initial in namechars: # ordinary name - print token, start token_list.append((tokens.NAME, token, lnum, start, line)) last_comment = '' elif initial == '\\': # continued stmt From benjamin at codespeak.net Thu Jul 23 00:53:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 00:53:02 +0200 (CEST) Subject: [pypy-svn] r66525 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090722225302.D9FBA168569@codespeak.net> Author: benjamin Date: Thu Jul 23 00:53:01 2009 New Revision: 66525 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Log: appease flow space Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Thu Jul 23 00:53:01 2009 @@ -85,6 +85,7 @@ line = '' pos = 0 lines.append("") + strstart = (0, 0) for line in lines: lnum = lnum + 1 pos, max = 0, len(line) From benjamin at codespeak.net Thu Jul 23 01:41:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 01:41:50 +0200 (CEST) Subject: [pypy-svn] r66526 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090722234150.B3720169E30@codespeak.net> Author: benjamin Date: Thu Jul 23 01:41:49 2009 New Revision: 66526 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: provide a reasonable default Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Thu Jul 23 01:41:49 2009 @@ -58,12 +58,12 @@ class CompileInfo(object): - def __init__(self, filename, mode="exec", flags=0, future_lineno=0): + def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0)): self.filename = filename self.mode = mode self.encoding = None self.flags = flags - self.last_future_import = future_lineno + self.last_future_import = future_pos _targets = { From benjamin at codespeak.net Thu Jul 23 03:07:43 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 03:07:43 +0200 (CEST) Subject: [pypy-svn] r66527 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090723010743.C983216803E@codespeak.net> Author: benjamin Date: Thu Jul 23 03:07:42 2009 New Revision: 66527 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Log: guard against DoneException Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/future.py Thu Jul 23 03:07:42 2009 @@ -73,6 +73,7 @@ self.col_offset = 0 self.docstringConsumed = False self.flags = 0 + self.got_features = 0 def getc(self, offset=0): try: @@ -190,17 +191,21 @@ raise DoneException self.pos += 6 self.consumeWhitespace() - if self.getc() == '(': - self.pos += 1 - self.consumeWhitespace() - self.setFlag(self.getName()) - # Set flag corresponding to name - self.getMore(parenList=True) - else: - self.setFlag(self.getName()) - self.getMore() - self.col_offset = col_offset - self.lineno = line + old_got = self.got_features + try: + if self.getc() == '(': + self.pos += 1 + self.consumeWhitespace() + self.setFlag(self.getName()) + # Set flag corresponding to name + self.getMore(parenList=True) + else: + self.setFlag(self.getName()) + self.getMore() + finally: + if self.got_features > old_got: + self.col_offset = col_offset + self.lineno = line self.consumeEmptyLine() else: return @@ -269,6 +274,7 @@ self.getMore(parenList=parenList) def setFlag(self, feature): + self.got_features += 1 try: self.flags |= self.futureFlags.compiler_features[feature] except KeyError: From benjamin at codespeak.net Thu Jul 23 03:13:28 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 03:13:28 +0200 (CEST) Subject: [pypy-svn] r66528 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090723011328.4DEE9169E1D@codespeak.net> Author: benjamin Date: Thu Jul 23 03:13:27 2009 New Revision: 66528 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py Log: add traceback Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_genexps.py Thu Jul 23 03:13:27 2009 @@ -137,12 +137,12 @@ >>> (y for y in (1,2)) = 10 #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: can't assign to generator expression + SyntaxError: can't assign to generator expression (, line 1) >>> (y for y in (1,2)) += 10 #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: can't assign to generator expression + SyntaxError: can't assign to generator expression (, line 1) From arigo at codespeak.net Thu Jul 23 10:08:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 10:08:36 +0200 (CEST) Subject: [pypy-svn] r66529 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723080836.7AE59169E29@codespeak.net> Author: arigo Date: Thu Jul 23 10:08:34 2009 New Revision: 66529 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (contents, props changed) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Implement virtuals in resume.py. Already a bit complicated, and that's without worrying too much about making the data as compact as possible... Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Thu Jul 23 10:08:34 2009 @@ -1,4 +1,6 @@ -from pypy.jit.metainterp.history import Box +import sys +from pypy.jit.metainterp.history import Box, Const +from pypy.jit.metainterp.resoperation import rop # Logic to encode the chain of frames and the state of the boxes at a # FAIL operation, and to decode it again. This is a bit advanced, @@ -6,7 +8,7 @@ # arbitrary cycles. # XXX I guess that building the data so that it is compact as possible -# would be a big win. +# on the 'storage' object would be a big win. class ResumeDataBuilder(object): @@ -41,33 +43,123 @@ storage.rd_frame_infos = self.frame_infos[:] storage.rd_nums = self.nums[:] storage.rd_consts = self.consts[:] + storage.rd_virtuals = None return self.liveboxes +VIRTUAL_FLAG = int((sys.maxint+1) // 2) +assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two + +class ResumeDataVirtualAdder(object): + + def __init__(self, storage, liveboxes): + self.storage = storage + self.nums = storage.rd_nums[:] + self.consts = storage.rd_consts[:] + assert storage.rd_virtuals is None + self.original_liveboxes = liveboxes + self.liveboxes = {} + for box in liveboxes: + self.liveboxes[box] = 0 + self.virtuals = [] + self.vfieldboxes = [] + + def make_virtual(self, virtualbox, known_class, sizedescr, + fielddescrs, fieldboxes): + assert self.liveboxes[virtualbox] == 0 + self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG + vinfo = VirtualInfo(known_class, sizedescr, fielddescrs) + self.virtuals.append(vinfo) + self.vfieldboxes.append(fieldboxes) + for box in fieldboxes: + if isinstance(box, Box): + self.liveboxes.setdefault(box, 0) + + def finish(self): + storage = self.storage + liveboxes = [] + for box, index in self.liveboxes.items(): + if index == 0: + self.liveboxes[box] = len(liveboxes) + liveboxes.append(box) + for i in range(len(storage.rd_nums)): + num = storage.rd_nums[i] + if num >= 0: + box = self.original_liveboxes[num] + storage.rd_nums[i] = self.liveboxes[box] + storage.rd_virtuals = self.virtuals[:] + for i in range(len(storage.rd_virtuals)): + vinfo = storage.rd_virtuals[i] + fieldboxes = self.vfieldboxes[i] + vinfo.fieldnums = [self._getboxindex(box) for box in fieldboxes] + storage.rd_consts = self.consts[:] + return liveboxes + + def _getboxindex(self, box): + if isinstance(box, Const): + result = -2 - len(self.consts) + self.consts.append(box) + return result + else: + return self.liveboxes[box] + + +class VirtualInfo(object): + def __init__(self, known_class, sizedescr, fielddescrs): + self.known_class = known_class + self.sizedescr = sizedescr + self.fielddescrs = fielddescrs + #self.fieldnums = ... + + def allocate(self, metainterp): + return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, + [self.known_class], + descr=self.sizedescr) + + def setfields(self, metainterp, box, fn_decode_box): + for i in range(len(self.fielddescrs)): + fieldbox = fn_decode_box(self.fieldnums[i]) + metainterp.execute_and_record(rop.SETFIELD_GC, + [box, fieldbox], + descr=self.fielddescrs[i]) + + class ResumeDataReader(object): i_frame_infos = 0 i_boxes = 0 - def __init__(self, storage, liveboxes): + def __init__(self, storage, liveboxes, metainterp=None): self.frame_infos = storage.rd_frame_infos self.nums = storage.rd_nums self.consts = storage.rd_consts self.liveboxes = liveboxes + if storage.rd_virtuals is not None: + self.prepare_virtuals(metainterp, storage.rd_virtuals) + + def prepare_virtuals(self, metainterp, virtuals): + self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + for i in range(len(virtuals)): + vinfo = virtuals[i] + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): boxes = [] while True: num = self.nums[self.i_boxes] self.i_boxes += 1 - if num >= 0: - box = self.liveboxes[num] - elif num != -1: - box = self.consts[-2 - num] - else: + if num == -1: break - boxes.append(box) + boxes.append(self._decode_box(num)) return boxes + def _decode_box(self, num): + if num < 0: + return self.consts[-2 - num] + elif num & VIRTUAL_FLAG: + return self.virtuals[num - VIRTUAL_FLAG] + else: + return self.liveboxes[num] + def has_more_frame_infos(self): return self.i_frame_infos < len(self.frame_infos) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 23 10:08:34 2009 @@ -53,6 +53,7 @@ nodesize2 = cpu.sizeof(NODE2) valuedescr = cpu.fielddescrof(NODE, 'value') nextdescr = cpu.fielddescrof(NODE, 'next') + otherdescr = cpu.fielddescrof(NODE2, 'other') cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2)} Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 10:08:34 2009 @@ -0,0 +1,115 @@ +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.jit.metainterp.resume import * +from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr +from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin +from pypy.jit.metainterp import executor + + +class Storage: + pass + +def make_demo_storage(): + b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + storage = Storage() + builder = ResumeDataBuilder() + builder.generate_boxes([b1, c1, b1, b2]) + builder.generate_boxes([c2, c3]) + builder.generate_boxes([b1, b2, b3]) + liveboxes = builder.finish(storage) + assert liveboxes == [b1, b2, b3] + return storage + +# ____________________________________________________________ + + +def test_simple(): + storage = make_demo_storage() + b1s, b2s, b3s = [BoxInt(), BoxPtr(), BoxInt()] + assert b1s != b3s + reader = ResumeDataReader(storage, [b1s, b2s, b3s]) + lst = reader.consume_boxes() + assert lst == [b1s, ConstInt(1), b1s, b2s] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1s, b2s, b3s] + + +def test_frame_info(): + storage = Storage() + # + builder = ResumeDataBuilder() + builder.generate_frame_info(1, 2) + builder.generate_frame_info(3, 4) + liveboxes = builder.finish(storage) + assert liveboxes == [] + # + reader = ResumeDataReader(storage, liveboxes) + assert reader.has_more_frame_infos() + fi = reader.consume_frame_info() + assert fi == (1, 2) + assert reader.has_more_frame_infos() + fi = reader.consume_frame_info() + assert fi == (3, 4) + assert not reader.has_more_frame_infos() + + +class MyMetaInterp: + def execute_and_record(self, opnum, argboxes, descr=None): + return executor.execute(LLtypeMixin.cpu, opnum, argboxes, descr) + + +def test_virtual_adder(): + storage = make_demo_storage() + b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3), + BoxPtr(), BoxPtr()] + c1s = ConstInt(111) + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + modifier.make_virtual(b2s, + ConstAddr(LLtypeMixin.node_vtable_adr, + LLtypeMixin.cpu), + LLtypeMixin.nodesize, + [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], + [b4s, c1s]) # new fields + modifier.make_virtual(b4s, + ConstAddr(LLtypeMixin.node_vtable_adr2, + LLtypeMixin.cpu), + LLtypeMixin.nodesize2, + [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, + LLtypeMixin.otherdescr], + [b2s, b3s, b5s]) # new fields + # done + liveboxes = modifier.finish() + # we get 'liveboxes' in an unspecified order + sortedliveboxes = liveboxes[:] + sortedliveboxes.sort(key=lambda box: box.value) + assert sortedliveboxes == [b1s, + #b2s -- virtual + b3s, + #b4s -- virtual + #b2s -- again, shared + #b3s -- again, shared + b5s] + # + demo55 = lltype.malloc(LLtypeMixin.NODE) + demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) + b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] + newliveboxes = [{b1s:b1t, b3s:b3t, b5s:b5t}[bs] for bs in liveboxes] + reader = ResumeDataReader(storage, newliveboxes, MyMetaInterp()) + lst = reader.consume_boxes() + b2t = lst[-1] + assert lst == [b1t, ConstInt(1), b1t, b2t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1t, b2t, b3t] + # + ptr = b2t.value._obj.container._as_ptr() + assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE) + assert ptr.value == 111 + ptr2 = ptr.next + ptr2 = lltype.cast_pointer(lltype.Ptr(LLtypeMixin.NODE2), ptr2) + assert ptr2.other == demo55 + assert ptr2.parent.value == 33 + assert ptr2.parent.next == ptr From arigo at codespeak.net Thu Jul 23 10:24:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 10:24:56 +0200 (CEST) Subject: [pypy-svn] r66530 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723082456.CF0AB16847C@codespeak.net> Author: arigo Date: Thu Jul 23 10:24:55 2009 New Revision: 66530 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Log: Revert r66529 for now. I will try instead of match the existing interface for now, instead of going down that path of complexity. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Thu Jul 23 10:24:55 2009 @@ -1,14 +1,10 @@ -import sys -from pypy.jit.metainterp.history import Box, Const -from pypy.jit.metainterp.resoperation import rop +from pypy.jit.metainterp.history import Box # Logic to encode the chain of frames and the state of the boxes at a -# FAIL operation, and to decode it again. This is a bit advanced, -# because it needs to support optimize.py which encodes virtuals with -# arbitrary cycles. +# FAIL operation, and to decode it again. # XXX I guess that building the data so that it is compact as possible -# on the 'storage' object would be a big win. +# would be a big win. class ResumeDataBuilder(object): @@ -43,123 +39,33 @@ storage.rd_frame_infos = self.frame_infos[:] storage.rd_nums = self.nums[:] storage.rd_consts = self.consts[:] - storage.rd_virtuals = None return self.liveboxes -VIRTUAL_FLAG = int((sys.maxint+1) // 2) -assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two - -class ResumeDataVirtualAdder(object): - - def __init__(self, storage, liveboxes): - self.storage = storage - self.nums = storage.rd_nums[:] - self.consts = storage.rd_consts[:] - assert storage.rd_virtuals is None - self.original_liveboxes = liveboxes - self.liveboxes = {} - for box in liveboxes: - self.liveboxes[box] = 0 - self.virtuals = [] - self.vfieldboxes = [] - - def make_virtual(self, virtualbox, known_class, sizedescr, - fielddescrs, fieldboxes): - assert self.liveboxes[virtualbox] == 0 - self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG - vinfo = VirtualInfo(known_class, sizedescr, fielddescrs) - self.virtuals.append(vinfo) - self.vfieldboxes.append(fieldboxes) - for box in fieldboxes: - if isinstance(box, Box): - self.liveboxes.setdefault(box, 0) - - def finish(self): - storage = self.storage - liveboxes = [] - for box, index in self.liveboxes.items(): - if index == 0: - self.liveboxes[box] = len(liveboxes) - liveboxes.append(box) - for i in range(len(storage.rd_nums)): - num = storage.rd_nums[i] - if num >= 0: - box = self.original_liveboxes[num] - storage.rd_nums[i] = self.liveboxes[box] - storage.rd_virtuals = self.virtuals[:] - for i in range(len(storage.rd_virtuals)): - vinfo = storage.rd_virtuals[i] - fieldboxes = self.vfieldboxes[i] - vinfo.fieldnums = [self._getboxindex(box) for box in fieldboxes] - storage.rd_consts = self.consts[:] - return liveboxes - - def _getboxindex(self, box): - if isinstance(box, Const): - result = -2 - len(self.consts) - self.consts.append(box) - return result - else: - return self.liveboxes[box] - - -class VirtualInfo(object): - def __init__(self, known_class, sizedescr, fielddescrs): - self.known_class = known_class - self.sizedescr = sizedescr - self.fielddescrs = fielddescrs - #self.fieldnums = ... - - def allocate(self, metainterp): - return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, - [self.known_class], - descr=self.sizedescr) - - def setfields(self, metainterp, box, fn_decode_box): - for i in range(len(self.fielddescrs)): - fieldbox = fn_decode_box(self.fieldnums[i]) - metainterp.execute_and_record(rop.SETFIELD_GC, - [box, fieldbox], - descr=self.fielddescrs[i]) - - class ResumeDataReader(object): i_frame_infos = 0 i_boxes = 0 - def __init__(self, storage, liveboxes, metainterp=None): + def __init__(self, storage, liveboxes): self.frame_infos = storage.rd_frame_infos self.nums = storage.rd_nums self.consts = storage.rd_consts self.liveboxes = liveboxes - if storage.rd_virtuals is not None: - self.prepare_virtuals(metainterp, storage.rd_virtuals) - - def prepare_virtuals(self, metainterp, virtuals): - self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] - for i in range(len(virtuals)): - vinfo = virtuals[i] - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): boxes = [] while True: num = self.nums[self.i_boxes] self.i_boxes += 1 - if num == -1: + if num >= 0: + box = self.liveboxes[num] + elif num != -1: + box = self.consts[-2 - num] + else: break - boxes.append(self._decode_box(num)) + boxes.append(box) return boxes - def _decode_box(self, num): - if num < 0: - return self.consts[-2 - num] - elif num & VIRTUAL_FLAG: - return self.virtuals[num - VIRTUAL_FLAG] - else: - return self.liveboxes[num] - def has_more_frame_infos(self): return self.i_frame_infos < len(self.frame_infos) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 10:24:55 2009 @@ -61,6 +61,8 @@ def test_virtual_adder(): + py.test.skip("not enabled") + # see r66529 in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp storage = make_demo_storage() b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] From arigo at codespeak.net Thu Jul 23 10:25:40 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 10:25:40 +0200 (CEST) Subject: [pypy-svn] r66531 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090723082540.EDDA616847C@codespeak.net> Author: arigo Date: Thu Jul 23 10:25:40 2009 New Revision: 66531 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Log: oups Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 10:25:40 2009 @@ -1,3 +1,4 @@ +import py from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr From fijal at codespeak.net Thu Jul 23 11:31:51 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 23 Jul 2009 11:31:51 +0200 (CEST) Subject: [pypy-svn] r66532 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090723093151.41CE3168489@codespeak.net> Author: fijal Date: Thu Jul 23 11:31:49 2009 New Revision: 66532 Modified: pypy/extradoc/talk/euroscipy2009/talk.txt Log: Some progress on talk Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Thu Jul 23 11:31:49 2009 @@ -19,17 +19,41 @@ PyPy - what's that? =================== -xxx +* A Python interpreter written in Python + +* A flexible compiler compiling restricted version + of Python to lower level platform + +* An open source project (MIT license) Motivation behind the project ============================= -xxx +* CPython is not flexible enough + +* Python is so much better to write in than C + +* Let compiler do the hard work + +* Psyco/stackless hard to maintain/extend + +* "Psyco consumes one brain per inch of progress" + +Motivation - user perspective +============================== + +* Noone should be ever forced to write in C for performance Status of PyPy today ==================== -xxx +* Very compliant python interpreter (passes CPython test suite) + +* Can run complex python applications (Django, twisted, ctypes) + +* Pretty good garbage collector + +* JIT - in progress (more later) Python's performance ==================== @@ -47,17 +71,36 @@ * Psyco hard to maintain and extend +* Python, contrary to popular belief, is a very complex language + +* We want to generate JIT out of interpreter's description + +* ... instead of writing it by hand + +JIT details +=========== + +* Interpreter + JIT generated from the interpreter + xxx JIT - status ============ -* JIT compiler generator, not JIT compiler +xxx + +xxx + +EULAG +===== +Cloud processes cover tremendous range of scales, from thousands of +kilometers to a fraction of a cm... -XXXX -EULAG slides -XXXX ++----------------------+------------------------+---------------------+ +| .. image:: earth.png | .. image:: clouds.png | .. image:: flow.png | +| :width: 100px | :width: 100px | :width: 100px | ++----------------------+------------------------+---------------------+ Data postprocessing =================== @@ -85,3 +128,16 @@ * Still about 4x slower than C or matrix operations +JIT & Numpy +=========== + +xxx + +Future directions +================= + +xxx + +Thank you! Questions? +===================== + From fijal at codespeak.net Thu Jul 23 11:40:50 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 23 Jul 2009 11:40:50 +0200 (CEST) Subject: [pypy-svn] r66533 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090723094050.A8AC61684B0@codespeak.net> Author: fijal Date: Thu Jul 23 11:40:49 2009 New Revision: 66533 Modified: pypy/extradoc/talk/euroscipy2009/talk.txt Log: some more... Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Thu Jul 23 11:40:49 2009 @@ -80,13 +80,23 @@ JIT details =========== -* Interpreter + JIT generated from the interpreter +* Interpreter + JIT automatically generated from the interpreter -xxx +* Tracing JIT (a-la tracemonkey) + +* XXX JIT - status ============ +* Simple numeric stuff - 20-30x performance boost + +* C-like speed for simple stuff + +* only x86 backend so far + +* More work needed + xxx xxx @@ -99,9 +109,14 @@ +----------------------+------------------------+---------------------+ | .. image:: earth.png | .. image:: clouds.png | .. image:: flow.png | -| :width: 100px | :width: 100px | :width: 100px | +| :width: 90px | :width: 100px | :width: 100px | +----------------------+------------------------+---------------------+ +EULAG +===== + +XXX + Data postprocessing =================== @@ -131,7 +146,13 @@ JIT & Numpy =========== -xxx +* Just started really + +* Specifically tailored programs see good speedups + +* Ongoing jit refactoring XXX + +* More work needed XXX Future directions ================= From david at codespeak.net Thu Jul 23 12:36:33 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 12:36:33 +0200 (CEST) Subject: [pypy-svn] r66534 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090723103633.7B7DB16846E@codespeak.net> Author: david Date: Thu Jul 23 12:36:32 2009 New Revision: 66534 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: For method on Object Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Thu Jul 23 12:36:32 2009 @@ -72,3 +72,24 @@ import pdb pdb.set_trace() return w_target + + at register_method('Object', 'for') +def object_for(space, w_target, w_message, w_context): + argcount = len(w_message.arguments) + assert argcount >= 4 and argcount <=5 + + body = w_message.arguments[-1] + start = int(w_message.arguments[1].name) + stop = int(w_message.arguments[2].name) + if argcount == 4: + step = 1 + else: + step = int(w_message.arguments[3].name) + + + key = w_message.arguments[0].name + + for i in range(start, stop, step): + w_context.slots[key] = W_Number(space, i) + t = body.eval(space, w_context, w_context) + return t \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Thu Jul 23 12:36:32 2009 @@ -63,4 +63,14 @@ assert res.value == -1 inp = '-"a"' - py.test.raises(Exception, "interpret(inp)") \ No newline at end of file + py.test.raises(Exception, "interpret(inp)") + +def test_object_for(): + inp = """a:= list(); + for(x, 0, 10, 3, a append(x)); + a""" + res, space = interpret(inp) + + assert len(res.items) == 4 + results = [t.value for t in res.items] + results == [0, 3, 6, 9] \ No newline at end of file From david at codespeak.net Thu Jul 23 12:37:07 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 12:37:07 +0200 (CEST) Subject: [pypy-svn] r66535 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090723103707.EB583168568@codespeak.net> Author: david Date: Thu Jul 23 12:37:07 2009 New Revision: 66535 Modified: pypy/branch/io-lang/pypy/lang/io/message.py pypy/branch/io-lang/pypy/lang/io/test/test_message.py Log: arguments on Message Modified: pypy/branch/io-lang/pypy/lang/io/message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/message.py Thu Jul 23 12:37:07 2009 @@ -6,7 +6,12 @@ if arg_num < len(w_message.arguments): return w_message.arguments[arg_num] return space.w_nil - + + at register_method('Message', 'arguments') +def message_arguments(space, w_receiver, w_message, w_context): + return space.w_list.clone_and_init(space, w_receiver.arguments) + + # @register_method('Message', 'setIsActivatable', unwrap_spec=[object, bool]) # def message_setIsActivatable(space, w_target, setting): # w_target.activateable = setting Modified: pypy/branch/io-lang/pypy/lang/io/test/test_message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_message.py Thu Jul 23 12:37:07 2009 @@ -1,5 +1,5 @@ from pypy.lang.io.parserhack import interpret -from pypy.lang.io.model import W_Message, W_Block +from pypy.lang.io.model import W_Message, W_Block, W_List import py @@ -14,7 +14,16 @@ inp = 'a := message(foo(2,3,4)); a argAt(1)' res, space = interpret(inp) assert res.name == '3' - + +def test_message_arguments(): + inp = """msg := message(B(C D, E)); + msg arguments""" + res, space = interpret(inp) + assert isinstance(res, W_List) + assert res[0].name == 'C' + assert res[0].next.name == 'D' + assert res[1].name == 'E' + # def test_setIsActivatable(): # inp = "a := block(1);a setIsActivateable(true); a" # res,space = interpret(inp) From benjamin at codespeak.net Thu Jul 23 14:58:34 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 14:58:34 +0200 (CEST) Subject: [pypy-svn] r66536 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723125834.81554169E98@codespeak.net> Author: benjamin Date: Thu Jul 23 14:58:33 2009 New Revision: 66536 Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (contents, props changed) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: implement simple jump optimizations Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Thu Jul 23 14:58:33 2009 @@ -214,6 +214,14 @@ offset += instr.size() if instr.has_jump: target, absolute = instr.jump + op = instr.opcode + if op == ops.JUMP_ABSOLUTE or op == ops.JUMP_FORWARD: + if target.instructions: + target_op = target.instructions[0].opcode + if target_op == ops.JUMP_ABSOLUTE: + target = target.instructions[0].jump[0] + instr.opcode = ops.JUMP_ABSOLUTE + absolute = True if absolute: jump_arg = target.offset else: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 14:58:33 2009 @@ -4,6 +4,7 @@ from pypy.interpreter.astcompiler import (ast2 as ast, assemble, symtable, consts, misc) +from pypy.interpreter.astcompiler import optimize # For side effects from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops from pypy.interpreter.pyparser import future @@ -306,8 +307,7 @@ def visit_Assert(self, asrt): self.update_position(asrt.lineno) end = self.new_block() - asrt.test.walkabout(self) - self.emit_jump(ops.JUMP_IF_TRUE, end) + asrt.test.accept_jump_if(self, True, end) self.emit_op(ops.POP_TOP) self.emit_op_name(ops.LOAD_GLOBAL, self.names, "AssertionError") if asrt.msg: @@ -378,8 +378,7 @@ self.visit_sequence(if_.body) else: next = self.new_block() - if_.test.walkabout(self) - self.emit_jump(ops.JUMP_IF_FALSE, next) + if_.test.accept_jump_if(self, False, next) self.emit_op(ops.POP_TOP) self.visit_sequence(if_.body) self.emit_jump(ops.JUMP_FORWARD, end) @@ -462,8 +461,7 @@ if test_constant == misc.CONST_NOT_CONST: # Force another lineno to be set for tracing purposes. self.lineno_set = False - wh.test.walkabout(self) - self.emit_jump(ops.JUMP_IF_FALSE, anchor) + wh.test.accept_jump_if(self, False, anchor) self.emit_op(ops.POP_TOP) self.visit_sequence(wh.body) self.emit_jump(ops.JUMP_ABSOLUTE, loop, True) @@ -777,8 +775,7 @@ self.update_position(ifexp.lineno) end = self.new_block() otherwise = self.new_block() - ifexp.test.walkabout(self) - self.emit_jump(ops.JUMP_IF_FALSE, otherwise) + ifexp.test.accept_jump_if(self, False, otherwise) self.emit_op(ops.POP_TOP) ifexp.body.walkabout(self) self.emit_jump(ops.JUMP_FORWARD, end) @@ -921,8 +918,7 @@ if gen.ifs: if_count = len(gen.ifs) for if_ in gen.ifs: - if_.walkabout(self) - self.emit_jump(ops.JUMP_IF_FALSE, if_cleanup) + if_.accept_jump_if(self, False, if_cleanup) self.use_next_block() self.emit_op(ops.POP_TOP) else: @@ -976,8 +972,7 @@ if gen.ifs: ifs_count = len(gen.ifs) for if_ in gen.ifs: - if_.walkabout(self) - self.emit_jump(ops.JUMP_IF_FALSE, if_cleanup) + if_.accept_jump_if(self, False, if_cleanup) self.use_next_block() self.emit_op(ops.POP_TOP) else: Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 14:58:33 2009 @@ -0,0 +1,41 @@ +from pypy.interpreter.astcompiler import ast2 as ast +from pypy.tool import stdlib_opcode as ops + + +class __extend__(ast.expr): + + def accept_jump_if(self, gen, condition, target): + self.walkabout(gen) + if condition: + gen.emit_jump(ops.JUMP_IF_TRUE, target) + else: + gen.emit_jump(ops.JUMP_IF_FALSE, target) + + +class __extend__(ast.UnaryOp): + + def accept_jump_if(self, gen, condition, target): + if self.op == ast.Not: + self.operand.accept_jump_if(gen, not condition, target) + else: + ast.expr.accept_jump_if(self, gen, condition, target) + + + +class __extend__(ast.BoolOp): + + def _accept_jump_if_any_is(self, gen, condition, target): + self.values[0].accept_jump_if(gen, condition, target) + for i in range(1, len(self.values)): + gen.emit_op(ops.POP_TOP) + self.values[i].accept_jump_if(gen, condition, target) + + def accept_jump_if(self, gen, condition, target): + if condition and self.op == ast.And or \ + (not condition and self.op == ast.Or): + end = gen.new_block() + self._accept_jump_if_any_is(gen, not condition, end) + gen.emit_jump(ops.JUMP_FORWARD, target) + gen.use_next_block(end) + else: + self._accept_jump_if_any_is(gen, condition, target) From benjamin at codespeak.net Thu Jul 23 15:20:41 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 15:20:41 +0200 (CEST) Subject: [pypy-svn] r66537 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723132041.44647169E40@codespeak.net> Author: benjamin Date: Thu Jul 23 15:20:39 2009 New Revision: 66537 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: make constant functions methods of ast Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 15:20:39 2009 @@ -370,11 +370,11 @@ def visit_If(self, if_): self.update_position(if_.lineno, True) end = self.new_block() - test_constant = misc.expr_constant(self.space, if_.test) - if test_constant == misc.CONST_FALSE: + test_constant = if_.test.as_constant_truth(self.space) + if test_constant == optimize.CONST_FALSE: if if_.orelse: self.visit_sequence(if_.orelse) - elif test_constant == misc.CONST_TRUE: + elif test_constant == optimize.CONST_TRUE: self.visit_sequence(if_.body) else: next = self.new_block() @@ -445,27 +445,27 @@ def visit_While(self, wh): self.update_position(wh.lineno, True) - test_constant = misc.expr_constant(self.space, wh.test) - if test_constant == misc.CONST_FALSE: + test_constant = wh.test.as_constant_truth(self.space) + if test_constant == optimize.CONST_FALSE: if wh.orelse: self.visit_sequence(wh.orelse) else: end = self.new_block() anchor = None - if test_constant == misc.CONST_NOT_CONST: + if test_constant == optimize.CONST_NOT_CONST: anchor = self.new_block() self.emit_jump(ops.SETUP_LOOP, end) loop = self.new_block() self.push_frame_block(F_BLOCK_LOOP, loop) self.use_next_block(loop) - if test_constant == misc.CONST_NOT_CONST: + if test_constant == optimize.CONST_NOT_CONST: # Force another lineno to be set for tracing purposes. self.lineno_set = False wh.test.accept_jump_if(self, False, anchor) self.emit_op(ops.POP_TOP) self.visit_sequence(wh.body) self.emit_jump(ops.JUMP_ABSOLUTE, loop, True) - if test_constant == misc.CONST_NOT_CONST: + if test_constant == optimize.CONST_NOT_CONST: self.use_next_block(anchor) self.emit_op(ops.POP_TOP) self.emit_op(ops.POP_BLOCK) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Thu Jul 23 15:20:39 2009 @@ -23,23 +23,6 @@ _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset) -CONST_NOT_CONST = -1 -CONST_FALSE = 0 -CONST_TRUE = 1 - -def expr_constant(space, expr): - if isinstance(expr, ast.Num): - return int(space.is_true(expr.n)) - elif isinstance(expr, ast.Str): - return int(space.is_true(expr.s)) - return CONST_NOT_CONST - -def is_constant(expr): - if isinstance(expr, (ast.Num, ast.Str)): - return True - return False - - def dict_to_switch(d): def lookup(query): if we_are_translated(): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 15:20:39 2009 @@ -2,8 +2,22 @@ from pypy.tool import stdlib_opcode as ops +CONST_NOT_CONST = -1 +CONST_FALSE = 0 +CONST_TRUE = 1 + + class __extend__(ast.expr): + def as_constant_truth(self, space): + const = self.as_constant() + if const is None: + return CONST_NOT_CONST + return int(space.is_true(const)) + + def as_constant(self): + return None + def accept_jump_if(self, gen, condition, target): self.walkabout(gen) if condition: @@ -12,6 +26,18 @@ gen.emit_jump(ops.JUMP_IF_FALSE, target) +class __extend__(ast.Num): + + def as_constant(self): + return self.n + + +class __extend__(ast.Str): + + def as_constant(self): + return self.s + + class __extend__(ast.UnaryOp): def accept_jump_if(self, gen, condition, target): From benjamin at codespeak.net Thu Jul 23 15:34:40 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 15:34:40 +0200 (CEST) Subject: [pypy-svn] r66538 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723133440.40AD41684A1@codespeak.net> Author: benjamin Date: Thu Jul 23 15:34:39 2009 New Revision: 66538 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Log: there's no need any more for a separate lnotab class Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Thu Jul 23 15:34:39 2009 @@ -282,14 +282,44 @@ return max_depth def _build_lnotab(self, blocks): - lineno_table_builder = LinenoTableBuilder(self.first_lineno) + current_line = self.first_lineno + current_off = 0 + table = [] + push = table.append for block in blocks: offset = block.offset for instr in block.instructions: if instr.lineno: - lineno_table_builder.note_lineno_here(offset, instr.lineno) + # compute deltas + line = instr.lineno - current_line + assert line >= 0 + addr = offset - current_off + # Python assumes that lineno always increases with + # increasing bytecode address (lnotab is unsigned char). + # Depending on when SET_LINENO instructions are emitted this + # is not always true. Consider the code: + # a = (1, + # b) + # In the bytecode stream, the assignment to "a" occurs after + # the loading of "b". This works with the C Python compiler + # because it only generates a SET_LINENO instruction for the + # assignment. + if line or addr: + while addr > 255: + push(chr(255)) + push(chr(0)) + addr -= 255 + while line > 255: + push(chr(addr)) + push(chr(255)) + line -= 255 + addr = 0 + push(chr(addr)) + push(chr(line)) + current_line = instr.lineno + current_off = offset offset += instr.size() - return lineno_table_builder.get_table() + return ''.join(table) def assemble(self): if not self.current_block.have_return: @@ -537,45 +567,3 @@ return _static_opcode_stack_effects[op] except KeyError: return _stack_effect_computers[op](arg) - - -class LinenoTableBuilder(object): - - def __init__(self, first_lineno): - self.first = self.current_line = first_lineno - self.current_off = 0 - self.table = [] - - def get_table(self): - return ''.join(self.table) - - def note_lineno_here(self, offset, lineno): - # compute deltas - line = lineno - self.current_line - assert line >= 0 - addr = offset - self.current_off - # Python assumes that lineno always increases with - # increasing bytecode address (lnotab is unsigned char). - # Depending on when SET_LINENO instructions are emitted - # this is not always true. Consider the code: - # a = (1, - # b) - # In the bytecode stream, the assignment to "a" occurs - # after the loading of "b". This works with the C Python - # compiler because it only generates a SET_LINENO instruction - # for the assignment. - if line or addr: - push = self.table.append - while addr > 255: - push(chr(255)) - push(chr(0)) - addr -= 255 - while line > 255: - push(chr(addr)) - push(chr(255)) - line -= 255 - addr = 0 - push(chr(addr)) - push(chr(line)) - self.current_line = lineno - self.current_off = offset From benjamin at codespeak.net Thu Jul 23 16:32:21 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 16:32:21 +0200 (CEST) Subject: [pypy-svn] r66539 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools Message-ID: <20090723143221.1989C169E85@codespeak.net> Author: benjamin Date: Thu Jul 23 16:32:20 2009 New Revision: 66539 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/Python.asdl Log: add a Const node to store the results of constant folding Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/Python.asdl ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/Python.asdl (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/Python.asdl Thu Jul 23 16:32:20 2009 @@ -77,6 +77,9 @@ | List(expr* elts, expr_context ctx) | Tuple(expr *elts, expr_context ctx) + -- PyPy modification + | Const(object value) + -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) From arigo at codespeak.net Thu Jul 23 17:30:03 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 17:30:03 +0200 (CEST) Subject: [pypy-svn] r66541 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723153003.70000169EAA@codespeak.net> Author: arigo Date: Thu Jul 23 17:30:01 2009 New Revision: 66541 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Log: Unrevert r66530, and tie virtuals to FAILs in optimizeopt.py. In-progress. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Thu Jul 23 17:30:01 2009 @@ -4,6 +4,7 @@ from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs +from pypy.jit.metainterp import resume from pypy.rlib.objectmodel import we_are_translated @@ -36,11 +37,9 @@ def force_box(self): return self.box - def get_args_for_fail(self, list, memo): - if not self.is_constant(): - if self.box not in memo: - list.append(self.box) - memo[self.box] = None + def get_args_for_fail(self, srcbox, modifier): + if self.is_constant(): + modifier.make_constant(srcbox, self.box) def is_constant(self): return self.level == LEVEL_CONSTANT @@ -120,10 +119,11 @@ newoperations.append(op) return self.box - def get_args_for_fail(self, list, memo): + def get_args_for_fail(self, srcbox, modifier): if self.box is not None: - InstanceValue.get_args_for_fail(self, list, memo) + InstanceValue.get_args_for_fail(self, srcbox, modifier) else: + xxxxx if self.source_op is not None: #otherwise, no loop detection needed keybox = self.source_op.result if keybox in memo: @@ -258,18 +258,19 @@ op_fail = op1.suboperations[0] assert op_fail.opnum == rop.FAIL # - memo = {} - newboxes = [] + if not we_are_translated() and op_fail.descr is None: # for tests + op_fail.descr = Storage() + resume.ResumeDataBuilder() + resume.generate_boxes(op_fail.args) + resume.finish(op_fail.descr) + modifier = resume.ResumeDataVirtualAdder(op_fail.descr, op_fail.args) for box in op_fail.args: - try: + if box in self.values: value = self.values[box] - except KeyError: - if box not in memo: - newboxes.append(box) - memo[box] = None - else: - value.get_args_for_fail(newboxes, memo) - # NB. we mutate op_fail in-place above. That's bad. Hopefully + value.get_args_for_fail(box, modifier) + newboxes = modifier.finish() + # NB. we mutate op_fail in-place below, as well as op_fail.descr + # via the ResumeDataVirtualAdder. That's bad. Hopefully # it does not really matter because no-one is going to look again # at its unoptimized version. We cannot really clone it because of # how the rest works (e.g. it is returned by cpu.execute_operations()). @@ -405,3 +406,6 @@ optimize_ops = _findall(Optimizer, 'optimize_') + +class Storage: + "for tests." Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Thu Jul 23 17:30:01 2009 @@ -1512,7 +1512,7 @@ if not we_are_translated(): self._debug_history.append(['guard_failure', None, None]) vinfo = self.staticdata.virtualizable_info - resumereader = resume.ResumeDataReader(resumedescr, newboxes) + resumereader = resume.ResumeDataReader(resumedescr, newboxes, self) if vinfo is not None: self.virtualizable_boxes = resumereader.consume_boxes() # just jumped away from assembler (case 4 in the comment in Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Thu Jul 23 17:30:01 2009 @@ -1,10 +1,14 @@ -from pypy.jit.metainterp.history import Box +import sys +from pypy.jit.metainterp.history import Box, Const +from pypy.jit.metainterp.resoperation import rop # Logic to encode the chain of frames and the state of the boxes at a -# FAIL operation, and to decode it again. +# FAIL operation, and to decode it again. This is a bit advanced, +# because it needs to support optimize.py which encodes virtuals with +# arbitrary cycles. # XXX I guess that building the data so that it is compact as possible -# would be a big win. +# on the 'storage' object would be a big win. class ResumeDataBuilder(object): @@ -39,33 +43,135 @@ storage.rd_frame_infos = self.frame_infos[:] storage.rd_nums = self.nums[:] storage.rd_consts = self.consts[:] + storage.rd_virtuals = None return self.liveboxes +VIRTUAL_FLAG = int((sys.maxint+1) // 2) +assert not (VIRTUAL_FLAG & (VIRTUAL_FLAG-1)) # a power of two + +class ResumeDataVirtualAdder(object): + + def __init__(self, storage, liveboxes): + self.storage = storage + self.nums = storage.rd_nums[:] + self.consts = storage.rd_consts[:] + assert storage.rd_virtuals is None + self.original_liveboxes = liveboxes + self.liveboxes = {} + for box in liveboxes: + self.liveboxes[box] = 0 + self.liveboxes_order = liveboxes[:] + self.virtuals = [] + self.vfieldboxes = [] + + def make_constant(self, box, const): + assert self.liveboxes[box] == 0 + self.liveboxes[box] = self._getconstindex(const) + + def make_virtual(self, virtualbox, known_class, sizedescr, + fielddescrs, fieldboxes): + assert self.liveboxes[virtualbox] == 0 + self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG + vinfo = VirtualInfo(known_class, sizedescr, fielddescrs) + self.virtuals.append(vinfo) + self.vfieldboxes.append(fieldboxes) + for box in fieldboxes: + if isinstance(box, Box): + self.liveboxes.setdefault(box, 0) + self.liveboxes_order.append(box) + + def is_virtual(self, virtualbox): + return self.liveboxes[virtualbox] >= VIRTUAL_FLAG + + def finish(self): + storage = self.storage + liveboxes = [] + for box in self.liveboxes_order: + if self.liveboxes[box] == 0: + self.liveboxes[box] = len(liveboxes) + liveboxes.append(box) + for i in range(len(storage.rd_nums)): + num = storage.rd_nums[i] + if num >= 0: + box = self.original_liveboxes[num] + storage.rd_nums[i] = self.liveboxes[box] + storage.rd_virtuals = self.virtuals[:] + for i in range(len(storage.rd_virtuals)): + vinfo = storage.rd_virtuals[i] + fieldboxes = self.vfieldboxes[i] + vinfo.fieldnums = [self._getboxindex(box) for box in fieldboxes] + storage.rd_consts = self.consts[:] + return liveboxes + + def _getboxindex(self, box): + if isinstance(box, Const): + return self._getconstindex(box) + else: + return self.liveboxes[box] + + def _getconstindex(self, const): + result = -2 - len(self.consts) + self.consts.append(const) + return result + + +class VirtualInfo(object): + def __init__(self, known_class, sizedescr, fielddescrs): + self.known_class = known_class + self.sizedescr = sizedescr + self.fielddescrs = fielddescrs + #self.fieldnums = ... + + def allocate(self, metainterp): + return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, + [self.known_class], + descr=self.sizedescr) + + def setfields(self, metainterp, box, fn_decode_box): + for i in range(len(self.fielddescrs)): + fieldbox = fn_decode_box(self.fieldnums[i]) + metainterp.execute_and_record(rop.SETFIELD_GC, + [box, fieldbox], + descr=self.fielddescrs[i]) + + class ResumeDataReader(object): i_frame_infos = 0 i_boxes = 0 - def __init__(self, storage, liveboxes): + def __init__(self, storage, liveboxes, metainterp=None): self.frame_infos = storage.rd_frame_infos self.nums = storage.rd_nums self.consts = storage.rd_consts self.liveboxes = liveboxes + if storage.rd_virtuals is not None: + self._prepare_virtuals(metainterp, storage.rd_virtuals) + + def _prepare_virtuals(self, metainterp, virtuals): + self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + for i in range(len(virtuals)): + vinfo = virtuals[i] + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): boxes = [] while True: num = self.nums[self.i_boxes] self.i_boxes += 1 - if num >= 0: - box = self.liveboxes[num] - elif num != -1: - box = self.consts[-2 - num] - else: + if num == -1: break - boxes.append(box) + boxes.append(self._decode_box(num)) return boxes + def _decode_box(self, num): + if num < 0: + return self.consts[-2 - num] + elif num & VIRTUAL_FLAG: + return self.virtuals[num - VIRTUAL_FLAG] + else: + return self.liveboxes[num] + def has_more_frame_infos(self): return self.i_frame_infos < len(self.frame_infos) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Thu Jul 23 17:30:01 2009 @@ -147,6 +147,8 @@ args.append(self.getvar(arg)) except KeyError: raise ParseError("Unknown var: %s" % arg) + if hasattr(descr, '_oparser_uses_descr'): + descr._oparser_uses_descr(self, args) return opnum, args, descr def parse_result_op(self, line): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 17:30:01 2009 @@ -1,8 +1,11 @@ import py +from pypy.jit.metainterp.test.test_resume import MyMetaInterp from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize +from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.metainterp import resume from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse @@ -570,28 +573,55 @@ self.optimize_loop(ops, 'Not', expected, i1=5, boxkinds={'myptr': self.nodebox.value}) - def test_expand_fail_arguments(self): + # ---------- + + def make_fail_descr(self, boxestext): + class FailDescr(AbstractDescr): + args_seen = [] + def _oparser_uses_descr(self, oparse, args): + # typically called twice, before and after optimization + if len(self.args_seen) == 0: + boxes = [oparse.box_for_var(boxtext.strip()) + for boxtext in boxestext.split(',')] + builder = resume.ResumeDataBuilder() + builder.generate_boxes(boxes) + liveboxes = builder.finish(fdescr) + assert liveboxes == args + self.args_seen.append((args, oparse)) + # + fdescr = FailDescr() + self.fdescr = fdescr + self.namespace['fdescr'] = fdescr + + def check_expanded_fail_descr(self, expectedtext): + fdescr = self.fdescr + args, oparse = fdescr.args_seen[-1] + reader = resume.ResumeDataReader(fdescr, args, MyMetaInterp()) + boxes = reader.consume_boxes() + expected = [oparse.getvar(boxtext.strip()) + for boxtext in expectedtext.split(',')] + assert boxes == expected + + def test_expand_fail_1(self): + self.make_fail_descr('i2, i3') ops = """ - [i1, i2, i3, p3] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p1, 1, descr=valuedescr) - setfield_gc(p1, p2, descr=nextdescr) - setfield_gc(p2, i2, descr=valuedescr) - setfield_gc(p2, p3, descr=nextdescr) + [i1, i3] + i2 = int_add(10, 5) guard_true(i1) - fail(p1, i3) - jump(i2, i1, i3, p3) + fail(i2, i3, descr=fdescr) + jump(i1, i3) """ expected = """ - [i1, i2, i3, p3] + [i1, i3] guard_true(i1) - fail(i2, p3, i3) - jump(i2, 1, i3, p3) + fail(i3, descr=fdescr) + jump(1, i3) """ - self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + self.optimize_loop(ops, 'Not, Not', expected, i1=1, i2=15) + self.check_expanded_fail_descr('15, i3') - def test_expand_fail_loop(self): + def test_expand_fail_2(self): + py.test.skip("in-progress") ops = """ [i1, i2] p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) @@ -604,12 +634,44 @@ expected = """ [i1, i2] guard_true(i1) - fail(i2) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, p1, descr=nextdescr) + fail(p1) jump(1, i2) """ self.optimize_loop(ops, 'Not, Not', expected, i1=1) - def test_expand_fail_duplicate(self): + def test_expand_fail_3(self): + py.test.skip("in-progress") + ops = """ + [i1, i2, i3, p3] + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, 1, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p3, descr=nextdescr) + guard_true(i1) + fail(p1, i3) + jump(i2, i1, i3, p3) + """ + expected = """ + [i1, i2, i3, p3] + guard_true(i1) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, 1, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p3, descr=nextdescr) + fail(p1, i3) + jump(i2, 1, i3, p3) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + + def test_expand_fail_4(self): + py.test.skip("in-progress") ops = """ [i1, i2] p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) @@ -624,17 +686,22 @@ expected = """ [i1, i2] guard_true(i1) - fail(i2) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + fail(%s) jump(1, i2) """ - self.optimize_loop(ops % 'p1', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'p1, i2', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'p1, i2, i2', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'i2, p1, i2', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'i2, i2, p1', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'p1, p1', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'p1, p2', 'Not, Not', expected, i1=1) - self.optimize_loop(ops % 'p2, p1', 'Not, Not', expected, i1=1) + self.optimize_loop(ops % 'p1', 'Not, Not', + expected % 'p1', i1=1) + self.optimize_loop(ops % 'p1, i2', 'Not, Not', + expected % 'p1, i2', i1=1) + self.optimize_loop(ops % 'i2, p1', 'Not, Not', + expected % 'i2, p1', i1=1) + self.optimize_loop(ops % 'p1, p2', 'Not, Not', + expected % 'p1, p2', i1=1) class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 17:30:01 2009 @@ -60,15 +60,40 @@ def execute_and_record(self, opnum, argboxes, descr=None): return executor.execute(LLtypeMixin.cpu, opnum, argboxes, descr) +demo55 = lltype.malloc(LLtypeMixin.NODE) +demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) -def test_virtual_adder(): - py.test.skip("not enabled") - # see r66529 in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp + +def test_virtual_adder_no_op(): + storage = make_demo_storage() + b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b1s) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + # done + liveboxes = modifier.finish() + assert liveboxes == [b1s, b2s, b3s] + # + b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] + reader = ResumeDataReader(storage, [b1t, b2t, b3t], MyMetaInterp()) + lst = reader.consume_boxes() + assert lst == [b1t, ConstInt(1), b1t, b2t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [b1t, b2t, b3t] + + +def test_virtual_adder_make_virtual(): storage = make_demo_storage() b1s, b2s, b3s, b4s, b5s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr(), BoxPtr()] c1s = ConstInt(111) modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b1s) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), @@ -82,24 +107,23 @@ [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, LLtypeMixin.otherdescr], [b2s, b3s, b5s]) # new fields + assert not modifier.is_virtual(b1s) + assert modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + assert modifier.is_virtual(b4s) + assert not modifier.is_virtual(b5s) # done liveboxes = modifier.finish() - # we get 'liveboxes' in an unspecified order - sortedliveboxes = liveboxes[:] - sortedliveboxes.sort(key=lambda box: box.value) - assert sortedliveboxes == [b1s, - #b2s -- virtual - b3s, - #b4s -- virtual - #b2s -- again, shared - #b3s -- again, shared - b5s] + assert liveboxes == [b1s, + #b2s -- virtual + b3s, + #b4s -- virtual + #b2s -- again, shared + #b3s -- again, shared + b5s] # - demo55 = lltype.malloc(LLtypeMixin.NODE) - demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] - newliveboxes = [{b1s:b1t, b3s:b3t, b5s:b5t}[bs] for bs in liveboxes] - reader = ResumeDataReader(storage, newliveboxes, MyMetaInterp()) + reader = ResumeDataReader(storage, [b1t, b3t, b5t], MyMetaInterp()) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] @@ -116,3 +140,26 @@ assert ptr2.other == demo55 assert ptr2.parent.value == 33 assert ptr2.parent.next == ptr + + +def test_virtual_adder_make_constant(): + storage = make_demo_storage() + b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + modifier.make_constant(b1s, ConstInt(111)) # <------- + assert not modifier.is_virtual(b1s) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + # done + liveboxes = modifier.finish() + assert liveboxes == [b2s, b3s] + # + b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] + reader = ResumeDataReader(storage, [b2t, b3t], MyMetaInterp()) + lst = reader.consume_boxes() + c1t = ConstInt(111) + assert lst == [c1t, ConstInt(1), c1t, b2t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [c1t, b2t, b3t] From david at codespeak.net Thu Jul 23 17:30:26 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 17:30:26 +0200 (CEST) Subject: [pypy-svn] r66542 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090723153026.4B208169EAF@codespeak.net> Author: david Date: Thu Jul 23 17:30:25 2009 New Revision: 66542 Modified: pypy/branch/io-lang/pypy/lang/io/call.py pypy/branch/io-lang/pypy/lang/io/test/test_call.py Log: evalArgAt method on Call Modified: pypy/branch/io-lang/pypy/lang/io/call.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/call.py (original) +++ pypy/branch/io-lang/pypy/lang/io/call.py Thu Jul 23 17:30:25 2009 @@ -6,4 +6,9 @@ return space.w_message.slots['argAt'].apply( space, w_target.slots['message'], - w_message, w_context) \ No newline at end of file + w_message, w_context) + + + at register_method('Call', 'evalArgAt') +def call_eval_arg_at(space, w_target, w_message, w_context): + return call_arg_at(space, w_target, w_message, w_context).eval(space, w_context, w_context) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_call.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_call.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_call.py Thu Jul 23 17:30:25 2009 @@ -5,4 +5,10 @@ def test_message_arg_at(): inp = 'a := method(x, y, z, call); a(1,2,3) argAt(1)' res, space = interpret(inp) - assert res.name == '2' \ No newline at end of file + assert res.name == '2' + + +def test_message_evalArgAt(): + inp = """t := 99; a := method(x, y, z, call); a(1,t,3) evalArgAt(1)""" + res, space = interpret(inp) + assert res.value == 99 \ No newline at end of file From david at codespeak.net Thu Jul 23 17:33:25 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 17:33:25 +0200 (CEST) Subject: [pypy-svn] r66543 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20090723153325.A40AD169EAE@codespeak.net> Author: david Date: Thu Jul 23 17:33:24 2009 New Revision: 66543 Modified: pypy/branch/io-lang/pypy/lang/io/object.py Log: Improved Object for Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Thu Jul 23 17:33:24 2009 @@ -79,16 +79,16 @@ assert argcount >= 4 and argcount <=5 body = w_message.arguments[-1] - start = int(w_message.arguments[1].name) - stop = int(w_message.arguments[2].name) + start = w_message.arguments[1].eval(space, w_target, w_context).value + stop = w_message.arguments[2].eval(space, w_target, w_context).value if argcount == 4: step = 1 else: - step = int(w_message.arguments[3].name) + step = w_message.arguments[3].eval(space, w_message, w_context).value key = w_message.arguments[0].name - + for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) From david at codespeak.net Thu Jul 23 17:33:43 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 17:33:43 +0200 (CEST) Subject: [pypy-svn] r66544 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20090723153343.34472169EB2@codespeak.net> Author: david Date: Thu Jul 23 17:33:42 2009 New Revision: 66544 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_map.py Log: Test for io-method with Modified: pypy/branch/io-lang/pypy/lang/io/test/test_map.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_map.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_map.py Thu Jul 23 17:33:42 2009 @@ -161,3 +161,10 @@ assert res.slots['2'].value == 99 assert res.slots['3'].value == 3 assert res.slots['4'].value == 234 + + +def test_map_with(): + inp = """Map with("a", 1, "b", 2) asObject""" + res, space = interpret(inp) + assert res.slots['a'].value == 1 + assert res.slots['b'].value == 2 \ No newline at end of file From david at codespeak.net Thu Jul 23 17:40:14 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 17:40:14 +0200 (CEST) Subject: [pypy-svn] r66545 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20090723154014.3CCCC169EAE@codespeak.net> Author: david Date: Thu Jul 23 17:40:13 2009 New Revision: 66545 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Ensure Object for leaks the index variable Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Thu Jul 23 17:40:13 2009 @@ -73,4 +73,12 @@ assert len(res.items) == 4 results = [t.value for t in res.items] - results == [0, 3, 6, 9] \ No newline at end of file + results == [0, 3, 6, 9] + +def test_object_leaks(): + inp = """a:= list(); + for(x, 0, 10, 3, a append(x)); + x""" + res, _ = interpret(inp) + + assert res.value == 9 \ No newline at end of file From david at codespeak.net Thu Jul 23 18:23:09 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 18:23:09 +0200 (CEST) Subject: [pypy-svn] r66546 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090723162309.38824169EB2@codespeak.net> Author: david Date: Thu Jul 23 18:23:07 2009 New Revision: 66546 Modified: pypy/branch/io-lang/pypy/lang/io/call.py pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/test/test_call.py Log: target and sender on Call objects Modified: pypy/branch/io-lang/pypy/lang/io/call.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/call.py (original) +++ pypy/branch/io-lang/pypy/lang/io/call.py Thu Jul 23 18:23:07 2009 @@ -11,4 +11,12 @@ @register_method('Call', 'evalArgAt') def call_eval_arg_at(space, w_target, w_message, w_context): - return call_arg_at(space, w_target, w_message, w_context).eval(space, w_context, w_context) \ No newline at end of file + return call_arg_at(space, w_target, w_message, w_context).eval(space, w_context, w_context) + + at register_method('Call', 'sender') +def call_sender(space, w_target, w_message, w_context): + return w_target.sender + + at register_method('Call', 'target') +def call_target(space, w_target, w_message, w_context): + return w_target.target \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Thu Jul 23 18:23:07 2009 @@ -219,7 +219,11 @@ def call(self, space, w_receiver, w_message, w_context): w_locals = self.space.w_locals.clone() + w_call = self.space.w_call.clone() + w_call.sender = w_context + w_call.target = w_receiver + assert w_locals is not None assert w_call is not None Modified: pypy/branch/io-lang/pypy/lang/io/test/test_call.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_call.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_call.py Thu Jul 23 18:23:07 2009 @@ -8,7 +8,21 @@ assert res.name == '2' -def test_message_evalArgAt(): +def test_call_evalArgAt(): inp = """t := 99; a := method(x, y, z, call); a(1,t,3) evalArgAt(1)""" res, space = interpret(inp) - assert res.value == 99 \ No newline at end of file + assert res.value == 99 + +def test_call_sender(): + inp = """foo := Object clone + foo a := method(x, y, z, call); + foo a(1,2,3) sender""" + res, space = interpret(inp) + assert res == space.w_lobby + +def test_call_receiver(): + inp = """foo := Object clone + foo a := method(x, y, z, call); + foo a(1,2,3) target""" + res, space = interpret(inp) + assert res == space.w_lobby.slots['foo'] \ No newline at end of file From david at codespeak.net Thu Jul 23 18:31:30 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 23 Jul 2009 18:31:30 +0200 (CEST) Subject: [pypy-svn] r66547 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090723163130.31FDD169EBD@codespeak.net> Author: david Date: Thu Jul 23 18:31:30 2009 New Revision: 66547 Modified: pypy/branch/io-lang/pypy/lang/io/message.py pypy/branch/io-lang/pypy/lang/io/test/test_message.py Log: name on Message Modified: pypy/branch/io-lang/pypy/lang/io/message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/message.py Thu Jul 23 18:31:30 2009 @@ -1,5 +1,5 @@ from pypy.lang.io.register import register_method -from pypy.lang.io.model import W_Message +from pypy.lang.io.model import W_Message, W_ImmutableSequence @register_method('Message', 'argAt', unwrap_spec=[object, int]) def message_arg_at(space, w_message, arg_num): @@ -10,7 +10,11 @@ @register_method('Message', 'arguments') def message_arguments(space, w_receiver, w_message, w_context): return space.w_list.clone_and_init(space, w_receiver.arguments) - + + at register_method('Message', 'name') +def message_name(space, w_receiver, w_message, w_context): + # XXX TODO clone from space + return W_ImmutableSequence(space, w_receiver.name) # @register_method('Message', 'setIsActivatable', unwrap_spec=[object, bool]) # def message_setIsActivatable(space, w_target, setting): Modified: pypy/branch/io-lang/pypy/lang/io/test/test_message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_message.py Thu Jul 23 18:31:30 2009 @@ -1,5 +1,5 @@ from pypy.lang.io.parserhack import interpret -from pypy.lang.io.model import W_Message, W_Block, W_List +from pypy.lang.io.model import W_Message, W_Block, W_List, W_ImmutableSequence import py @@ -23,7 +23,14 @@ assert res[0].name == 'C' assert res[0].next.name == 'D' assert res[1].name == 'E' - + +def test_message_name(): + inp = """msg := message(B(C D, E)); + msg name""" + res, space = interpret(inp) + assert isinstance(res, W_ImmutableSequence) + assert res.value == 'B' + # def test_setIsActivatable(): # inp = "a := block(1);a setIsActivateable(true); a" # res,space = interpret(inp) From benjamin at codespeak.net Thu Jul 23 18:54:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 18:54:02 +0200 (CEST) Subject: [pypy-svn] r66548 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723165402.AFECF169ED3@codespeak.net> Author: benjamin Date: Thu Jul 23 18:54:01 2009 New Revision: 66548 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: support visiting constants Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 18:54:01 2009 @@ -724,6 +724,10 @@ self.update_position(string.lineno) self.load_const(string.s) + def visit_Const(self, const): + self.update_position(const.lineno) + self.load_const(const.value) + def visit_UnaryOp(self, op): self.update_position(op.lineno) op.operand.walkabout(self) From benjamin at codespeak.net Thu Jul 23 18:54:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 18:54:35 +0200 (CEST) Subject: [pypy-svn] r66549 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler test Message-ID: <20090723165435.55CD6169EDD@codespeak.net> Author: benjamin Date: Thu Jul 23 18:54:34 2009 New Revision: 66549 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Log: add a AST visitor which folds constants Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 18:54:34 2009 @@ -1,5 +1,14 @@ -from pypy.interpreter.astcompiler import ast2 as ast +import sys +import itertools + +from pypy.interpreter.astcompiler import ast2 as ast, consts, misc from pypy.tool import stdlib_opcode as ops +from pypy.interpreter.error import OperationError +from pypy.rlib.unroll import unrolling_iterable + + +def optimize_ast(space, tree, compile_info): + return tree.mutate_over(OptimizingVisitor(space, compile_info)) CONST_NOT_CONST = -1 @@ -38,6 +47,12 @@ return self.s +class __extend__(ast.Const): + + def as_constant(self): + return self.value + + class __extend__(ast.UnaryOp): def accept_jump_if(self, gen, condition, target): @@ -65,3 +80,164 @@ gen.use_next_block(end) else: self._accept_jump_if_any_is(gen, condition, target) + + +def _binary_fold(name): + def do_fold(space, left, right): + return getattr(space, name)(left, right) + return do_fold + +def _unary_fold(name): + def do_fold(space, operand): + return getattr(space, name)(operand) + return do_fold + +def _fold_pow(space, left, right): + return space.pow(left, right, space.w_None) + +def _fold_not(space, operand): + return space.wrap(not space.is_true(operand)) + + +binary_folders = { + ast.Add : _binary_fold("add"), + ast.Sub : _binary_fold("sub"), + ast.Mult : _binary_fold("mul"), + ast.Div : _binary_fold("truediv"), + ast.FloorDiv : _binary_fold("floordiv"), + ast.Mod : _binary_fold("mod"), + ast.Pow : _fold_pow, + ast.LShift : _binary_fold("lshift"), + ast.RShift : _binary_fold("rshift"), + ast.BitOr : _binary_fold("or_"), + ast.BitXor : _binary_fold("xor"), + ast.BitAnd : _binary_fold("and_") +} +unrolling_binary_folders = unrolling_iterable(binary_folders.items()) + +unary_folders = { + ast.Not : _fold_not, + ast.USub : _unary_fold("neg"), + ast.UAdd : _unary_fold("pos"), + ast.Invert : _unary_fold("invert") +} +unrolling_unary_folders = unrolling_iterable(unary_folders.items()) + +for folder in itertools.chain(binary_folders.itervalues(), + unary_folders.itervalues()): + folder._always_inline_ = True +del folder + +opposite_compare_operations = misc.dict_to_switch({ + ast.Is : ast.IsNot, + ast.IsNot : ast.Is, + ast.In : ast.NotIn, + ast.NotIn : ast.In +}) + + +class OptimizingVisitor(ast.ASTVisitor): + + def __init__(self, space, compile_info): + self.space = space + self.compile_info = compile_info + + def default_visitor(self, node): + return node + + def visit_BinOp(self, binop): + left = binop.left.as_constant() + if left is not None: + right = binop.right.as_constant() + if right is not None: + op = binop.op + if op == ast.Div and \ + not self.compile_info.flags & consts.CO_FUTURE_DIVISION: + return binop + try: + for op_kind, folder in unrolling_binary_folders: + if op_kind == op: + w_const = folder(self.space, left, right) + except OperationError: + pass + else: + try: + w_len = self.space.len(w_const) + except OperationError: + pass + else: + if self.space.int_w(w_len) > 20: + return binop + return ast.Const(w_const, binop.lineno, binop.col_offset) + return binop + + def visit_UnaryOp(self, unary): + w_operand = unary.operand.as_constant() + op = unary.op + if w_operand is not None: + try: + for op_kind, folder in unrolling_unary_folders: + if op_kind == op: + w_const = folder(self.space, w_operand) + w_minint = self.space.wrap(-sys.maxint - 1) + # This makes sure the result is an integer. + if self.space.eq_w(w_minint, w_const): + w_const = w_minint + except OperationError: + pass + else: + return ast.Const(w_const, unary.lineno, unary.col_offset) + elif op == ast.Not: + compare = unary.operand + if isinstance(compare, ast.Compare) and len(compare.ops) == 1: + cmp_op = compare.ops[0] + try: + opposite = opposite_compare_operations(cmp_op) + except KeyError: + pass + else: + compare.ops[0] = opposite + return compare + return unary + + def visit_BoolOp(self, bop): + values = bop.values + we_are_and = bop.op == ast.And + i = 0 + while i < len(values) - 1: + truth = values[i].as_constant_truth(self.space) + if truth != CONST_NOT_CONST: + if truth == CONST_FALSE == we_are_and: + del values[i + 1:] + break + else: + del values[i] + else: + i += 1 + if len(values) == 1: + return values[0] + return bop + + def visit_Repr(self, rep): + w_const = rep.value.as_constant() + if w_const is not None: + w_repr = self.space.repr(w_const) + return ast.Const(w_repr, rep.lineno, rep.col_offset) + return rep + + def visit_Name(self, name): + if name.id == "None": + assert name.ctx == ast.Load + return ast.Const(self.space.w_None, name.lineno, name.col_offset) + return name + + def visit_Tuple(self, tup): + consts_w = [] + if tup.elts: + for node in tup.elts: + w_const = node.as_constant() + if w_const is None: + return tup + consts_w.append(w_const) + w_consts = self.space.newtuple(consts_w) + return ast.Const(w_consts, tup.lineno, tup.col_offset) Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Thu Jul 23 18:54:34 2009 @@ -231,7 +231,7 @@ from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError from pypy.interpreter.astcompiler.astbuilder import ast_from_node from pypy.interpreter.astcompiler.codegen import compile_ast - from pypy.interpreter.astcompiler import consts + from pypy.interpreter.astcompiler import consts, optimize space = self.space @@ -247,6 +247,7 @@ info = CompileInfo(filename, mode, flags, future_lineno) parse_tree = self.parser.parse_source(source, info) module = ast_from_node(space, parse_tree, info) + module = optimize.optimize_ast(space, module, info) code = compile_ast(space, module, info) except IndentationError, e: raise OperationError(space.w_IndentationError, Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Thu Jul 23 18:54:34 2009 @@ -723,42 +723,6 @@ ## py.test.skip("unsupported") class AppTestOptimizer: - def test_constant_fold_add(self): - import parser - class Folder(object): - def defaultvisit(self, node): - return node - - def __getattr__(self, attrname): - if attrname.startswith('visit'): - return self.defaultvisit - raise AttributeError(attrname) - - def visitAdd(self, node): - left = node.left - right = node.right - if isinstance(left, parser.ASTConst) and \ - isinstance(right, parser.ASTConst): - if type(left.value) == type(right.value): - return parser.ASTConst(left.value + right.value) - return node - - def hook(ast, enc, filename): - return ast.mutate(Folder()) - - parser.install_compiler_hook(hook) - code = compile("1+2", "", "eval") - parser.install_compiler_hook(None) - import dis, sys, StringIO - s = StringIO.StringIO() - so = sys.stdout - sys.stdout = s - try: - dis.dis(code) - finally: - sys.stdout = so - output = s.getvalue() - assert 'BINARY_ADD' not in output def test_remove_ending(self): source = """def f(): @@ -777,7 +741,41 @@ output = s.getvalue() assert output.count('LOAD_CONST') == 1 - + def test_none_constant(self): + import opcode + co = compile("def f(): return None", "", "exec").co_consts[0] + co = co.co_code + op = ord(co[0]) + (ord(co[1]) << 8) + assert op == opcode.opmap["LOAD_CONST"] + + def test_division_folding(self): + def code(source): + return compile(source, "", "exec") + co = code("x = 10//4") + assert len(co.co_consts) == 2 + assert co.co_consts[0] == 2 + co = code("x = 10/4") + assert len(co.co_consts) == 3 + assert co.co_consts[:2] == (10, 4) + co = code("from __future__ import division\nx = 10/4") + assert co.co_consts[2] == 2.5 + + def test_tuple_folding(self): + co = compile("(1, 2, 3)", "", "exec") + assert co.co_consts == ((1, 2, 3), None) + co = compile("()", "", "exec") + assert co.co_consts == ((), None) + + def test_unary_folding(self): + co = compile("-(3)", "", "exec") + assert co.co_consts[0] == -3 + co = compile("~3", "", "exec") + assert co.co_consts[0] == ~3 + co = compile("+(-3)", "", "exec") + assert co.co_consts[0] == -3 + co = compile("not None", "", "exec") + assert co.co_consts[0] is True + def test_folding_of_binops_on_constants(self): def disassemble(func): from StringIO import StringIO From arigo at codespeak.net Thu Jul 23 19:11:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 19:11:09 +0200 (CEST) Subject: [pypy-svn] r66550 - in pypy/branch/pyjitpl5-optimize4/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20090723171109.42AB6169F11@codespeak.net> Author: arigo Date: Thu Jul 23 19:11:08 2009 New Revision: 66550 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/heaptracker.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Log: In-progress. Main change: NEW_WITH_VTABLE no longer takes a 'descr', because it can more easily fish it from a global table when needed. It avoids adding a 'descr' everywhere starting from the SpecNodes. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py Thu Jul 23 19:11:08 2009 @@ -268,19 +268,23 @@ return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) def do_strlen(self, args, descr=None): + assert descr is None string = args[0].getptr_base() return history.BoxInt(llimpl.do_strlen(0, string)) def do_strgetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() return history.BoxInt(llimpl.do_strgetitem(0, string, index)) def do_unicodelen(self, args, descr=None): + assert descr is None string = args[0].getptr_base() return history.BoxInt(llimpl.do_unicodelen(0, string)) def do_unicodegetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() return history.BoxInt(llimpl.do_unicodegetitem(0, string, index)) @@ -319,8 +323,10 @@ def do_new(self, args, size): return history.BoxPtr(llimpl.do_new(size.ofs)) - def do_new_with_vtable(self, args, size): + def do_new_with_vtable(self, args, descr=None): + assert descr is None vtable = args[0].getint() + size = self.class_sizes[vtable] # attribute set by codewriter.py result = llimpl.do_new(size.ofs) llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, vtable, self.memo_cast) @@ -365,20 +371,24 @@ self.memo_cast) def do_newstr(self, args, descr=None): + assert descr is None length = args[0].getint() return history.BoxPtr(llimpl.do_newstr(0, length)) def do_newunicode(self, args, descr=None): + assert descr is None length = args[0].getint() return history.BoxPtr(llimpl.do_newunicode(0, length)) def do_strsetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() newvalue = args[2].getint() llimpl.do_strsetitem(0, string, index, newvalue) def do_unicodesetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() newvalue = args[2].getint() @@ -401,11 +411,13 @@ llimpl.do_call_void(func, self.memo_cast) def do_cast_int_to_ptr(self, args, descr=None): + assert descr is None return history.BoxPtr(llimpl.cast_from_int(llmemory.GCREF, args[0].getint(), self.memo_cast)) def do_cast_ptr_to_int(self, args, descr=None): + assert descr is None return history.BoxInt(llimpl.cast_to_int(args[0].getptr_base(), self.memo_cast)) @@ -511,7 +523,7 @@ # XXX: return None if RESULT is Void return x - def do_oosend(self, args, descr=None): + def do_oosend(self, args, descr): assert isinstance(descr, MethDescr) selfbox = args[0] argboxes = args[1:] Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py Thu Jul 23 19:11:08 2009 @@ -134,7 +134,7 @@ if self.portal_graph is None or graph is self.portal_graph: return () fnptr = self.rtyper.getcallable(graph) - if self.metainterp_sd.cpu.is_oo: + if self.cpu.is_oo: if oosend_methdescr: return (None, oosend_methdescr) else: @@ -742,8 +742,8 @@ # store the vtable as an address -- that's fine, because the # GC doesn't need to follow them self.emit('new_with_vtable', - self.get_position(self.cpu.sizeof(STRUCT)), self.const_position(vtable)) + self.codewriter.register_known_gctype(vtable, STRUCT) else: self.emit('new', self.get_position(self.cpu.sizeof(STRUCT))) self.register_var(op.result) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/heaptracker.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/heaptracker.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/heaptracker.py Thu Jul 23 19:11:08 2009 @@ -86,35 +86,4 @@ vtable.name[i] = namez[i] testing_gcstruct2vtable[GCSTRUCT] = vtable -def populate_type_cache(graphs, cpu): - if not cpu.translate_support_code: - cache = {} - else: - cache = [] - for graph in graphs: - for block in graph.iterblocks(): - for op in block.operations: - if not cpu.is_oo and op.opname == 'malloc': - STRUCT = op.args[0].value - if isinstance(STRUCT, lltype.GcStruct): - vtable = get_vtable_for_gcstruct(cpu, STRUCT) - if vtable: - if not cpu.translate_support_code: - vt = cpu.cast_adr_to_int( - llmemory.cast_ptr_to_adr(vtable)) - cache[vt] = cpu.sizeof(STRUCT) - else: - vt = llmemory.cast_ptr_to_adr(vtable) - cache.append((vt, cpu.sizeof(STRUCT))) - elif cpu.is_oo and op.opname == 'new': - TYPE = op.args[0].value - if isinstance(TYPE, ootype.OOType): - cls = ootype.cast_to_object(ootype.runtimeClass(TYPE)) - typedescr = cpu.typedescrof(TYPE) - if not cpu.translate_support_code: - cache[cls] = typedescr - else: - cache.append((cls, typedescr)) - return cache - testing_gcstruct2vtable = {} Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Thu Jul 23 19:11:08 2009 @@ -37,9 +37,11 @@ def force_box(self): return self.box - def get_args_for_fail(self, srcbox, modifier): - if self.is_constant(): - modifier.make_constant(srcbox, self.box) + def get_key_box(self): + return self.box + + def get_args_for_fail(self, modifier): + pass def is_constant(self): return self.level == LEVEL_CONSTANT @@ -91,8 +93,10 @@ box = None level = LEVEL_KNOWNCLASS - def __init__(self, optimizer, source_op=None): + def __init__(self, optimizer, known_class, keybox, source_op=None): self.optimizer = optimizer + self.known_class = known_class + self.keybox = keybox # only used as a key in dictionaries self.source_op = source_op # the NEW_WITH_VTABLE operation building it self._fields = av_newdict() @@ -119,20 +123,21 @@ newoperations.append(op) return self.box - def get_args_for_fail(self, srcbox, modifier): - if self.box is not None: - InstanceValue.get_args_for_fail(self, srcbox, modifier) - else: - xxxxx - if self.source_op is not None: #otherwise, no loop detection needed - keybox = self.source_op.result - if keybox in memo: - return - memo[keybox] = None + def get_key_box(self): + if self.box is None: + return self.keybox + return self.box + + def get_args_for_fail(self, modifier): + if self.box is None and not modifier.is_virtual(self.keybox): lst = self._fields.keys() sort_descrs(lst) + fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] + modifier.make_virtual(self.keybox, self.known_class, + lst, fieldboxes) for ofs in lst: - self._fields[ofs].get_args_for_fail(list, memo) + fieldvalue = self._fields[ofs] + fieldvalue.get_args_for_fail(modifier) class __extend__(SpecNode): @@ -143,7 +148,7 @@ class __extend__(VirtualInstanceSpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): - vvalue = optimizer.make_virtual(box) + vvalue = optimizer.make_virtual(self.known_class, box) for ofs, subspecnode in self.fields: subbox = optimizer.new_box(ofs) vvalue.setfield(ofs, optimizer.getvalue(subbox)) @@ -187,8 +192,8 @@ def known_nonnull(self, box): return self.getvalue(box).is_nonnull() - def make_virtual(self, box, source_op=None): - vvalue = VirtualValue(self, source_op) + def make_virtual(self, known_class, box, source_op=None): + vvalue = VirtualValue(self, known_class, box, source_op) self.make_equal_to(box, vvalue) return vvalue @@ -259,21 +264,28 @@ assert op_fail.opnum == rop.FAIL # if not we_are_translated() and op_fail.descr is None: # for tests - op_fail.descr = Storage() - resume.ResumeDataBuilder() - resume.generate_boxes(op_fail.args) - resume.finish(op_fail.descr) - modifier = resume.ResumeDataVirtualAdder(op_fail.descr, op_fail.args) + descr = Storage() + builder = resume.ResumeDataBuilder() + builder.generate_boxes(op_fail.args) + builder.finish(descr) + else: + descr = op_fail.descr + oldboxes = [] + for box in op_fail.args: + if box in self.values: + box = self.values[box].get_key_box() # may be a Const, too + oldboxes.append(box) + modifier = resume.ResumeDataVirtualAdder(descr, oldboxes) for box in op_fail.args: if box in self.values: value = self.values[box] - value.get_args_for_fail(box, modifier) + value.get_args_for_fail(modifier) newboxes = modifier.finish() - # NB. we mutate op_fail in-place below, as well as op_fail.descr + # XXX we mutate op_fail in-place below, as well as op_fail.descr # via the ResumeDataVirtualAdder. That's bad. Hopefully # it does not really matter because no-one is going to look again - # at its unoptimized version. We cannot really clone it because of - # how the rest works (e.g. it is returned by cpu.execute_operations()). + # at its unoptimized version. We should really clone it (and + # the descr too). op_fail.args = newboxes op2.suboperations = op1.suboperations op1.optimized = op2 @@ -402,7 +414,7 @@ self.optimize_default(op) def optimize_NEW_WITH_VTABLE(self, op): - self.make_virtual(op.result, op) + self.make_virtual(op.args[0], op.result, op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Thu Jul 23 19:11:08 2009 @@ -7,7 +7,6 @@ from pypy.jit.metainterp import history, compile, resume from pypy.jit.metainterp.history import Const, ConstInt, Box from pypy.jit.metainterp.resoperation import rop -from pypy.jit.metainterp.heaptracker import populate_type_cache from pypy.jit.metainterp import codewriter, executor from pypy.jit.metainterp import typesystem from pypy.rlib.rarithmetic import intmask @@ -310,9 +309,9 @@ def opimpl_new(self, size): self.execute(rop.NEW, [], descr=size) - @arguments("descr", "constbox") - def opimpl_new_with_vtable(self, size, vtablebox): - self.execute(rop.NEW_WITH_VTABLE, [vtablebox], descr=size) + @arguments("constbox") + def opimpl_new_with_vtable(self, vtablebox): + self.execute(rop.NEW_WITH_VTABLE, [vtablebox]) @arguments("box") def opimpl_runtimenew(self, classbox): @@ -932,11 +931,6 @@ self.opcode_implementations = [] self.opcode_names = [] self.opname_to_index = {} - self._class_sizes = populate_type_cache(graphs, self.cpu) - if not cpu.translate_support_code: - self.cpu.class_sizes = self._class_sizes - else: - self.cpu.class_sizes = None if optimizer is None: from pypy.jit.metainterp import optimize4 as optimizer self.optimize_loop = optimizer.optimize_loop @@ -979,11 +973,7 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: - if self.cpu.class_sizes is None: - cs = {} - for key, value in self._class_sizes: - cs[key] = value - self.cpu.class_sizes = cs + xxxxxxxx self.cpu.setup_once() log(self.jit_starting_line) if not self.profiler.initialized: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Thu Jul 23 19:11:08 2009 @@ -59,24 +59,26 @@ assert storage.rd_virtuals is None self.original_liveboxes = liveboxes self.liveboxes = {} - for box in liveboxes: - self.liveboxes[box] = 0 - self.liveboxes_order = liveboxes[:] + self.liveboxes_order = [] + self._register_boxes(liveboxes) self.virtuals = [] self.vfieldboxes = [] def make_constant(self, box, const): - assert self.liveboxes[box] == 0 - self.liveboxes[box] = self._getconstindex(const) + # this part of the interface is not used so far by optimizeopt.py + if self.liveboxes[box] == 0: + self.liveboxes[box] = self._getconstindex(const) - def make_virtual(self, virtualbox, known_class, sizedescr, - fielddescrs, fieldboxes): + def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): assert self.liveboxes[virtualbox] == 0 self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG - vinfo = VirtualInfo(known_class, sizedescr, fielddescrs) + vinfo = VirtualInfo(known_class, fielddescrs) self.virtuals.append(vinfo) self.vfieldboxes.append(fieldboxes) - for box in fieldboxes: + self._register_boxes(fieldboxes) + + def _register_boxes(self, boxes): + for box in boxes: if isinstance(box, Box): self.liveboxes.setdefault(box, 0) self.liveboxes_order.append(box) @@ -95,7 +97,7 @@ num = storage.rd_nums[i] if num >= 0: box = self.original_liveboxes[num] - storage.rd_nums[i] = self.liveboxes[box] + storage.rd_nums[i] = self._getboxindex(box) storage.rd_virtuals = self.virtuals[:] for i in range(len(storage.rd_virtuals)): vinfo = storage.rd_virtuals[i] @@ -117,16 +119,14 @@ class VirtualInfo(object): - def __init__(self, known_class, sizedescr, fielddescrs): + def __init__(self, known_class, fielddescrs): self.known_class = known_class - self.sizedescr = sizedescr self.fielddescrs = fielddescrs #self.fieldnums = ... def allocate(self, metainterp): return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, - [self.known_class], - descr=self.sizedescr) + [self.known_class]) def setfields(self, metainterp, box, fn_decode_box): for i in range(len(self.fielddescrs)): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 23 19:11:08 2009 @@ -191,7 +191,7 @@ def test_find_nodes_new_1(self): ops = """ [p1] - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable)) jump(p2) """ boxes, getnode = self.find_nodes(ops, 'Virtual(node_vtable)') @@ -215,8 +215,8 @@ def test_find_nodes_new_2(self): ops = """ [i1, p1] - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p3 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p2 = new_with_vtable(ConstClass(node_vtable)) + p3 = new_with_vtable(ConstClass(node_vtable2)) setfield_gc(p2, p3, descr=nextdescr) setfield_gc(p3, i1, descr=valuedescr) jump(i1, p2) @@ -235,9 +235,9 @@ i1 = getfield_gc(p1, descr=valuedescr) i2 = int_sub(i1, 1) sum2 = int_add(sum, i1) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, i2, descr=valuedescr) - p3 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p3 = new_with_vtable(ConstClass(node_vtable2)) setfield_gc(p2, p3, descr=nextdescr) jump(sum2, p2) """ @@ -273,7 +273,7 @@ def test_find_nodes_new_aliasing_0(self): ops = """ [p1, p2] - p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p3 = new_with_vtable(ConstClass(node_vtable)) jump(p3, p3) """ # both p1 and p2 must be NotSpecNodes; it's not possible to pass @@ -291,7 +291,7 @@ i1 = getfield_gc(p1, descr=valuedescr) i2 = int_sub(i1, 1) sum2 = int_add(sum, i1) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p2, descr=nextdescr) jump(sum2, p2) @@ -305,7 +305,7 @@ ops = """ [p1, p2] escape(p2) - p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p3 = new_with_vtable(ConstClass(node_vtable)) jump(p3, p3) """ # both p1 and p2 must be NotSpecNodes; it's not possible to pass @@ -318,7 +318,7 @@ [p1] guard_class(p1, ConstClass(node_vtable)) fail() - p2 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p2 = new_with_vtable(ConstClass(node_vtable2)) jump(p2) """ self.find_nodes(ops, 'Not') @@ -330,7 +330,7 @@ fail() guard_class(p1, ConstClass(node_vtable2)) fail() - p2 = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p2 = new_with_vtable(ConstClass(node_vtable2)) jump(p2, p2) """ self.find_nodes(ops, 'Not, Not') @@ -339,7 +339,7 @@ ops = """ [p0] escape(p0) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) jump(p1) """ self.find_nodes(ops, 'Not') @@ -347,9 +347,9 @@ def test_find_nodes_new_unused(self): ops = """ [p0] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) + p2 = new_with_vtable(ConstClass(node_vtable)) + p3 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, p3, descr=nextdescr) jump(p1) @@ -362,8 +362,8 @@ def test_find_nodes_oois(self): ops = """ [p3, p4, p2] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable)) i1 = oononnull(p0) guard_true(i1) fail() @@ -409,7 +409,7 @@ def test_find_nodes_call(self): ops = """ [i0, p2] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(ConstClass(node_vtable)) i1 = call_pure(i0, p0) # forces p0 to not be virtual jump(i1, p0) """ @@ -421,7 +421,7 @@ i0 = getfield_gc(p0, descr=valuedescr) guard_value(i0, 0) fail() - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) # the field 'value' has its default value of 0 jump(p1) """ @@ -468,7 +468,7 @@ def test_bridge_simple_virtual_1(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, i0, descr=valuedescr) jump(p0) """ @@ -517,7 +517,7 @@ def test_bridge_virtual_mismatch_1(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p0, i0, descr=valuedescr) jump(p0, p0) """ @@ -551,9 +551,9 @@ def test_bridge_unused(self): ops = """ [] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p3 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) + p2 = new_with_vtable(ConstClass(node_vtable)) + p3 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, p3, descr=nextdescr) jump(p1) @@ -588,7 +588,7 @@ def test_bridge_virtual_to_fail(self): ops = """ [i1] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i1, descr=valuedescr) fail(p1) """ Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 19:11:08 2009 @@ -4,7 +4,7 @@ OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.metainterp.history import AbstractDescr, ConstInt from pypy.jit.metainterp import resume from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse @@ -354,7 +354,7 @@ [i, p0] i0 = getfield_gc(p0, descr=valuedescr) i1 = int_add(i0, i) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i1, descr=valuedescr) jump(i, p1) """ @@ -470,7 +470,7 @@ i0 = getfield_gc(p0, descr=valuedescr) guard_value(i0, 0) fail() - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) # the field 'value' has its default value of 0 jump(p1) """ @@ -488,7 +488,7 @@ def test_virtual_3(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -504,7 +504,7 @@ def test_nonvirtual_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -515,7 +515,7 @@ expected = """ [i] i1 = int_add(i, 1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i, descr=valuedescr) escape(p1) escape(p1) @@ -529,7 +529,7 @@ i0 = getfield_gc(p0, descr=valuedescr) escape(p0) i1 = int_add(i0, i) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i1, descr=valuedescr) jump(i, p1) """ @@ -539,7 +539,7 @@ def test_getfield_gc_pure_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_pure(p1, descr=valuedescr) jump(i1) @@ -575,16 +575,14 @@ # ---------- - def make_fail_descr(self, boxestext): + def make_fail_descr(self): class FailDescr(AbstractDescr): args_seen = [] def _oparser_uses_descr(self, oparse, args): # typically called twice, before and after optimization if len(self.args_seen) == 0: - boxes = [oparse.box_for_var(boxtext.strip()) - for boxtext in boxestext.split(',')] builder = resume.ResumeDataBuilder() - builder.generate_boxes(boxes) + builder.generate_boxes(args) liveboxes = builder.finish(fdescr) assert liveboxes == args self.args_seen.append((args, oparse)) @@ -593,23 +591,36 @@ self.fdescr = fdescr self.namespace['fdescr'] = fdescr + def unpack_expected_fail_args(self, oparse, text): + import re + r = re.compile(r"\bwhere\s+(\w+)\s+is a\s+(\w+)") + parts = list(r.finditer(text)) + if parts: + text = text[:parts[0].start()] + xxx + return [oparse.getvar(s.strip()) for s in text.split(',')] + def check_expanded_fail_descr(self, expectedtext): fdescr = self.fdescr args, oparse = fdescr.args_seen[-1] reader = resume.ResumeDataReader(fdescr, args, MyMetaInterp()) boxes = reader.consume_boxes() - expected = [oparse.getvar(boxtext.strip()) - for boxtext in expectedtext.split(',')] + expected = self.unpack_expected_fail_args(oparse, expectedtext) assert boxes == expected def test_expand_fail_1(self): - self.make_fail_descr('i2, i3') + self.make_fail_descr() ops = """ [i1, i3] + # first rename i3 into i4 + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i3, descr=valuedescr) + i4 = getfield_gc(p1, descr=valuedescr) + # i2 = int_add(10, 5) guard_true(i1) - fail(i2, i3, descr=fdescr) - jump(i1, i3) + fail(i2, i4, descr=fdescr) + jump(i1, i4) """ expected = """ [i1, i3] @@ -622,32 +633,33 @@ def test_expand_fail_2(self): py.test.skip("in-progress") + self.make_fail_descr() ops = """ [i1, i2] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p1, descr=nextdescr) guard_true(i1) - fail(p1) + fail(p1, descr=fdescr) jump(i1, i2) """ expected = """ [i1, i2] guard_true(i1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p1, i2, descr=valuedescr) - setfield_gc(p1, p1, descr=nextdescr) - fail(p1) + fail(i2, descr=fdescr) jump(1, i2) """ self.optimize_loop(ops, 'Not, Not', expected, i1=1) + self.check_expanded_fail_descr('''ptr + where ptr is a node_vtable, valuedescr=i2 + ''') def test_expand_fail_3(self): py.test.skip("in-progress") ops = """ [i1, i2, i3, p3] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) + p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, 1, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 19:11:08 2009 @@ -97,13 +97,11 @@ modifier.make_virtual(b2s, ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu), - LLtypeMixin.nodesize, [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr], [b4s, c1s]) # new fields modifier.make_virtual(b4s, ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu), - LLtypeMixin.nodesize2, [LLtypeMixin.nextdescr, LLtypeMixin.valuedescr, LLtypeMixin.otherdescr], [b2s, b3s, b5s]) # new fields @@ -143,23 +141,31 @@ def test_virtual_adder_make_constant(): - storage = make_demo_storage() - b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] - modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) - modifier.make_constant(b1s, ConstInt(111)) # <------- - assert not modifier.is_virtual(b1s) - assert not modifier.is_virtual(b2s) - assert not modifier.is_virtual(b3s) - # done - liveboxes = modifier.finish() - assert liveboxes == [b2s, b3s] - # - b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] - reader = ResumeDataReader(storage, [b2t, b3t], MyMetaInterp()) - lst = reader.consume_boxes() - c1t = ConstInt(111) - assert lst == [c1t, ConstInt(1), c1t, b2t] - lst = reader.consume_boxes() - assert lst == [ConstInt(2), ConstInt(3)] - lst = reader.consume_boxes() - assert lst == [c1t, b2t, b3t] + for testnumber in [0, 1]: + storage = make_demo_storage() + b1s, b2s, b3s = [BoxInt(1), BoxPtr(), BoxInt(3)] + if testnumber == 0: + # I. making a constant with make_constant() + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + modifier.make_constant(b1s, ConstInt(111)) + assert not modifier.is_virtual(b1s) + else: + # II. making a constant by directly specifying a constant in + # the list of liveboxes + b1s = ConstInt(111) + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + # done + liveboxes = modifier.finish() + assert liveboxes == [b2s, b3s] + # + b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] + reader = ResumeDataReader(storage, [b2t, b3t], MyMetaInterp()) + lst = reader.consume_boxes() + c1t = ConstInt(111) + assert lst == [c1t, ConstInt(1), c1t, b2t] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + lst = reader.consume_boxes() + assert lst == [c1t, b2t, b3t] From benjamin at codespeak.net Thu Jul 23 19:24:00 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 19:24:00 +0200 (CEST) Subject: [pypy-svn] r66551 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723172400.7D227169ECE@codespeak.net> Author: benjamin Date: Thu Jul 23 19:23:59 2009 New Revision: 66551 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: help the flow space with variables Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 19:23:59 2009 @@ -158,6 +158,9 @@ for op_kind, folder in unrolling_binary_folders: if op_kind == op: w_const = folder(self.space, left, right) + break + else: + raise AssertionError("unknown binary operation") except OperationError: pass else: @@ -179,6 +182,9 @@ for op_kind, folder in unrolling_unary_folders: if op_kind == op: w_const = folder(self.space, w_operand) + break + else: + raise AssertionError("unknown unary operation") w_minint = self.space.wrap(-sys.maxint - 1) # This makes sure the result is an integer. if self.space.eq_w(w_minint, w_const): From benjamin at codespeak.net Thu Jul 23 19:45:40 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 19:45:40 +0200 (CEST) Subject: [pypy-svn] r66552 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723174540.9615B169EB1@codespeak.net> Author: benjamin Date: Thu Jul 23 19:45:39 2009 New Revision: 66552 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: optimize unpacking in assignment Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Thu Jul 23 19:45:39 2009 @@ -8,6 +8,7 @@ from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops from pypy.interpreter.pyparser import future +from pypy.interpreter.error import OperationError from pypy.module.__builtin__.__init__ import BUILTIN_TO_INDEX @@ -619,6 +620,8 @@ def visit_Assign(self, assign): self.update_position(assign.lineno, True) + if self._optimize_unpacking(assign): + return assign.value.walkabout(self) duplications = len(assign.targets) - 1 for i in range(len(assign.targets)): @@ -626,6 +629,62 @@ self.emit_op(ops.DUP_TOP) assign.targets[i].walkabout(self) + def _optimize_unpacking(self, assign): + if len(assign.targets) != 1: + return False + targets = self._list_from_sequence_node(assign.targets[0], False) + if targets is None: + return False + values = self._list_from_sequence_node(assign.value, True) + if values is None: + return False + targets_count = len(targets) + values_count = len(values) + if targets_count != values_count: + return False + for target in targets: + if not isinstance(target, ast.Name): + break + else: + self.visit_sequence(values) + seen_names = {} + for i in range(targets_count - 1, -1, -1): + target = targets[i] + assert isinstance(target, ast.Name) + if target.id not in seen_names: + seen_names[target.id] = True + self.name_op(target.id, ast.Store) + else: + self.emit_op(ops.POP_TOP) + return True + if values_count > 3: + return False + self.visit_sequence(values) + if values_count == 2: + self.emit_op(ops.ROT_TWO) + elif values_count == 3: + self.emit_op(ops.ROT_THREE) + self.emit_op(ops.ROT_TWO) + self.visit_sequence(targets) + return True + + def _list_from_sequence_node(self, node, const_possible): + if isinstance(node, ast.Tuple): + nodes = node.elts + elif isinstance(node, ast.List): + nodes = node.elts + elif const_possible and isinstance(node, ast.Const): + try: + values_w = self.space.unpackiterable(node.value) + except OperationError: + return None + line = node.lineno + column = node.col_offset + nodes = [ast.Const(obj, line, column) for obj in values_w] + else: + nodes = None + return nodes + def visit_With(self, wih): self.update_position(wih.lineno, True) body_block = self.new_block() From arigo at codespeak.net Thu Jul 23 19:52:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 19:52:31 +0200 (CEST) Subject: [pypy-svn] r66553 - in pypy/branch/pyjitpl5-optimize4/pypy/jit: backend/llgraph metainterp/test Message-ID: <20090723175231.E975A169EA1@codespeak.net> Author: arigo Date: Thu Jul 23 19:52:31 2009 New Revision: 66553 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Log: Progress; the next test passes (also on ootype). Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py Thu Jul 23 19:52:31 2009 @@ -462,9 +462,11 @@ else: return ootype.NULL - def do_new_with_vtable(self, args, typedescr): - assert isinstance(typedescr, TypeDescr) - assert len(args) == 1 # but we don't need it, so ignore + def do_new_with_vtable(self, args, descr=None): + assert descr is None + assert len(args) == 1 + cls = args[0].getobj() + typedescr = self.class_sizes[cls] return typedescr.create() def do_new_array(self, args, typedescr): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 19:52:31 2009 @@ -1,11 +1,13 @@ import py +from pypy.rpython.lltypesystem import rclass +from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.test.test_resume import MyMetaInterp from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize -from pypy.jit.metainterp.history import AbstractDescr, ConstInt -from pypy.jit.metainterp import resume +from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxPtr +from pypy.jit.metainterp import resume, executor from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse @@ -591,22 +593,63 @@ self.fdescr = fdescr self.namespace['fdescr'] = fdescr - def unpack_expected_fail_args(self, oparse, text): + def _verify_fail_args(self, boxes, oparse, text): import re r = re.compile(r"\bwhere\s+(\w+)\s+is a\s+(\w+)") parts = list(r.finditer(text)) - if parts: - text = text[:parts[0].start()] - xxx - return [oparse.getvar(s.strip()) for s in text.split(',')] + ends = [match.start() for match in parts] + [len(text)] + # + virtuals = {} + for match, end in zip(parts, ends[1:]): + pvar = match.group(1) + cls_vtable = self.namespace[match.group(2)] + fieldstext = text[match.end():end] + virtuals[pvar] = (cls_vtable, None, fieldstext) + # + def _variables_equal(box, varname, strict): + if varname not in virtuals: + if strict: + assert box == oparse.getvar(varname) + else: + assert box.value == oparse.getvar(varname).value + else: + cls_vtable, resolved, fieldstext = virtuals[varname] + if not self.cpu.is_oo: + assert box.getptr(rclass.OBJECTPTR).typeptr == cls_vtable + else: + root = ootype.cast_from_object(ootype.ROOT, box.getobj()) + assert ootype.classof(root) == cls_vtable + if resolved is not None: + assert resolved.value == box.value + else: + virtuals[varname] = cls_vtable, box, fieldstext + # + basetext = text[:ends[0]] + varnames = [s.strip() for s in basetext.split(',')] + assert len(boxes) == len(varnames) + for box, varname in zip(boxes, varnames): + _variables_equal(box, varname, strict=True) + # + for cls_vtable, resolved, fieldstext in virtuals.values(): + assert resolved is not None + for fieldtext in fieldstext.split(','): + fieldtext = fieldtext.strip() + if not fieldtext: + continue + fieldname, fieldvalue = fieldtext.split('=') + fielddescr = self.namespace[fieldname.strip()] + fieldbox = executor.execute(self.cpu, + rop.GETFIELD_GC, + [resolved], + descr=fielddescr) + _variables_equal(fieldbox, fieldvalue.strip(), strict=False) def check_expanded_fail_descr(self, expectedtext): fdescr = self.fdescr args, oparse = fdescr.args_seen[-1] - reader = resume.ResumeDataReader(fdescr, args, MyMetaInterp()) + reader = resume.ResumeDataReader(fdescr, args, MyMetaInterp(self.cpu)) boxes = reader.consume_boxes() - expected = self.unpack_expected_fail_args(oparse, expectedtext) - assert boxes == expected + self._verify_fail_args(boxes, oparse, expectedtext) def test_expand_fail_1(self): self.make_fail_descr() @@ -632,7 +675,6 @@ self.check_expanded_fail_descr('15, i3') def test_expand_fail_2(self): - py.test.skip("in-progress") self.make_fail_descr() ops = """ [i1, i2] Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py Thu Jul 23 19:52:31 2009 @@ -57,8 +57,10 @@ class MyMetaInterp: + def __init__(self, cpu): + self.cpu = cpu def execute_and_record(self, opnum, argboxes, descr=None): - return executor.execute(LLtypeMixin.cpu, opnum, argboxes, descr) + return executor.execute(self.cpu, opnum, argboxes, descr) demo55 = lltype.malloc(LLtypeMixin.NODE) demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) @@ -76,7 +78,8 @@ assert liveboxes == [b1s, b2s, b3s] # b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] - reader = ResumeDataReader(storage, [b1t, b2t, b3t], MyMetaInterp()) + reader = ResumeDataReader(storage, [b1t, b2t, b3t], + MyMetaInterp(LLtypeMixin.cpu)) lst = reader.consume_boxes() assert lst == [b1t, ConstInt(1), b1t, b2t] lst = reader.consume_boxes() @@ -121,7 +124,8 @@ b5s] # b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] - reader = ResumeDataReader(storage, [b1t, b3t, b5t], MyMetaInterp()) + reader = ResumeDataReader(storage, [b1t, b3t, b5t], + MyMetaInterp(LLtypeMixin.cpu)) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] @@ -161,7 +165,8 @@ assert liveboxes == [b2s, b3s] # b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] - reader = ResumeDataReader(storage, [b2t, b3t], MyMetaInterp()) + reader = ResumeDataReader(storage, [b2t, b3t], + MyMetaInterp(LLtypeMixin.cpu)) lst = reader.consume_boxes() c1t = ConstInt(111) assert lst == [c1t, ConstInt(1), c1t, b2t] From benjamin at codespeak.net Thu Jul 23 20:05:20 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 20:05:20 +0200 (CEST) Subject: [pypy-svn] r66554 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723180520.5F5B6168461@codespeak.net> Author: benjamin Date: Thu Jul 23 20:05:19 2009 New Revision: 66554 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: don't resize list Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 20:05:19 2009 @@ -238,12 +238,15 @@ return name def visit_Tuple(self, tup): - consts_w = [] if tup.elts: - for node in tup.elts: + consts_w = []*len(tup.elts) + for i in range(len(tup.elts)): + node = tup.elts[i] w_const = node.as_constant() if w_const is None: return tup - consts_w.append(w_const) + consts_w[i] = node + else: + consts_w = [] w_consts = self.space.newtuple(consts_w) return ast.Const(w_consts, tup.lineno, tup.col_offset) From benjamin at codespeak.net Thu Jul 23 20:08:49 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 20:08:49 +0200 (CEST) Subject: [pypy-svn] r66555 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723180849.67722168467@codespeak.net> Author: benjamin Date: Thu Jul 23 20:08:45 2009 New Revision: 66555 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: fill list initially with None Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 20:08:45 2009 @@ -239,7 +239,7 @@ def visit_Tuple(self, tup): if tup.elts: - consts_w = []*len(tup.elts) + consts_w = [None]*len(tup.elts) for i in range(len(tup.elts)): node = tup.elts[i] w_const = node.as_constant() From arigo at codespeak.net Thu Jul 23 20:21:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 20:21:32 +0200 (CEST) Subject: [pypy-svn] r66556 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090723182132.14DCC168459@codespeak.net> Author: arigo Date: Thu Jul 23 20:21:31 2009 New Revision: 66556 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: test_expand_fail_3 passes, after fixing some issue of sharing old data in global dicts that get confused if we run both the lltype and the ootype tests. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Thu Jul 23 20:21:31 2009 @@ -4,12 +4,13 @@ """ from pypy.jit.metainterp.history import TreeLoop, BoxInt, BoxPtr, ConstInt,\ - ConstAddr, ConstObj, ConstPtr, Box + ConstAddr, ConstObj, ConstPtr, Box, BoxObj from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype _cache = {} +_default_namespace = {'lltype': {}, 'ootype': {}} class ParseError(Exception): pass @@ -58,7 +59,7 @@ def box_for_var(self, elem): try: - return _cache[elem] + return _cache[self.type_system, elem] except KeyError: pass if elem.startswith('i'): @@ -67,7 +68,10 @@ _box_counter_more_than(elem[1:]) elif elem.startswith('p'): # pointer - box = BoxPtr() + if self.cpu.is_oo: + box = BoxObj() + else: + box = BoxPtr() _box_counter_more_than(elem[1:]) else: for prefix, boxclass in self.boxkinds.iteritems(): @@ -76,7 +80,7 @@ break else: raise ParseError("Unknown variable type: %s" % elem) - _cache[elem] = box + _cache[self.type_system, elem] = box box._str = elem return box @@ -220,8 +224,10 @@ inpargs = self.parse_header_line(line[1:-1]) return base_indent, inpargs -def parse(descr, cpu=None, namespace={}, type_system='lltype', +def parse(descr, cpu=None, namespace=None, type_system='lltype', boxkinds=None): + if namespace is None: + namespace = _default_namespace[type_system] return OpParser(descr, cpu, namespace, type_system, boxkinds).parse() def _box_counter_more_than(s): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 23 20:21:31 2009 @@ -82,6 +82,10 @@ nodesize = cpu.typedescrof(NODE) nodesize2 = cpu.typedescrof(NODE2) + # force a consistent order + valuedescr.sort_key() + nextdescr.sort_key() + cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), node_vtable_adr2: cpu.typedescrof(NODE2)} namespace = locals() Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 20:21:31 2009 @@ -6,7 +6,7 @@ OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize -from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxPtr +from pypy.jit.metainterp.history import AbstractDescr, ConstInt from pypy.jit.metainterp import resume, executor from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse @@ -630,7 +630,9 @@ for box, varname in zip(boxes, varnames): _variables_equal(box, varname, strict=True) # - for cls_vtable, resolved, fieldstext in virtuals.values(): + for match in parts: + pvar = match.group(1) + cls_vtable, resolved, fieldstext = virtuals[pvar] assert resolved is not None for fieldtext in fieldstext.split(','): fieldtext = fieldtext.strip() @@ -697,7 +699,7 @@ ''') def test_expand_fail_3(self): - py.test.skip("in-progress") + self.make_fail_descr() ops = """ [i1, i2, i3, p3] p1 = new_with_vtable(ConstClass(node_vtable)) @@ -707,22 +709,20 @@ setfield_gc(p2, i2, descr=valuedescr) setfield_gc(p2, p3, descr=nextdescr) guard_true(i1) - fail(p1, i3) + fail(p1, i3, descr=fdescr) jump(i2, i1, i3, p3) """ expected = """ [i1, i2, i3, p3] guard_true(i1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p1, 1, descr=valuedescr) - setfield_gc(p1, p2, descr=nextdescr) - setfield_gc(p2, i2, descr=valuedescr) - setfield_gc(p2, p3, descr=nextdescr) - fail(p1, i3) + fail(i3, i2, p3, descr=fdescr) jump(i2, 1, i3, p3) """ self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + self.check_expanded_fail_descr('''p1, i3 + where p1 is a node_vtable, valuedescr=1, nextdescr=p2 + where p2 is a node_vtable, valuedescr=i2, nextdescr=p3 + ''') def test_expand_fail_4(self): py.test.skip("in-progress") From arigo at codespeak.net Thu Jul 23 21:23:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 21:23:46 +0200 (CEST) Subject: [pypy-svn] r66557 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723192346.3B955169E30@codespeak.net> Author: arigo Date: Thu Jul 23 21:23:44 2009 New Revision: 66557 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Fourth test. Small fix in resume.py. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Thu Jul 23 21:23:44 2009 @@ -79,8 +79,8 @@ def _register_boxes(self, boxes): for box in boxes: - if isinstance(box, Box): - self.liveboxes.setdefault(box, 0) + if isinstance(box, Box) and box not in self.liveboxes: + self.liveboxes[box] = 0 self.liveboxes_order.append(box) def is_virtual(self, virtualbox): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 21:23:44 2009 @@ -725,37 +725,33 @@ ''') def test_expand_fail_4(self): - py.test.skip("in-progress") - ops = """ - [i1, i2] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) - setfield_gc(p1, i2, descr=valuedescr) - setfield_gc(p1, p2, descr=nextdescr) - setfield_gc(p2, i2, descr=valuedescr) - guard_true(i1) - fail(%s) - jump(i1, i2) - """ - expected = """ - [i1, i2] - guard_true(i1) + for arg in ['p1', 'p1,i2', 'i2,p1', 'p1,p2', 'p2,p1', + 'p1,p2,i2', 'p1,i2,p2', 'p2,p1,i2', + 'p2,i2,p1', 'i2,p1,p2', 'i2,p2,p1']: + self.make_fail_descr() + ops = """ + [i1, i2, i3] p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i3, descr=valuedescr) + i4 = getfield_gc(p1, descr=valuedescr) # copy of i3 p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) - fail(%s) - jump(1, i2) - """ - self.optimize_loop(ops % 'p1', 'Not, Not', - expected % 'p1', i1=1) - self.optimize_loop(ops % 'p1, i2', 'Not, Not', - expected % 'p1, i2', i1=1) - self.optimize_loop(ops % 'i2, p1', 'Not, Not', - expected % 'i2, p1', i1=1) - self.optimize_loop(ops % 'p1, p2', 'Not, Not', - expected % 'p1, p2', i1=1) + guard_true(i1) + fail(i4, %s, i3, descr=fdescr) + jump(i1, i2, i3) + """ + expected = """ + [i1, i2, i3] + guard_true(i1) + fail(i3, i2, descr=fdescr) + jump(1, i2, i3) + """ + self.optimize_loop(ops % arg, 'Not, Not, Not', expected, i1=1) + self.check_expanded_fail_descr('''i3, %s, i3 + where p1 is a node_vtable, valuedescr=i2, nextdescr=p2 + where p2 is a node_vtable, valuedescr=i2''' % arg) class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): From arigo at codespeak.net Thu Jul 23 21:27:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 21:27:55 +0200 (CEST) Subject: [pypy-svn] r66558 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090723192755.3A85D168066@codespeak.net> Author: arigo Date: Thu Jul 23 21:27:54 2009 New Revision: 66558 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Next test, passes. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 21:27:54 2009 @@ -754,6 +754,33 @@ where p2 is a node_vtable, valuedescr=i2''' % arg) + def test_expand_fail_5(self): + self.make_fail_descr() + ops = """ + [i1, i2, i3, i4] + p1 = new_with_vtable(ConstClass(node_vtable)) + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i4, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + setfield_gc(p2, i2, descr=valuedescr) + setfield_gc(p2, p1, descr=nextdescr) # a cycle + guard_true(i1) + fail(p1, i3, p2, i4, descr=fdescr) + jump(i2, i1, i3, i4) + """ + expected = """ + [i1, i2, i3, i4] + guard_true(i1) + fail(i3, i4, i2, descr=fdescr) + jump(i2, 1, i3, i4) + """ + self.optimize_loop(ops, 'Not, Not, Not, Not', expected, i1=1) + self.check_expanded_fail_descr('''p1, i3, p2, i4 + where p1 is a node_vtable, valuedescr=i4, nextdescr=p2 + where p2 is a node_vtable, valuedescr=i2, nextdescr=p1 + ''') + + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Thu Jul 23 21:33:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 21:33:11 +0200 (CEST) Subject: [pypy-svn] r66559 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723193311.E0255169EBD@codespeak.net> Author: arigo Date: Thu Jul 23 21:33:11 2009 New Revision: 66559 Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize2.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize2.py Log: Remove this file for now. Missing feature in optimize.py: virtuals allocated with NEW instead of NEW_WITH_VTABLE. From arigo at codespeak.net Thu Jul 23 21:41:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 21:41:01 +0200 (CEST) Subject: [pypy-svn] r66560 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723194101.B573A169E8B@codespeak.net> Author: arigo Date: Thu Jul 23 21:41:01 2009 New Revision: 66560 Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize3.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize3.py Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Port the remaining tests from test_optimize3, and remove it. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 21:41:01 2009 @@ -153,7 +153,7 @@ # ---------- - def test_remove_guard_class(self): + def test_remove_guard_class_1(self): ops = """ [p0] guard_class(p0, ConstClass(node_vtable)) @@ -170,6 +170,23 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_remove_guard_class_2(self): + ops = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable)) + escape(p0) + guard_class(p0, ConstClass(node_vtable)) + fail() + jump(i0) + """ + expected = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable)) + escape(p0) + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_remove_consecutive_guard_value_constfold(self): ops = """ [i0] @@ -503,6 +520,27 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_virtual_4(self): + ops = """ + [i0, p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + i1 = getfield_gc(p0, descr=valuedescr) + i2 = int_sub(i1, 1) + i3 = int_add(i0, i1) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i2, descr=valuedescr) + jump(i3, p1) + """ + expected = """ + [i0, i1] + i2 = int_sub(i1, 1) + i3 = int_add(i0, i1) + jump(i3, i2) + """ + self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', + expected) + def test_nonvirtual_1(self): ops = """ [i] @@ -538,6 +576,28 @@ expected = ops self.optimize_loop(ops, 'Not, Not', expected) + def test_nonvirtual_later(self): + ops = """ + [i] + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i, descr=valuedescr) + i1 = getfield_gc(p1, descr=valuedescr) + escape(p1) + i2 = getfield_gc(p1, descr=valuedescr) + i3 = int_add(i1, i2) + jump(i3) + """ + expected = """ + [i] + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i, descr=valuedescr) + escape(p1) + i2 = getfield_gc(p1, descr=valuedescr) + i3 = int_add(i, i2) + jump(i3) + """ + self.optimize_loop(ops, 'Not', expected) + def test_getfield_gc_pure_1(self): ops = """ [i] From arigo at codespeak.net Thu Jul 23 21:52:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 21:52:04 +0200 (CEST) Subject: [pypy-svn] r66561 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723195204.59D96169ED0@codespeak.net> Author: arigo Date: Thu Jul 23 21:52:03 2009 New Revision: 66561 Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimize4.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizebridge4.py Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Kill optimize4, porting a last test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 21:52:03 2009 @@ -840,6 +840,28 @@ where p2 is a node_vtable, valuedescr=i2, nextdescr=p1 ''') + def test_expand_fail_6(self): + self.make_fail_descr() + ops = """ + [p0, i0, i1] + guard_true(i0) + fail(p0, descr=fdescr) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + jump(p1, i1, i1) + """ + expected = """ + [i1b, i0, i1] + guard_true(i0) + fail(i1b, descr=fdescr) + jump(i1, i1, i1) + """ + self.optimize_loop(ops, '''Virtual(node_vtable, valuedescr=Not), + Not, Not''', expected, i0=1) + self.check_expanded_fail_descr('''p0 + where p0 is a node_vtable, valuedescr=i1b + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Thu Jul 23 22:19:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 22:19:09 +0200 (CEST) Subject: [pypy-svn] r66562 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090723201909.420541684A1@codespeak.net> Author: arigo Date: Thu Jul 23 22:19:08 2009 New Revision: 66562 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (contents, props changed) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_basic.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Link optimize.py with the rest of the code. Still some failures to investigate. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py Thu Jul 23 22:19:08 2009 @@ -94,6 +94,7 @@ self.ts = ts self.counter = 0 self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) + self.class_sizes = [] def make_portal_bytecode(self, graph): log.info("making JitCodes...") Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Thu Jul 23 22:19:08 2009 @@ -0,0 +1,37 @@ +# ____________________________________________________________ + +from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder +from pypy.jit.metainterp.optimizeopt import optimize_loop_1 +from pypy.jit.metainterp.specnode import equals_specnodes + +def optimize_loop(options, old_loops, loop, cpu): + if not options.specialize: # for tests only + if old_loops: + return old_loops[0] + else: + return None + finder = PerfectSpecializationFinder() + finder.find_nodes_loop(loop) + for old_loop in old_loops: + if equals_specnodes(old_loop.specnodes, loop.specnodes): + return old_loops + optimize_loop_1(cpu, loop) + return None + +# ____________________________________________________________ + +from pypy.jit.metainterp.optimizefindnode import BridgeSpecializationFinder +from pypy.jit.metainterp.optimizeopt import optimize_bridge_1 + +def optimize_bridge(options, old_loops, bridge, cpu): + if not options.specialize: # for tests only + return old_loops[0] + finder = BridgeSpecializationFinder() + finder.find_nodes_bridge(bridge) + for old_loop in old_loops: + if finder.bridge_matches(old_loop.specnodes): + optimize_bridge_1(cpu, bridge) + return old_loop + return None + +# ____________________________________________________________ Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py Thu Jul 23 22:19:08 2009 @@ -160,7 +160,7 @@ def find_nodes_loop(self, loop): self.setup_input_nodes(loop.inputargs) self.find_nodes(loop.operations) - self.build_result_specnodes(loop.operations[-1]) + self.build_result_specnodes(loop) def setup_input_nodes(self, inputargs): inputnodes = [] @@ -170,9 +170,10 @@ self.nodes[box] = instnode self.inputnodes = inputnodes - def build_result_specnodes(self, op): + def build_result_specnodes(self, loop): # Build the list of specnodes based on the result # computed by NodeFinder.find_nodes(). + op = loop.operations[-1] assert op.opnum == rop.JUMP specnodes = [] assert len(self.inputnodes) == len(op.args) @@ -180,7 +181,7 @@ inputnode = self.inputnodes[i] exitnode = self.getnode(op.args[i]) specnodes.append(self.intersect(inputnode, exitnode)) - self.specnodes = specnodes + loop.specnodes = specnodes def intersect(self, inputnode, exitnode): assert inputnode.fromstart @@ -272,10 +273,11 @@ class BridgeSpecializationFinder(NodeFinder): - def find_nodes_bridge(self, bridge, specnodes): - if specnodes is not None: + def find_nodes_bridge(self, bridge, specnodes=None): + if specnodes is not None: # not used actually self.setup_bridge_input_nodes(specnodes, bridge.inputargs) self.find_nodes(bridge.operations) + self.jump_op = bridge.operations[-1] def setup_bridge_input_nodes(self, specnodes, inputargs): assert len(specnodes) == len(inputargs) @@ -284,7 +286,8 @@ box = inputargs[i] self.nodes[box] = instnode - def bridge_matches(self, jump_op, nextloop_specnodes): + def bridge_matches(self, nextloop_specnodes): + jump_op = self.jump_op assert len(jump_op.args) == len(nextloop_specnodes) for i in range(len(nextloop_specnodes)): exitnode = self.getnode(jump_op.args[i]) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Thu Jul 23 22:19:08 2009 @@ -8,7 +8,7 @@ from pypy.rlib.objectmodel import we_are_translated -def optimize(cpu, loop): +def optimize_loop_1(cpu, loop): """Optimize loop.operations to make it match the input of loop.specnodes and to remove internal overheadish operations. Note that loop.specnodes must be applicable to the loop; you will probably get an AssertionError @@ -18,6 +18,13 @@ optimizer.setup_virtuals() optimizer.propagate_forward() +def optimize_bridge_1(cpu, bridge): + """The same, but for a bridge. The only difference is that we don't + expect 'specnodes' on the bridge. + """ + optimizer = Optimizer(cpu, bridge) + optimizer.propagate_forward() + # ____________________________________________________________ LEVEL_UNKNOWN = 0 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Thu Jul 23 22:19:08 2009 @@ -932,7 +932,7 @@ self.opcode_names = [] self.opname_to_index = {} if optimizer is None: - from pypy.jit.metainterp import optimize4 as optimizer + from pypy.jit.metainterp import optimize as optimizer self.optimize_loop = optimizer.optimize_loop self.optimize_bridge = optimizer.optimize_bridge @@ -973,7 +973,9 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: - xxxxxxxx + self.cpu.class_sizes = class_sizes = {} + for vtable, sizedescr in self._class_sizes: + class_sizes[vtable] = sizedescr self.cpu.setup_once() log(self.jit_starting_line) if not self.profiler.initialized: @@ -985,6 +987,7 @@ self._codewriter = codewriter.CodeWriter(self, policy, ts) self.portal_code = self._codewriter.make_portal_bytecode( self.portal_graph) + self._class_sizes = self._codewriter.class_sizes # ---------- construction-time interface ---------- Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Thu Jul 23 22:19:08 2009 @@ -12,8 +12,8 @@ class NotSpecNode(SpecNode): __slots__ = () - def _equals(self, other): # for tests only - return type(other) is NotSpecNode + def equals(self, other): + return isinstance(other, NotSpecNode) prebuiltNotSpecNode = NotSpecNode() @@ -24,11 +24,22 @@ self.known_class = known_class self.fields = fields # list: [(fieldofs, subspecnode)] - def _equals(self, other): # for tests only - ok = (type(other) is VirtualInstanceSpecNode and - self.known_class.equals(other.known_class) and - len(self.fields) == len(other.fields)) - if ok: - for (o1, s1), (o2, s2) in zip(self.fields, other.fields): - ok = ok and o1 == o2 and s1._equals(s2) - return ok + def equals(self, other): + if not (isinstance(other, VirtualInstanceSpecNode) and + self.known_class.equals(other.known_class) and + len(self.fields) == len(other.fields)): + return False + for i in range(len(self.fields)): + o1, s1 = self.fields[i] + o2, s2 = other.fields[i] + if not (o1.sort_key() == o2.sort_key() and s1.equals(s2)): + return False + return True + + +def equals_specnodes(specnodes1, specnodes2): + assert len(specnodes1) == len(specnodes2) + for i in range(len(specnodes1)): + if not specnodes1[i].equals(specnodes2[i]): + return False + return True Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_basic.py Thu Jul 23 22:19:08 2009 @@ -75,6 +75,7 @@ graph_key, called_from = cw.unfinished_graphs.pop() cw.make_one_bytecode(graph_key, False, called_from) metainterp.staticdata.portal_code = maingraph + metainterp.staticdata._class_sizes = cw.class_sizes metainterp.staticdata.state = FakeWarmRunnerDesc() metainterp.staticdata.DoneWithThisFrameInt = DoneWithThisFrame metainterp.staticdata.DoneWithThisFramePtr = DoneWithThisFrame Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 23 22:19:08 2009 @@ -121,7 +121,7 @@ lst = self.unpack_specnodes(text) assert len(specnodes) == len(lst) for x, y in zip(specnodes, lst): - assert x._equals(y) + assert x.equals(y) return True # ____________________________________________________________ @@ -133,7 +133,7 @@ loop = self.parse(ops, boxkinds=boxkinds) perfect_specialization_finder = PerfectSpecializationFinder() perfect_specialization_finder.find_nodes_loop(loop) - self.check_specnodes(perfect_specialization_finder.specnodes, spectext) + self.check_specnodes(loop.specnodes, spectext) return (loop.getboxes(), perfect_specialization_finder.getnode) def test_find_nodes_simple(self): @@ -444,9 +444,7 @@ bridge = self.parse(ops, boxkinds=boxkinds) bridge_specialization_finder = BridgeSpecializationFinder() bridge_specialization_finder.find_nodes_bridge(bridge, inputspecnodes) - matches = bridge_specialization_finder.bridge_matches( - bridge.operations[-1], - outputspecnodes) + matches = bridge_specialization_finder.bridge_matches(outputspecnodes) if mismatch: assert not matches else: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 23 22:19:08 2009 @@ -5,7 +5,7 @@ from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, OOtypeMixin, BaseTest) -from pypy.jit.metainterp.optimizeopt import optimize +from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.history import AbstractDescr, ConstInt from pypy.jit.metainterp import resume, executor from pypy.jit.metainterp.resoperation import rop, opname @@ -99,7 +99,7 @@ assert loop.operations[-1].opnum == rop.JUMP loop.operations[-1].jump_target = loop # - optimize(self.cpu, loop) + optimize_loop_1(self.cpu, loop) # expected = self.parse(optops, boxkinds=boxkinds) self.assert_equal(loop, expected) From arigo at codespeak.net Thu Jul 23 22:32:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 22:32:13 +0200 (CEST) Subject: [pypy-svn] r66563 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090723203213.98501169F1C@codespeak.net> Author: arigo Date: Thu Jul 23 22:32:13 2009 New Revision: 66563 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Log: Fix the building of class_sizes. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py Thu Jul 23 22:32:13 2009 @@ -95,6 +95,7 @@ self.counter = 0 self.raise_analyzer = RaiseAnalyzer(self.rtyper.annotator.translator) self.class_sizes = [] + self._class_sizes_seen = {} def make_portal_bytecode(self, graph): log.info("making JitCodes...") @@ -210,6 +211,21 @@ calldescr = self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT) return calldescr, non_void_args + def register_known_gctype(self, vtable, STRUCT): + # lltype only + key = vtable._as_obj() + if key not in self._class_sizes_seen: + self._class_sizes_seen[key] = True + sizedescr = self.cpu.sizeof(STRUCT) + self.class_sizes.append((vtable, sizedescr)) + + def register_known_ooclass(self, cls, CLASS): + # ootype only + if cls not in self._class_sizes_seen: + self._class_sizes_seen[cls] = True + typedescr = self.cpu.typedescrof(CLASS) + self.class_sizes.append((cls, typedescr)) + if 0: # disabled def fixed_list_descr_for_tp(self, TP): @@ -768,8 +784,8 @@ TYPE = op.args[0].value cls = ootype.runtimeClass(TYPE) self.emit('new_with_vtable', - self.get_position(self.cpu.typedescrof(TYPE)), self.const_position(cls)) + self.codewriter.register_known_ooclass(cls, TYPE) self.register_var(op.result) def serialize_op_oonewarray(self, op): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Thu Jul 23 22:32:13 2009 @@ -1,4 +1,5 @@ import py +from pypy.rpython.lltypesystem import llmemory from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict from pypy.rlib.unroll import unrolling_iterable @@ -973,9 +974,7 @@ def _setup_once(self): """Runtime setup needed by the various components of the JIT.""" if not self.globaldata.initialized: - self.cpu.class_sizes = class_sizes = {} - for vtable, sizedescr in self._class_sizes: - class_sizes[vtable] = sizedescr + self._setup_class_sizes() self.cpu.setup_once() log(self.jit_starting_line) if not self.profiler.initialized: @@ -983,6 +982,16 @@ self.profiler.initialized = True self.globaldata.initialized = True + def _setup_class_sizes(self): + self.cpu.class_sizes = class_sizes = {} + for vtable, sizedescr in self._class_sizes: + if not self.cpu.is_oo: + vtable = llmemory.cast_ptr_to_adr(vtable) + vtable = self.cpu.cast_adr_to_int(vtable) + else: + vtable = ootype.cast_to_object(vtable) + class_sizes[vtable] = sizedescr + def generate_bytecode(self, policy, ts): self._codewriter = codewriter.CodeWriter(self, policy, ts) self.portal_code = self._codewriter.make_portal_bytecode( From arigo at codespeak.net Thu Jul 23 22:34:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 23 Jul 2009 22:34:10 +0200 (CEST) Subject: [pypy-svn] r66564 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090723203410.B45D4169F19@codespeak.net> Author: arigo Date: Thu Jul 23 22:34:10 2009 New Revision: 66564 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Log: Pass the next test. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Thu Jul 23 22:34:10 2009 @@ -30,6 +30,7 @@ finder.find_nodes_bridge(bridge) for old_loop in old_loops: if finder.bridge_matches(old_loop.specnodes): + bridge.operations[-1].jump_target = old_loop optimize_bridge_1(cpu, bridge) return old_loop return None From benjamin at codespeak.net Thu Jul 23 23:49:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Thu, 23 Jul 2009 23:49:16 +0200 (CEST) Subject: [pypy-svn] r66565 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090723214916.8E63616845C@codespeak.net> Author: benjamin Date: Thu Jul 23 23:49:15 2009 New Revision: 66565 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: fix test Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Thu Jul 23 23:49:15 2009 @@ -245,7 +245,7 @@ w_const = node.as_constant() if w_const is None: return tup - consts_w[i] = node + consts_w[i] = w_const else: consts_w = [] w_consts = self.space.newtuple(consts_w) From benjamin at codespeak.net Fri Jul 24 02:40:26 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 02:40:26 +0200 (CEST) Subject: [pypy-svn] r66566 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090724004026.437CA169EC5@codespeak.net> Author: benjamin Date: Fri Jul 24 02:40:24 2009 New Revision: 66566 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: add raising default implementations to AST Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Fri Jul 24 02:40:24 2009 @@ -16,6 +16,18 @@ CONST_TRUE = 1 +class __extend__(ast.AST): + + def as_constant_truth(self, space): + raise AssertionError("only for expressions") + + def as_constant(self): + raise AssertionError("only for expressions") + + def accept_jump_if(self, gen, condition, target): + raise AssertionError("only for expressions") + + class __extend__(ast.expr): def as_constant_truth(self, space): From benjamin at codespeak.net Fri Jul 24 03:18:16 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 03:18:16 +0200 (CEST) Subject: [pypy-svn] r66568 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090724011816.03F0116845E@codespeak.net> Author: benjamin Date: Fri Jul 24 03:18:15 2009 New Revision: 66568 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Log: fix complicated conditions for folding bool ops Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Fri Jul 24 03:18:15 2009 @@ -225,7 +225,7 @@ while i < len(values) - 1: truth = values[i].as_constant_truth(self.space) if truth != CONST_NOT_CONST: - if truth == CONST_FALSE == we_are_and: + if (truth != CONST_TRUE) == we_are_and: del values[i + 1:] break else: From benjamin at codespeak.net Fri Jul 24 03:31:54 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 03:31:54 +0200 (CEST) Subject: [pypy-svn] r66569 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090724013154.D0BCA169EBD@codespeak.net> Author: benjamin Date: Fri Jul 24 03:31:54 2009 New Revision: 66569 Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py (contents, props changed) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Log: move a few global helpers to methods of the ast Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py Fri Jul 24 03:31:54 2009 @@ -0,0 +1,46 @@ +from pypy.interpreter.astcompiler import ast2 as ast +from pypy.interpreter.error import OperationError + + +class __extend__(ast.expr): + + constant = False + + def as_node_list(self, space): + return None + + +class __extend__(ast.List): + + def as_node_list(self, space): + return self.elts + + +class __extend__(ast.Tuple): + + def as_list_w(self, space): + return self.elts + + +class __extend__(ast.Const): + + constant = True + + def as_node_list(self, space): + try: + values_w = space.unpackiterable(self.value) + except OperationError: + return None + line = self.lineno + column = self.col_offset + return [ast.Const(w_obj, line, column) for w_obj in values_w] + + +class __extend__(ast.Str): + + constant = True + + +class __extend__(ast.Num): + + constant = True Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 24 03:31:54 2009 @@ -4,7 +4,7 @@ from pypy.interpreter.astcompiler import (ast2 as ast, assemble, symtable, consts, misc) -from pypy.interpreter.astcompiler import optimize # For side effects +from pypy.interpreter.astcompiler import optimize, asthelpers # For side effects from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops from pypy.interpreter.pyparser import future @@ -632,10 +632,10 @@ def _optimize_unpacking(self, assign): if len(assign.targets) != 1: return False - targets = self._list_from_sequence_node(assign.targets[0], False) + targets = assign.targets[0].as_node_list(self.space) if targets is None: return False - values = self._list_from_sequence_node(assign.value, True) + values = assign.value.as_node_list(self.space) if values is None: return False targets_count = len(targets) @@ -668,23 +668,6 @@ self.visit_sequence(targets) return True - def _list_from_sequence_node(self, node, const_possible): - if isinstance(node, ast.Tuple): - nodes = node.elts - elif isinstance(node, ast.List): - nodes = node.elts - elif const_possible and isinstance(node, ast.Const): - try: - values_w = self.space.unpackiterable(node.value) - except OperationError: - return None - line = node.lineno - column = node.col_offset - nodes = [ast.Const(obj, line, column) for obj in values_w] - else: - nodes = None - return nodes - def visit_With(self, wih): self.update_position(wih.lineno, True) body_block = self.new_block() @@ -762,8 +745,7 @@ if self.interactive: expr.value.walkabout(self) self.emit_op(ops.PRINT_EXPR) - elif not isinstance(expr.value, ast.Num) and \ - not isinstance(expr.value, ast.Str): + elif not expr.value.constant: expr.value.walkabout(self) self.emit_op(ops.POP_TOP) From benjamin at codespeak.net Fri Jul 24 04:07:06 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 04:07:06 +0200 (CEST) Subject: [pypy-svn] r66570 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090724020706.5EA08169ED3@codespeak.net> Author: benjamin Date: Fri Jul 24 04:07:05 2009 New Revision: 66570 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Log: use a switch for the operator map Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Fri Jul 24 04:07:05 2009 @@ -1,4 +1,4 @@ -from pypy.interpreter.astcompiler import ast2 as ast +from pypy.interpreter.astcompiler import ast2 as ast, misc from pypy.interpreter import error from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError @@ -25,7 +25,7 @@ '**=' : ast.Pow } -operator_map = { +operator_map = misc.dict_to_switch({ tokens.VBAR : ast.BitOr, tokens.CIRCUMFLEX : ast.BitXor, tokens.AMPER : ast.BitAnd, @@ -37,7 +37,7 @@ tokens.SLASH : ast.Div, tokens.DOUBLESLASH : ast.FloorDiv, tokens.PERCENT : ast.Mod -} +}) class ASTBuilder(object): @@ -857,13 +857,13 @@ def handle_binop(self, binop_node): left = self.handle_expr(binop_node.children[0]) right = self.handle_expr(binop_node.children[2]) - op = operator_map[binop_node.children[1].type] + op = operator_map(binop_node.children[1].type) result = ast.BinOp(left, op, right, binop_node.lineno, binop_node.column) number_of_ops = (len(binop_node.children) - 1) / 2 for i in range(1, number_of_ops): op_node = binop_node.children[i * 2 + 1] - op = operator_map[op_node.type] + op = operator_map(op_node.type) sub_right = self.handle_expr(binop_node.children[i * 2 + 2]) result = ast.BinOp(result, op, sub_right, op_node.lineno, op_node.column) From benjamin at codespeak.net Fri Jul 24 04:25:50 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 04:25:50 +0200 (CEST) Subject: [pypy-svn] r66571 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090724022550.23C6A16845B@codespeak.net> Author: benjamin Date: Fri Jul 24 04:25:49 2009 New Revision: 66571 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: raise encoding syntax errors with filename Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Fri Jul 24 04:25:49 2009 @@ -88,12 +88,13 @@ # If an encoding is explicitly given check that it is utf-8. decl_enc = _check_for_encoding(textsrc) if decl_enc and decl_enc != "utf-8": - raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie") + raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie", + filename=compile_info.filename) elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8: enc = 'utf-8' - if _check_for_encoding(textsrc) is not None: - raise error.SyntaxError("coding declaration in unicode string") + raise error.SyntaxError("coding declaration in unicode string", + filename=compile_info.filename) else: enc = _normalize_encoding(_check_for_encoding(textsrc)) if enc is not None and enc not in ('utf-8', 'iso-8859-1'): @@ -105,7 +106,8 @@ # KeyError space = self.space if space.is_w(e.w_type, space.w_LookupError): - raise error.SyntaxError("Unknown encoding: %s" % enc) + raise error.SyntaxError("Unknown encoding: %s" % enc, + filename=compile_info.filename) raise flags = compile_info.flags From benjamin at codespeak.net Fri Jul 24 05:13:57 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 05:13:57 +0200 (CEST) Subject: [pypy-svn] r66572 - pypy/branch/parser-compiler/pypy/interpreter/test Message-ID: <20090724031357.8859E1684A1@codespeak.net> Author: benjamin Date: Fri Jul 24 05:13:54 2009 New Revision: 66572 Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Log: prevent compiler from optimizing expression out in test Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Fri Jul 24 05:13:54 2009 @@ -761,19 +761,19 @@ assert co.co_consts[2] == 2.5 def test_tuple_folding(self): - co = compile("(1, 2, 3)", "", "exec") + co = compile("x = (1, 2, 3)", "", "exec") assert co.co_consts == ((1, 2, 3), None) - co = compile("()", "", "exec") + co = compile("x = ()", "", "exec") assert co.co_consts == ((), None) def test_unary_folding(self): - co = compile("-(3)", "", "exec") + co = compile("x = -(3)", "", "exec") assert co.co_consts[0] == -3 - co = compile("~3", "", "exec") + co = compile("x = ~3", "", "exec") assert co.co_consts[0] == ~3 - co = compile("+(-3)", "", "exec") + co = compile("x = +(-3)", "", "exec") assert co.co_consts[0] == -3 - co = compile("not None", "", "exec") + co = compile("x = not None", "", "exec") assert co.co_consts[0] is True def test_folding_of_binops_on_constants(self): From arigo at codespeak.net Fri Jul 24 12:52:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 12:52:42 +0200 (CEST) Subject: [pypy-svn] r66573 - in pypy/branch/pyjitpl5-optimize4/pypy/jit: backend/llgraph metainterp Message-ID: <20090724105242.7C79C498425@codespeak.net> Author: arigo Date: Fri Jul 24 12:52:41 2009 New Revision: 66573 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Log: Fix the llgraph backend for NEW_WITH_VTABLE not receiving a descr any more. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py Fri Jul 24 12:52:41 2009 @@ -624,8 +624,10 @@ def op_new(self, size): return do_new(size.ofs) - def op_new_with_vtable(self, size, vtable): - result = do_new(size.ofs) + def op_new_with_vtable(self, descr, vtable): + assert descr is None + size = get_class_size(self.memocast, vtable) + result = do_new(size) value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable, self.memocast) return result @@ -685,8 +687,9 @@ OPHANDLERS = [None] * (rop._LAST+1) - def op_new_with_vtable(self, typedescr, vtable): - from pypy.jit.backend.llgraph import runner + def op_new_with_vtable(self, descr, vtable): + assert descr is None + typedescr = get_class_size(self.memocast, vtable) return ootype.cast_to_object(ootype.new(typedescr.TYPE)) def op_new_array(self, typedescr, count): @@ -916,6 +919,7 @@ def __init__(self): self.addresses = [llmemory.NULL] self.rev_cache = {} + self.vtable_to_size = {} def new_memo_cast(): memocast = MemoCast() @@ -938,6 +942,14 @@ assert 0 <= int < len(memocast.addresses) return memocast.addresses[int] +def get_class_size(memocast, vtable): + memocast = _from_opaque(memocast) + return memocast.vtable_to_size[vtable] + +def set_class_size(memocast, vtable, size): + memocast = _from_opaque(memocast) + memocast.vtable_to_size[vtable] = size + class GuardFailed(Exception): pass @@ -1234,6 +1246,7 @@ setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) setannotation(cast_int_to_adr, annmodel.SomeAddress()) +setannotation(set_class_size, annmodel.s_None) setannotation(do_arraylen_gc, annmodel.SomeInteger()) setannotation(do_strlen, annmodel.SomeInteger()) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py Fri Jul 24 12:52:41 2009 @@ -91,6 +91,13 @@ assert self.translate_support_code return False + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + for vtable, size in class_sizes.items(): + if not self.is_oo: + size = size.ofs + llimpl.set_class_size(self.memo_cast, vtable, size) + def compile_operations(self, loop, bridge=None): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop @@ -326,7 +333,7 @@ def do_new_with_vtable(self, args, descr=None): assert descr is None vtable = args[0].getint() - size = self.class_sizes[vtable] # attribute set by codewriter.py + size = self.class_sizes[vtable] result = llimpl.do_new(size.ofs) llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, vtable, self.memo_cast) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Fri Jul 24 12:52:41 2009 @@ -983,7 +983,7 @@ self.globaldata.initialized = True def _setup_class_sizes(self): - self.cpu.class_sizes = class_sizes = {} + class_sizes = {} for vtable, sizedescr in self._class_sizes: if not self.cpu.is_oo: vtable = llmemory.cast_ptr_to_adr(vtable) @@ -991,6 +991,7 @@ else: vtable = ootype.cast_to_object(vtable) class_sizes[vtable] = sizedescr + self.cpu.set_class_sizes(class_sizes) def generate_bytecode(self, policy, ts): self._codewriter = codewriter.CodeWriter(self, policy, ts) From arigo at codespeak.net Fri Jul 24 13:33:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 13:33:38 +0200 (CEST) Subject: [pypy-svn] r66574 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090724113338.5D97B169F23@codespeak.net> Author: arigo Date: Fri Jul 24 13:33:37 2009 New Revision: 66574 Removed: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_loop_optimize2.py Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Log: Kill test_loop_optimize2. Small fix in oparser.py. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py Fri Jul 24 13:33:37 2009 @@ -68,7 +68,7 @@ _box_counter_more_than(elem[1:]) elif elem.startswith('p'): # pointer - if self.cpu.is_oo: + if getattr(self.cpu, 'is_oo', False): box = BoxObj() else: box = BoxPtr() From arigo at codespeak.net Fri Jul 24 13:34:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 13:34:05 +0200 (CEST) Subject: [pypy-svn] r66575 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090724113405.B08DE169F23@codespeak.net> Author: arigo Date: Fri Jul 24 13:34:05 2009 New Revision: 66575 Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_specnode.py (contents, props changed) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Log: Add the method 'extract_runtime_data' to VirtualInstanceSpecNode, and write tests for specnode.py. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py Fri Jul 24 13:34:05 2009 @@ -5,9 +5,6 @@ __metaclass__ = extendabletype # extended in optimizefindnode.py __slots__ = () - def extract_runtime_data(self, cpu, valuebox, resultlist): - resultlist.append(valuebox) - class NotSpecNode(SpecNode): __slots__ = () @@ -15,6 +12,9 @@ def equals(self, other): return isinstance(other, NotSpecNode) + def extract_runtime_data(self, cpu, valuebox, resultlist): + resultlist.append(valuebox) + prebuiltNotSpecNode = NotSpecNode() @@ -36,6 +36,14 @@ return False return True + def extract_runtime_data(self, cpu, valuebox, resultlist): + from pypy.jit.metainterp import executor, history, resoperation + for ofs, subspecnode in self.fields: + assert isinstance(ofs, history.AbstractDescr) + fieldbox = executor.execute(cpu, resoperation.rop.GETFIELD_GC, + [valuebox], ofs) + subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) + def equals_specnodes(specnodes1, specnodes2): assert len(specnodes1) == len(specnodes2) Added: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_specnode.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_specnode.py Fri Jul 24 13:34:05 2009 @@ -0,0 +1,39 @@ +from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.jit.metainterp.history import AbstractDescr, BoxPtr, ConstInt +from pypy.jit.metainterp.specnode import prebuiltNotSpecNode +from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode +from pypy.jit.metainterp.specnode import equals_specnodes +from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin + +def _get_vspecnode(classnum=123): + return VirtualInstanceSpecNode(ConstInt(classnum), + [(LLtypeMixin.valuedescr, prebuiltNotSpecNode), + (LLtypeMixin.nextdescr, prebuiltNotSpecNode)]) + +def test_equals_specnodes(): + assert equals_specnodes([prebuiltNotSpecNode, prebuiltNotSpecNode], + [prebuiltNotSpecNode, prebuiltNotSpecNode]) + vspecnode1 = _get_vspecnode(1) + vspecnode2 = _get_vspecnode(2) + assert equals_specnodes([vspecnode1], [vspecnode1]) + assert not equals_specnodes([vspecnode1], [vspecnode2]) + assert not equals_specnodes([vspecnode1], [prebuiltNotSpecNode]) + assert not equals_specnodes([prebuiltNotSpecNode], [vspecnode2]) + +def test_extract_runtime_data_1(): + res = [] + prebuiltNotSpecNode.extract_runtime_data("cpu", "box1", res) + prebuiltNotSpecNode.extract_runtime_data("cpu", "box2", res) + assert res == ["box1", "box2"] + +def test_extract_runtime_data_2(): + structure = lltype.malloc(LLtypeMixin.NODE) + structure.value = 515 + structure.next = lltype.malloc(LLtypeMixin.NODE) + structbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, structure)) + vspecnode = _get_vspecnode() + res = [] + vspecnode.extract_runtime_data(LLtypeMixin.cpu, structbox, res) + assert len(res) == 2 + assert res[0].value == structure.value + assert res[1].value._obj.container._as_ptr() == structure.next From arigo at codespeak.net Fri Jul 24 14:16:44 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 14:16:44 +0200 (CEST) Subject: [pypy-svn] r66576 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090724121644.7FC6E169F35@codespeak.net> Author: arigo Date: Fri Jul 24 14:16:42 2009 New Revision: 66576 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Test and fix. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Fri Jul 24 14:16:42 2009 @@ -331,7 +331,7 @@ def optimize_GUARD_VALUE(self, op): assert isinstance(op.args[1], Const) - assert op.args[0].getint() == op.args[1].getint() + assert op.args[0].get_() == op.args[1].get_() self.optimize_guard(op) def optimize_GUARD_TRUE(self, op): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Fri Jul 24 14:16:42 2009 @@ -321,6 +321,17 @@ """ self.optimize_loop(ops, '', expected, i=8) + def test_constptr_guard_value(self): + ops = """ + [] + p1 = escape() + guard_value(p1, ConstPtr(myptr)) + fail() + jump() + """ + self.optimize_loop(ops, '', ops, p1=self.nodebox.value, + boxkinds={'myptr': self.nodebox.value}) + # ---------- def test_fold_guard_no_exception(self): From arigo at codespeak.net Fri Jul 24 14:36:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 14:36:07 +0200 (CEST) Subject: [pypy-svn] r66577 - in pypy/branch/pyjitpl5-optimize4/pypy/jit: backend/llgraph metainterp Message-ID: <20090724123607.4717B169F39@codespeak.net> Author: arigo Date: Fri Jul 24 14:36:05 2009 New Revision: 66577 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Log: Minor fixes and assert improvements. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py Fri Jul 24 14:36:05 2009 @@ -822,6 +822,8 @@ frame.loop = loop frame.env = {} for i in range(len(loop.inputargs)): + expected_type = loop.inputargs[i].concretetype + assert lltype.typeOf(_future_values[i]) == expected_type frame.env[loop.inputargs[i]] = _future_values[i] del _future_values[:] Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py Fri Jul 24 14:36:05 2009 @@ -1151,7 +1151,7 @@ if not canfold: op = self.history.record(opnum, argboxes, resbox, descr) if (not we_are_translated() and op is not None - and self.framestack): + and getattr(self, 'framestack', None)): op.pc = self.framestack[-1].pc op.name = self.framestack[-1].jitcode.name if require_attention: From arigo at codespeak.net Fri Jul 24 14:43:04 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 14:43:04 +0200 (CEST) Subject: [pypy-svn] r66578 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090724124304.F3F30169EC5@codespeak.net> Author: arigo Date: Fri Jul 24 14:43:04 2009 New Revision: 66578 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_tl.py Log: Force backendopts for these tests. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_tl.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_tl.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_tl.py Fri Jul 24 14:43:04 2009 @@ -93,14 +93,18 @@ cls.main = main def test_tl_base(self): - res = self.meta_interp(self.main.im_func, [0, 6], listops=True) + # 'backendopt=True' is used on lltype to kill unneeded access + # to the class, which generates spurious 'guard_class' + res = self.meta_interp(self.main.im_func, [0, 6], listops=True, + backendopt=True) assert res == 5040 self.check_loops({'int_mul':1, 'jump':1, 'int_sub':1, 'int_is_true':1, 'int_le':1, 'guard_false':1, 'guard_value':1}) def test_tl_2(self): - res = self.meta_interp(self.main.im_func, [1, 10], listops=True) + res = self.meta_interp(self.main.im_func, [1, 10], listops=True, + backendopt=True) assert res == self.main.im_func(1, 10) self.check_loops({'int_sub':1, 'int_le':1, 'int_is_true':1, 'guard_false':1, 'jump':1, @@ -142,7 +146,7 @@ return interp(codes[num], inputarg=arg) res = self.meta_interp(main, [0, 20], optimizer=simple_optimize, - listops=listops) + listops=listops, backendopt=True) assert res == 0 def test_tl_call_full_of_residuals(self): From benjamin at codespeak.net Fri Jul 24 14:45:24 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 14:45:24 +0200 (CEST) Subject: [pypy-svn] r66579 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler test Message-ID: <20090724124524.D5A54169F17@codespeak.net> Author: benjamin Date: Fri Jul 24 14:45:24 2009 New Revision: 66579 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Log: store the type of the contents of the tuple in the constant array Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Fri Jul 24 14:45:24 2009 @@ -180,17 +180,18 @@ container[name] = index return index - def add_const(self, obj): + def add_const(self, obj, w_key=None): space = self.space - w_key = space.newtuple([obj, space.type(obj)]) + if w_key is None: + w_key = space.newtuple([obj, space.type(obj)]) w_len = space.finditem(self.w_consts, w_key) if w_len is None: w_len = space.len(self.w_consts) space.setitem(self.w_consts, w_key, w_len) return space.int_w(w_len) - def load_const(self, obj): - index = self.add_const(obj) + def load_const(self, obj, w_key=None): + index = self.add_const(obj, w_key) self.emit_op_arg(ops.LOAD_CONST, index) def update_position(self, lineno, force=False): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 24 14:45:24 2009 @@ -767,7 +767,22 @@ def visit_Const(self, const): self.update_position(const.lineno) - self.load_const(const.value) + space = self.space + value = const.value + # Constant tuples are tricky because we have to avoid letting equal + # values of the tuple that are not the same types being held together in + # the constant dict. So we make the key include the type of each item + # in the sequence. + if space.is_true(space.isinstance(value, space.w_tuple)): + length = space.int_w(space.len(value)) + key_w = [None]*(length + 2) + key_w[0] = value + for i in range(1, length + 1): + key_w[i] = space.type(space.getitem(value, space.wrap(i - 1))) + key_w[-1] = space.w_tuple + self.load_const(value, space.newtuple(key_w)) + else: + self.load_const(value) def visit_UnaryOp(self, op): self.update_position(op.lineno) Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Fri Jul 24 14:45:24 2009 @@ -748,6 +748,12 @@ op = ord(co[0]) + (ord(co[1]) << 8) assert op == opcode.opmap["LOAD_CONST"] + def test_tuple_constants(self): + ns = {} + exec "x = (1, 0); y = (1L, 0L)" in ns + assert isinstance(ns["x"][0], int) + assert isinstance(ns["y"][0], long) + def test_division_folding(self): def code(source): return compile(source, "", "exec") From arigo at codespeak.net Fri Jul 24 15:01:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:01:53 +0200 (CEST) Subject: [pypy-svn] r66580 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090724130153.C3625169EAF@codespeak.net> Author: arigo Date: Fri Jul 24 15:01:52 2009 New Revision: 66580 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: Test and fix (swap two lines :-) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Fri Jul 24 15:01:52 2009 @@ -158,8 +158,8 @@ vvalue = optimizer.make_virtual(self.known_class, box) for ofs, subspecnode in self.fields: subbox = optimizer.new_box(ofs) - vvalue.setfield(ofs, optimizer.getvalue(subbox)) subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) + vvalue.setfield(ofs, optimizer.getvalue(subbox)) def teardown_virtual_node(self, optimizer, value, newexitargs): assert value.is_virtual() for ofs, subspecnode in self.fields: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Fri Jul 24 15:01:52 2009 @@ -539,7 +539,7 @@ i1 = getfield_gc(p0, descr=valuedescr) i2 = int_sub(i1, 1) i3 = int_add(i0, i1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i2, descr=valuedescr) jump(i3, p1) """ @@ -552,6 +552,34 @@ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected) + def test_virtual_5(self): + ops = """ + [i0, p0] + guard_class(p0, ConstClass(node_vtable)) + fail() + i1 = getfield_gc(p0, descr=valuedescr) + i2 = int_sub(i1, 1) + i3 = int_add(i0, i1) + p2 = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2, i1, descr=valuedescr) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, p2, descr=nextdescr) + jump(i3, p1) + """ + expected = """ + [i0, i1, i1bis] + i2 = int_sub(i1, 1) + i3 = int_add(i0, i1) + jump(i3, i2, i1) + """ + self.optimize_loop(ops, + '''Not, Virtual(node_vtable, + valuedescr=Not, + nextdescr=Virtual(node_vtable2, + valuedescr=Not))''', + expected) + def test_nonvirtual_1(self): ops = """ [i] @@ -802,10 +830,10 @@ self.make_fail_descr() ops = """ [i1, i2, i3] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i3, descr=valuedescr) i4 = getfield_gc(p1, descr=valuedescr) # copy of i3 - p2 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p2 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p1, i2, descr=valuedescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p2, i2, descr=valuedescr) From arigo at codespeak.net Fri Jul 24 15:02:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:02:37 +0200 (CEST) Subject: [pypy-svn] r66581 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090724130237.5E1EB169EAF@codespeak.net> Author: arigo Date: Fri Jul 24 15:02:36 2009 New Revision: 66581 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py Log: Make this test pass. I think what I get now is fine, so let's accept the number 2 here... Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py Fri Jul 24 15:02:36 2009 @@ -126,7 +126,7 @@ res = self.meta_interp(f, [10], policy=StopAtXPolicy(externfn)) assert res == f(10) self.check_loop_count(2) - self.check_loops(**{self._new_op: 1}) + self.check_loops(**{self._new_op: 2}) # XXX was 1 self.check_loops(int_mul=0, call=1) def test_two_virtuals(self): From arigo at codespeak.net Fri Jul 24 15:04:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:04:35 +0200 (CEST) Subject: [pypy-svn] r66582 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090724130435.CB393169F28@codespeak.net> Author: arigo Date: Fri Jul 24 15:04:35 2009 New Revision: 66582 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py Log: Remove these tests. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py Fri Jul 24 15:04:35 2009 @@ -327,24 +327,3 @@ p = lltype.malloc(NODE2) p.parent.typeptr = vtable2 return p - -# ____________________________________________________________ - - -class Optimize3Mixin(object): - - def meta_interp(self, *args, **kwds): - from pypy.jit.metainterp import optimize3 - kwds['optimizer'] = optimize3 - return super(Optimize3Mixin, self).meta_interp(*args, **kwds) - -class TestLLtype_Instance3(Optimize3Mixin, TestLLtype_Instance): - - def skip(self): - py.test.skip('in progress') - - test_two_loops_with_virtual = skip - test_two_loops_with_escaping_virtual = skip - test_immutable_constant_getfield = skip - test_virtual_on_virtual = skip - test_bridge_from_interpreter = skip From david at codespeak.net Fri Jul 24 15:05:54 2009 From: david at codespeak.net (david at codespeak.net) Date: Fri, 24 Jul 2009 15:05:54 +0200 (CEST) Subject: [pypy-svn] r66583 - pypy/branch/io-lang/pypy/lang/io/test Message-ID: <20090724130554.12AB9169F28@codespeak.net> Author: david Date: Fri Jul 24 15:05:54 2009 New Revision: 66583 Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Test for improved Object for method Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Fri Jul 24 15:05:54 2009 @@ -75,6 +75,19 @@ results = [t.value for t in res.items] results == [0, 3, 6, 9] +def test_improved_object_for(): + + inp = """a:= list(); + x := 2 + y := 11 + for(x, 0, y - 1, x + 1, a append(x)); + a""" + res, space = interpret(inp) + + assert len(res.items) == 4 + results = [t.value for t in res.items] + results == [0, 3, 6, 9] + def test_object_leaks(): inp = """a:= list(); for(x, 0, 10, 3, a append(x)); From arigo at codespeak.net Fri Jul 24 15:06:48 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:06:48 +0200 (CEST) Subject: [pypy-svn] r66584 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test Message-ID: <20090724130648.43857169F28@codespeak.net> Author: arigo Date: Fri Jul 24 15:06:47 2009 New Revision: 66584 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_zrpy_basic.py Log: Fix imports in the tests. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtualizable.py Fri Jul 24 15:06:47 2009 @@ -9,7 +9,7 @@ from pypy.rpython.lltypesystem.rvirtualizable2 import VirtualizableAccessor from pypy.jit.metainterp.warmspot import get_stats from pypy.jit.metainterp import history, heaptracker -from pypy.jit.metainterp.test.test_optimize import NODE +from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin promote_virtualizable = lloperation.llop.promote_virtualizable debug_print = lloperation.llop.debug_print @@ -24,7 +24,7 @@ ('vable_base', llmemory.Address), ('vable_rti', VABLERTIPTR), ('inst_x', lltype.Signed), - ('inst_node', lltype.Ptr(NODE)), + ('inst_node', lltype.Ptr(LLtypeMixin.NODE)), hints = {'virtualizable2_accessor': VirtualizableAccessor()}) XY._hints['virtualizable2_accessor'].initialize( XY, ['inst_x', 'inst_node']) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_zrpy_basic.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_zrpy_basic.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_zrpy_basic.py Fri Jul 24 15:06:47 2009 @@ -46,10 +46,10 @@ res = ll_meta_interp(f, [17], CPUClass=self.CPUClass, type_system=self.type_system) assert res == (17+14+11+8+7+6+5+4) * 10 - from pypy.jit.metainterp import optimize4 + from pypy.jit.metainterp import optimize res = rpython_ll_meta_interp(f, [17], loops=2, CPUClass=self.CPUClass, type_system=self.type_system, - optimizer=optimize4) + optimizer=optimize) assert res == (17+14+11+8+7+6+5+4) * 10 From arigo at codespeak.net Fri Jul 24 15:23:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:23:20 +0200 (CEST) Subject: [pypy-svn] r66585 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090724132320.AD858169F39@codespeak.net> Author: arigo Date: Fri Jul 24 15:23:19 2009 New Revision: 66585 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Log: Argh. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py Fri Jul 24 15:23:19 2009 @@ -14,7 +14,7 @@ finder.find_nodes_loop(loop) for old_loop in old_loops: if equals_specnodes(old_loop.specnodes, loop.specnodes): - return old_loops + return old_loop optimize_loop_1(cpu, loop) return None From benjamin at codespeak.net Fri Jul 24 15:53:55 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 15:53:55 +0200 (CEST) Subject: [pypy-svn] r66586 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090724135355.81D37169F33@codespeak.net> Author: benjamin Date: Fri Jul 24 15:53:54 2009 New Revision: 66586 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_peepholer.py Log: prevent thing we're testing for being optimized out Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py Fri Jul 24 15:53:54 2009 @@ -293,6 +293,8 @@ self.assertRaises(TypeError, eval, '%s(a, b)' % op, {'a': a, 'b': b}) else: + print ia, a, ib, b, op + print eval("%s(a, b)" % op) self.assertEquals(format_result(res), format_result(eval('%s(a, b)' % op)), '%s(%s, %s) == %s failed' % (op, a, b, res)) Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_peepholer.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_peepholer.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_peepholer.py Fri Jul 24 15:53:54 2009 @@ -43,7 +43,7 @@ def test_none_as_constant(self): # LOAD_GLOBAL None --> LOAD_CONST None def f(x): - None + x = None return x asm = disassemble(f) for elem in ('LOAD_GLOBAL',): From benjamin at codespeak.net Fri Jul 24 15:56:08 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 15:56:08 +0200 (CEST) Subject: [pypy-svn] r66587 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler Message-ID: <20090724135608.A052D169F3F@codespeak.net> Author: benjamin Date: Fri Jul 24 15:56:07 2009 New Revision: 66587 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py Log: fix method name Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py Fri Jul 24 15:56:07 2009 @@ -18,7 +18,7 @@ class __extend__(ast.Tuple): - def as_list_w(self, space): + def as_node_list(self, space): return self.elts From arigo at codespeak.net Fri Jul 24 15:56:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 15:56:55 +0200 (CEST) Subject: [pypy-svn] r66588 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp: . test Message-ID: <20090724135655.BBBD2169F32@codespeak.net> Author: arigo Date: Fri Jul 24 15:56:55 2009 New Revision: 66588 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Log: RPythonification. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py Fri Jul 24 15:56:55 2009 @@ -1,5 +1,5 @@ from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, llmemory, rstr +from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass from pypy.rpython.ootypesystem import ootype from pypy.rpython import rlist from pypy.objspace.flow.model import Variable, Constant, Link, c_last_exception @@ -106,11 +106,25 @@ graph_key, called_from = self.unfinished_graphs.pop() self.make_one_bytecode(graph_key, False, called_from) log.info("there are %d JitCode instances." % len(self.all_graphs)) - # xxx annotation hack: make sure there is at least one ConstAddr around - if self.rtyper.type_system.name == 'lltype': - jitcode.constants.append(history.ConstAddr(llmemory.NULL, self.cpu)) + self.annotation_hacks(jitcode) return jitcode + def annotation_hacks(self, jitcode): + # xxx annotation hack: make sure there is at least one ConstAddr around + #if self.rtyper.type_system.name == 'lltypesystem': + # jitcode.constants.append(history.ConstAddr(llmemory.NULL, + # self.cpu)) + # xxx annotation hack: make sure class_sizes is not empty + if not self.class_sizes: + if self.rtyper.type_system.name == 'lltypesystem': + STRUCT = lltype.GcStruct('empty') + vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + self.register_known_gctype(vtable, STRUCT) + else: + TYPE = ootype.Instance('empty', ootype.ROOT) + cls = ootype.runtimeClass(TYPE) + self.register_known_ooclass(cls, TYPE) + def make_one_bytecode(self, graph_key, portal, called_from=None): maker = BytecodeMaker(self, graph_key, portal) if not hasattr(maker.bytecode, 'code'): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py Fri Jul 24 15:56:55 2009 @@ -3,8 +3,8 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode -from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs -from pypy.jit.metainterp import resume +from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs +from pypy.jit.metainterp import resume, compile from pypy.rlib.objectmodel import we_are_translated @@ -82,6 +82,9 @@ self.level = LEVEL_NONNULL def is_virtual(self): + # Don't check this with 'isinstance(_, VirtualValue)'! + # Even if it is a VirtualValue, the 'box' can be non-None, + # meaning it has been forced. return self.box is None @@ -105,7 +108,7 @@ self.known_class = known_class self.keybox = keybox # only used as a key in dictionaries self.source_op = source_op # the NEW_WITH_VTABLE operation building it - self._fields = av_newdict() + self._fields = av_newdict2() def getfield(self, ofs): return self._fields[ofs] @@ -114,6 +117,7 @@ return self._fields.get(ofs, default) def setfield(self, ofs, fieldvalue): + assert isinstance(fieldvalue, InstanceValue) self._fields[ofs] = fieldvalue def force_box(self): @@ -277,6 +281,7 @@ builder.finish(descr) else: descr = op_fail.descr + assert isinstance(descr, compile.ResumeGuardDescr) oldboxes = [] for box in op_fail.args: if box in self.values: Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py Fri Jul 24 15:56:55 2009 @@ -14,6 +14,11 @@ def av_newdict(): return r_dict(av_eq, av_hash) +def av_newdict2(): + # another implementation of av_newdict(), allowing different types for + # the values... + return r_dict(av_eq, av_hash) + def _findall(Class, name_prefix): result = [] for value, name in resoperation.opname.items(): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Fri Jul 24 15:56:55 2009 @@ -139,20 +139,21 @@ class ResumeDataReader(object): i_frame_infos = 0 i_boxes = 0 + virtuals = None def __init__(self, storage, liveboxes, metainterp=None): self.frame_infos = storage.rd_frame_infos self.nums = storage.rd_nums self.consts = storage.rd_consts self.liveboxes = liveboxes - if storage.rd_virtuals is not None: - self._prepare_virtuals(metainterp, storage.rd_virtuals) + self._prepare_virtuals(metainterp, storage.rd_virtuals) def _prepare_virtuals(self, metainterp, virtuals): - self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] - for i in range(len(virtuals)): - vinfo = virtuals[i] - vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) + if virtuals: + self.virtuals = [vinfo.allocate(metainterp) for vinfo in virtuals] + for i in range(len(virtuals)): + vinfo = virtuals[i] + vinfo.setfields(metainterp, self.virtuals[i], self._decode_box) def consume_boxes(self): boxes = [] Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py Fri Jul 24 15:56:55 2009 @@ -1,13 +1,14 @@ import py from pypy.rpython.lltypesystem import rclass from pypy.rpython.ootypesystem import ootype +from pypy.rlib.objectmodel import instantiate from pypy.jit.metainterp.test.test_resume import MyMetaInterp from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, OOtypeMixin, BaseTest) from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.history import AbstractDescr, ConstInt -from pypy.jit.metainterp import resume, executor +from pypy.jit.metainterp import resume, executor, compile from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.metainterp.test.oparser import parse @@ -677,7 +678,7 @@ # ---------- def make_fail_descr(self): - class FailDescr(AbstractDescr): + class FailDescr(compile.ResumeGuardDescr): args_seen = [] def _oparser_uses_descr(self, oparse, args): # typically called twice, before and after optimization @@ -688,7 +689,7 @@ assert liveboxes == args self.args_seen.append((args, oparse)) # - fdescr = FailDescr() + fdescr = instantiate(FailDescr) self.fdescr = fdescr self.namespace['fdescr'] = fdescr From benjamin at codespeak.net Fri Jul 24 15:57:36 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 15:57:36 +0200 (CEST) Subject: [pypy-svn] r66589 - pypy/branch/parser-compiler/lib-python/modified-2.5.2/test Message-ID: <20090724135736.56226169F33@codespeak.net> Author: benjamin Date: Fri Jul 24 15:57:35 2009 New Revision: 66589 Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py Log: revert unintended change Modified: pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py ============================================================================== --- pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py (original) +++ pypy/branch/parser-compiler/lib-python/modified-2.5.2/test/test_coercion.py Fri Jul 24 15:57:35 2009 @@ -293,8 +293,6 @@ self.assertRaises(TypeError, eval, '%s(a, b)' % op, {'a': a, 'b': b}) else: - print ia, a, ib, b, op - print eval("%s(a, b)" % op) self.assertEquals(format_result(res), format_result(eval('%s(a, b)' % op)), '%s(%s, %s) == %s failed' % (op, a, b, res)) From arigo at codespeak.net Fri Jul 24 16:01:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:01:33 +0200 (CEST) Subject: [pypy-svn] r66590 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/backend: llgraph/test test Message-ID: <20090724140133.6061F169F33@codespeak.net> Author: arigo Date: Fri Jul 24 16:01:32 2009 New Revision: 66590 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/test/test_llgraph.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py Log: Fix for NEW_WITH_VTABLE not taking the descr any more. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/test/test_llgraph.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/test/test_llgraph.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/test/test_llgraph.py Fri Jul 24 16:01:32 2009 @@ -189,9 +189,9 @@ # descrsize2 = cpu.sizeof(rclass.OBJECT) vtable2 = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) - x = cpu.do_new_with_vtable( - [BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)))], - descrsize2) + vtable2_int = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)) + cpu.set_class_sizes({vtable2_int: descrsize2}) + x = cpu.do_new_with_vtable([ConstInt(vtable2_int)]) assert isinstance(x, BoxPtr) assert x.getptr(rclass.OBJECTPTR).typeptr == vtable2 # Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py Fri Jul 24 16:01:32 2009 @@ -618,11 +618,9 @@ def test_new_with_vtable(self): cpu = self.cpu t_box, T_box = self.alloc_instance(self.T) - sizedescr = cpu.sizeof(self.T) - r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ptr', - descr=sizedescr) - r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ptr', - descr=sizedescr) + cpu.set_class_sizes({T_box.value: cpu.sizeof(self.T)}) + r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ptr') + r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [T_box], 'ptr') assert r1.value != r2.value descr1 = cpu.fielddescrof(self.S, 'chr1') descr2 = cpu.fielddescrof(self.S, 'chr2') @@ -763,9 +761,9 @@ # descrsize2 = cpu.sizeof(rclass.OBJECT) vtable2 = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) - x = cpu.do_new_with_vtable( - [BoxInt(cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)))], - descrsize2) + vtable2_int = cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(vtable2)) + cpu.set_class_sizes({vtable2_int: descrsize2}) + x = cpu.do_new_with_vtable([ConstInt(vtable2_int)]) assert isinstance(x, BoxPtr) # well... #assert x.getptr(rclass.OBJECTPTR).typeptr == vtable2 From arigo at codespeak.net Fri Jul 24 16:07:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:07:33 +0200 (CEST) Subject: [pypy-svn] r66591 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/backend: . x86 Message-ID: <20090724140733.8623C169F20@codespeak.net> Author: arigo Date: Fri Jul 24 16:07:33 2009 New Revision: 66591 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/model.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/regalloc.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py Log: Fix x86 for NEW_WITH_VTABLE not taking the descr any more. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/model.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/model.py Fri Jul 24 16:07:33 2009 @@ -1,5 +1,8 @@ class AbstractCPU(object): + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + def setup_once(self): """Called once by the front-end when the program starts.""" pass Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/regalloc.py Fri Jul 24 16:07:33 2009 @@ -919,7 +919,9 @@ return self._call(op, arglocs) def consider_new_with_vtable(self, op, ignored): - args = self.assembler.cpu.gc_ll_descr.args_for_new(op.descr) + classint = op.args[0].getint() + descrsize = self.assembler.cpu.class_sizes[classint] + args = self.assembler.cpu.gc_ll_descr.args_for_new(descrsize) arglocs = [imm(x) for x in args] arglocs.append(self.loc(op.args[0])) return self._call(op, arglocs) Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py Fri Jul 24 16:07:33 2009 @@ -89,6 +89,9 @@ 'typeptr', translate_support_code) + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + def _setup_prebuilt_error(self, prefix, Class): if self.rtyper is not None: # normal case bk = self.rtyper.annotator.bookkeeper @@ -424,7 +427,7 @@ raise NotImplementedError("size = %d" % size) def _new_do_len(TP): - def do_strlen(self, args, descr=0): + def do_strlen(self, args, descr=None): basesize, itemsize, ofs_length = symbolic.get_array_token(TP, self.translate_support_code) gcref = args[0].getptr(llmemory.GCREF) @@ -435,7 +438,7 @@ do_strlen = _new_do_len(rstr.STR) do_unicodelen = _new_do_len(rstr.UNICODE) - def do_strgetitem(self, args, descr=0): + def do_strgetitem(self, args, descr=None): basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.translate_support_code) gcref = args[0].getptr(llmemory.GCREF) @@ -443,7 +446,7 @@ v = rffi.cast(rffi.CArrayPtr(lltype.Char), gcref)[basesize + i] return BoxInt(ord(v)) - def do_unicodegetitem(self, args, descr=0): + def do_unicodegetitem(self, args, descr=None): basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code) gcref = args[0].getptr(llmemory.GCREF) @@ -507,10 +510,13 @@ res = self.gc_ll_descr.gc_malloc(descrsize) return BoxPtr(res) - def do_new_with_vtable(self, args, descrsize): + def do_new_with_vtable(self, args, descr=None): + assert descr is None + classint = args[0].getint() + descrsize = self.class_sizes[classint] res = self.gc_ll_descr.gc_malloc(descrsize) as_array = rffi.cast(rffi.CArrayPtr(lltype.Signed), res) - as_array[self.vtable_offset/WORD] = args[0].getint() + as_array[self.vtable_offset/WORD] = classint return BoxPtr(res) def do_new_array(self, args, arraydescr): @@ -518,19 +524,19 @@ res = self.gc_ll_descr.gc_malloc_array(arraydescr, num_elem) return BoxPtr(self.cast_adr_to_gcref(res)) - def do_newstr(self, args, descr=0): + def do_newstr(self, args, descr=None): num_elem = args[0].getint() tsc = self.translate_support_code res = self.gc_ll_descr.gc_malloc_str(num_elem, tsc) return BoxPtr(self.cast_adr_to_gcref(res)) - def do_newunicode(self, args, descr=0): + def do_newunicode(self, args, descr=None): num_elem = args[0].getint() tsc = self.translate_support_code res = self.gc_ll_descr.gc_malloc_unicode(num_elem, tsc) return BoxPtr(self.cast_adr_to_gcref(res)) - def do_strsetitem(self, args, descr=0): + def do_strsetitem(self, args, descr=None): basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.translate_support_code) index = args[1].getint() @@ -538,7 +544,7 @@ a = args[0].getptr(llmemory.GCREF) rffi.cast(rffi.CArrayPtr(lltype.Char), a)[index + basesize] = chr(v) - def do_unicodesetitem(self, args, descr=0): + def do_unicodesetitem(self, args, descr=None): basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code) index = args[1].getint() From arigo at codespeak.net Fri Jul 24 16:12:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:12:25 +0200 (CEST) Subject: [pypy-svn] r66592 - pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test Message-ID: <20090724141225.D836416856A@codespeak.net> Author: arigo Date: Fri Jul 24 16:12:25 2009 New Revision: 66592 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/test_ll_random.py Log: Fix for NEW_WITH_VTABLE not taking the descr any more. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/test_ll_random.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/test_ll_random.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/test_ll_random.py Fri Jul 24 16:12:25 2009 @@ -18,6 +18,7 @@ self.runicodes = [] self.structure_types = [] self.structure_types_and_vtables = [] + self.class_sizes_cache = [] def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct): while True: @@ -97,6 +98,13 @@ vtable.name[i] = name[i] vtable.name[len(name)] = '\x00' self.structure_types_and_vtables.append((S, vtable)) + # + vtable_adr = llmemory.cast_ptr_to_adr(vtable) + vtable_int = self.cpu.cast_adr_to_int(vtable_adr) + descr = self.cpu.sizeof(S) + self.class_sizes_cache.append((vtable_int, descr)) + self.cpu.set_class_sizes(dict(self.class_sizes_cache)) + # return S, vtable def get_random_structure(self, r, has_vtable=False): @@ -231,10 +239,12 @@ if self.opnum == rop.NEW_WITH_VTABLE: S, vtable = builder.get_random_structure_type_and_vtable(r) args = [ConstAddr(llmemory.cast_ptr_to_adr(vtable), builder.cpu)] + descr = None else: S = builder.get_random_structure_type(r) args = [] - v_ptr = builder.do(self.opnum, args, self.size_descr(builder, S)) + descr = self.size_descr(builder, S) + v_ptr = builder.do(self.opnum, args, descr) builder.ptrvars.append((v_ptr, S)) class ArrayOperation(test_random.AbstractOperation): From arigo at codespeak.net Fri Jul 24 16:13:56 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:13:56 +0200 (CEST) Subject: [pypy-svn] r66593 - pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86 Message-ID: <20090724141356.3C95C169F29@codespeak.net> Author: arigo Date: Fri Jul 24 16:13:54 2009 New Revision: 66593 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py Log: Add this new method. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/runner.py Fri Jul 24 16:13:54 2009 @@ -39,6 +39,9 @@ def sort_key(self): return self.v0 # the ofs field for fielddescrs + def is_pointer_field(self): + return self.flag2 # for fielddescrs + def equals(self, other): if not isinstance(other, ConstDescr3): return False From arigo at codespeak.net Fri Jul 24 16:26:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:26:37 +0200 (CEST) Subject: [pypy-svn] r66594 - pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal Message-ID: <20090724142637.D9947169F29@codespeak.net> Author: arigo Date: Fri Jul 24 16:26:36 2009 New Revision: 66594 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal/runner.py Log: Fix for NEW_WITH_VTABLE not taking the descr any more. Add the new is_pointer_field() method. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal/runner.py Fri Jul 24 16:26:36 2009 @@ -4,7 +4,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr, rclass from pypy.rpython.ootypesystem import ootype from pypy.jit.metainterp.history import AbstractDescr, AbstractMethDescr -from pypy.jit.metainterp.history import Box, BoxInt, BoxPtr, BoxObj +from pypy.jit.metainterp.history import Box, BoxInt, BoxPtr, BoxObj, getkind from pypy.jit.metainterp import executor from pypy.jit.metainterp.resoperation import rop, opname from pypy.jit.backend import model @@ -209,7 +209,11 @@ setattr(p, %(name)r, x) """ % dict).compile() in dict2 sort_key = self._count_sort_key(T, name) - return FieldDescr(dict2['getfield'], dict2['setfield'], sort_key) + if getkind(FIELDTYPE) in 'po': # pointer or object + Class = PtrFieldDescr + else: + Class = NonPtrFieldDescr + return Class(dict2['getfield'], dict2['setfield'], sort_key) # ---------- @@ -424,8 +428,9 @@ p = sizedescr.alloc() return BoxPtr(p) - def do_new_with_vtable(self, args, sizedescr): - assert isinstance(sizedescr, SizeDescr) + def do_new_with_vtable(self, args, descr=None): + assert descr is None + sizedescr = self.class_sizes[args[0].getint()] assert sizedescr.alloc is not None p = sizedescr.alloc() classadr = args[0].getaddr(self) @@ -614,8 +619,9 @@ # ---------------- - def do_new_with_vtable(self, args, sizedescr): - assert isinstance(sizedescr, SizeDescr) + def do_new_with_vtable(self, args, descr): + assert descr is None + sizedescr = self.class_sizes[args[0].getobj()] assert sizedescr.alloc is not None obj = sizedescr.alloc() return BoxObj(obj) @@ -658,6 +664,14 @@ def sort_key(self): return self._sort_key +class PtrFieldDescr(FieldDescr): + def is_pointer_field(self): + return True + +class NonPtrFieldDescr(FieldDescr): + def is_pointer_field(self): + return False + class ArrayDescr(AbstractDescr): new = None length = None From arigo at codespeak.net Fri Jul 24 16:45:47 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:45:47 +0200 (CEST) Subject: [pypy-svn] r66595 - in pypy/branch/pyjitpl5-optimize4/pypy/jit/backend: llvm test Message-ID: <20090724144547.63666169F4F@codespeak.net> Author: arigo Date: Fri Jul 24 16:45:46 2009 New Revision: 66595 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/compile.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/runner.py pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py Log: Partial fix of the llvm backend. The next failure is Really Obscure(tm). Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/compile.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/compile.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/compile.py Fri Jul 24 16:45:46 2009 @@ -667,7 +667,9 @@ self.vars[op.result] = res def generate_NEW_WITH_VTABLE(self, op): - self.generate_NEW(op) + sizedescr = self.cpu.class_sizes[op.args[0].getint()] + res = self._generate_new(self.cpu._make_const_int(sizedescr.size)) + self.vars[op.result] = res loc = self._generate_field_gep(op.result, self.cpu.vtable_descr) value_ref = self.getintarg(op.args[0]) llvm_rffi.LLVMBuildStore(self.builder, value_ref, loc, "") Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/runner.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/runner.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/runner.py Fri Jul 24 16:45:46 2009 @@ -5,6 +5,7 @@ from pypy.rlib import runicode from pypy.jit.metainterp.history import AbstractDescr, INT from pypy.jit.metainterp.history import BoxInt, BoxPtr +from pypy.jit.backend.model import AbstractCPU from pypy.jit.backend.llvm import llvm_rffi from pypy.jit.metainterp import history from pypy.jit.metainterp.resoperation import rop, ResOperation @@ -82,6 +83,9 @@ _nowrapper=True) assert rffi.sizeof(rffi.SIZE_T) == self.size_of_int + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + def setup_once(self): if not we_are_translated(): llvm_rffi.teardown_now() @@ -532,8 +536,9 @@ res = self.malloc_fn_ptr(rffi.cast(rffi.SIZE_T, sizedescr.size)) return BoxPtr(res) - def do_new_with_vtable(self, args, sizedescr): - assert isinstance(sizedescr, SizeDescr) + def do_new_with_vtable(self, args, descr=None): + assert descr is None + sizedescr = self.class_sizes[args[0].getint()] res = self.malloc_fn_ptr(rffi.cast(rffi.SIZE_T, sizedescr.size)) self._do_setfield(res, args[0], self.vtable_descr) return BoxPtr(res) @@ -692,6 +697,8 @@ def __init__(self, offset, size_index): self.offset = offset self.size_index = size_index # index in cpu.types_by_index + def is_pointer_field(self): + return self.size_index == LLVMCPU.SIZE_GCPTR class ArrayDescr(AbstractDescr): def __init__(self, itemsize, itemsize_index): Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/runner_test.py Fri Jul 24 16:45:46 2009 @@ -274,6 +274,7 @@ def test_field_basic(self): t_box, T_box = self.alloc_instance(self.T) fielddescr = self.cpu.fielddescrof(self.S, 'value') + assert not fielddescr.is_pointer_field() res = self.execute_operation(rop.SETFIELD_GC, [t_box, BoxInt(39082)], 'void', descr=fielddescr) assert res is None @@ -296,6 +297,7 @@ # u_box, U_box = self.alloc_instance(self.U) fielddescr2 = self.cpu.fielddescrof(self.S, 'next') + assert fielddescr2.is_pointer_field() res = self.execute_operation(rop.SETFIELD_GC, [t_box, u_box], 'void', descr=fielddescr2) assert res is None From arigo at codespeak.net Fri Jul 24 16:51:10 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 16:51:10 +0200 (CEST) Subject: [pypy-svn] r66596 - pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp Message-ID: <20090724145110.4FE20169F54@codespeak.net> Author: arigo Date: Fri Jul 24 16:51:09 2009 New Revision: 66596 Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Log: RPythonification. Modified: pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py Fri Jul 24 16:51:09 2009 @@ -169,7 +169,9 @@ if num < 0: return self.consts[-2 - num] elif num & VIRTUAL_FLAG: - return self.virtuals[num - VIRTUAL_FLAG] + virtuals = self.virtuals + assert virtuals is not None + return virtuals[num - VIRTUAL_FLAG] else: return self.liveboxes[num] From benjamin at codespeak.net Fri Jul 24 17:26:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 17:26:31 +0200 (CEST) Subject: [pypy-svn] r66597 - in pypy/branch/parser-compiler/pypy/interpreter: astcompiler pyparser test Message-ID: <20090724152631.4447C3180F8@codespeak.net> Author: benjamin Date: Fri Jul 24 17:26:29 2009 New Revision: 66597 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Log: add lots of comments and docstrings Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Fri Jul 24 17:26:29 2009 @@ -11,6 +11,7 @@ class Instruction(object): + """Represents a single opcode.""" def __init__(self, opcode, arg=0): self.opcode = opcode @@ -19,6 +20,7 @@ self.has_jump = False def size(self): + """Return the size of bytes of this instruction when it is encoded.""" if self.opcode >= ops.HAVE_ARGUMENT: if self.arg > 0xFFFF: return 6 @@ -28,6 +30,10 @@ return 1 def jump_to(self, target, absolute=False): + """Indicate the target this jump instruction. + + The opcode must be a JUMP opcode. + """ self.jump = (target, absolute) self.has_jump = True @@ -45,6 +51,12 @@ class Block(object): + """A basic control flow block. + + It has one entry point and several possible exit points. Its instructions + may be jumps to other blocks, or if control flow reaches the end of the + block, it continues to next_block. + """ def __init__(self): self.instructions = [] @@ -65,18 +77,21 @@ self.marked = True def post_order(self): + """Return this block and its children in post order.""" blocks = [] self._post_order(blocks) blocks.reverse() return blocks def code_size(self): + """Return the encoded size of all the instructions in this block.""" i = 0 for instr in self.instructions: i += instr.size() return i def get_code(self): + """Encode the instructions in this block into bytecode.""" code = [] for instr in self.instructions: opcode = instr.opcode @@ -115,6 +130,7 @@ class PythonCodeMaker(ast.ASTVisitor): + """Knows how to assemble a PyCode object.""" def __init__(self, space, name, first_lineno, scope, compile_info): self.space = space @@ -138,10 +154,12 @@ return Block() def use_block(self, block): + """Start emitting bytecode into block.""" self.current_block = block self.instrs = block.instructions def use_next_block(self, block=None): + """Set this block as the next_block for the last and use it.""" if block is None: block = self.new_block() self.current_block.next_block = block @@ -149,6 +167,7 @@ return block def emit_op(self, op): + """Emit an opcode without an argument.""" instr = Instruction(op) if not self.lineno_set: instr.lineno = self.lineno @@ -159,6 +178,7 @@ return instr def emit_op_arg(self, op, arg): + """Emit an opcode with an integer argument.""" instr = Instruction(op, arg) if not self.lineno_set: instr.lineno = self.lineno @@ -166,12 +186,15 @@ self.instrs.append(instr) def emit_op_name(self, op, container, name): + """Emit an opcode referencing a name.""" self.emit_op_arg(op, self.add_name(container, name)) def emit_jump(self, op, block_to, absolute=False): + """Emit a jump opcode to another block.""" self.emit_op(op).jump_to(block_to, absolute) def add_name(self, container, name): + """Get the index of a name in container.""" name = self.scope.mangle(name) try: index = container[name] @@ -181,7 +204,10 @@ return index def add_const(self, obj, w_key=None): + """Add a W_Root to the constant array and return its location.""" space = self.space + # To avoid confusing equal but separate types, we hash store the type of + # the constant in the dictionary. if w_key is None: w_key = space.newtuple([obj, space.type(obj)]) w_len = space.finditem(self.w_consts, w_key) @@ -195,17 +221,23 @@ self.emit_op_arg(ops.LOAD_CONST, index) def update_position(self, lineno, force=False): + """Possibly change the lineno for the next instructions.""" if force or lineno > self.lineno: - if force and lineno < self.lineno: - raise AssertionError self.lineno = lineno self.lineno_set = False def _resolve_block_targets(self, blocks): + """"Compute the arguments of jump instructions.""" last_extended_arg_count = 0 + # The reason for this loop is extended jumps. EXTENDED_ARG extends the + # bytecode size, so it might invalidate the offsets we've already given. + # Thus we have to loop until the number of extended args is stable. Any + # extended jump at all is extremely rare, so performance is not too + # concerning. while True: extended_arg_count = 0 offset = 0 + # Calculate the code offset of each block. for block in blocks: block.offset = offset offset += block.code_size() @@ -216,6 +248,8 @@ if instr.has_jump: target, absolute = instr.jump op = instr.opcode + # Optimize an unconditional jump going to another + # unconditional jump. if op == ops.JUMP_ABSOLUTE or op == ops.JUMP_FORWARD: if target.instructions: target_op = target.instructions[0].opcode @@ -236,6 +270,7 @@ last_extended_arg_count = extended_arg_count def _build_consts_array(self): + """Turn the applevel constants dictionary into a list.""" w_consts = self.w_consts space = self.space consts_w = [space.w_None] * space.int_w(space.len(w_consts)) @@ -253,9 +288,11 @@ return consts_w def _get_code_flags(self): + """Get an extra flags that should be attached to the code object.""" raise NotImplementedError def _stacksize(self, blocks): + """Compute co_stacksize.""" for block in blocks: block.marked = False block.initial_depth = -1000 @@ -275,6 +312,7 @@ depth, max_depth) if instr.opcode == ops.JUMP_ABSOLUTE or \ instr.opcode == ops.JUMP_FORWARD: + # Nothing more can occur. break if block.next_block: max_depth = self._recursive_stack_depth_walk(block.next_block, @@ -283,6 +321,7 @@ return max_depth def _build_lnotab(self, blocks): + """Build the line number table for tracebacks and tracing.""" current_line = self.first_lineno current_off = 0 table = [] @@ -323,11 +362,14 @@ return ''.join(table) def assemble(self): + """Build a PyCode object.""" + # Unless it's interactive, every code object must in an a return. if not self.current_block.have_return: self.use_next_block() if self.add_none_to_final_return: self.load_const(self.space.w_None) self.emit_op(ops.RETURN_VALUE) + # Set the first lineno if it is not already explicitly set. if self.first_lineno == -1: if self.first_block.instructions: self.first_lineno = self.first_block.instructions[0].lineno @@ -555,8 +597,10 @@ def _opcode_stack_effect(op, arg): + """Return the stack effect of a opcode an its argument.""" if we_are_translated(): for possible_op in ops.unrolling_opcode_descs: + # EXTENDED_ARG should never get in here. if possible_op.index == ops.EXTENDED_ARG: continue if op == possible_op.index: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Fri Jul 24 17:26:29 2009 @@ -6,8 +6,9 @@ from pypy.rlib.objectmodel import specialize -def ast_from_node(space, n, compile_info): - return ASTBuilder(space, n, compile_info).build_ast() +def ast_from_node(space, node, compile_info): + """Turn a parse tree, node, to AST.""" + return ASTBuilder(space, node, compile_info).build_ast() augassign_operator_map = { @@ -48,6 +49,7 @@ self.root_node = n def build_ast(self): + """Convert an top level parse tree node into an AST mod.""" n = self.root_node if n.type == syms.file_input: stmts = [] @@ -88,6 +90,7 @@ raise AssertionError("unknown root node") def number_of_statements(self, n): + """Compute the number of AST statements contained in a node.""" stmt_type = n.type if stmt_type == syms.compound_stmt: return 1 @@ -100,10 +103,12 @@ raise AssertionError("non-statement node") def error(self, msg, n): + """Raise a SyntaxError with the lineno and column set to n's.""" raise SyntaxError(msg, n.lineno, n.column, filename=self.compile_info.filename) def check_forbidden_name(self, name, node): + """Raise an error if the name cannot be assigned to.""" if name == "None": self.error("assignment to None", node) if name == "__debug__": @@ -111,6 +116,7 @@ # XXX Warn about using True and False def set_context(self, expr, ctx, node): + """Set the context of an expression to Store or Del if possible.""" error = None sequence = None if isinstance(expr, ast.Attribute): @@ -227,6 +233,8 @@ if import_name_type == syms.import_as_name: name = import_name.children[0].value if len(import_name.children) == 3: + # 'as' is not yet a keyword in Python 2.5, so the grammar + # just specifies a NAME token. We check it manually here. if import_name.children[1].value != "as": self.error("must use 'as' in import", import_name) as_name = import_name.children[2].value @@ -492,6 +500,7 @@ return ast.With(test, target, body, with_node.lineno, with_node.column) def handle_with_var(self, with_var_node): + # The grammar doesn't require 'as', so check it manually. if with_var_node.children[0].value != "as": self.error("expected \"with [expr] as [var]\"", with_var_node) return self.handle_expr(with_var_node.children[1]) @@ -870,7 +879,7 @@ return result def handle_factor(self, factor_node): - # Fold '-' on constants. + # Fold '-' on constant numbers. if factor_node.children[0].type == tokens.MINUS and \ len(factor_node.children) == 2: factor = factor_node.children[1] @@ -1100,6 +1109,7 @@ encoding = self.compile_info.encoding sub_strings_w = [parsestring.parsestr(space, encoding, s.value) for s in atom_node.children] + # This implements implicit string concatenation. if len(sub_strings_w) > 1: w_sub_strings = space.newlist(sub_strings_w) w_join = space.getattr(space.wrap(""), space.wrap("join")) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 24 17:26:29 2009 @@ -2,6 +2,11 @@ Generate Python bytecode from a Abstract Syntax Tree. """ +# NOTE TO READERS: All the ugly and "obvious" isinstance assertions here are to +# help the annotator. To it, unfortunately, everything is not so obvious. If +# you figure out a way to remove them, great, but try a translation first, +# please. + from pypy.interpreter.astcompiler import (ast2 as ast, assemble, symtable, consts, misc) from pypy.interpreter.astcompiler import optimize, asthelpers # For side effects @@ -13,6 +18,7 @@ def compile_ast(space, module, info): + """Generate a code object from AST.""" symbols = symtable.SymtableBuilder(space, module, info) return TopLevelCodeGenerator(space, module, symbols, info).assemble() @@ -106,6 +112,7 @@ }) +# These are frame blocks. F_BLOCK_LOOP = 0 F_BLOCK_EXCEPT = 1 F_BLOCK_FINALLY = 2 @@ -113,6 +120,11 @@ class PythonCodeGenerator(assemble.PythonCodeMaker): + """Base code generator. + + A subclass of this is created for every scope to be compiled. It walks + across the AST tree generating bytecode as needed. + """ def __init__(self, space, name, tree, lineno, symbols, compile_info): self.scope = symbols.find_scope(tree) @@ -125,15 +137,21 @@ self._compile(tree) def _compile(self, tree): + """Override in subclasses to compile a scope.""" raise NotImplementedError def current_temporary_name(self): + """Return the name of the current temporary variable. + + This must be in sync with the one during symbol table building. + """ name = "_[%d]" % (self.temporary_name_counter,) self.temporary_name_counter += 1 assert self.scope.lookup(name) != symtable.SCOPE_UNKNOWN return name def sub_scope(self, kind, name, node, lineno): + """Convenience function for compiling a sub scope.""" generator = kind(self.space, name, node, lineno, self.symbols, self.compile_info) return generator.assemble() @@ -151,6 +169,7 @@ filename=self.compile_info.filename) def name_op(self, identifier, ctx): + """Generate an operation appropiate for the scope of the identifier.""" scope = self.scope.lookup(identifier) op = ops.NOP container = self.names @@ -182,9 +201,11 @@ return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str) def _get_code_flags(self): + # Default for everything but module scopes. return consts.CO_NEWLOCALS def _handle_body(self, body): + """Compile a list of statements, handling doc strings if needed.""" if body: start = 0 if self.is_docstring(body[0]): @@ -212,8 +233,10 @@ mod.body.walkabout(self) def _make_function(self, code, num_defaults=0): + """Emit the opcodes to turn a code object into a function.""" code_index = self.add_const(code) if code.co_freevars: + # Load cell and free vars to pass on. for free in code.co_freevars: free_scope = self.scope.lookup(free) if free_scope == symtable.SCOPE_CELL: @@ -230,6 +253,7 @@ def visit_FunctionDef(self, func): self.update_position(func.lineno, True) + # Load decorators first, but apply them after the function is created. if func.decorators: self.visit_sequence(func.decorators) if func.args.defaults: @@ -240,6 +264,7 @@ code = self.sub_scope(FunctionCodeGenerator, func.name, func, func.lineno) self._make_function(code, num_defaults) + # Apply decorators. if func.decorators: for i in range(len(func.decorators)): self.emit_op_arg(ops.CALL_FUNCTION, 1) @@ -403,6 +428,7 @@ if not self.frame_blocks: self.error("'continue' not properly in loop", cont) current_block, block = self.frame_blocks[-1] + # Continue cannot be in a finally block. if current_block == F_BLOCK_LOOP: self.emit_jump(ops.JUMP_ABSOLUTE, block, True) elif current_block == F_BLOCK_EXCEPT or \ @@ -524,6 +550,7 @@ self.visit_sequence(tf.body) self.emit_op(ops.POP_BLOCK) self.pop_frame_block(F_BLOCK_FINALLY, body) + # Indicates there was no exception. self.load_const(self.space.w_None) self.use_next_block(end) self.push_frame_block(F_BLOCK_FINALLY_END, end) @@ -559,6 +586,8 @@ self.load_const(self.space.wrap(level)) self.load_const(self.space.w_None) self.emit_op_name(ops.IMPORT_NAME, self.names, alias.name) + # If there's no asname then we store the root module. If there is + # an asname, _import_as stores the last module of the chain into it. if alias.asname: self._import_as(alias) else: @@ -575,6 +604,7 @@ first = imp.names[0] assert isinstance(first, ast.alias) star_import = len(imp.names) == 1 and first.name == "*" + # Various error checking for future imports. if imp.module == "__future__": last_line, last_offset = self.compile_info.last_future_import if imp.lineno > last_line or \ @@ -603,6 +633,7 @@ if imp.module: mod_name = imp.module else: + # In the case of a relative import. mod_name = "" self.emit_op_name(ops.IMPORT_NAME, self.names, mod_name) if star_import: @@ -630,6 +661,7 @@ assign.targets[i].walkabout(self) def _optimize_unpacking(self, assign): + """Try to optimize out BUILD_TUPLE and UNPACK_SEQUENCE opcodes.""" if len(assign.targets) != 1: return False targets = assign.targets[0].as_node_list(self.space) @@ -745,6 +777,7 @@ if self.interactive: expr.value.walkabout(self) self.emit_op(ops.PRINT_EXPR) + # Only compile if the expression isn't constant. elif not expr.value.constant: expr.value.walkabout(self) self.emit_op(ops.POP_TOP) @@ -1230,6 +1263,7 @@ def _compile(self, func): assert isinstance(func, ast.FunctionDef) + # If there's a docstring, store it as the first constant. if self.is_docstring(func.body[0]): doc_string = func.body[0] assert isinstance(doc_string, ast.Expr) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Fri Jul 24 17:26:29 2009 @@ -16,6 +16,10 @@ del app def syntax_warning(space, msg, fn, lineno, offset): + """Raise an applevel SyntaxWarning. + + If the user has set this warning to raise an error, a SyntaxError will be + raised.""" w_msg = space.wrap(msg) w_filename = space.wrap(fn) w_lineno = space.wrap(lineno) @@ -24,6 +28,7 @@ def dict_to_switch(d): + """Convert of dictionary with integer keys to a switch statement.""" def lookup(query): if we_are_translated(): for key, value in unrolling_iteritems: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Fri Jul 24 17:26:29 2009 @@ -1,3 +1,4 @@ +"""codegen helpers and AST constant folding.""" import sys import itertools @@ -19,9 +20,11 @@ class __extend__(ast.AST): def as_constant_truth(self, space): + """Return the truth of this node if known.""" raise AssertionError("only for expressions") def as_constant(self): + """Return the value of this node as a wrapped constant if possible.""" raise AssertionError("only for expressions") def accept_jump_if(self, gen, condition, target): @@ -149,6 +152,7 @@ class OptimizingVisitor(ast.ASTVisitor): + """Constant folds AST.""" def __init__(self, space, compile_info): self.space = space @@ -163,6 +167,9 @@ right = binop.right.as_constant() if right is not None: op = binop.op + # Can't fold straight division without "from __future_ import + # division" because it might be affected at runtime by the -Q + # flag. if op == ast.Div and \ not self.compile_info.flags & consts.CO_FUTURE_DIVISION: return binop @@ -173,9 +180,12 @@ break else: raise AssertionError("unknown binary operation") + # Let all errors be found at runtime. except OperationError: pass else: + # To avoid blowing up the size of pyc files, we only fold + # reasonably sized sequences. try: w_len = self.space.len(w_const) except OperationError: @@ -244,12 +254,15 @@ return rep def visit_Name(self, name): + # Turn loading None into a constant lookup. Eventaully, we can do this + # for True and False, too. if name.id == "None": assert name.ctx == ast.Load return ast.Const(self.space.w_None, name.lineno, name.col_offset) return name def visit_Tuple(self, tup): + """Try to turn tuple building into a constant.""" if tup.elts: consts_w = [None]*len(tup.elts) for i in range(len(tup.elts)): Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 24 17:26:29 2009 @@ -44,16 +44,22 @@ self.nested = False def lookup(self, name): + """Find the scope of identifier 'name'.""" return self.symbols.get(self.mangle(name), SCOPE_UNKNOWN) def lookup_role(self, name): return self.roles.get(self.mangle(name), SYM_BLANK) def new_temporary_name(self): + """Return the next temporary name. + + This must be in sync with PythonCodeGenerator's counter. + """ self.note_symbol("_[%d]" % (self.temp_name_counter,), SYM_ASSIGNED) self.temp_name_counter += 1 def note_symbol(self, identifier, role): + """Record that identifier occurs in this scope.""" mangled = self.mangle(identifier) new_role = role if mangled in self.roles: @@ -69,17 +75,21 @@ return mangled def note_yield(self, yield_node): + """Called when a yield is found.""" raise SyntaxError("'yield' outside function", yield_node.lineno, yield_node.col_offset) def note_return(self, ret): + """Called when a return statement is found.""" raise SyntaxError("return outside function", ret.lineno, ret.col_offset) def note_exec(self, exc): + """Called when an exec statement is found.""" self.has_exec = True def note_import_star(self, imp): + """Called when a start import is found.""" pass def mangle(self, name): @@ -89,10 +99,12 @@ return name def add_child(self, child_scope): + """Note a new child scope.""" child_scope.parent = self self.children.append(child_scope) def _finalize_name(self, name, flags, local, bound, free, globs): + """Decide on the scope of a name.""" if flags & SYM_GLOBAL: if flags & SYM_PARAM: err = "name '%s' is both local and global" % (name,) @@ -124,11 +136,13 @@ self.symbols[name] = SCOPE_GLOBAL_IMPLICIT def _pass_on_bindings(self, local, bound, globs, new_bound, new_globs): + """Allow child scopes to see names bound here and in outer scopes.""" new_globs.update(globs) if bound: new_bound.update(bound) def _finalize_cells(self, free): + """Hook for FunctionScope.""" pass def _check_optimization(self): @@ -137,6 +151,7 @@ _hide_bound_from_nested_scopes = False def finalize(self, bound, free, globs): + """Enter final bookeeping data in to self.symbols.""" self.symbols = {} local = {} new_globs = {} @@ -150,6 +165,8 @@ self._pass_on_bindings(local, bound, globs, new_bound, new_globs) child_frees = {} for child in self.children: + # Symbol dictionaries are copied to avoid having child scopes + # pollute each other's. child_free = new_free.copy() child.finalize(new_bound.copy(), child_free, new_globs.copy()) child_frees.update(child_free) @@ -279,6 +296,7 @@ class SymtableBuilder(ast.GenericASTVisitor): + """Find symbol information from AST.""" def __init__(self, space, module, compile_info): self.space = space @@ -300,6 +318,7 @@ assert not self.stack def push_scope(self, scope, node): + """Push a child scope.""" if self.stack: self.stack[-1].add_child(scope) self.stack.append(scope) @@ -315,13 +334,16 @@ self.scope = None def find_scope(self, scope_node): + """Lookup the scope for a given AST node.""" return self.scopes[scope_node] def implicit_arg(self, pos): + """Note a implicit arg for implicit tuple unpacking.""" name = ".%d" % (pos,) self.note_symbol(name, SYM_PARAM) def note_symbol(self, identifier, role): + """Note the identifer on the current scope.""" mangled = self.scope.note_symbol(identifier, role) if role & SYM_GLOBAL: if mangled in self.globs: @@ -330,6 +352,7 @@ def visit_FunctionDef(self, func): self.note_symbol(func.name, SYM_ASSIGNED) + # Function defaults and decorators happen in the outer scope. if func.args.defaults: self.visit_sequence(func.args.defaults) if func.decorators: @@ -449,6 +472,8 @@ if isinstance(arg, ast.Name): self.note_symbol(arg.id, SYM_PARAM) elif isinstance(arg, ast.Tuple): + # Tuple unpacking in the argument list. Add a secret variable + # name to recieve the tuple with. if is_toplevel: self.implicit_arg(i) else: Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/parser.py Fri Jul 24 17:26:29 2009 @@ -89,6 +89,10 @@ self.stack = None def prepare(self, start=-1): + """Setup the parser for parsing. + + Takes the starting symbol as an argument. + """ if start == -1: start = self.grammar.start self.root = None @@ -106,28 +110,37 @@ for i, next_state in arcs: sym_id = self.grammar.labels[i] if label_index == i: + # We matched a non-terminal. self.shift(next_state, token_type, value, lineno, column) state = states[next_state] + # While the only possible action is to accept, pop nodes off + # the stack. while state[1] and not state[0]: self.pop() if not self.stack: + # Parsing is done. return True dfa, state_index, node = self.stack[-1] state = dfa[0][state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id] + # Check if this token can start a child node. if label_index in sub_node_dfa[1]: self.push(sub_node_dfa, next_state, sym_id, lineno, column) break else: + # We failed to find any arcs to another state, so unless this + # state is accepting, it's invalid input. if is_accepting: self.pop() if not self.stack: raise ParseError("too much input", token_type, value, lineno, column, line) else: + # If only one possible input would satisfy, attach it to the + # error. if len(arcs) == 1: expected = sym_id else: @@ -136,6 +149,7 @@ column, line, expected) def classify(self, token_type, value, lineno, column, line): + """Find the label for a token.""" if token_type == self.grammar.KEYWORD_TOKEN: label_index = self.grammar.keyword_ids.get(value, -1) if label_index != -1: @@ -147,18 +161,21 @@ return label_index def shift(self, next_state, token_type, value, lineno, column): + """Shift a non-terminal and prepare for the next state.""" dfa, state, node = self.stack[-1] new_node = Node(token_type, value, None, lineno, column) node.children.append(new_node) self.stack[-1] = (dfa, next_state, node) def push(self, next_dfa, next_state, node_type, lineno, column): + """Push a terminal and adjust the current state.""" dfa, state, node = self.stack[-1] new_node = Node(node_type, None, [], lineno, column) self.stack[-1] = (dfa, next_state, node) self.stack.append((next_dfa, 0, new_node)) def pop(self): + """Pop an entry off the stack and make its node a child of the last.""" dfa, state, node = self.stack.pop() if self.stack: self.stack[-1][2].children.append(node) Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Fri Jul 24 17:26:29 2009 @@ -57,6 +57,15 @@ class CompileInfo(object): + """Stores information about the source being compiled. + + * filename: The filename of the source. + * mode: The parse mode to use. ('exec', 'eval', or 'single') + * flags: Parser and compiler flags. + * encoding: The source encoding. + * last_future_import: The line number and offset of the last __future__ + import. + """ def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0)): self.filename = filename @@ -79,7 +88,11 @@ self.space = space def parse_source(self, textsrc, compile_info): - """Parse a python source according to goal""" + """Main entry point for parsing Python source. + + Everything from decoding the source to tokenizing to building the parse + tree is handled here. + """ # Detect source encoding. enc = None if textsrc.startswith("\xEF\xBB\xBF"): @@ -111,10 +124,18 @@ raise flags = compile_info.flags + + # In order to not raise errors when 'as' or 'with' are used as names in + # code that does not explicitly enable the with statement, we have two + # grammars. One with 'as' and 'with' and keywords and one without. + # This is far better than CPython, where the parser is hacked up to + # check for __future__ imports and recognize new keywords accordingly. if flags & consts.CO_FUTURE_WITH_STATEMENT: self.grammar = pygram.python_grammar else: self.grammar = pygram.python_grammar_no_with_statement + + # The tokenizer is very picky about how it wants its input. source_lines = textsrc.splitlines(True) if source_lines and not source_lines[-1].endswith("\n"): source_lines[-1] += '\n' @@ -129,6 +150,8 @@ if self.add_token(tp, value, lineno, column, line): break except parser.ParseError, e: + # Catch parse errors, pretty them up and reraise them as a + # SyntaxError. new_err = error.IndentationError if tp == pygram.tokens.INDENT: msg = "unexpected indent" @@ -142,6 +165,7 @@ else: tree = self.root finally: + # Avoid hanging onto the tree. self.root = None if enc is not None: compile_info.encoding = enc Modified: pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/test/test_compiler.py Fri Jul 24 17:26:29 2009 @@ -744,6 +744,7 @@ def test_none_constant(self): import opcode co = compile("def f(): return None", "", "exec").co_consts[0] + assert "None" not in co.co_names co = co.co_code op = ord(co[0]) + (ord(co[1]) << 8) assert op == opcode.opmap["LOAD_CONST"] From benjamin at codespeak.net Fri Jul 24 17:58:31 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 17:58:31 +0200 (CEST) Subject: [pypy-svn] r66598 - in pypy/branch/parser-compiler/pypy/interpreter: . astcompiler astcompiler/test pyparser pyparser/test Message-ID: <20090724155831.0E746168457@codespeak.net> Author: benjamin Date: Fri Jul 24 17:58:30 2009 New Revision: 66598 Removed: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.txt pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astgen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/future.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/opt.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/pyassem.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/pycodegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symbols.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/visitor.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/asthelper.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnfgrammar.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnflexer.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/ebnfparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/grammar.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonlexer.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonparse.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pythonutil.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/symbol.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/syntaxtree.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/support.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_lookahead.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pytokenizer.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/tuplebuilder.py Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py Log: die, old compiler, DIE!!!! Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Fri Jul 24 17:58:30 2009 @@ -43,24 +43,6 @@ return lookup -def flatten(tup): - elts = [] - for elt in tup: - if type(elt) == tuple: - elts = elts + flatten(elt) - else: - elts.append(elt) - return elts - -class Counter: - def __init__(self, initial): - self.count = initial - - def next(self): - i = self.count - self.count += 1 - return i - MANGLE_LEN = 256 # magic constant from compile.c def mangle(name, klass): @@ -87,32 +69,3 @@ klass = klass[:end] return "_%s%s" % (klass, name) - -class Queue(object): - def __init__(self, item): - self.head = [item] - self.tail = [] - - def pop(self): - if self.head: - return self.head.pop() - else: - for i in range(len(self.tail)-1, -1, -1): - self.head.append(self.tail[i]) - self.tail = [] - return self.head.pop() - - def extend(self, items): - self.tail.extend(items) - - def nonempty(self): - return self.tail or self.head - -def set_filename(filename, tree): - """Set the filename attribute to filename on every node in tree""" - worklist = Queue(tree) - while worklist.nonempty(): - node = worklist.pop() - assert isinstance(node, ast.Node) - node.filename = filename - worklist.extend(node.getChildNodes()) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_compiler.py Fri Jul 24 17:58:30 2009 @@ -1,9 +1,6 @@ import py from pypy.interpreter.astcompiler import codegen, astbuilder from pypy.interpreter.pyparser import pyparse - -from pypy.interpreter.astcompiler import misc, pycodegen, opt -from pypy.interpreter.pyparser.test.support import source2ast from pypy.interpreter.pyparser.test import expressions from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyparser.error import SyntaxError, IndentationError Modified: pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pycompiler.py Fri Jul 24 17:58:30 2009 @@ -224,11 +224,12 @@ self.compiler_flags = self.futureFlags.allowed_flags def compile(self, source, filename, mode, flags): - from pypy.interpreter.pyparser.error import SyntaxError, IndentationError + from pypy.interpreter.pyparser.error import (SyntaxError, + IndentationError, + TokenIndentationError) from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyparser.pyparse import CompileInfo from pypy.interpreter.pyparser.future import getFutures - from pypy.interpreter.pyparser.pythonlexer import TokenIndentationError from pypy.interpreter.astcompiler.astbuilder import ast_from_node from pypy.interpreter.astcompiler.codegen import compile_ast from pypy.interpreter.astcompiler import consts, optimize Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytoken.py Fri Jul 24 17:58:30 2009 @@ -1,76 +1,4 @@ -# A replacement for the token module -# -# adds a new map token_values to avoid doing getattr on the module -# from PyPy RPython - -N_TOKENS = 0 - -# This is used to replace None -NULLTOKEN = -1 - -# tok_rpunct = {} - -def setup_tokens(parser): - # global tok_rpunct -# For compatibility, this produces the same constant values as Python 2.4. - parser.add_token( 'ENDMARKER' ) - parser.add_token( 'NAME' ) - parser.add_token( 'NUMBER' ) - parser.add_token( 'STRING' ) - parser.add_token( 'NEWLINE' ) - parser.add_token( 'INDENT' ) - parser.add_token( 'DEDENT' ) - parser.add_token( 'LPAR', "(" ) - parser.add_token( 'RPAR', ")" ) - parser.add_token( 'LSQB', "[" ) - parser.add_token( 'RSQB', "]" ) - parser.add_token( 'COLON', ":" ) - parser.add_token( 'COMMA', "," ) - parser.add_token( 'SEMI', ";" ) - parser.add_token( 'PLUS', "+" ) - parser.add_token( 'MINUS', "-" ) - parser.add_token( 'STAR', "*" ) - parser.add_token( 'SLASH', "/" ) - parser.add_token( 'VBAR', "|" ) - parser.add_token( 'AMPER', "&" ) - parser.add_token( 'LESS', "<" ) - parser.add_token( 'GREATER', ">" ) - parser.add_token( 'EQUAL', "=" ) - parser.add_token( 'DOT', "." ) - parser.add_token( 'PERCENT', "%" ) - parser.add_token( 'BACKQUOTE', "`" ) - parser.add_token( 'LBRACE', "{" ) - parser.add_token( 'RBRACE', "}" ) - parser.add_token( 'EQEQUAL', "==" ) - ne = parser.add_token( 'NOTEQUAL', "!=" ) - parser.tok_values["<>"] = ne - parser.add_token( 'LESSEQUAL', "<=" ) - parser.add_token( 'GREATEREQUAL', ">=" ) - parser.add_token( 'TILDE', "~" ) - parser.add_token( 'CIRCUMFLEX', "^" ) - parser.add_token( 'LEFTSHIFT', "<<" ) - parser.add_token( 'RIGHTSHIFT', ">>" ) - parser.add_token( 'DOUBLESTAR', "**" ) - parser.add_token( 'PLUSEQUAL', "+=" ) - parser.add_token( 'MINEQUAL', "-=" ) - parser.add_token( 'STAREQUAL', "*=" ) - parser.add_token( 'SLASHEQUAL', "/=" ) - parser.add_token( 'PERCENTEQUAL', "%=" ) - parser.add_token( 'AMPEREQUAL', "&=" ) - parser.add_token( 'VBAREQUAL', "|=" ) - parser.add_token( 'CIRCUMFLEXEQUAL', "^=" ) - parser.add_token( 'LEFTSHIFTEQUAL', "<<=" ) - parser.add_token( 'RIGHTSHIFTEQUAL', ">>=" ) - parser.add_token( 'DOUBLESTAREQUAL', "**=" ) - parser.add_token( 'DOUBLESLASH', "//" ) - parser.add_token( 'DOUBLESLASHEQUAL',"//=" ) - parser.add_token( 'AT', "@" ) - parser.add_token( 'OP' ) - parser.add_token( 'ERRORTOKEN' ) - -# extra PyPy-specific tokens - parser.add_token( "COMMENT" ) - parser.add_token( "NL" ) +"""Python token definitions.""" python_tokens = {} python_opmap = {} Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_parser.py Fri Jul 24 17:58:30 2009 @@ -1,50 +1,3 @@ -from pypy.interpreter.pyparser.asthelper import get_atoms -from pypy.interpreter.pyparser.grammar import Parser -from pypy.interpreter.pyparser import error - - -def test_symbols(): - p = Parser() - x1 = p.add_symbol('sym') - x2 = p.add_token('tok') - x3 = p.add_anon_symbol(':sym') - x4 = p.add_anon_symbol(':sym1') - # test basic numbering assumption - # symbols and tokens are attributed sequentially - # using the same counter - assert x2 == x1 + 1 - # anon symbols have negative value - assert x3 != x2 + 1 - assert x4 == x3 - 1 - assert x3 < 0 - y1 = p.add_symbol('sym') - assert y1 == x1 - y2 = p.add_token('tok') - assert y2 == x2 - y3 = p.add_symbol(':sym') - assert y3 == x3 - y4 = p.add_symbol(':sym1') - assert y4 == x4 - - -def test_load(): - d = { 5 : 'sym1', - 6 : 'sym2', - 9 : 'sym3', - } - p = Parser() - p.load_symbols( d ) - v = p.add_symbol('sym4') - # check that we avoid numbering conflicts - assert v>9 - v = p.add_symbol( 'sym1' ) - assert v == 5 - v = p.add_symbol( 'sym2' ) - assert v == 6 - v = p.add_symbol( 'sym3' ) - assert v == 9 - - # New parser tests. import py import tokenize From benjamin at codespeak.net Fri Jul 24 18:05:37 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 18:05:37 +0200 (CEST) Subject: [pypy-svn] r66599 - in pypy/branch/parser-compiler/pypy/interpreter/astcompiler: . test tools Message-ID: <20090724160537.A18A7169F44@codespeak.net> Author: benjamin Date: Fri Jul 24 18:05:33 2009 New Revision: 66599 Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.py (contents, props changed) Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Log: move ast2.py to ast.py and add it the repo Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/assemble.py Fri Jul 24 18:05:33 2009 @@ -2,7 +2,7 @@ Python control flow graph generation and bytecode assembly. """ -from pypy.interpreter.astcompiler import ast2 as ast, symtable +from pypy.interpreter.astcompiler import ast, symtable from pypy.interpreter import pycode from pypy.tool import stdlib_opcode as ops Added: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.py ============================================================================== --- (empty file) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/ast.py Fri Jul 24 18:05:33 2009 @@ -0,0 +1,1448 @@ +# Generated by tools/asdl_py.py +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter import typedef +from pypy.tool.pairtype import extendabletype + +class AST(Wrappable): + + __slots__ = () + + __metaclass__ = extendabletype + + def walkabout(self, visitor): + raise AssertionError("walkabout() implementation not provided") + + def mutate_over(self, visitor): + raise AssertionError("mutate_over() implementation not provided") + +class NodeVisitorNotImplemented(Exception): + pass + +class mod(AST): + + __slots__ = () + +class Module(mod): + + __slots__ = ('body') + + def __init__(self, body): + self.body = body + + def walkabout(self, visitor): + visitor.visit_Module(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_Module(self) + +class Interactive(mod): + + __slots__ = ('body') + + def __init__(self, body): + self.body = body + + def walkabout(self, visitor): + visitor.visit_Interactive(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_Interactive(self) + +class Expression(mod): + + __slots__ = ('body') + + def __init__(self, body): + self.body = body + + def walkabout(self, visitor): + visitor.visit_Expression(self) + + def mutate_over(self, visitor): + self.body = self.body.mutate_over(visitor) + return visitor.visit_Expression(self) + +class Suite(mod): + + __slots__ = ('body') + + def __init__(self, body): + self.body = body + + def walkabout(self, visitor): + visitor.visit_Suite(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_Suite(self) + +class stmt(AST): + + __slots__ = ('lineno', 'col_offset') + +class FunctionDef(stmt): + + __slots__ = ('name', 'args', 'body', 'decorators') + + def __init__(self, name, args, body, decorators, lineno, col_offset): + self.name = name + self.args = args + self.body = body + self.decorators = decorators + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_FunctionDef(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + if self.decorators: + visitor._mutate_sequence(self.decorators) + return visitor.visit_FunctionDef(self) + +class ClassDef(stmt): + + __slots__ = ('name', 'bases', 'body') + + def __init__(self, name, bases, body, lineno, col_offset): + self.name = name + self.bases = bases + self.body = body + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_ClassDef(self) + + def mutate_over(self, visitor): + if self.bases: + visitor._mutate_sequence(self.bases) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_ClassDef(self) + +class Return(stmt): + + __slots__ = ('value') + + def __init__(self, value, lineno, col_offset): + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Return(self) + + def mutate_over(self, visitor): + if self.value: + self.value = self.value.mutate_over(visitor) + return visitor.visit_Return(self) + +class Delete(stmt): + + __slots__ = ('targets') + + def __init__(self, targets, lineno, col_offset): + self.targets = targets + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Delete(self) + + def mutate_over(self, visitor): + if self.targets: + visitor._mutate_sequence(self.targets) + return visitor.visit_Delete(self) + +class Assign(stmt): + + __slots__ = ('targets', 'value') + + def __init__(self, targets, value, lineno, col_offset): + self.targets = targets + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Assign(self) + + def mutate_over(self, visitor): + if self.targets: + visitor._mutate_sequence(self.targets) + self.value = self.value.mutate_over(visitor) + return visitor.visit_Assign(self) + +class AugAssign(stmt): + + __slots__ = ('target', 'op', 'value') + + def __init__(self, target, op, value, lineno, col_offset): + self.target = target + self.op = op + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_AugAssign(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.value = self.value.mutate_over(visitor) + return visitor.visit_AugAssign(self) + +class Print(stmt): + + __slots__ = ('dest', 'values', 'nl') + + def __init__(self, dest, values, nl, lineno, col_offset): + self.dest = dest + self.values = values + self.nl = nl + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Print(self) + + def mutate_over(self, visitor): + if self.dest: + self.dest = self.dest.mutate_over(visitor) + if self.values: + visitor._mutate_sequence(self.values) + return visitor.visit_Print(self) + +class For(stmt): + + __slots__ = ('target', 'iter', 'body', 'orelse') + + def __init__(self, target, iter, body, orelse, lineno, col_offset): + self.target = target + self.iter = iter + self.body = body + self.orelse = orelse + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_For(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.iter = self.iter.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + if self.orelse: + visitor._mutate_sequence(self.orelse) + return visitor.visit_For(self) + +class While(stmt): + + __slots__ = ('test', 'body', 'orelse') + + def __init__(self, test, body, orelse, lineno, col_offset): + self.test = test + self.body = body + self.orelse = orelse + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_While(self) + + def mutate_over(self, visitor): + self.test = self.test.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + if self.orelse: + visitor._mutate_sequence(self.orelse) + return visitor.visit_While(self) + +class If(stmt): + + __slots__ = ('test', 'body', 'orelse') + + def __init__(self, test, body, orelse, lineno, col_offset): + self.test = test + self.body = body + self.orelse = orelse + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_If(self) + + def mutate_over(self, visitor): + self.test = self.test.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + if self.orelse: + visitor._mutate_sequence(self.orelse) + return visitor.visit_If(self) + +class With(stmt): + + __slots__ = ('context_expr', 'optional_vars', 'body') + + def __init__(self, context_expr, optional_vars, body, lineno, col_offset): + self.context_expr = context_expr + self.optional_vars = optional_vars + self.body = body + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_With(self) + + def mutate_over(self, visitor): + self.context_expr = self.context_expr.mutate_over(visitor) + if self.optional_vars: + self.optional_vars = self.optional_vars.mutate_over(visitor) + if self.body: + visitor._mutate_sequence(self.body) + return visitor.visit_With(self) + +class Raise(stmt): + + __slots__ = ('type', 'inst', 'tback') + + def __init__(self, type, inst, tback, lineno, col_offset): + self.type = type + self.inst = inst + self.tback = tback + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Raise(self) + + def mutate_over(self, visitor): + if self.type: + self.type = self.type.mutate_over(visitor) + if self.inst: + self.inst = self.inst.mutate_over(visitor) + if self.tback: + self.tback = self.tback.mutate_over(visitor) + return visitor.visit_Raise(self) + +class TryExcept(stmt): + + __slots__ = ('body', 'handlers', 'orelse') + + def __init__(self, body, handlers, orelse, lineno, col_offset): + self.body = body + self.handlers = handlers + self.orelse = orelse + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_TryExcept(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + if self.orelse: + visitor._mutate_sequence(self.orelse) + return visitor.visit_TryExcept(self) + +class TryFinally(stmt): + + __slots__ = ('body', 'finalbody') + + def __init__(self, body, finalbody, lineno, col_offset): + self.body = body + self.finalbody = finalbody + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_TryFinally(self) + + def mutate_over(self, visitor): + if self.body: + visitor._mutate_sequence(self.body) + if self.finalbody: + visitor._mutate_sequence(self.finalbody) + return visitor.visit_TryFinally(self) + +class Assert(stmt): + + __slots__ = ('test', 'msg') + + def __init__(self, test, msg, lineno, col_offset): + self.test = test + self.msg = msg + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Assert(self) + + def mutate_over(self, visitor): + self.test = self.test.mutate_over(visitor) + if self.msg: + self.msg = self.msg.mutate_over(visitor) + return visitor.visit_Assert(self) + +class Import(stmt): + + __slots__ = ('names') + + def __init__(self, names, lineno, col_offset): + self.names = names + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Import(self) + + def mutate_over(self, visitor): + return visitor.visit_Import(self) + +class ImportFrom(stmt): + + __slots__ = ('module', 'names', 'level') + + def __init__(self, module, names, level, lineno, col_offset): + self.module = module + self.names = names + self.level = level + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_ImportFrom(self) + + def mutate_over(self, visitor): + return visitor.visit_ImportFrom(self) + +class Exec(stmt): + + __slots__ = ('body', 'globals', 'locals') + + def __init__(self, body, globals, locals, lineno, col_offset): + self.body = body + self.globals = globals + self.locals = locals + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Exec(self) + + def mutate_over(self, visitor): + self.body = self.body.mutate_over(visitor) + if self.globals: + self.globals = self.globals.mutate_over(visitor) + if self.locals: + self.locals = self.locals.mutate_over(visitor) + return visitor.visit_Exec(self) + +class Global(stmt): + + __slots__ = ('names') + + def __init__(self, names, lineno, col_offset): + self.names = names + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Global(self) + + def mutate_over(self, visitor): + return visitor.visit_Global(self) + +class Expr(stmt): + + __slots__ = ('value') + + def __init__(self, value, lineno, col_offset): + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Expr(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Expr(self) + +class Pass(stmt): + + __slots__ = () + + def __init__(self, lineno, col_offset): + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Pass(self) + + def mutate_over(self, visitor): + return visitor.visit_Pass(self) + +class Break(stmt): + + __slots__ = () + + def __init__(self, lineno, col_offset): + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Break(self) + + def mutate_over(self, visitor): + return visitor.visit_Break(self) + +class Continue(stmt): + + __slots__ = () + + def __init__(self, lineno, col_offset): + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Continue(self) + + def mutate_over(self, visitor): + return visitor.visit_Continue(self) + +class expr(AST): + + __slots__ = ('lineno', 'col_offset') + +class BoolOp(expr): + + __slots__ = ('op', 'values') + + def __init__(self, op, values, lineno, col_offset): + self.op = op + self.values = values + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_BoolOp(self) + + def mutate_over(self, visitor): + if self.values: + visitor._mutate_sequence(self.values) + return visitor.visit_BoolOp(self) + +class BinOp(expr): + + __slots__ = ('left', 'op', 'right') + + def __init__(self, left, op, right, lineno, col_offset): + self.left = left + self.op = op + self.right = right + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_BinOp(self) + + def mutate_over(self, visitor): + self.left = self.left.mutate_over(visitor) + self.right = self.right.mutate_over(visitor) + return visitor.visit_BinOp(self) + +class UnaryOp(expr): + + __slots__ = ('op', 'operand') + + def __init__(self, op, operand, lineno, col_offset): + self.op = op + self.operand = operand + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_UnaryOp(self) + + def mutate_over(self, visitor): + self.operand = self.operand.mutate_over(visitor) + return visitor.visit_UnaryOp(self) + +class Lambda(expr): + + __slots__ = ('args', 'body') + + def __init__(self, args, body, lineno, col_offset): + self.args = args + self.body = body + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Lambda(self) + + def mutate_over(self, visitor): + self.body = self.body.mutate_over(visitor) + return visitor.visit_Lambda(self) + +class IfExp(expr): + + __slots__ = ('test', 'body', 'orelse') + + def __init__(self, test, body, orelse, lineno, col_offset): + self.test = test + self.body = body + self.orelse = orelse + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_IfExp(self) + + def mutate_over(self, visitor): + self.test = self.test.mutate_over(visitor) + self.body = self.body.mutate_over(visitor) + self.orelse = self.orelse.mutate_over(visitor) + return visitor.visit_IfExp(self) + +class Dict(expr): + + __slots__ = ('keys', 'values') + + def __init__(self, keys, values, lineno, col_offset): + self.keys = keys + self.values = values + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Dict(self) + + def mutate_over(self, visitor): + if self.keys: + visitor._mutate_sequence(self.keys) + if self.values: + visitor._mutate_sequence(self.values) + return visitor.visit_Dict(self) + +class ListComp(expr): + + __slots__ = ('elt', 'generators') + + def __init__(self, elt, generators, lineno, col_offset): + self.elt = elt + self.generators = generators + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_ListComp(self) + + def mutate_over(self, visitor): + self.elt = self.elt.mutate_over(visitor) + return visitor.visit_ListComp(self) + +class GeneratorExp(expr): + + __slots__ = ('elt', 'generators') + + def __init__(self, elt, generators, lineno, col_offset): + self.elt = elt + self.generators = generators + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_GeneratorExp(self) + + def mutate_over(self, visitor): + self.elt = self.elt.mutate_over(visitor) + return visitor.visit_GeneratorExp(self) + +class Yield(expr): + + __slots__ = ('value') + + def __init__(self, value, lineno, col_offset): + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Yield(self) + + def mutate_over(self, visitor): + if self.value: + self.value = self.value.mutate_over(visitor) + return visitor.visit_Yield(self) + +class Compare(expr): + + __slots__ = ('left', 'ops', 'comparators') + + def __init__(self, left, ops, comparators, lineno, col_offset): + self.left = left + self.ops = ops + self.comparators = comparators + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Compare(self) + + def mutate_over(self, visitor): + self.left = self.left.mutate_over(visitor) + if self.comparators: + visitor._mutate_sequence(self.comparators) + return visitor.visit_Compare(self) + +class Call(expr): + + __slots__ = ('func', 'args', 'keywords', 'starargs', 'kwargs') + + def __init__(self, func, args, keywords, starargs, kwargs, lineno, col_offset): + self.func = func + self.args = args + self.keywords = keywords + self.starargs = starargs + self.kwargs = kwargs + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Call(self) + + def mutate_over(self, visitor): + self.func = self.func.mutate_over(visitor) + if self.args: + visitor._mutate_sequence(self.args) + if self.starargs: + self.starargs = self.starargs.mutate_over(visitor) + if self.kwargs: + self.kwargs = self.kwargs.mutate_over(visitor) + return visitor.visit_Call(self) + +class Repr(expr): + + __slots__ = ('value') + + def __init__(self, value, lineno, col_offset): + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Repr(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Repr(self) + +class Num(expr): + + __slots__ = ('n') + + def __init__(self, n, lineno, col_offset): + self.n = n + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Num(self) + + def mutate_over(self, visitor): + return visitor.visit_Num(self) + +class Str(expr): + + __slots__ = ('s') + + def __init__(self, s, lineno, col_offset): + self.s = s + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Str(self) + + def mutate_over(self, visitor): + return visitor.visit_Str(self) + +class Attribute(expr): + + __slots__ = ('value', 'attr', 'ctx') + + def __init__(self, value, attr, ctx, lineno, col_offset): + self.value = value + self.attr = attr + self.ctx = ctx + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Attribute(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Attribute(self) + +class Subscript(expr): + + __slots__ = ('value', 'slice', 'ctx') + + def __init__(self, value, slice, ctx, lineno, col_offset): + self.value = value + self.slice = slice + self.ctx = ctx + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Subscript(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + self.slice = self.slice.mutate_over(visitor) + return visitor.visit_Subscript(self) + +class Name(expr): + + __slots__ = ('id', 'ctx') + + def __init__(self, id, ctx, lineno, col_offset): + self.id = id + self.ctx = ctx + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Name(self) + + def mutate_over(self, visitor): + return visitor.visit_Name(self) + +class List(expr): + + __slots__ = ('elts', 'ctx') + + def __init__(self, elts, ctx, lineno, col_offset): + self.elts = elts + self.ctx = ctx + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_List(self) + + def mutate_over(self, visitor): + if self.elts: + visitor._mutate_sequence(self.elts) + return visitor.visit_List(self) + +class Tuple(expr): + + __slots__ = ('elts', 'ctx') + + def __init__(self, elts, ctx, lineno, col_offset): + self.elts = elts + self.ctx = ctx + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Tuple(self) + + def mutate_over(self, visitor): + if self.elts: + visitor._mutate_sequence(self.elts) + return visitor.visit_Tuple(self) + +class Const(expr): + + __slots__ = ('value') + + def __init__(self, value, lineno, col_offset): + self.value = value + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_Const(self) + + def mutate_over(self, visitor): + return visitor.visit_Const(self) + +Load = 1 +Store = 2 +Del = 3 +AugLoad = 4 +AugStore = 5 +Param = 6 + +class slice(AST): + + __slots__ = () + +class Ellipsis(slice): + + __slots__ = () + + def __init__(self): + pass + + def walkabout(self, visitor): + visitor.visit_Ellipsis(self) + + def mutate_over(self, visitor): + return visitor.visit_Ellipsis(self) + +class Slice(slice): + + __slots__ = ('lower', 'upper', 'step') + + def __init__(self, lower, upper, step): + self.lower = lower + self.upper = upper + self.step = step + + def walkabout(self, visitor): + visitor.visit_Slice(self) + + def mutate_over(self, visitor): + if self.lower: + self.lower = self.lower.mutate_over(visitor) + if self.upper: + self.upper = self.upper.mutate_over(visitor) + if self.step: + self.step = self.step.mutate_over(visitor) + return visitor.visit_Slice(self) + +class ExtSlice(slice): + + __slots__ = ('dims') + + def __init__(self, dims): + self.dims = dims + + def walkabout(self, visitor): + visitor.visit_ExtSlice(self) + + def mutate_over(self, visitor): + if self.dims: + visitor._mutate_sequence(self.dims) + return visitor.visit_ExtSlice(self) + +class Index(slice): + + __slots__ = ('value') + + def __init__(self, value): + self.value = value + + def walkabout(self, visitor): + visitor.visit_Index(self) + + def mutate_over(self, visitor): + self.value = self.value.mutate_over(visitor) + return visitor.visit_Index(self) + +And = 1 +Or = 2 + +Add = 1 +Sub = 2 +Mult = 3 +Div = 4 +Mod = 5 +Pow = 6 +LShift = 7 +RShift = 8 +BitOr = 9 +BitXor = 10 +BitAnd = 11 +FloorDiv = 12 + +Invert = 1 +Not = 2 +UAdd = 3 +USub = 4 + +Eq = 1 +NotEq = 2 +Lt = 3 +LtE = 4 +Gt = 5 +GtE = 6 +Is = 7 +IsNot = 8 +In = 9 +NotIn = 10 + +class comprehension(AST): + + __slots__ = ('target', 'iter', 'ifs') + + def __init__(self, target, iter, ifs): + self.target = target + self.iter = iter + self.ifs = ifs + + def walkabout(self, visitor): + visitor.visit_comprehension(self) + +class excepthandler(AST): + + __slots__ = ('type', 'name', 'body', 'lineno', 'col_offset') + + def __init__(self, type, name, body, lineno, col_offset): + self.type = type + self.name = name + self.body = body + self.lineno = lineno + self.col_offset = col_offset + + def walkabout(self, visitor): + visitor.visit_excepthandler(self) + +class arguments(AST): + + __slots__ = ('args', 'vararg', 'kwarg', 'defaults') + + def __init__(self, args, vararg, kwarg, defaults): + self.args = args + self.vararg = vararg + self.kwarg = kwarg + self.defaults = defaults + + def walkabout(self, visitor): + visitor.visit_arguments(self) + +class keyword(AST): + + __slots__ = ('arg', 'value') + + def __init__(self, arg, value): + self.arg = arg + self.value = value + + def walkabout(self, visitor): + visitor.visit_keyword(self) + +class alias(AST): + + __slots__ = ('name', 'asname') + + def __init__(self, name, asname): + self.name = name + self.asname = asname + + def walkabout(self, visitor): + visitor.visit_alias(self) + +class ASTVisitor(object): + + def visit_sequence(self, seq): + for node in seq: + node.walkabout(self) + + def default_visitor(self, node): + raise NodeVisitorNotImplemented + + def _mutate_sequence(self, seq): + for i in range(len(seq)): + seq[i] = seq[i].mutate_over(self) + + def visit_Module(self, node): + return self.default_visitor(node) + def visit_Interactive(self, node): + return self.default_visitor(node) + def visit_Expression(self, node): + return self.default_visitor(node) + def visit_Suite(self, node): + return self.default_visitor(node) + def visit_FunctionDef(self, node): + return self.default_visitor(node) + def visit_ClassDef(self, node): + return self.default_visitor(node) + def visit_Return(self, node): + return self.default_visitor(node) + def visit_Delete(self, node): + return self.default_visitor(node) + def visit_Assign(self, node): + return self.default_visitor(node) + def visit_AugAssign(self, node): + return self.default_visitor(node) + def visit_Print(self, node): + return self.default_visitor(node) + def visit_For(self, node): + return self.default_visitor(node) + def visit_While(self, node): + return self.default_visitor(node) + def visit_If(self, node): + return self.default_visitor(node) + def visit_With(self, node): + return self.default_visitor(node) + def visit_Raise(self, node): + return self.default_visitor(node) + def visit_TryExcept(self, node): + return self.default_visitor(node) + def visit_TryFinally(self, node): + return self.default_visitor(node) + def visit_Assert(self, node): + return self.default_visitor(node) + def visit_Import(self, node): + return self.default_visitor(node) + def visit_ImportFrom(self, node): + return self.default_visitor(node) + def visit_Exec(self, node): + return self.default_visitor(node) + def visit_Global(self, node): + return self.default_visitor(node) + def visit_Expr(self, node): + return self.default_visitor(node) + def visit_Pass(self, node): + return self.default_visitor(node) + def visit_Break(self, node): + return self.default_visitor(node) + def visit_Continue(self, node): + return self.default_visitor(node) + def visit_BoolOp(self, node): + return self.default_visitor(node) + def visit_BinOp(self, node): + return self.default_visitor(node) + def visit_UnaryOp(self, node): + return self.default_visitor(node) + def visit_Lambda(self, node): + return self.default_visitor(node) + def visit_IfExp(self, node): + return self.default_visitor(node) + def visit_Dict(self, node): + return self.default_visitor(node) + def visit_ListComp(self, node): + return self.default_visitor(node) + def visit_GeneratorExp(self, node): + return self.default_visitor(node) + def visit_Yield(self, node): + return self.default_visitor(node) + def visit_Compare(self, node): + return self.default_visitor(node) + def visit_Call(self, node): + return self.default_visitor(node) + def visit_Repr(self, node): + return self.default_visitor(node) + def visit_Num(self, node): + return self.default_visitor(node) + def visit_Str(self, node): + return self.default_visitor(node) + def visit_Attribute(self, node): + return self.default_visitor(node) + def visit_Subscript(self, node): + return self.default_visitor(node) + def visit_Name(self, node): + return self.default_visitor(node) + def visit_List(self, node): + return self.default_visitor(node) + def visit_Tuple(self, node): + return self.default_visitor(node) + def visit_Const(self, node): + return self.default_visitor(node) + def visit_Ellipsis(self, node): + return self.default_visitor(node) + def visit_Slice(self, node): + return self.default_visitor(node) + def visit_ExtSlice(self, node): + return self.default_visitor(node) + def visit_Index(self, node): + return self.default_visitor(node) + def visit_comprehension(self, node): + return self.default_visitor(node) + def visit_excepthandler(self, node): + return self.default_visitor(node) + def visit_arguments(self, node): + return self.default_visitor(node) + def visit_keyword(self, node): + return self.default_visitor(node) + def visit_alias(self, node): + return self.default_visitor(node) + +class GenericASTVisitor(ASTVisitor): + + def visit_Module(self, node): + if node.body: + self.visit_sequence(node.body) + + def visit_Interactive(self, node): + if node.body: + self.visit_sequence(node.body) + + def visit_Expression(self, node): + node.body.walkabout(self) + + def visit_Suite(self, node): + if node.body: + self.visit_sequence(node.body) + + def visit_FunctionDef(self, node): + node.args.walkabout(self) + if node.body: + self.visit_sequence(node.body) + if node.decorators: + self.visit_sequence(node.decorators) + + def visit_ClassDef(self, node): + if node.bases: + self.visit_sequence(node.bases) + if node.body: + self.visit_sequence(node.body) + + def visit_Return(self, node): + if node.value: + node.value.walkabout(self) + + def visit_Delete(self, node): + if node.targets: + self.visit_sequence(node.targets) + + def visit_Assign(self, node): + if node.targets: + self.visit_sequence(node.targets) + node.value.walkabout(self) + + def visit_AugAssign(self, node): + node.target.walkabout(self) + node.value.walkabout(self) + + def visit_Print(self, node): + if node.dest: + node.dest.walkabout(self) + if node.values: + self.visit_sequence(node.values) + + def visit_For(self, node): + node.target.walkabout(self) + node.iter.walkabout(self) + if node.body: + self.visit_sequence(node.body) + if node.orelse: + self.visit_sequence(node.orelse) + + def visit_While(self, node): + node.test.walkabout(self) + if node.body: + self.visit_sequence(node.body) + if node.orelse: + self.visit_sequence(node.orelse) + + def visit_If(self, node): + node.test.walkabout(self) + if node.body: + self.visit_sequence(node.body) + if node.orelse: + self.visit_sequence(node.orelse) + + def visit_With(self, node): + node.context_expr.walkabout(self) + if node.optional_vars: + node.optional_vars.walkabout(self) + if node.body: + self.visit_sequence(node.body) + + def visit_Raise(self, node): + if node.type: + node.type.walkabout(self) + if node.inst: + node.inst.walkabout(self) + if node.tback: + node.tback.walkabout(self) + + def visit_TryExcept(self, node): + if node.body: + self.visit_sequence(node.body) + if node.handlers: + self.visit_sequence(node.handlers) + if node.orelse: + self.visit_sequence(node.orelse) + + def visit_TryFinally(self, node): + if node.body: + self.visit_sequence(node.body) + if node.finalbody: + self.visit_sequence(node.finalbody) + + def visit_Assert(self, node): + node.test.walkabout(self) + if node.msg: + node.msg.walkabout(self) + + def visit_Import(self, node): + if node.names: + self.visit_sequence(node.names) + + def visit_ImportFrom(self, node): + if node.names: + self.visit_sequence(node.names) + + def visit_Exec(self, node): + node.body.walkabout(self) + if node.globals: + node.globals.walkabout(self) + if node.locals: + node.locals.walkabout(self) + + def visit_Global(self, node): + pass + + def visit_Expr(self, node): + node.value.walkabout(self) + + def visit_Pass(self, node): + pass + + def visit_Break(self, node): + pass + + def visit_Continue(self, node): + pass + + def visit_BoolOp(self, node): + if node.values: + self.visit_sequence(node.values) + + def visit_BinOp(self, node): + node.left.walkabout(self) + node.right.walkabout(self) + + def visit_UnaryOp(self, node): + node.operand.walkabout(self) + + def visit_Lambda(self, node): + node.args.walkabout(self) + node.body.walkabout(self) + + def visit_IfExp(self, node): + node.test.walkabout(self) + node.body.walkabout(self) + node.orelse.walkabout(self) + + def visit_Dict(self, node): + if node.keys: + self.visit_sequence(node.keys) + if node.values: + self.visit_sequence(node.values) + + def visit_ListComp(self, node): + node.elt.walkabout(self) + if node.generators: + self.visit_sequence(node.generators) + + def visit_GeneratorExp(self, node): + node.elt.walkabout(self) + if node.generators: + self.visit_sequence(node.generators) + + def visit_Yield(self, node): + if node.value: + node.value.walkabout(self) + + def visit_Compare(self, node): + node.left.walkabout(self) + if node.comparators: + self.visit_sequence(node.comparators) + + def visit_Call(self, node): + node.func.walkabout(self) + if node.args: + self.visit_sequence(node.args) + if node.keywords: + self.visit_sequence(node.keywords) + if node.starargs: + node.starargs.walkabout(self) + if node.kwargs: + node.kwargs.walkabout(self) + + def visit_Repr(self, node): + node.value.walkabout(self) + + def visit_Num(self, node): + pass + + def visit_Str(self, node): + pass + + def visit_Attribute(self, node): + node.value.walkabout(self) + + def visit_Subscript(self, node): + node.value.walkabout(self) + node.slice.walkabout(self) + + def visit_Name(self, node): + pass + + def visit_List(self, node): + if node.elts: + self.visit_sequence(node.elts) + + def visit_Tuple(self, node): + if node.elts: + self.visit_sequence(node.elts) + + def visit_Const(self, node): + pass + + def visit_Ellipsis(self, node): + pass + + def visit_Slice(self, node): + if node.lower: + node.lower.walkabout(self) + if node.upper: + node.upper.walkabout(self) + if node.step: + node.step.walkabout(self) + + def visit_ExtSlice(self, node): + if node.dims: + self.visit_sequence(node.dims) + + def visit_Index(self, node): + node.value.walkabout(self) + + def visit_comprehension(self, node): + node.target.walkabout(self) + node.iter.walkabout(self) + if node.ifs: + self.visit_sequence(node.ifs) + + def visit_excepthandler(self, node): + if node.type: + node.type.walkabout(self) + if node.name: + node.name.walkabout(self) + if node.body: + self.visit_sequence(node.body) + + def visit_arguments(self, node): + if node.args: + self.visit_sequence(node.args) + if node.defaults: + self.visit_sequence(node.defaults) + + def visit_keyword(self, node): + node.value.walkabout(self) + + def visit_alias(self, node): + pass + + Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/astbuilder.py Fri Jul 24 18:05:33 2009 @@ -1,4 +1,4 @@ -from pypy.interpreter.astcompiler import ast2 as ast, misc +from pypy.interpreter.astcompiler import ast, misc from pypy.interpreter import error from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/asthelpers.py Fri Jul 24 18:05:33 2009 @@ -1,4 +1,4 @@ -from pypy.interpreter.astcompiler import ast2 as ast +from pypy.interpreter.astcompiler import ast from pypy.interpreter.error import OperationError Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/codegen.py Fri Jul 24 18:05:33 2009 @@ -7,8 +7,7 @@ # you figure out a way to remove them, great, but try a translation first, # please. -from pypy.interpreter.astcompiler import (ast2 as ast, assemble, symtable, - consts, misc) +from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc from pypy.interpreter.astcompiler import optimize, asthelpers # For side effects from pypy.interpreter.pyparser.error import SyntaxError from pypy.tool import stdlib_opcode as ops Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/misc.py Fri Jul 24 18:05:33 2009 @@ -1,5 +1,5 @@ from pypy.interpreter import gateway -from pypy.interpreter.astcompiler import ast2 as ast +from pypy.interpreter.astcompiler import ast from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/optimize.py Fri Jul 24 18:05:33 2009 @@ -2,7 +2,7 @@ import sys import itertools -from pypy.interpreter.astcompiler import ast2 as ast, consts, misc +from pypy.interpreter.astcompiler import ast, consts, misc from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError from pypy.rlib.unroll import unrolling_iterable Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/symtable.py Fri Jul 24 18:05:33 2009 @@ -2,7 +2,7 @@ Symbol tabling building. """ -from pypy.interpreter.astcompiler import ast2 as ast, misc +from pypy.interpreter.astcompiler import ast, misc from pypy.interpreter.pyparser.error import SyntaxError # These are for internal use only: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_astbuilder.py Fri Jul 24 18:05:33 2009 @@ -7,7 +7,7 @@ from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.astcompiler.astbuilder import ast_from_node -from pypy.interpreter.astcompiler import ast2 as ast, consts +from pypy.interpreter.astcompiler import ast, consts class TestAstBuilder: Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/tools/asdl_py.py Fri Jul 24 18:05:33 2009 @@ -247,7 +247,7 @@ print "Assuming default values of Python.asdl and ast.py" here = os.path.dirname(__file__) def_file = os.path.join(here, "Python.asdl") - out_file = os.path.join(here, "..", "ast2.py") + out_file = os.path.join(here, "..", "ast.py") else: print >> sys.stderr, "invalid arguments" return 2 From arigo at codespeak.net Fri Jul 24 18:07:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:07:01 +0200 (CEST) Subject: [pypy-svn] r66600 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090724160701.3CE63169F49@codespeak.net> Author: arigo Date: Fri Jul 24 18:07:00 2009 New Revision: 66600 Removed: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize2.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize3.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize2.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize3.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimize4.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizebridge4.py Log: Move these files away, in preparation for the merge. From benjamin at codespeak.net Fri Jul 24 18:10:35 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 18:10:35 +0200 (CEST) Subject: [pypy-svn] r66601 - pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test Message-ID: <20090724161035.EF14C169F4C@codespeak.net> Author: benjamin Date: Fri Jul 24 18:10:33 2009 New Revision: 66601 Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Log: ast2 -> ast Modified: pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/astcompiler/test/test_symtable.py Fri Jul 24 18:10:33 2009 @@ -1,7 +1,6 @@ import string import py -from pypy.interpreter.astcompiler import (ast2 as ast, astbuilder, symtable, - consts) +from pypy.interpreter.astcompiler import ast astbuilder, symtable, consts from pypy.interpreter.pyparser import pyparse from pypy.interpreter.pyparser.error import SyntaxError From arigo at codespeak.net Fri Jul 24 18:16:24 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:16:24 +0200 (CEST) Subject: [pypy-svn] r66602 - pypy/branch/pyjitpl5/pypy/jit/backend Message-ID: <20090724161624.A9CCC169F4E@codespeak.net> Author: arigo Date: Fri Jul 24 18:16:19 2009 New Revision: 66602 Added: pypy/branch/pyjitpl5/pypy/jit/backend/model.py.merge.tmp - copied, changed from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/model.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/model.py revisions 66237 to 66600: ------------------------------------------------------------------------ r66591 | arigo | 2009-07-24 16:07:33 +0200 (Fri, 24 Jul 2009) | 2 lines Fix x86 for NEW_WITH_VTABLE not taking the descr any more. ------------------------------------------------------------------------ r66241 | arigo | 2009-07-15 18:36:01 +0200 (Wed, 15 Jul 2009) | 4 lines A branch in which to attempt refactoring of optimize4 to take optimize3's idea of separating InstanceNodes and InstanceValues. ------------------------------------------------------------------------ Copied: pypy/branch/pyjitpl5/pypy/jit/backend/model.py.merge.tmp (from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/model.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/model.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/model.py.merge.tmp Fri Jul 24 18:16:19 2009 @@ -1,5 +1,8 @@ class AbstractCPU(object): + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + def setup_once(self): """Called once by the front-end when the program starts.""" pass From arigo at codespeak.net Fri Jul 24 18:16:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:16:25 +0200 (CEST) Subject: [pypy-svn] r66603 - pypy/branch/pyjitpl5/pypy/jit/backend/llgraph Message-ID: <20090724161625.235C9169F4E@codespeak.net> Author: arigo Date: Fri Jul 24 18:16:23 2009 New Revision: 66603 Added: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py.merge.tmp - copied, changed from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/llimpl.py revisions 66237 to 66600: ------------------------------------------------------------------------ r66577 | arigo | 2009-07-24 14:36:05 +0200 (Fri, 24 Jul 2009) | 2 lines Minor fixes and assert improvements. ------------------------------------------------------------------------ r66573 | arigo | 2009-07-24 12:52:41 +0200 (Fri, 24 Jul 2009) | 2 lines Fix the llgraph backend for NEW_WITH_VTABLE not receiving a descr any more. ------------------------------------------------------------------------ r66241 | arigo | 2009-07-15 18:36:01 +0200 (Wed, 15 Jul 2009) | 4 lines A branch in which to attempt refactoring of optimize4 to take optimize3's idea of separating InstanceNodes and InstanceValues. ------------------------------------------------------------------------ Copied: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py.merge.tmp (from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py.merge.tmp Fri Jul 24 18:16:23 2009 @@ -624,8 +624,10 @@ def op_new(self, size): return do_new(size.ofs) - def op_new_with_vtable(self, size, vtable): - result = do_new(size.ofs) + def op_new_with_vtable(self, descr, vtable): + assert descr is None + size = get_class_size(self.memocast, vtable) + result = do_new(size) value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable, self.memocast) return result @@ -685,8 +687,9 @@ OPHANDLERS = [None] * (rop._LAST+1) - def op_new_with_vtable(self, typedescr, vtable): - from pypy.jit.backend.llgraph import runner + def op_new_with_vtable(self, descr, vtable): + assert descr is None + typedescr = get_class_size(self.memocast, vtable) return ootype.cast_to_object(ootype.new(typedescr.TYPE)) def op_new_array(self, typedescr, count): @@ -821,6 +824,8 @@ frame.loop = loop frame.env = {} for i in range(len(loop.inputargs)): + expected_type = loop.inputargs[i].concretetype + assert lltype.typeOf(_future_values[i]) == expected_type frame.env[loop.inputargs[i]] = _future_values[i] del _future_values[:] @@ -918,6 +923,7 @@ def __init__(self): self.addresses = [llmemory.NULL] self.rev_cache = {} + self.vtable_to_size = {} def new_memo_cast(): memocast = MemoCast() @@ -940,6 +946,14 @@ assert 0 <= int < len(memocast.addresses) return memocast.addresses[int] +def get_class_size(memocast, vtable): + memocast = _from_opaque(memocast) + return memocast.vtable_to_size[vtable] + +def set_class_size(memocast, vtable, size): + memocast = _from_opaque(memocast) + memocast.vtable_to_size[vtable] = size + class GuardFailed(Exception): pass @@ -1236,6 +1250,7 @@ setannotation(new_memo_cast, s_MemoCast) setannotation(cast_adr_to_int, annmodel.SomeInteger()) setannotation(cast_int_to_adr, annmodel.SomeAddress()) +setannotation(set_class_size, annmodel.s_None) setannotation(do_arraylen_gc, annmodel.SomeInteger()) setannotation(do_strlen, annmodel.SomeInteger()) From arigo at codespeak.net Fri Jul 24 18:16:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:16:29 +0200 (CEST) Subject: [pypy-svn] r66604 - pypy/branch/pyjitpl5/pypy/jit/backend/llgraph Message-ID: <20090724161629.4D3FC169F4E@codespeak.net> Author: arigo Date: Fri Jul 24 18:16:28 2009 New Revision: 66604 Added: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py.merge.tmp - copied, changed from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/runner.py revisions 66237 to 66600: ------------------------------------------------------------------------ r66573 | arigo | 2009-07-24 12:52:41 +0200 (Fri, 24 Jul 2009) | 2 lines Fix the llgraph backend for NEW_WITH_VTABLE not receiving a descr any more. ------------------------------------------------------------------------ r66553 | arigo | 2009-07-23 19:52:31 +0200 (Thu, 23 Jul 2009) | 2 lines Progress; the next test passes (also on ootype). ------------------------------------------------------------------------ r66550 | arigo | 2009-07-23 19:11:08 +0200 (Thu, 23 Jul 2009) | 4 lines In-progress. Main change: NEW_WITH_VTABLE no longer takes a 'descr', because it can more easily fish it from a global table when needed. It avoids adding a 'descr' everywhere starting from the SpecNodes. ------------------------------------------------------------------------ r66451 | arigo | 2009-07-20 18:49:56 +0200 (Mon, 20 Jul 2009) | 2 lines Forgot to check this in. A small extension needed. ------------------------------------------------------------------------ r66241 | arigo | 2009-07-15 18:36:01 +0200 (Wed, 15 Jul 2009) | 4 lines A branch in which to attempt refactoring of optimize4 to take optimize3's idea of separating InstanceNodes and InstanceValues. ------------------------------------------------------------------------ Copied: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py.merge.tmp (from r66600, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py.merge.tmp Fri Jul 24 18:16:28 2009 @@ -43,6 +43,9 @@ def sort_key(self): return self.ofs + def is_pointer_field(self): + return self.typeinfo == 'p' + def equals(self, other): if not isinstance(other, Descr): return False @@ -88,6 +91,13 @@ assert self.translate_support_code return False + def set_class_sizes(self, class_sizes): + self.class_sizes = class_sizes + for vtable, size in class_sizes.items(): + if not self.is_oo: + size = size.ofs + llimpl.set_class_size(self.memo_cast, vtable, size) + def compile_operations(self, loop, bridge=None): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop @@ -265,19 +275,23 @@ return history.BoxInt(llimpl.do_arraylen_gc(arraydescr, array)) def do_strlen(self, args, descr=None): + assert descr is None string = args[0].getptr_base() return history.BoxInt(llimpl.do_strlen(0, string)) def do_strgetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() return history.BoxInt(llimpl.do_strgetitem(0, string, index)) def do_unicodelen(self, args, descr=None): + assert descr is None string = args[0].getptr_base() return history.BoxInt(llimpl.do_unicodelen(0, string)) def do_unicodegetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() return history.BoxInt(llimpl.do_unicodegetitem(0, string, index)) @@ -316,8 +330,10 @@ def do_new(self, args, size): return history.BoxPtr(llimpl.do_new(size.ofs)) - def do_new_with_vtable(self, args, size): + def do_new_with_vtable(self, args, descr=None): + assert descr is None vtable = args[0].getint() + size = self.class_sizes[vtable] result = llimpl.do_new(size.ofs) llimpl.do_setfield_gc_int(result, self.fielddescrof_vtable.ofs, vtable, self.memo_cast) @@ -362,20 +378,24 @@ self.memo_cast) def do_newstr(self, args, descr=None): + assert descr is None length = args[0].getint() return history.BoxPtr(llimpl.do_newstr(0, length)) def do_newunicode(self, args, descr=None): + assert descr is None length = args[0].getint() return history.BoxPtr(llimpl.do_newunicode(0, length)) def do_strsetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() newvalue = args[2].getint() llimpl.do_strsetitem(0, string, index, newvalue) def do_unicodesetitem(self, args, descr=None): + assert descr is None string = args[0].getptr_base() index = args[1].getint() newvalue = args[2].getint() @@ -398,11 +418,13 @@ llimpl.do_call_void(func, self.memo_cast) def do_cast_int_to_ptr(self, args, descr=None): + assert descr is None return history.BoxPtr(llimpl.cast_from_int(llmemory.GCREF, args[0].getint(), self.memo_cast)) def do_cast_ptr_to_int(self, args, descr=None): + assert descr is None return history.BoxInt(llimpl.cast_to_int(args[0].getptr_base(), self.memo_cast)) @@ -447,9 +469,11 @@ else: return ootype.NULL - def do_new_with_vtable(self, args, typedescr): - assert isinstance(typedescr, TypeDescr) - assert len(args) == 1 # but we don't need it, so ignore + def do_new_with_vtable(self, args, descr=None): + assert descr is None + assert len(args) == 1 + cls = args[0].getobj() + typedescr = self.class_sizes[cls] return typedescr.create() def do_new_array(self, args, typedescr): @@ -500,7 +524,7 @@ # XXX: return None if RESULT is Void return x - def do_oosend(self, args, descr=None): + def do_oosend(self, args, descr): assert isinstance(descr, MethDescr) selfbox = args[0] argboxes = args[1:] @@ -671,10 +695,14 @@ self.getfield = getfield self.setfield = setfield + self._is_pointer_field = (history.getkind(T) == 'obj') def sort_key(self): return self._keys.getkey((self.TYPE, self.fieldname)) + def is_pointer_field(self): + return self._is_pointer_field + def equals(self, other): return self.TYPE == other.TYPE and \ self.fieldname == other.fieldname From arigo at codespeak.net Fri Jul 24 18:16:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:16:37 +0200 (CEST) Subject: [pypy-svn] r66605 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090724161637.96D30169F44@codespeak.net> Author: arigo Date: Fri Jul 24 18:16:35 2009 New Revision: 66605 Added: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py.merge.tmp - copied, changed from r66600, pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_basic.py revisions 66237 to 66600: ------------------------------------------------------------------------ r66562 | arigo | 2009-07-23 22:19:08 +0200 (Thu, 23 Jul 2009) | 3 lines Link optimize.py with the rest of the code. Still some failures to investigate. ------------------------------------------------------------------------ r66241 | arigo | 2009-07-15 18:36:01 +0200 (Wed, 15 Jul 2009) | 4 lines A branch in which to attempt refactoring of optimize4 to take optimize3's idea of separating InstanceNodes and InstanceValues. ------------------------------------------------------------------------ Copied: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py.merge.tmp (from r66600, pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py) ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py.merge.tmp Fri Jul 24 18:16:35 2009 @@ -75,6 +75,7 @@ graph_key, called_from = cw.unfinished_graphs.pop() cw.make_one_bytecode(graph_key, False, called_from) metainterp.staticdata.portal_code = maingraph + metainterp.staticdata._class_sizes = cw.class_sizes metainterp.staticdata.state = FakeWarmRunnerDesc() metainterp.staticdata.DoneWithThisFrameInt = DoneWithThisFrame metainterp.staticdata.DoneWithThisFramePtr = DoneWithThisFrame From arigo at codespeak.net Fri Jul 24 18:18:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:18:00 +0200 (CEST) Subject: [pypy-svn] r66606 - in pypy/branch/pyjitpl5/pypy/jit: backend backend/llgraph backend/llgraph/test backend/llvm backend/minimal backend/test backend/x86 metainterp metainterp/test Message-ID: <20090724161800.03B50168071@codespeak.net> Author: arigo Date: Fri Jul 24 18:17:59 2009 New Revision: 66606 Added: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py - copied unchanged from r66605, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py - copied unchanged from r66605, pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/test/ - copied from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llgraph/test/ pypy/branch/pyjitpl5/pypy/jit/backend/llvm/ - copied from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/llvm/ pypy/branch/pyjitpl5/pypy/jit/backend/minimal/ - copied from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/minimal/ pypy/branch/pyjitpl5/pypy/jit/backend/model.py - copied unchanged from r66605, pypy/branch/pyjitpl5/pypy/jit/backend/model.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/backend/test/ - copied from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/test/ pypy/branch/pyjitpl5/pypy/jit/backend/x86/ - copied from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/backend/x86/ pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/compile.py pypy/branch/pyjitpl5/pypy/jit/metainterp/heaptracker.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/heaptracker.py pypy/branch/pyjitpl5/pypy/jit/metainterp/history.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/history.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimize.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeutil.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/optimizeutil.py pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/oparser.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/oparser.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py - copied unchanged from r66605, pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_resume.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_specnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_tl.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_tl.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtual.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtual.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_zrpy_basic.py - copied unchanged from r66605, pypy/branch/pyjitpl5-optimize4/pypy/jit/metainterp/test/test_zrpy_basic.py Removed: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/backend/model.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py.merge.tmp pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_loop_optimize2.py Log: Merge the pyjitpl5-optimize4 branch: a clean-up and rewrite of optimize.py. From arigo at codespeak.net Fri Jul 24 18:20:11 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 24 Jul 2009 18:20:11 +0200 (CEST) Subject: [pypy-svn] r66607 - pypy/branch/pyjitpl5-optimize4 Message-ID: <20090724162011.1566F169F21@codespeak.net> Author: arigo Date: Fri Jul 24 18:20:08 2009 New Revision: 66607 Removed: pypy/branch/pyjitpl5-optimize4/ Log: Remove the merged branch. From fijal at codespeak.net Fri Jul 24 19:35:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 24 Jul 2009 19:35:18 +0200 (CEST) Subject: [pypy-svn] r66609 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090724173518.06D89169F53@codespeak.net> Author: fijal Date: Fri Jul 24 19:35:17 2009 New Revision: 66609 Added: pypy/extradoc/talk/euroscipy2009/author.latex pypy/extradoc/talk/euroscipy2009/beamerdefs.txt (contents, props changed) pypy/extradoc/talk/euroscipy2009/clouds.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/dump.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/earth.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/flow.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/makepdf (contents, props changed) pypy/extradoc/talk/euroscipy2009/output.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/pypy-logo.png (contents, props changed) pypy/extradoc/talk/euroscipy2009/stylesheet.latex pypy/extradoc/talk/euroscipy2009/world.png (contents, props changed) Modified: pypy/extradoc/talk/euroscipy2009/talk.txt Log: Update on talk Added: pypy/extradoc/talk/euroscipy2009/author.latex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/euroscipy2009/author.latex Fri Jul 24 19:35:17 2009 @@ -0,0 +1,7 @@ +\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} + +\title[PyPy status talk]{PyPy status talk} +\author[Maciej Fijalkowski, Dorota Jarecka]{Maciej Fijalkowski \and +Dorota Jarecka} +\institute[EuroSciPy 2009]{EuroSciPy 2009} +\date{July 25, 2009} Added: pypy/extradoc/talk/euroscipy2009/beamerdefs.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/euroscipy2009/beamerdefs.txt Fri Jul 24 19:35:17 2009 @@ -0,0 +1,77 @@ +.. colors +.. =========================== + +.. role:: green +.. role:: red + + +.. general useful commands +.. =========================== + +.. |pause| raw:: latex + + \pause + +.. |small| raw:: latex + + {\small + +.. |end_small| raw:: latex + + } + + +.. closed bracket +.. =========================== + +.. |>| raw:: latex + + } + + +.. example block +.. =========================== + +.. |example<| raw:: latex + + \begin{exampleblock}{ + + +.. |end_example| raw:: latex + + \end{exampleblock} + + + +.. alert block +.. =========================== + +.. |alert<| raw:: latex + + \begin{alertblock}{ + + +.. |end_alert| raw:: latex + + \end{alertblock} + + + +.. columns +.. =========================== + +.. |column1| raw:: latex + + \begin{columns} + \begin{column}{0\textwidth} + +.. |column2| raw:: latex + + \end{column} + \begin{column}{0\textwidth} + + +.. |end_columns| raw:: latex + + \end{column} + \end{columns} Added: pypy/extradoc/talk/euroscipy2009/clouds.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/dump.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/earth.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/flow.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/makepdf ============================================================================== --- (empty file) +++ pypy/extradoc/talk/euroscipy2009/makepdf Fri Jul 24 19:35:17 2009 @@ -0,0 +1,9 @@ +#!/bin/bash + +# WARNING: to work, it needs this patch for docutils +# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414 + +rst2beamer.py --stylesheet=stylesheet.latex --documentoptions=14pt talk.txt talk.latex || exit +sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit +pdflatex talk.latex || exit + Added: pypy/extradoc/talk/euroscipy2009/output.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/pypy-logo.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/euroscipy2009/stylesheet.latex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/euroscipy2009/stylesheet.latex Fri Jul 24 19:35:17 2009 @@ -0,0 +1,10 @@ +\usetheme{Boadilla} +\setbeamercovered{transparent} +\setbeamertemplate{navigation symbols}{} + +\definecolor{darkgreen}{rgb}{0, 0.5, 0.0} +\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor} + +\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor} +\newcommand{\red}[1]{\color{red}#1\normalcolor} Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Fri Jul 24 19:35:17 2009 @@ -1,3 +1,4 @@ +.. include:: beamerdefs.txt ============================================= PyPy's JIT for scientific python computations @@ -10,7 +11,7 @@ * Introduction to JIT -* A bit about EULAG model +* A bit about cloud modelling & EULAG * JIT & numpy integration @@ -19,6 +20,10 @@ PyPy - what's that? =================== +.. image:: pypy-logo.png + :width: 50px + :align: right + * A Python interpreter written in Python * A flexible compiler compiling restricted version @@ -37,6 +42,8 @@ * Psyco/stackless hard to maintain/extend +|pause| + * "Psyco consumes one brain per inch of progress" Motivation - user perspective @@ -44,6 +51,10 @@ * Noone should be ever forced to write in C for performance +|pause| + +* ... with maybe some exceptions + Status of PyPy today ==================== @@ -75,6 +86,8 @@ * We want to generate JIT out of interpreter's description +|pause| + * ... instead of writing it by hand JIT details @@ -84,25 +97,27 @@ * Tracing JIT (a-la tracemonkey) -* XXX +* Written in a high-level language JIT - status ============ * Simple numeric stuff - 20-30x performance boost -* C-like speed for simple stuff +|pause| -* only x86 backend so far +* ... which is similar to unoptimized C (gcc -O0) -* More work needed +|pause| -xxx +* only x86 backend so far (ie no x87) -xxx +* More work needed -EULAG -===== +* Only about 2 man-year put so far + +Cloud modelling +=============== Cloud processes cover tremendous range of scales, from thousands of kilometers to a fraction of a cm... @@ -112,10 +127,74 @@ | :width: 90px | :width: 100px | :width: 100px | +----------------------+------------------------+---------------------+ +Global climate model +==================== + +Horizontal grid size of Global Climate Models is ~ 100km + +.. image:: world.png + :width: 300px + :align: center + +Clouds have to be parametrized + +XXX +=== + +Large Eddy Simulation (LES) Models +================================== + +* processes, which scales are range of 100m could be calculated explicitly + +* processes of smaller spatial scale, (e.g. turbulent mixing) + have to be still parametrized + +.. image:: dump.png + :width: 300px + :align: center + EULAG ===== -XXX +* developed in National Center for Atmospheric Science (NCAR), + Boulder, Colorado, since '80 + +* written in Fortran 77 + +* many different developers and users + +* lack of version control, good documentation and bug tracking system + +EULAG - data size +================= + +* model time step - 3s, total time of simulations - 6h + +* horizontal/vertical grid size 25m/25m + number of horizontal/vertical points - 128/121 + +Analyzing model data +==================== + +.. image:: output.png + :width: 200px + +* checkin amount of liquid water in each box + +* checkin for each "dry" box, distance to the cloud edge + +* calculating relative humidity in each "dry" + +Data used for analyzis +====================== + +* written every 4 minutes + +* last 4 hours of the simulation + +* around 8 basic variables, e.g. velocities, temperature + +* For each variable: 60*121*128*128 values Data postprocessing =================== @@ -137,28 +216,88 @@ * Implement minimal version of numeric for PyPy -* Reuse the JIT for speedups +* Use the JIT for speedups -* ~10x speedups over CPython version +Results +======= + +* ~10x speedups over pure-python version on top of CPython * Still about 4x slower than C or matrix operations +|pause| + +* ... but not everything can be expressed using matrix operations + +* Due to limitations, programs needs to be carefully crafted + JIT & Numpy =========== * Just started really -* Specifically tailored programs see good speedups +* 138 LOC + +* only ints and simple operations + +JIT & Numpy status +================== + +* No support from JIT **at all** + +* General JIT progress gives speedups with numpy + +* More work on numpy needed, but it's all very simple + +JIT & Numpy - future +==================== + +* Possible to optimize beyond C implementation + +* Matrix operations like: a + b + c should not allocate immediate results -* Ongoing jit refactoring XXX +* Use of SSE operations can yield even bigger speedups + (or whatever architecture permits) -* More work needed XXX +* Make sure that unmodified programs run faster + +* Future looking pretty good Future directions ================= -xxx +* Work more on JIT in general (inlining, x86_64 backend, etc.) + +* Produce a release by the end of the year + +* Release should speed up real-world program + +* Work on numpy as time permits + +* Seek more funding + +How you can help? +================= + +* PyPy is open source after all + +* Very friendly to newcomers + +* A lot of promising fields to work on + +* We also seek funding or research grants Thank you! Questions? ===================== +* **PyPy**: + +http://codespeak.net/pypy + +http://morepypy.blogspot.com + +pypy-dev at codespeak.net + +* **Cloud modelling**: + +XXX Added: pypy/extradoc/talk/euroscipy2009/world.png ============================================================================== Binary file. No diff available. From fijal at codespeak.net Fri Jul 24 21:17:00 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 24 Jul 2009 21:17:00 +0200 (CEST) Subject: [pypy-svn] r66611 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090724191700.6BDC5169F53@codespeak.net> Author: fijal Date: Fri Jul 24 21:16:56 2009 New Revision: 66611 Added: pypy/extradoc/talk/euroscipy2009/talk.pdf (contents, props changed) Modified: pypy/extradoc/talk/euroscipy2009/author.latex pypy/extradoc/talk/euroscipy2009/talk.txt Log: Update and upload pdf Modified: pypy/extradoc/talk/euroscipy2009/author.latex ============================================================================== --- pypy/extradoc/talk/euroscipy2009/author.latex (original) +++ pypy/extradoc/talk/euroscipy2009/author.latex Fri Jul 24 21:16:56 2009 @@ -1,7 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} \title[PyPy status talk]{PyPy status talk} -\author[Maciej Fijalkowski, Dorota Jarecka]{Maciej Fijalkowski \and -Dorota Jarecka} +\author[Maciej Fijalkowski, Dorota Jarecka]{Maciej Fijalkowski (merlinux GmbH) \and \\ +Dorota Jarecka (University of Warsaw)} \institute[EuroSciPy 2009]{EuroSciPy 2009} \date{July 25, 2009} Added: pypy/extradoc/talk/euroscipy2009/talk.pdf ============================================================================== Binary file. No diff available. Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Fri Jul 24 21:16:56 2009 @@ -1,3 +1,4 @@ + .. include:: beamerdefs.txt ============================================= @@ -122,10 +123,18 @@ Cloud processes cover tremendous range of scales, from thousands of kilometers to a fraction of a cm... -+----------------------+------------------------+---------------------+ -| .. image:: earth.png | .. image:: clouds.png | .. image:: flow.png | -| :width: 90px | :width: 100px | :width: 100px | -+----------------------+------------------------+---------------------+ +.. +----------------------+------------------------+---------------------+ +.. | .. image:: earth.png | .. image:: clouds.png | .. image:: flow.png | +.. | :width: 90px | :width: 100px | :width: 100px | +.. +----------------------+------------------------+---------------------+ + +.. raw:: latex + + \begin{figure}[!h] + \subfigure{\includegraphics[width=90px]{earth.png}} + \subfigure{\includegraphics[width=100px]{clouds.png}} + \subfigure{\includegraphics[width=100px]{flow.png}} + \end{figure} Global climate model ==================== @@ -138,9 +147,6 @@ Clouds have to be parametrized -XXX -=== - Large Eddy Simulation (LES) Models ================================== @@ -177,7 +183,8 @@ ==================== .. image:: output.png - :width: 200px + :width: 170px + :align: center * checkin amount of liquid water in each box @@ -300,4 +307,4 @@ * **Cloud modelling**: -XXX +dorota at igf.fuw.edu.pl From fijal at codespeak.net Fri Jul 24 22:16:42 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 24 Jul 2009 22:16:42 +0200 (CEST) Subject: [pypy-svn] r66613 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090724201642.35780169F5C@codespeak.net> Author: fijal Date: Fri Jul 24 22:16:40 2009 New Revision: 66613 Modified: pypy/extradoc/talk/euroscipy2009/author.latex pypy/extradoc/talk/euroscipy2009/talk.pdf pypy/extradoc/talk/euroscipy2009/talk.txt Log: Some minor adjustments, add two slides about tracing jits Modified: pypy/extradoc/talk/euroscipy2009/author.latex ============================================================================== --- pypy/extradoc/talk/euroscipy2009/author.latex (original) +++ pypy/extradoc/talk/euroscipy2009/author.latex Fri Jul 24 22:16:40 2009 @@ -1,7 +1,7 @@ \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0} -\title[PyPy status talk]{PyPy status talk} -\author[Maciej Fijalkowski, Dorota Jarecka]{Maciej Fijalkowski (merlinux GmbH) \and \\ +\title[PyPy's JIT for scientific python computations]{PyPy's JIT for scientific python computations} +\author[M Fijalkowski, D Jarecka]{Maciej Fijalkowski (merlinux GmbH) \and \\ Dorota Jarecka (University of Warsaw)} -\institute[EuroSciPy 2009]{EuroSciPy 2009} +\institute[EuroSciPy]{EuroSciPy 2009} \date{July 25, 2009} Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Fri Jul 24 22:16:40 2009 @@ -25,7 +25,7 @@ :width: 50px :align: right -* A Python interpreter written in Python +* A Python interpreter written in (a restricted subset of) Python * A flexible compiler compiling restricted version of Python to lower level platform @@ -100,6 +100,31 @@ * Written in a high-level language +Tracing JIT +=========== + +* Modeled after dynamo VM by Michael Franz et al + +* Looks at the single loop of a program while its executed + +* Assumes the same path is commonly taken (a bit more complex than that) + +* Compiles relatively little assembler + +Tracing the interpreter +======================= + +* Follows interpretation of each bytecode via internal interpreter + calls + +* Automatically inlines everything + +* Creates assembler corresponding to a real interpreter path + +* ... plus a couple of guards + +* Works for any interpreter + JIT - status ============ @@ -139,7 +164,9 @@ Global climate model ==================== -Horizontal grid size of Global Climate Models is ~ 100km +.. raw:: latex + + Horizontal grid size of Global Climate Models is $\sim$ 100km .. image:: world.png :width: 300px @@ -152,15 +179,15 @@ * processes, which scales are range of 100m could be calculated explicitly -* processes of smaller spatial scale, (e.g. turbulent mixing) - have to be still parametrized +* processes of smaller spatial scale, (e.g. turbulent mixing of a cloud + with air from the environment) have to be still parametrized .. image:: dump.png :width: 300px :align: center -EULAG -===== +EULAG - numerical solver for all-scale geophysical flows +======================================================== * developed in National Center for Atmospheric Science (NCAR), Boulder, Colorado, since '80 @@ -171,13 +198,7 @@ * lack of version control, good documentation and bug tracking system -EULAG - data size -================= - -* model time step - 3s, total time of simulations - 6h - -* horizontal/vertical grid size 25m/25m - number of horizontal/vertical points - 128/121 +* http://www.mmm.ucar.edu/eulag Analyzing model data ==================== @@ -206,7 +227,7 @@ Data postprocessing =================== -* Short programs (~200 python LOC) +* Short programs (about 200 lines of code each) * Relatively simple logic @@ -228,7 +249,7 @@ Results ======= -* ~10x speedups over pure-python version on top of CPython +* roughly 10x speedups over pure-python version on top of CPython * Still about 4x slower than C or matrix operations @@ -261,7 +282,7 @@ * Possible to optimize beyond C implementation -* Matrix operations like: a + b + c should not allocate immediate results +* Matrix operations like: a + b + c should not allocate immediate values * Use of SSE operations can yield even bigger speedups (or whatever architecture permits) From benjamin at codespeak.net Fri Jul 24 22:19:02 2009 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Fri, 24 Jul 2009 22:19:02 +0200 (CEST) Subject: [pypy-svn] r66614 - pypy/branch/parser-compiler/pypy/interpreter/pyparser Message-ID: <20090724201902.67ACB169F73@codespeak.net> Author: benjamin Date: Fri Jul 24 22:19:01 2009 New Revision: 66614 Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Log: 2.4 fixes Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/metaparser.py Fri Jul 24 22:19:01 2009 @@ -4,7 +4,6 @@ Inspired by Guido van Rossum's pgen2. """ -import collections import StringIO import tokenize import token @@ -79,11 +78,11 @@ start.find_unlabeled_states(base_nfas) state_stack = [DFA(base_nfas, end)] for state in state_stack: - arcs = collections.defaultdict(set) + arcs = {} for nfa in state.nfas: for label, sub_nfa in nfa.arcs: if label is not None: - sub_nfa.find_unlabeled_states(arcs[label]) + sub_nfa.find_unlabeled_states(arcs.setdefault(label, set())) for label, nfa_set in arcs.iteritems(): for st in state_stack: if st.nfas == nfa_set: Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py ============================================================================== --- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original) +++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Fri Jul 24 22:19:01 2009 @@ -145,25 +145,26 @@ self.prepare(_targets[compile_info.mode]) tp = 0 try: - tokens = pytokenizer.generate_tokens(source_lines, flags) - for tp, value, lineno, column, line in tokens: - if self.add_token(tp, value, lineno, column, line): - break - except parser.ParseError, e: - # Catch parse errors, pretty them up and reraise them as a - # SyntaxError. - new_err = error.IndentationError - if tp == pygram.tokens.INDENT: - msg = "unexpected indent" - elif e.expected == pygram.tokens.INDENT: - msg = "expected an indented block" + try: + tokens = pytokenizer.generate_tokens(source_lines, flags) + for tp, value, lineno, column, line in tokens: + if self.add_token(tp, value, lineno, column, line): + break + except parser.ParseError, e: + # Catch parse errors, pretty them up and reraise them as a + # SyntaxError. + new_err = error.IndentationError + if tp == pygram.tokens.INDENT: + msg = "unexpected indent" + elif e.expected == pygram.tokens.INDENT: + msg = "expected an indented block" + else: + new_err = error.SyntaxError + msg = "invalid syntax" + raise new_err(msg, e.lineno, e.column, e.line, + compile_info.filename) else: - new_err = error.SyntaxError - msg = "invalid syntax" - raise new_err(msg, e.lineno, e.column, e.line, - compile_info.filename) - else: - tree = self.root + tree = self.root finally: # Avoid hanging onto the tree. self.root = None From fijal at codespeak.net Fri Jul 24 23:08:51 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 24 Jul 2009 23:08:51 +0200 (CEST) Subject: [pypy-svn] r66615 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090724210851.400C4169F66@codespeak.net> Author: fijal Date: Fri Jul 24 23:08:49 2009 New Revision: 66615 Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf pypy/extradoc/talk/euroscipy2009/talk.txt Log: last-minute changes Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Fri Jul 24 23:08:49 2009 @@ -177,7 +177,7 @@ Large Eddy Simulation (LES) Models ================================== -* processes, which scales are range of 100m could be calculated explicitly +* processes, which scales are in a range of 100m could be calculated explicitly * processes of smaller spatial scale, (e.g. turbulent mixing of a cloud with air from the environment) have to be still parametrized @@ -189,7 +189,7 @@ EULAG - numerical solver for all-scale geophysical flows ======================================================== -* developed in National Center for Atmospheric Science (NCAR), +* developed in National Center for Atmospheric Research (NCAR), Boulder, Colorado, since '80 * written in Fortran 77 @@ -200,6 +200,17 @@ * http://www.mmm.ucar.edu/eulag +Data used for analyzis +====================== + +* written every 4 minutes + +* last 4 hours of the simulation + +* around 8 basic variables, e.g. velocities, temperature + +* For each variable: 60*121*128*128=100M values + Analyzing model data ==================== @@ -209,20 +220,9 @@ * checkin amount of liquid water in each box -* checkin for each "dry" box, distance to the cloud edge - -* calculating relative humidity in each "dry" - -Data used for analyzis -====================== - -* written every 4 minutes - -* last 4 hours of the simulation - -* around 8 basic variables, e.g. velocities, temperature +* checkin for each "dry" grid box, distance to the cloud edge -* For each variable: 60*121*128*128 values +* calculating relative humidity in each "dry" grid box Data postprocessing =================== @@ -233,7 +233,7 @@ * Numpy for operations, matplotlib for output -* Massive datasets +* Massive datasets (about 100M values) * Sometimes require walking array elements From hpk at codespeak.net Sat Jul 25 09:47:21 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 25 Jul 2009 09:47:21 +0200 (CEST) Subject: [pypy-svn] r66617 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090725074721.BF2AC169F9F@codespeak.net> Author: hpk Date: Sat Jul 25 09:47:20 2009 New Revision: 66617 Modified: pypy/extradoc/talk/euroscipy2009/talk.txt Log: making "how you can help" a bit more concrete (discussing currently with maciej) Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Sat Jul 25 09:47:20 2009 @@ -307,13 +307,14 @@ How you can help? ================= -* PyPy is open source after all +* contribute extension modules -* Very friendly to newcomers +* organise a PyPy sprint, travel funding a plus -* A lot of promising fields to work on +* fund feature or ext module development + +* PyPy is open source, newcomer friendly! -* We also seek funding or research grants Thank you! Questions? ===================== From fijal at codespeak.net Sat Jul 25 09:48:32 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 25 Jul 2009 09:48:32 +0200 (CEST) Subject: [pypy-svn] r66618 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090725074832.05FFE169F9D@codespeak.net> Author: fijal Date: Sat Jul 25 09:48:31 2009 New Revision: 66618 Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf pypy/extradoc/talk/euroscipy2009/talk.txt Log: mention JIT backends as well Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Sat Jul 25 09:48:31 2009 @@ -309,6 +309,8 @@ * contribute extension modules +* contribute JIT backend + * organise a PyPy sprint, travel funding a plus * fund feature or ext module development From hpk at codespeak.net Sat Jul 25 09:54:40 2009 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 25 Jul 2009 09:54:40 +0200 (CEST) Subject: [pypy-svn] r66619 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090725075440.CF093169FA1@codespeak.net> Author: hpk Date: Sat Jul 25 09:54:38 2009 New Revision: 66619 Modified: pypy/extradoc/talk/euroscipy2009/talk.txt Log: adding a concrete point with respect to the Numpy topic before Modified: pypy/extradoc/talk/euroscipy2009/talk.txt ============================================================================== --- pypy/extradoc/talk/euroscipy2009/talk.txt (original) +++ pypy/extradoc/talk/euroscipy2009/talk.txt Sat Jul 25 09:54:38 2009 @@ -307,6 +307,8 @@ How you can help? ================= +* take over or fund numpy-on-PyPy work + * contribute extension modules * contribute JIT backend From arigo at codespeak.net Sat Jul 25 09:56:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 09:56:18 +0200 (CEST) Subject: [pypy-svn] r66620 - pypy/branch/pyjitpl5/pypy/jit/backend/minimal Message-ID: <20090725075618.A731D169FA1@codespeak.net> Author: arigo Date: Sat Jul 25 09:56:17 2009 New Revision: 66620 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py Log: Oups, fix test. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py Sat Jul 25 09:56:17 2009 @@ -209,7 +209,7 @@ setattr(p, %(name)r, x) """ % dict).compile() in dict2 sort_key = self._count_sort_key(T, name) - if getkind(FIELDTYPE) in 'po': # pointer or object + if getkind(FIELDTYPE) in ('ptr', 'obj'): Class = PtrFieldDescr else: Class = NonPtrFieldDescr From arigo at codespeak.net Sat Jul 25 09:57:05 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 09:57:05 +0200 (CEST) Subject: [pypy-svn] r66621 - pypy/branch/pyjitpl5/pypy/jit/tl/tla Message-ID: <20090725075705.78AB9169FA1@codespeak.net> Author: arigo Date: Sat Jul 25 09:57:04 2009 New Revision: 66621 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py Log: Fix test. Modified: pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/tla/test_tla.py Sat Jul 25 09:57:04 2009 @@ -156,7 +156,7 @@ # ____________________________________________________________ from pypy.jit.metainterp.test.test_basic import LLJitMixin -from pypy.jit.metainterp import optimize4 +from pypy.jit.metainterp import optimize class TestLLtype(LLJitMixin): def test_loop(self): @@ -172,5 +172,5 @@ assert isinstance(w_result, tla.W_IntObject) return w_result.intvalue res = self.meta_interp(interp_w, [42], listops=True, - optimizer=optimize4) + optimizer=optimize) assert res == 0 From fijal at codespeak.net Sat Jul 25 12:05:07 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 25 Jul 2009 12:05:07 +0200 (CEST) Subject: [pypy-svn] r66622 - pypy/branch/pyjitpl5/pypy/jit/tl/spli/test Message-ID: <20090725100507.BB134169F6F@codespeak.net> Author: fijal Date: Sat Jul 25 12:05:06 2009 New Revision: 66622 Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Log: Add some asserts and a comment Modified: pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/tl/spli/test/test_jit.py Sat Jul 25 12:05:06 2009 @@ -37,6 +37,7 @@ i = i + 1 return i self.interpret(f, []) + self.check_loops(new_with_vtable=0) def test_bridge(self): def f(a, b): @@ -51,6 +52,7 @@ return total self.interpret(f, [1, 10]) + self.check_loops(new_with_vtable=0) def test_bridge_bad_case(self): def f(a, b): @@ -64,4 +66,8 @@ return a + b self.interpret(f, [1, 10]) - + self.check_loops(new_with_vtable=1) # XXX should eventually be 0? + # I think it should be either 0 or 2, 1 makes little sense + # If the loop after entering goes first time to the bridge, a + # is rewrapped again, without preserving the identity. I'm not + # sure how bad it is From arigo at codespeak.net Sat Jul 25 12:07:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 12:07:43 +0200 (CEST) Subject: [pypy-svn] r66623 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090725100743.7FB9E169F85@codespeak.net> Author: arigo Date: Sat Jul 25 12:07:43 2009 New Revision: 66623 Added: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_del.py (contents, props changed) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py pypy/branch/pyjitpl5/pypy/jit/metainterp/support.py Log: Move mallocs with dels out of the JIT loop, by encoding them as residual calls. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py Sat Jul 25 12:07:43 2009 @@ -770,6 +770,11 @@ STRUCT = op.args[0].value vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT) if vtable: + # do we have a __del__? + rtti = lltype.getRuntimeTypeInfo(STRUCT) + if hasattr(rtti._obj, 'destructor_funcptr'): + self.handle_builtin_call(op) + return # store the vtable as an address -- that's fine, because the # GC doesn't need to follow them self.emit('new_with_vtable', @@ -797,6 +802,7 @@ def serialize_op_new(self, op): TYPE = op.args[0].value cls = ootype.runtimeClass(TYPE) + # XXX detect finalizers self.emit('new_with_vtable', self.const_position(cls)) self.codewriter.register_known_ooclass(cls, TYPE) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/support.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/support.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/support.py Sat Jul 25 12:07:43 2009 @@ -183,6 +183,12 @@ _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode _ll_5_unicode_copy_contents = ll_rstr.copy_unicode_contents + # ---------- malloc with del ---------- + + def _ll_0_alloc_with_del(RESULT): + return lltype.malloc(RESULT) + _ll_0_alloc_with_del.need_result_type = True + class OOtypeHelpers: @@ -334,6 +340,10 @@ else: raise Exception("oohash() of type %r" % (T,)) +def get_malloc_oopspec(op): + assert op.args[1].value == {'flavor': 'gc'} + return 'alloc_with_del', [] + RENAMED_ADT_NAME = { 'list': { @@ -356,14 +366,16 @@ if op.opname == 'oosend': SELFTYPE, name, opargs = decompose_oosend(op) return get_send_oopspec(SELFTYPE, name), opargs - elif op.opname in ('oostring', 'oounicode'): - return get_oostring_oopspec(op) - elif op.opname == 'oohash': - return get_oohash_oopspec(op) elif op.opname == 'direct_call': fnobj = get_funcobj(op.args[0].value) opargs = op.args[1:] return get_call_oopspec_opargs(fnobj, opargs) + elif op.opname in ('oostring', 'oounicode'): + return get_oostring_oopspec(op) + elif op.opname == 'oohash': + return get_oohash_oopspec(op) + elif op.opname == 'malloc': # for malloc with a __del__ + return get_malloc_oopspec(op) else: raise ValueError(op.opname) Added: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_del.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_del.py Sat Jul 25 12:07:43 2009 @@ -0,0 +1,36 @@ +import py +from pypy.rlib.jit import JitDriver +from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin + + +class DelTests: + + def test_del_keep_obj(self): + myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + class Foo: + def __del__(self): + pass + def f(n): + x = None + while n > 0: + myjitdriver.can_enter_jit(x=x, n=n) + myjitdriver.jit_merge_point(x=x, n=n) + x = Foo() + Foo() + n -= 1 + self.meta_interp(f, [20]) + self.check_loops({'call': 2, # calls to a helper function + 'guard_no_exception': 2, # follows the calls + 'int_sub': 1, + 'int_gt': 1, + 'guard_true': 1, + 'jump': 1}) + + +class TestLLtype(DelTests, LLJitMixin): + pass + +class TestOOtype(DelTests, OOJitMixin): + def setup_class(cls): + py.test.skip("XXX dels are not implemented in the" + " static CLI or JVM backend") From arigo at codespeak.net Sat Jul 25 12:12:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 12:12:55 +0200 (CEST) Subject: [pypy-svn] r66624 - pypy/branch/pyjitpl5/pypy/jit/backend/x86/test Message-ID: <20090725101255.954D8169F8B@codespeak.net> Author: arigo Date: Sat Jul 25 12:12:54 2009 New Revision: 66624 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py Log: Add debug prints. The test passes again starting from r66623. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_zrpy_gc.py Sat Jul 25 12:12:54 2009 @@ -191,6 +191,7 @@ def test_compile_hybrid_4(): # Fourth version of the test, with __del__. + from pypy.rlib.debug import debug_print class Counter: cnt = 0 counter = Counter() @@ -199,6 +200,7 @@ counter.cnt -= 1 myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) def main(n, x): + debug_print('counter.cnt =', counter.cnt) assert counter.cnt < 5 counter.cnt = n // x.foo while n > 0: From arigo at codespeak.net Sat Jul 25 12:32:36 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 12:32:36 +0200 (CEST) Subject: [pypy-svn] r66625 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090725103236.8FE2E169E4F@codespeak.net> Author: arigo Date: Sat Jul 25 12:32:35 2009 New Revision: 66625 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py Log: Fix (shown by test_tl and test_virtual). Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/codewriter.py Sat Jul 25 12:32:35 2009 @@ -771,10 +771,14 @@ vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, STRUCT) if vtable: # do we have a __del__? - rtti = lltype.getRuntimeTypeInfo(STRUCT) - if hasattr(rtti._obj, 'destructor_funcptr'): - self.handle_builtin_call(op) - return + try: + rtti = lltype.getRuntimeTypeInfo(STRUCT) + except ValueError: + pass + else: + if hasattr(rtti._obj, 'destructor_funcptr'): + self.handle_builtin_call(op) + return # store the vtable as an address -- that's fine, because the # GC doesn't need to follow them self.emit('new_with_vtable', From fijal at codespeak.net Sat Jul 25 14:46:18 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 25 Jul 2009 14:46:18 +0200 (CEST) Subject: [pypy-svn] r66626 - pypy/extradoc/talk/euroscipy2009 Message-ID: <20090725124618.DD61C169F5E@codespeak.net> Author: fijal Date: Sat Jul 25 14:46:17 2009 New Revision: 66626 Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf Log: talk as it went Modified: pypy/extradoc/talk/euroscipy2009/talk.pdf ============================================================================== Binary files. No diff available. From david at codespeak.net Sat Jul 25 19:05:06 2009 From: david at codespeak.net (david at codespeak.net) Date: Sat, 25 Jul 2009 19:05:06 +0200 (CEST) Subject: [pypy-svn] r66629 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20090725170506.39322169F56@codespeak.net> Author: david Date: Sat Jul 25 19:05:04 2009 New Revision: 66629 Modified: pypy/branch/io-lang/pypy/lang/io/object.py Log: appendProto on Object Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sat Jul 25 19:05:04 2009 @@ -92,4 +92,9 @@ for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) - return t \ No newline at end of file + return t + + at register_method('Object', 'appendProto', unwrap_spec=[object, object]) +def object_append_proto(space, w_target, w_proto): + w_target.protos.append(w_proto) + return w_target \ No newline at end of file From arigo at codespeak.net Sat Jul 25 19:36:18 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 25 Jul 2009 19:36:18 +0200 (CEST) Subject: [pypy-svn] r66630 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090725173618.8F6C5169F5E@codespeak.net> Author: arigo Date: Sat Jul 25 19:36:18 2009 New Revision: 66630 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py Log: Check an invariant. It is known to fail on pypy-c-jit running lib-python/2.5.2/test/test_zlib.py: history.inputargs contains Consts. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/compile.py Sat Jul 25 19:36:18 2009 @@ -95,6 +95,8 @@ loop = create_empty_loop(metainterp) loop.greenkey = greenkey loop.inputargs = history.inputargs + for box in loop.inputargs: + assert isinstance(box, Box) if start > 0: loop.operations = history.operations[start:] else: @@ -114,6 +116,8 @@ return loop def send_loop_to_backend(metainterp, loop, guard_op, type): + for box in loop.inputargs: + assert isinstance(box, Box) metainterp.staticdata.profiler.start_backend() metainterp.cpu.compile_operations(loop, guard_op) metainterp.staticdata.profiler.end_backend() From david at codespeak.net Sat Jul 25 20:40:26 2009 From: david at codespeak.net (david at codespeak.net) Date: Sat, 25 Jul 2009 20:40:26 +0200 (CEST) Subject: [pypy-svn] r66631 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090725184026.8C641169F62@codespeak.net> Author: david Date: Sat Jul 25 20:40:25 2009 New Revision: 66631 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Object doMessage Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sat Jul 25 20:40:25 2009 @@ -97,4 +97,13 @@ @register_method('Object', 'appendProto', unwrap_spec=[object, object]) def object_append_proto(space, w_target, w_proto): w_target.protos.append(w_proto) - return w_target \ No newline at end of file + return w_target + + at register_method('Object', 'doMessage',) +def object_do_message(space, w_target, w_message, w_context): + w_msg = w_message.arguments[0].eval(space, w_context, w_context) + w_receiver = w_target + if len(w_message.arguments) == 2: + w_receiver = w_message.arguments[1].eval(space, w_context, w_context) + + return w_msg.eval(space, w_receiver, w_receiver) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sat Jul 25 20:40:25 2009 @@ -94,4 +94,26 @@ x""" res, _ = interpret(inp) - assert res.value == 9 \ No newline at end of file + assert res.value == 9 + +def test_object_append_proto(): + inp = """a := Object clone + b := Object clone + a appendProto(b)""" + res, space = interpret(inp) + assert res.protos == [space.w_object, space.w_lobby.slots['b']] + +def test_object_doMessage(): + inp = """m := message(asNumber + 123) + 1 doMessage(m)""" + res, space = interpret(inp) + + assert res.value == 124 + + +def test_object_doMessage_optional_context(): + inp = """m := message(asNumber + 123) + 1 doMessage(m, 2)""" + res, space = interpret(inp) + + assert res.value == 125 \ No newline at end of file From david at codespeak.net Sat Jul 25 21:10:52 2009 From: david at codespeak.net (david at codespeak.net) Date: Sat, 25 Jul 2009 21:10:52 +0200 (CEST) Subject: [pypy-svn] r66632 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090725191052.9E98A498429@codespeak.net> Author: david Date: Sat Jul 25 21:10:48 2009 New Revision: 66632 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Some work on control flow, support for break in for loops Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sat Jul 25 21:10:48 2009 @@ -92,6 +92,9 @@ for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) + if not space.is_normal_status(): + space.normal_status() + break return t @register_method('Object', 'appendProto', unwrap_spec=[object, object]) @@ -106,4 +109,14 @@ if len(w_message.arguments) == 2: w_receiver = w_message.arguments[1].eval(space, w_context, w_context) - return w_msg.eval(space, w_receiver, w_receiver) \ No newline at end of file + return w_msg.eval(space, w_receiver, w_receiver) + + + at register_method('Object', 'break') +def object_break(space, w_target, w_message, w_context): + w_result = space.w_nil + if len(w_message.arguments) > 0: + w_result = w_message.arguments[0].eval(space, w_context, w_context) + space.break_status(w_result) + return w_target + \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sat Jul 25 21:10:48 2009 @@ -87,6 +87,11 @@ assert len(res.items) == 4 results = [t.value for t in res.items] results == [0, 3, 6, 9] + +def test_object_for_returns_nil(): + inp = """for(x, 1, 2, nil)""" + res, space = interpret(inp) + assert res == space.w_nil def test_object_leaks(): inp = """a:= list(); @@ -116,4 +121,17 @@ 1 doMessage(m, 2)""" res, space = interpret(inp) - assert res.value == 125 \ No newline at end of file + assert res.value == 125 + +def test_object_break(): + inp = """for(x, 7, 1000, break) + x + """ + res, _ = interpret(inp) + assert res.value == 7 + +def test_object_break_return_value(): + inp = """for(x, 7, 1000, break(-1)) + """ + res, _ = interpret(inp) + assert res.value == -1 \ No newline at end of file From david at codespeak.net Sat Jul 25 21:16:01 2009 From: david at codespeak.net (david at codespeak.net) Date: Sat, 25 Jul 2009 21:16:01 +0200 (CEST) Subject: [pypy-svn] r66633 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090725191601.7A75A169EB3@codespeak.net> Author: david Date: Sat Jul 25 21:16:01 2009 New Revision: 66633 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Support for continue in for loops Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sat Jul 25 21:16:01 2009 @@ -119,4 +119,8 @@ w_result = w_message.arguments[0].eval(space, w_context, w_context) space.break_status(w_result) return w_target - \ No newline at end of file + + + at register_method('Object', 'continue') +def object_continue(space, w_target, w_message, w_context): + space.continue_status() \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sat Jul 25 21:16:01 2009 @@ -134,4 +134,11 @@ inp = """for(x, 7, 1000, break(-1)) """ res, _ = interpret(inp) - assert res.value == -1 \ No newline at end of file + assert res.value == -1 + +def test_object_continue(): + inp = """a := list() + for(x, 1, 10, continue; a append(x)) + a""" + res, space = interpret(inp) + assert len(res.items) == 0 \ No newline at end of file From fijal at codespeak.net Sun Jul 26 12:35:14 2009 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 26 Jul 2009 12:35:14 +0200 (CEST) Subject: [pypy-svn] r66634 - pypy/branch/pyjitpl5/pypy/jit/backend/x86 Message-ID: <20090726103514.4F026168575@codespeak.net> Author: fijal Date: Sun Jul 26 12:35:12 2009 New Revision: 66634 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Log: Add a sanity check. The reason is that I called previous interface execute_operations(loop, [args]) and it was understood as verbose Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Sun Jul 26 12:35:12 2009 @@ -258,6 +258,7 @@ return loop def execute_operations(self, loop, verbose=False): + assert isinstance(verbose, bool) func = self.get_bootstrap_code(loop) # debug info #if self.debug and not we_are_translated(): From david at codespeak.net Sun Jul 26 17:22:11 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 17:22:11 +0200 (CEST) Subject: [pypy-svn] r66635 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726152211.6AE6B169E2A@codespeak.net> Author: david Date: Sun Jul 26 17:22:10 2009 New Revision: 66635 Modified: pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/objspace.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Ojbect return method and missing control flow code Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Sun Jul 26 17:22:10 2009 @@ -198,6 +198,8 @@ w_method = w_receiver.lookup(self.name) assert w_method is not None, 'Method "%s" not found in "%s"' % (self.name, w_receiver.__class__) w_result = w_method.apply(space, w_receiver, self, w_context) + if not space.is_normal_status(): + return space.w_return_value if self.next: #TODO: optimize return self.next.eval(space, w_result, w_context) Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sun Jul 26 17:22:10 2009 @@ -89,6 +89,7 @@ key = w_message.arguments[0].name + space.normal_status() for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) @@ -123,4 +124,14 @@ @register_method('Object', 'continue') def object_continue(space, w_target, w_message, w_context): - space.continue_status() \ No newline at end of file + space.continue_status() + return w_target + + at register_method('Object', 'return') +def object_return(space, w_target, w_message, w_context): + w_value = space.w_nil + if len(w_message.arguments) > 0: + w_value = w_message.arguments[0].eval(space, w_context, w_context) + + space.return_status(w_value) + return w_target \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/objspace.py (original) +++ pypy/branch/io-lang/pypy/lang/io/objspace.py Sun Jul 26 17:22:10 2009 @@ -1,5 +1,5 @@ from pypy.rlib.objectmodel import instantiate -from pypy.lang.io.model import W_Number, W_Object, W_CFunction, W_Block, W_Message, W_List, W_Map +from pypy.lang.io.model import W_Number, W_Object, W_CFunction, W_Block, W_Message, W_List, W_Map, W_ImmutableSequence from pypy.lang.io.register import cfunction_definitions import pypy.lang.io.number @@ -23,9 +23,18 @@ self.w_false = W_Object(self, [self.w_object]) self.w_nil = W_Object(self, [self.w_object]) self.w_list = W_List(self, [self.w_object]) - self.w_call = W_Object(self) + self.w_call = W_Object(self, [self.w_object]) self.w_map = W_Map(self, [self.w_object]) + # flow control objects + self.w_normal = W_Object(self, [self.w_object]) + self.w_break = W_Object(self, [self.w_object]) + self.w_continue = W_Object(self, [self.w_object]) + self.w_return = W_Object(self, [self.w_object]) + self.w_eol = W_Object(self, [self.w_object]) + # default stop state + self.stop_status = self.w_normal + self.w_return_value = self.w_nil self.init_w_object() self.init_w_protos() @@ -48,6 +57,8 @@ self.init_w_map() + self.init_w_flow_objects() + def init_w_map(self): for key, function in cfunction_definitions['Map'].items(): self.w_map.slots[key] = W_CFunction(self, function) @@ -104,7 +115,46 @@ self.w_lobby.slots['Protos'] = self.w_protos def init_w_object(self): - - for key, function in cfunction_definitions['Object'].items(): - self.w_object.slots[key] = W_CFunction(self, function) \ No newline at end of file + self.w_object.slots[key] = W_CFunction(self, function) + + def init_w_flow_objects(self): + # Flow control: Normal + self.w_core.slots['Normal'] = self.w_normal + self.w_core.slots['Normal'].slots['type'] = W_ImmutableSequence(self, 'Normal') + + # Flow control: Break + self.w_core.slots['Break'] = self.w_block + self.w_core.slots['Break'].slots['type'] = W_ImmutableSequence(self, 'Break') + + # Flow control: Continue + self.w_core.slots['Continue'] = self.w_continue + self.w_core.slots['Continue'].slots['type'] = W_ImmutableSequence(self, 'Continue') + + # Flow control: Return + self.w_core.slots['Return'] = self.w_return + self.w_core.slots['Return'].slots['type'] = W_ImmutableSequence(self, 'Return') + + # Flow control: Eol + self.w_core.slots['Eol'] = self.w_eol + self.w_core.slots['Eol'].slots['type'] = W_ImmutableSequence(self, 'Eol') + + + def break_status(self, result): + self.stop_status = self.w_break + self.w_return_value = result + + + def normal_status(self): + self.stop_status = self.w_normal + self.w_return_value = self.w_nil + + def is_normal_status(self): + return self.stop_status == self.w_normal + + def continue_status(self): + self.stop_status = self.w_continue + + def return_status(self, result): + self.stop_status = self.w_return + self.w_return_value = result \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sun Jul 26 17:22:10 2009 @@ -141,4 +141,28 @@ for(x, 1, 10, continue; a append(x)) a""" res, space = interpret(inp) - assert len(res.items) == 0 \ No newline at end of file + assert len(res.items) == 0 + +def test_object_return(): + inp = """x := method(y, return) + x(99)""" + res, space = interpret(inp) + assert res == space.w_nil + +def test_object_return2(): + inp = """x := method(y, return; 666) + x(99)""" + res, space = interpret(inp) + assert res.value == 666 + +def test_object_return_value(): + inp = """x := method(y, return 42) + x(99)""" + res, space = interpret(inp) + assert res.value == 42 + +def test_object_return_value2(): + inp = """x := method(y, return(1024); 666) + x(99)""" + res, space = interpret(inp) + assert res.value == 1024 \ No newline at end of file From david at codespeak.net Sun Jul 26 17:43:57 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 17:43:57 +0200 (CEST) Subject: [pypy-svn] r66636 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726154357.81396169E2A@codespeak.net> Author: david Date: Sun Jul 26 17:43:56 2009 New Revision: 66636 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/objspace.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: improved Object continue Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sun Jul 26 17:43:56 2009 @@ -80,7 +80,7 @@ body = w_message.arguments[-1] start = w_message.arguments[1].eval(space, w_target, w_context).value - stop = w_message.arguments[2].eval(space, w_target, w_context).value + stop = 1 + w_message.arguments[2].eval(space, w_target, w_context).value if argcount == 4: step = 1 else: @@ -93,7 +93,7 @@ for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) - if not space.is_normal_status(): + if not space.is_normal_status() and not space.is_continue_status(): space.normal_status() break return t Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/objspace.py (original) +++ pypy/branch/io-lang/pypy/lang/io/objspace.py Sun Jul 26 17:43:56 2009 @@ -154,7 +154,11 @@ def continue_status(self): self.stop_status = self.w_continue - + + def is_continue_status(self): + return self.stop_status == self.w_continue + def return_status(self, result): self.stop_status = self.w_return - self.w_return_value = result \ No newline at end of file + self.w_return_value = result + Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sun Jul 26 17:43:56 2009 @@ -139,9 +139,10 @@ def test_object_continue(): inp = """a := list() for(x, 1, 10, continue; a append(x)) - a""" + """ res, space = interpret(inp) - assert len(res.items) == 0 + assert space.w_lobby.slots['x'].value == 10 + assert len(space.w_lobby.slots['a'].items) == 0 def test_object_return(): inp = """x := method(y, return) @@ -165,4 +166,4 @@ inp = """x := method(y, return(1024); 666) x(99)""" res, space = interpret(inp) - assert res.value == 1024 \ No newline at end of file + assert res.value == 1024 From david at codespeak.net Sun Jul 26 17:55:02 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 17:55:02 +0200 (CEST) Subject: [pypy-svn] r66637 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726155502.3F7BE169E25@codespeak.net> Author: david Date: Sun Jul 26 17:54:59 2009 New Revision: 66637 Modified: pypy/branch/io-lang/pypy/lang/io/number.py pypy/branch/io-lang/pypy/lang/io/test/test_number.py Log: Number compare and == methods Modified: pypy/branch/io-lang/pypy/lang/io/number.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/number.py (original) +++ pypy/branch/io-lang/pypy/lang/io/number.py Sun Jul 26 17:54:59 2009 @@ -57,4 +57,16 @@ @register_method('Number', '*', unwrap_spec=[float, float]) def w_number_multiply(space, a, b): - return W_Number(space, a*b) \ No newline at end of file + return W_Number(space, a*b) + + at register_method('Number', '==', unwrap_spec=[float, float]) +def w_number_equals(space, a, b): + w_value = space.w_false + if w_number_compare(space, a, b) == 0: + w_value = space.w_true + return w_value + + at register_method('Number', 'compare', unwrap_spec=[float, float]) +def w_number_compare(space, a, b): + return W_Number(space, cmp(a,b), [space.w_number]) + \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_number.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_number.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_number.py Sun Jul 26 17:54:59 2009 @@ -146,4 +146,25 @@ inp = '6*7' res, _ = interpret(inp) assert res.value == 42 - \ No newline at end of file + +def test_equals(): + inp = '3 == 5' + res, space = interpret(inp) + assert res == space.w_false + + inp = '5 == 5' + res, space = interpret(inp) + assert res == space.w_true + +def test_compare(): + inp = '7 compare(7)' + res, space = interpret(inp) + assert res.value == 0 + + inp = '7 compare(8)' + res, space = interpret(inp) + assert res.value == -1 + + inp = '7 compare(6)' + res, space = interpret(inp) + assert res.value == 1 \ No newline at end of file From pedronis at codespeak.net Sun Jul 26 18:11:28 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 26 Jul 2009 18:11:28 +0200 (CEST) Subject: [pypy-svn] r66638 - pypy/trunk Message-ID: <20090726161128.4E28716851B@codespeak.net> Author: pedronis Date: Sun Jul 26 18:11:27 2009 New Revision: 66638 Modified: pypy/trunk/LICENSE Log: oh well, we didn't yet bump this Modified: pypy/trunk/LICENSE ============================================================================== --- pypy/trunk/LICENSE (original) +++ pypy/trunk/LICENSE Sun Jul 26 18:11:27 2009 @@ -27,7 +27,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2007 +PyPy Copyright holders 2003-2009 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at From pedronis at codespeak.net Sun Jul 26 18:21:36 2009 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 26 Jul 2009 18:21:36 +0200 (CEST) Subject: [pypy-svn] r66639 - pypy/trunk/pypy/tool/test Message-ID: <20090726162136.7BC54169E00@codespeak.net> Author: pedronis Date: Sun Jul 26 18:21:35 2009 New Revision: 66639 Added: pypy/trunk/pypy/tool/test/test_license.py (contents, props changed) Log: in the spirit of testing all which can be, test that the copyright year in the license is up to date Added: pypy/trunk/pypy/tool/test/test_license.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/tool/test/test_license.py Sun Jul 26 18:21:35 2009 @@ -0,0 +1,13 @@ +import py, datetime + +def test_license(): + lic = (py.path.local(__file__).dirpath().dirpath() + .dirpath().dirpath().join('LICENSE')) + text = lic.read() + COPYRIGHT_HOLDERS="PyPy Copyright holders 2003-" + assert COPYRIGHT_HOLDERS in text + pos = text.find(COPYRIGHT_HOLDERS) + year2 = text[pos+len(COPYRIGHT_HOLDERS):pos+len(COPYRIGHT_HOLDERS)+5] + copyright_year = int(year2) + cur_year = datetime.date.today().year + assert copyright_year == cur_year From david at codespeak.net Sun Jul 26 19:12:15 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 19:12:15 +0200 (CEST) Subject: [pypy-svn] r66640 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726171215.4FD9116855A@codespeak.net> Author: david Date: Sun Jul 26 19:12:13 2009 New Revision: 66640 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: Object if method Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sun Jul 26 19:12:13 2009 @@ -134,4 +134,17 @@ w_value = w_message.arguments[0].eval(space, w_context, w_context) space.return_status(w_value) - return w_target \ No newline at end of file + return w_target + + at register_method('Object', 'if') +def object_if(space, w_target, w_message, w_context): + w_condition = w_message.arguments[0].eval(space, w_context, w_context) + + if w_condition is space.w_true: + index = 1 + else: + index = 2 + + if index < len(w_message.arguments): + return w_message.arguments[index].eval(space, w_context, w_context) + return w_condition \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sun Jul 26 19:12:13 2009 @@ -167,3 +167,32 @@ x(99)""" res, space = interpret(inp) assert res.value == 1024 + +def test_object_if(): + inp = """a := list() + for(i, 1, 10, + if(i == 3, continue) + a append(i)) + a + """ + res, space = interpret(inp) + values = [x.value for x in res.items] + assert values == [1, 2 ,3, 4, 5, 6, 7, 8, 9, 10] + +def test_object_if2(): + inp = """if(false, 1, 2)""" + res, _ = interpret(inp) + assert res.value == 2 + + inp = """if(true, 1, 2)""" + res, _ = interpret(inp) + assert res.value == 1 + +def test_object_if3(): + inp = 'if(true)' + res, space = interpret(inp) + assert res is space.w_true + + inp = 'if(false)' + res, space = interpret(inp) + assert res is space.w_false \ No newline at end of file From david at codespeak.net Sun Jul 26 19:31:11 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 19:31:11 +0200 (CEST) Subject: [pypy-svn] r66641 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726173111.7E3F9169E1C@codespeak.net> Author: david Date: Sun Jul 26 19:31:11 2009 New Revision: 66641 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: stopStatus method on Object to retrieve the current stopStatus or the stopStatus of a given message Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Sun Jul 26 19:31:11 2009 @@ -147,4 +147,12 @@ if index < len(w_message.arguments): return w_message.arguments[index].eval(space, w_context, w_context) - return w_condition \ No newline at end of file + return w_condition + + at register_method('Object', 'stopStatus') +def object_stopstatus(space, w_target, w_message, w_context): + if len(w_message.arguments) > 0: + w_message.arguments[0].eval(space, w_context, w_context) + w = space.stop_status + space.normal_status() + return w \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Sun Jul 26 19:31:11 2009 @@ -195,4 +195,21 @@ inp = 'if(false)' res, space = interpret(inp) - assert res is space.w_false \ No newline at end of file + assert res is space.w_false + +def test_object_stopStatus(): + inp = 'stopStatus' + res, space = interpret(inp) + assert res is space.w_normal + + inp = 'stopStatus(break)' + res, space = interpret(inp) + assert res is space.w_break + + inp = 'stopStatus(continue)' + res, space = interpret(inp) + assert res is space.w_continue + + inp = 'stopStatus(return 42)' + res, space = interpret(inp) + assert res is space.w_return \ No newline at end of file From david at codespeak.net Sun Jul 26 19:52:09 2009 From: david at codespeak.net (david at codespeak.net) Date: Sun, 26 Jul 2009 19:52:09 +0200 (CEST) Subject: [pypy-svn] r66642 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090726175209.5BE35169E6C@codespeak.net> Author: david Date: Sun Jul 26 19:52:08 2009 New Revision: 66642 Modified: pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/test/test_map.py Log: fix for Map hasValue Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Sun Jul 26 19:52:08 2009 @@ -130,7 +130,7 @@ def has_value(self, w_value): for x in self.items.values(): - if x.value == x: + if x.value == w_value: return True return False Modified: pypy/branch/io-lang/pypy/lang/io/test/test_map.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_map.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_map.py Sun Jul 26 19:52:08 2009 @@ -74,13 +74,13 @@ assert values == ['nil', 3, 234] def test_has_value(): - inp = 'Map clone atPut("1", "nil") atPut("2", "lorem") atPut("3", 3) atPut("4", 234) hasValue("234")' + inp = 'Map clone atPut("1", "nil") atPut("2", "lorem") atPut("3", 3) atPut("4", 234) hasValue(234)' res, space = interpret(inp) - assert res == space.w_true + assert res is space.w_true inp = 'Map clone atPut("1", "nil") atPut("2", "lorem") atPut("3", 3) atPut("4", 234) hasValue("1234567890")' res, space = interpret(inp) - assert res == space.w_false + assert res is space.w_false def test_values(): inp = 'Map clone atPut("1", 12345) atPut("2", 99) atPut("3", 3) atPut("4", 234) values' @@ -167,4 +167,4 @@ inp = """Map with("a", 1, "b", 2) asObject""" res, space = interpret(inp) assert res.slots['a'].value == 1 - assert res.slots['b'].value == 2 \ No newline at end of file + assert res.slots['b'].value == 2 From antocuni at codespeak.net Mon Jul 27 11:41:46 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 27 Jul 2009 11:41:46 +0200 (CEST) Subject: [pypy-svn] r66648 - pypy/branch/pyjitpl5/pypy/jit/backend/cli/test Message-ID: <20090727094146.1A34C168563@codespeak.net> Author: antocuni Date: Mon Jul 27 11:41:45 2009 New Revision: 66648 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_basic.py pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_runner.py Log: skip tests that cannot be run on top of pythonnet Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_basic.py Mon Jul 27 11:41:45 2009 @@ -39,6 +39,8 @@ test_free_object = skip test_stopatxpolicy = skip test_residual_call_pure = skip + test_div_overflow = skip + test_subclassof = skip def test_fielddescr_ootype(): Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/test/test_runner.py Mon Jul 27 11:41:45 2009 @@ -27,6 +27,8 @@ test_failing_guard_class = skip # GUARD_CLASS test_call = skip test_field = skip + test_field_basic = skip + test_ooops = skip def test_ovf_operations(self, reversed=False): self.skip() From antocuni at codespeak.net Mon Jul 27 12:08:31 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 27 Jul 2009 12:08:31 +0200 (CEST) Subject: [pypy-svn] r66649 - pypy/branch/pyjitpl5/pypy/jit/backend/cli Message-ID: <20090727100831.0BBFA1684EA@codespeak.net> Author: antocuni Date: Mon Jul 27 12:08:30 2009 New Revision: 66649 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py pypy/branch/pyjitpl5/pypy/jit/backend/cli/runner.py Log: fix most of the failing cli jit tests: new_with_vtable no longer takes a typedescr, but you need to fish it using the first constant argument instead Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/method.py Mon Jul 27 12:08:30 2009 @@ -477,8 +477,10 @@ self.il.Emit(OpCodes.Ret) def emit_op_new_with_vtable(self, op): - assert isinstance(op.args[0], ConstObj) # ignored, using the descr instead - descr = op.descr + clsbox = op.args[0] + assert isinstance(clsbox, ConstObj) + cls = clsbox.getobj() + descr = self.cpu.class_sizes[cls] assert isinstance(descr, runner.TypeDescr) clitype = descr.get_clitype() ctor_info = descr.get_constructor_info() Modified: pypy/branch/pyjitpl5/pypy/jit/backend/cli/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/cli/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/cli/runner.py Mon Jul 27 12:08:30 2009 @@ -144,9 +144,13 @@ # ---------------------- - def do_new_with_vtable(self, args, typedescr): - assert isinstance(typedescr, TypeDescr) - assert len(args) == 1 # but we don't need it, so ignore + def do_new_with_vtable(self, args, descr): + #assert isinstance(typedescr, TypeDescr) + #assert len(args) == 1 # but we don't need it, so ignore + assert descr is None + assert len(args) == 1 + cls = args[0].getobj() + typedescr = self.class_sizes[cls] return typedescr.create() def do_new_array(self, args, typedescr): From arigo at codespeak.net Mon Jul 27 14:03:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 14:03:25 +0200 (CEST) Subject: [pypy-svn] r66650 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090727120325.4CADC169E03@codespeak.net> Author: arigo Date: Mon Jul 27 14:03:23 2009 New Revision: 66650 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: A failing test, showing the same problem as test_zlib when run with pypy-c-jit. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Mon Jul 27 14:03:23 2009 @@ -730,6 +730,22 @@ self.meta_interp(f, [40, 0]) + def test_const_inputargs(self): + py.test.skip("in-progress") + myjitdriver = JitDriver(greens = ['m'], reds = ['n', 'x']) + def f(n, x): + m = 0x7FFFFFFF + while n > 0: + myjitdriver.can_enter_jit(m=m, n=n, x=x) + myjitdriver.jit_merge_point(m=m, n=n, x=x) + x = 42 + n -= 1 + m = m >> 1 + return x + + res = self.meta_interp(f, [50, 1]) + assert res == 42 + class TestOOtype(BasicTests, OOJitMixin): From arigo at codespeak.net Mon Jul 27 14:58:49 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 14:58:49 +0200 (CEST) Subject: [pypy-svn] r66651 - in pypy/branch/pyjitpl5/pypy/jit: backend/llgraph backend/llvm backend/test backend/x86 backend/x86/test metainterp metainterp/test Message-ID: <20090727125849.B7993168563@codespeak.net> Author: arigo Date: Mon Jul 27 14:58:46 2009 New Revision: 66651 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/llvm/compile.py pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py pypy/branch/pyjitpl5/pypy/jit/backend/test/test_random.py pypy/branch/pyjitpl5/pypy/jit/backend/x86/assembler.py pypy/branch/pyjitpl5/pypy/jit/backend/x86/regalloc.py pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_runner.py pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: Add rop.SAME_AS. Normally not used, because it is just removed by optimize.py, but it's useful to have it during tracing (see test). Implemented it anyway in the backends, for completeness and for tests running without optimize.py. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Mon Jul 27 14:58:46 2009 @@ -83,6 +83,7 @@ 'uint_ge' : (('int', 'int'), 'bool'), 'uint_xor' : (('int', 'int'), 'int'), 'uint_rshift' : (('int', 'int'), 'int'), + 'same_as' : (('int',), 'int'), # could also be ptr=>ptr 'new_with_vtable' : (('ptr',), 'ptr'), 'new' : ((), 'ptr'), 'new_array' : (('int',), 'ptr'), Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Mon Jul 27 14:58:46 2009 @@ -377,6 +377,9 @@ llimpl.do_setfield_raw_int(struct, fielddescr.ofs, newvalue, self.memo_cast) + def do_same_as(self, args, descr=None): + return args[0].clonebox() + def do_newstr(self, args, descr=None): assert descr is None length = args[0].getint() Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llvm/compile.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llvm/compile.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llvm/compile.py Mon Jul 27 14:58:46 2009 @@ -563,6 +563,12 @@ self.getptrarg(op.args[0]), self.cpu.const_null_charptr, "") + def generate_SAME_AS(self, op): + if op.args[0].type == INT: + self.vars[op.result] = self.getintarg(op.args[0]) + else: + self.vars[op.result] = self.getptrarg(op.args[0]) + def _generate_len_gep(self, array_ref, ty, const_index_length): array = llvm_rffi.LLVMBuildBitCast(self.builder, array_ref, ty, "") Modified: pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py Mon Jul 27 14:58:46 2009 @@ -506,6 +506,13 @@ 'int') assert r.value == 31313 + def test_same_as(self): + r = self.execute_operation(rop.SAME_AS, [ConstInt(5)], 'int') + assert r.value == 5 + u_box = self.alloc_unicode(u"hello\u1234") + r = self.execute_operation(rop.SAME_AS, [u_box.constbox()], 'ptr') + assert r.value == u_box.value + class LLtypeBackendTest(BaseBackendTest): Modified: pypy/branch/pyjitpl5/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/test/test_random.py Mon Jul 27 14:58:46 2009 @@ -196,6 +196,10 @@ v = builder.get_bool_var(r) self.put(builder, [v]) +class ConstUnaryOperation(UnaryOperation): + def produce_into(self, builder, r): + self.put(builder, [ConstInt(r.random_integer())]) + class BinaryOperation(AbstractOperation): def __init__(self, opnum, and_mask=-1, or_mask=0, boolres=False): AbstractOperation.__init__(self, opnum, boolres=boolres) @@ -318,6 +322,7 @@ OPERATIONS.append(UnaryOperation(rop.INT_IS_TRUE, boolres=True)) OPERATIONS.append(BooleanUnaryOperation(rop.BOOL_NOT, boolres=True)) +OPERATIONS.append(ConstUnaryOperation(rop.SAME_AS, boolres=True)) for _op in [rop.INT_ADD_OVF, rop.INT_SUB_OVF, Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/assembler.py Mon Jul 27 14:58:46 2009 @@ -449,6 +449,9 @@ self.mc.MOV(resloc, imm8(0)) self.mc.SETE(lower_byte(resloc)) + def genop_same_as(self, op, arglocs, resloc): + self.mc.MOV(resloc, arglocs[0]) + def genop_int_mod(self, op, arglocs, resloc): self.mc.CDQ() self.mc.IDIV(ecx) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/regalloc.py Mon Jul 27 14:58:46 2009 @@ -1110,6 +1110,12 @@ consider_ooisnull = _consider_nullity consider_oononnull = _consider_nullity + def consider_same_as(self, op, ignored): + argloc = self.loc(op.args[0]) + self.eventually_free_var(op.args[0]) + resloc = self.force_allocate_reg(op.result, []) + self.Perform(op, [argloc], resloc) + def consider_strlen(self, op, ignored): base_loc = self.make_sure_var_in_reg(op.args[0], op.args) self.eventually_free_vars(op.args) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/test/test_runner.py Mon Jul 27 14:58:46 2009 @@ -107,18 +107,6 @@ ]: assert self.execute_operation(op, args, 'int').value == res - def test_same_as(self): - py.test.skip("rewrite") - u = lltype.malloc(U) - uadr = lltype.cast_opaque_ptr(llmemory.GCREF, u) - for op, args, tp, res in [ - ('same_as', [BoxInt(7)], 'int', 7), - ('same_as', [ConstInt(7)], 'int', 7), - ('same_as', [BoxPtr(uadr)], 'ptr', uadr), - ('same_as', [ConstPtr(uadr)], 'ptr', uadr), - ]: - assert self.execute_operation(op, args, tp).value == res - def test_unicode(self): ofs = symbolic.get_field_token(rstr.UNICODE, 'chars', False)[0] u = rstr.mallocunicode(13) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/executor.py Mon Jul 27 14:58:46 2009 @@ -99,6 +99,9 @@ def do_bool_not(cpu, args, descr=None): return ConstInt(not args[0].getint()) +def do_same_as(cpu, args, descr=None): + return args[0] + def do_oononnull(cpu, args, descr=None): tp = args[0].type if tp == INT: Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/pyjitpl.py Mon Jul 27 14:58:46 2009 @@ -780,7 +780,7 @@ self.generate_merge_point(pc, self.env) if self.metainterp.seen_can_enter_jit: self.metainterp.seen_can_enter_jit = False - self.metainterp.reached_can_enter_jit(self.env[:]) + self.metainterp.reached_can_enter_jit(self.env) @arguments("jumptarget") def opimpl_setup_exception_block(self, exception_target): @@ -1223,10 +1223,22 @@ except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) + def forget_consts(self, boxes, startindex=0): + for i in range(startindex, len(boxes)): + box = boxes[i] + if isinstance(box, Const): + constbox = box + box = constbox.clonebox() + boxes[i] = box + self.history.record(rop.SAME_AS, [constbox], box) + def reached_can_enter_jit(self, live_arg_boxes): + self.forget_consts(live_arg_boxes, self.staticdata.num_green_args) + live_arg_boxes = live_arg_boxes[:] if self.staticdata.virtualizable_info is not None: # we use ':-1' to remove the last item, which is the virtualizable # itself + self.forget_consts(self.virtualizable_boxes) live_arg_boxes += self.virtualizable_boxes[:-1] # Called whenever we reach the 'can_enter_jit' hint. # First, attempt to make a bridge: Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py Mon Jul 27 14:58:46 2009 @@ -154,6 +154,8 @@ INT_INVERT = 62 BOOL_NOT = 63 # + SAME_AS = 64 # gets a Const, turns it into a Box + # _NOSIDEEFFECT_PTR_FIRST = 70 # -- start of no_side_effect_ptr operations -- OONONNULL = 70 OOISNULL = 71 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Mon Jul 27 14:58:46 2009 @@ -731,7 +731,6 @@ self.meta_interp(f, [40, 0]) def test_const_inputargs(self): - py.test.skip("in-progress") myjitdriver = JitDriver(greens = ['m'], reds = ['n', 'x']) def f(n, x): m = 0x7FFFFFFF From arigo at codespeak.net Mon Jul 27 15:41:08 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 15:41:08 +0200 (CEST) Subject: [pypy-svn] r66652 - in pypy/branch/pyjitpl5/pypy/rpython: . ootypesystem Message-ID: <20090727134108.9A796169E74@codespeak.net> Author: arigo Date: Mon Jul 27 15:41:08 2009 New Revision: 66652 Modified: pypy/branch/pyjitpl5/pypy/rpython/ootypesystem/ootype.py pypy/branch/pyjitpl5/pypy/rpython/rlist.py Log: Add a few _annenforceargs_, to fix jit/backend/cli/test_zrpy_vlist. That's all a bit annoying; maybe we should at some point find a better way to approach the issue of SomeInteger(nonneg=True) vs SomeInteger(nonneg=False). Modified: pypy/branch/pyjitpl5/pypy/rpython/ootypesystem/ootype.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rpython/ootypesystem/ootype.py (original) +++ pypy/branch/pyjitpl5/pypy/rpython/ootypesystem/ootype.py Mon Jul 27 15:41:08 2009 @@ -561,6 +561,7 @@ def ll_newlist(self, length): from pypy.rpython.ootypesystem import rlist return rlist.ll_newlist(self, length) + ll_newlist._annenforceargs_ = (None, int) # NB: We are expecting Lists of the same ITEMTYPE to compare/hash # equal. We don't redefine __eq__/__hash__ since the implementations @@ -690,6 +691,7 @@ def ll_newlist(self, length): from pypy.rpython.ootypesystem import rlist return rlist.ll_newarray(self, length) + ll_newlist._annenforceargs_ = (None, int) def ll_convert_from_array(self, array): return array Modified: pypy/branch/pyjitpl5/pypy/rpython/rlist.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rpython/rlist.py (original) +++ pypy/branch/pyjitpl5/pypy/rpython/rlist.py Mon Jul 27 15:41:08 2009 @@ -858,6 +858,7 @@ i += 1 j += 1 return l +ll_listslice_startonly._annenforceargs_ = (None, None, int) def ll_listslice_startstop(RESLIST, l1, start, stop): length = l1.ll_length() From arigo at codespeak.net Mon Jul 27 15:46:01 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 15:46:01 +0200 (CEST) Subject: [pypy-svn] r66653 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090727134601.B12071684C0@codespeak.net> Author: arigo Date: Mon Jul 27 15:46:00 2009 New Revision: 66653 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/TODO Log: A note: we are still missing threads. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/TODO ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/TODO (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/TODO Mon Jul 27 15:46:00 2009 @@ -9,3 +9,5 @@ * long term: memory management of the compiled code (free old code) * 'resume_info' should be shared (see pyjitpl.py) + +* support threads again From antocuni at codespeak.net Mon Jul 27 16:10:35 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 27 Jul 2009 16:10:35 +0200 (CEST) Subject: [pypy-svn] r66654 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090727141035.1CD3C168551@codespeak.net> Author: antocuni Date: Mon Jul 27 16:10:34 2009 New Revision: 66654 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Log: use simple_optimize for this test, so that the generated same_as is not removed away Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_basic.py Mon Jul 27 16:10:34 2009 @@ -731,6 +731,7 @@ self.meta_interp(f, [40, 0]) def test_const_inputargs(self): + from pypy.jit.metainterp import simple_optimize myjitdriver = JitDriver(greens = ['m'], reds = ['n', 'x']) def f(n, x): m = 0x7FFFFFFF @@ -742,7 +743,8 @@ m = m >> 1 return x - res = self.meta_interp(f, [50, 1]) + res = self.meta_interp(f, [50, 1], + optimizer=simple_optimize) assert res == 42 From arigo at codespeak.net Mon Jul 27 20:34:39 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 20:34:39 +0200 (CEST) Subject: [pypy-svn] r66655 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090727183439.CDAF0168556@codespeak.net> Author: arigo Date: Mon Jul 27 20:34:37 2009 New Revision: 66655 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: A skipped test corresponding to a bug seen on pypy-c-jit. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Jul 27 20:34:37 2009 @@ -433,6 +433,20 @@ # we might get incorrect results: when tracing, maybe i0 was not 0. self.find_nodes(ops, 'Virtual(node_vtable, valuedescr=Not)') + def test_find_nodes_p123(self): + py.test.skip("in-progress") + ops = """ + [i1, p2, p3] + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + jump(i1, p1, p2) + """ + self.find_nodes(ops, '''Not, + Virtual(node_vtable, valuedescr=Not), + Virtual(node_vtable, valuedescr=Not)''') + # ------------------------------ # Bridge tests From arigo at codespeak.net Mon Jul 27 21:07:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 21:07:23 +0200 (CEST) Subject: [pypy-svn] r66656 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090727190723.622B916C161@codespeak.net> Author: arigo Date: Mon Jul 27 21:07:20 2009 New Revision: 66656 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/history.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py Log: An RPython-friendly way to dump a loop. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/history.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/history.py Mon Jul 27 21:07:20 2009 @@ -595,6 +595,22 @@ if operations[-1].opnum == rop.JUMP: assert isinstance(operations[-1].jump_target, TreeLoop) + def dump(self): + # RPython-friendly + print '%r: inputargs =' % self, self._dump_args(self.inputargs) + for op in self.operations: + print '\t', op.getopname(), self._dump_args(op.args), \ + self._dump_box(op.result) + + def _dump_args(self, boxes): + return '[' + ', '.join([self._dump_box(box) for box in boxes]) + ']' + + def _dump_box(self, box): + if box is None: + return 'None' + else: + return box.repr_rpython() + def __repr__(self): return '<%s>' % (self.name,) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimize.py Mon Jul 27 21:07:20 2009 @@ -10,6 +10,7 @@ return old_loops[0] else: return None + #loop.dump() finder = PerfectSpecializationFinder() finder.find_nodes_loop(loop) for old_loop in old_loops: From arigo at codespeak.net Mon Jul 27 22:22:42 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 27 Jul 2009 22:22:42 +0200 (CEST) Subject: [pypy-svn] r66657 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090727202242.013F316845E@codespeak.net> Author: arigo Date: Mon Jul 27 22:22:40 2009 New Revision: 66657 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Bug fix in optimize*.py, originally shown by 'pypy-c-jit test_decimal.py'. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Mon Jul 27 22:22:40 2009 @@ -12,16 +12,16 @@ UNIQUE_NO = '\x02' class InstanceNode(object): - """For the first phase: InstanceNode is used to match the start and + """An instance of this class is used to match the start and the end of the loop, so it contains both 'origfields' that represents the field's status at the start and 'curfields' that represents it - at the current point (== the end when the first phase is complete). + at the current point (== the end when optimizefindnode is complete). """ escaped = False # if True, then all the rest of the info is pointless unique = UNIQUE_UNKNOWN # for find_unique_nodes() # fields used to store the shape of the potential Virtual - knownclsbox = None + knownclsbox = None # set only on freshly-allocated structures origfields = None # optimization; equivalent to an empty dict curfields = None # optimization; equivalent to an empty dict dependencies = None @@ -46,7 +46,8 @@ box.mark_escaped() def set_unique_nodes(self): - if (self.escaped or self.fromstart or self.knownclsbox is None + if (self.escaped or self.knownclsbox is None + # or self.fromstart, but this is implied by 'knownclsbox==None' or self.unique != UNIQUE_UNKNOWN): # this node is not suitable for being a virtual, or we # encounter it more than once when doing the recursion Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Mon Jul 27 22:22:40 2009 @@ -44,6 +44,9 @@ def force_box(self): return self.box + def prepare_force_box(self): + pass + def get_key_box(self): return self.box @@ -134,6 +137,18 @@ newoperations.append(op) return self.box + def prepare_force_box(self): + # This logic is not included in force_box() for safety reasons. + # It should only be used from teardown_virtual_node(); if we + # call force_box() from somewhere else and we get source_op=None, + # it is really a bug. + if self.box is None and self.source_op is None: + # rare case (shown by test_p123_simple) to force a Virtual + # from a specnode computed by optimizefindnode. + self.source_op = ResOperation(rop.NEW_WITH_VTABLE, + [self.known_class], + self.optimizer.new_ptr_box()) + def get_key_box(self): if self.box is None: return self.keybox @@ -155,6 +170,7 @@ def setup_virtual_node(self, optimizer, box, newinputargs): newinputargs.append(box) def teardown_virtual_node(self, optimizer, value, newexitargs): + value.prepare_force_box() newexitargs.append(value.force_box()) class __extend__(VirtualInstanceSpecNode): @@ -208,12 +224,15 @@ self.make_equal_to(box, vvalue) return vvalue + def new_ptr_box(self): + if not self.cpu.is_oo: + return BoxPtr() + else: + return BoxObj() + def new_box(self, fieldofs): if fieldofs.is_pointer_field(): - if not self.cpu.is_oo: - return BoxPtr() - else: - return BoxObj() + return self.new_ptr_box() else: return BoxInt() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Mon Jul 27 22:22:40 2009 @@ -433,8 +433,16 @@ # we might get incorrect results: when tracing, maybe i0 was not 0. self.find_nodes(ops, 'Virtual(node_vtable, valuedescr=Not)') - def test_find_nodes_p123(self): - py.test.skip("in-progress") + def test_find_nodes_nonvirtual_guard_class(self): + ops = """ + [p1] + guard_class(p1, ConstClass(node_vtable)) + fail(p1) + jump(p1) + """ + self.find_nodes(ops, 'Not') + + def test_find_nodes_p123_simple(self): ops = """ [i1, p2, p3] i3 = getfield_gc(p3, descr=valuedescr) @@ -443,9 +451,39 @@ setfield_gc(p1, i1, descr=valuedescr) jump(i1, p1, p2) """ + # We cannot track virtuals that survive for more than two iterations. + self.find_nodes(ops, 'Not, Virtual(node_vtable, valuedescr=Not), Not') + + def test_find_nodes_p123_guard_class(self): + ops = """ + [i1, p2, p3] + guard_class(p3, ConstClass(node_vtable)) + fail(i1, p2, p3) + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + jump(i1, p1, p2) + """ + # We cannot track virtuals that survive for more than two iterations. + self.find_nodes(ops, 'Not, Virtual(node_vtable, valuedescr=Not), Not') + + def test_find_nodes_p123_rec(self): + ops = """ + [i1, p2, p0d] + p3 = getfield_gc(p0d, descr=nextdescr) + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + p0c = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p0c, p2, descr=nextdescr) + jump(i1, p1, p0c) + """ + # We cannot track virtuals that survive for more than two iterations. self.find_nodes(ops, '''Not, Virtual(node_vtable, valuedescr=Not), - Virtual(node_vtable, valuedescr=Not)''') + Virtual(node_vtable, nextdescr=Not)''') # ------------------------------ # Bridge tests Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Mon Jul 27 22:22:40 2009 @@ -27,7 +27,10 @@ assert len(op1.args) == len(op2.args) for x, y in zip(op1.args, op2.args): assert x == remap.get(y, y) - assert op1.result == remap.get(op2.result, op2.result) + if op2.result in remap: + assert op1.result == remap[op2.result] + else: + remap[op2.result] = op1.result assert op1.descr == op2.descr if op1.suboperations: assert equaloplists(op1.suboperations, op2.suboperations, remap) @@ -333,6 +336,28 @@ self.optimize_loop(ops, '', ops, p1=self.nodebox.value, boxkinds={'myptr': self.nodebox.value}) + def test_p123_simple(self): + ops = """ + [i1, p2, p3] + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p1, i1, descr=valuedescr) + jump(i1, p1, p2) + """ + expected = """ + [i1, i2, p3] + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p2b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2b, i2, descr=valuedescr) + jump(i1, i1, p2b) + """ + # We cannot track virtuals that survive for more than two iterations. + self.optimize_loop(ops, + 'Not, Virtual(node_vtable, valuedescr=Not), Not', + expected) + # ---------- def test_fold_guard_no_exception(self): From david at codespeak.net Wed Jul 29 10:00:24 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 10:00:24 +0200 (CEST) Subject: [pypy-svn] r66659 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20090729080024.75CB5169E16@codespeak.net> Author: david Date: Wed Jul 29 10:00:22 2009 New Revision: 66659 Added: pypy/branch/io-lang/pypy/lang/io/autopath.py - copied unchanged from r66655, pypy/branch/io-lang/pypy/bin/autopath.py Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.py Log: make it possible to use parserhack from the commandline Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parserhack.py (original) +++ pypy/branch/io-lang/pypy/lang/io/parserhack.py Wed Jul 29 10:00:22 2009 @@ -1,3 +1,4 @@ +import autopath import py import os import glob @@ -12,7 +13,7 @@ child_in.write(input) child_in.close() s = child_out_err.read().strip() - # print s + print s return eval(s) def interpret(code): @@ -30,4 +31,10 @@ def load_io_files(space): files = glob.glob('io/*.io') for f in files: - parse_file(f, space).eval(space, space.w_lobby, space.w_lobby) \ No newline at end of file + parse_file(f, space).eval(space, space.w_lobby, space.w_lobby) + + +if __name__ == '__main__': + import sys + space = ObjSpace() + parse(py.path.local(sys.argv[1]).read(), space) \ No newline at end of file From david at codespeak.net Wed Jul 29 11:10:35 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 11:10:35 +0200 (CEST) Subject: [pypy-svn] r66660 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090729091035.0CEC8169E38@codespeak.net> Author: david Date: Wed Jul 29 11:10:34 2009 New Revision: 66660 Added: pypy/branch/io-lang/pypy/lang/io/coroutine.py pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py pypy/branch/io-lang/pypy/lang/io/test/test_coro.py Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py Log: start working on coroutine implementation Added: pypy/branch/io-lang/pypy/lang/io/coroutine.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/coroutine.py Wed Jul 29 11:10:34 2009 @@ -0,0 +1,12 @@ +from pypy.lang.io.register import register_method +from pypy.lang.io.model import W_Object +from pypy.lang.io.coroutinemodel import W_Coroutine + + + at register_method('Coroutine', 'currentCoroutine') +def coroutine_get_current(space, w_target, w_message, w_context): + return W_Coroutine.w_getcurrent(space) + + at register_method('Coroutine', 'isCurrent') +def coroutine_is_current(space, w_target, w_message, w_context): + return space.newbool(w_target is W_Coroutine.w_getcurrent(space)) \ No newline at end of file Added: pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/coroutinemodel.py Wed Jul 29 11:10:34 2009 @@ -0,0 +1,49 @@ +from pypy.lang.io.model import W_Object +from pypy.rlib.rcoroutine import make_coroutine_classes +d = make_coroutine_classes(W_Object) + +Coroutine = d['Coroutine'] +AbstractThunk = d['AbstractThunk'] +BaseCoState = d['BaseCoState'] + + +class W_Coroutine(Coroutine): + def __init__(self, space, state, protos): + Coroutine.__init__(self, state) + + W_Object.__init__(self, space, protos) + + def clone(self): + return W_Coroutine(self.space, self.state, [self]) + + @staticmethod + def w_getcurrent(space): + return W_Coroutine._get_state(space).current + + @staticmethod + def _get_state(space): + # XXX: Need a propper caching machinery + if not hasattr(space, '_coroutine_state'): + space._coroutine_state = AppCoState(space) + space._coroutine_state.post_install() + return space._coroutine_state + +class AppCoState(BaseCoState): + def __init__(self, space): + BaseCoState.__init__(self) + self.w_tempval = space.w_nil + self.space = space + + def post_install(self): + self.current = self.main = W_Coroutine(self.space, self, [self.space.w_object]) + # self.main.subctx.framestack = None # wack + +class IoThunk(AbstractThunk): + def __init__(self, space, w_message, w_receiver, w_context): + self.space = space + self.w_message = w_message + self.w_receiver = w_receiver + self.w_context = w_context + + def call(self): + self.w_message.eval(self.space, self.w_receiver, self.w_context) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/objspace.py (original) +++ pypy/branch/io-lang/pypy/lang/io/objspace.py Wed Jul 29 11:10:34 2009 @@ -1,5 +1,6 @@ from pypy.rlib.objectmodel import instantiate from pypy.lang.io.model import W_Number, W_Object, W_CFunction, W_Block, W_Message, W_List, W_Map, W_ImmutableSequence +from pypy.lang.io.coroutinemodel import W_Coroutine from pypy.lang.io.register import cfunction_definitions import pypy.lang.io.number @@ -9,6 +10,7 @@ import pypy.lang.io.call import pypy.lang.io.message import pypy.lang.io.map +import pypy.lang.io.coroutine class ObjSpace(object): """docstring for ObjSpace""" @@ -25,7 +27,7 @@ self.w_list = W_List(self, [self.w_object]) self.w_call = W_Object(self, [self.w_object]) self.w_map = W_Map(self, [self.w_object]) - + self.w_coroutine = W_Coroutine.w_getcurrent(self) # flow control objects self.w_normal = W_Object(self, [self.w_object]) self.w_break = W_Object(self, [self.w_object]) @@ -56,9 +58,13 @@ self.init_w_call() self.init_w_map() + + self.init_w_coroutine() self.init_w_flow_objects() + + def init_w_map(self): for key, function in cfunction_definitions['Map'].items(): self.w_map.slots[key] = W_CFunction(self, function) @@ -91,6 +97,7 @@ self.w_core.slots['Call'] = self.w_call self.w_core.slots['Map'] = self.w_map self.w_core.slots['Number'] = self.w_number + self.w_core.slots['Coroutine'] = self.w_coroutine def init_w_number(self): self.w_number = instantiate(W_Number) @@ -140,11 +147,14 @@ self.w_core.slots['Eol'].slots['type'] = W_ImmutableSequence(self, 'Eol') + def init_w_coroutine(self): + for key, function in cfunction_definitions['Coroutine'].items(): + self.w_coroutine.slots[key] = W_CFunction(self, function) + def break_status(self, result): self.stop_status = self.w_break self.w_return_value = result - def normal_status(self): self.stop_status = self.w_normal self.w_return_value = self.w_nil @@ -162,3 +172,7 @@ self.stop_status = self.w_return self.w_return_value = result + def newbool(self, value): + if value: + return self.w_true + return self.w_false \ No newline at end of file Added: pypy/branch/io-lang/pypy/lang/io/test/test_coro.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_coro.py Wed Jul 29 11:10:34 2009 @@ -0,0 +1,20 @@ +from pypy.lang.io.parserhack import parse, interpret +from pypy.lang.io.model import W_Object +import py + +def test_isCurrent(): + inp = """Coroutine currentCoroutine isCurrent""" + result, space = interpret(inp) + assert result is space.w_true + +def test_coro_resume(): + py.test.skip() + inp = """ + b := message(currentCoro setResult(23); currentCoro parentCoroutine resume; "bye" print) + a := Coroutine currentCoroutine clone + a setRunMessage(b) run + a result + """ + res,space = interpret(inp) + assert res.value == 23 + From arigo at codespeak.net Wed Jul 29 14:09:17 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 14:09:17 +0200 (CEST) Subject: [pypy-svn] r66661 - in pypy/branch/pyjitpl5/pypy/lib: . app_test Message-ID: <20090729120917.042B416846E@codespeak.net> Author: arigo Date: Wed Jul 29 14:09:15 2009 New Revision: 66661 Added: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py (contents, props changed) Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: Fix: d.get(key) without a default should be equivalent to d.get(key, None). It should not raise KeyError. Added: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py ============================================================================== --- (empty file) +++ pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Wed Jul 29 14:09:15 2009 @@ -0,0 +1,9 @@ +from pypy.lib import dbm +from pypy.tool.udir import udir + +def test_get(): + path = str(udir.join('test_dbm_extra.test_get')) + d = dbm.open(path, 'c') + x = d.get("42") + assert x is None + d.close() Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Wed Jul 29 14:09:15 2009 @@ -2,9 +2,6 @@ import ctypes.util import os, sys -class _singleton(object): - pass - class error(Exception): def __init__(self, msg): self.msg = msg @@ -36,7 +33,7 @@ k = getattr(lib, funcs['nextkey'])(self._aobj) return allkeys - def get(self, key, default=_singleton): + def get(self, key, default=None): if not self._aobj: raise error('DBM object has already been closed') dat = datum() @@ -45,8 +42,6 @@ k = getattr(lib, funcs['fetch'])(self._aobj, dat) if k.dptr: return k.dptr[:k.dsize] - if default is _singleton: - raise KeyError if getattr(lib, funcs['error'])(self._aobj): getattr(lib, funcs['clearerr'])(self._aobj) raise error("") From arigo at codespeak.net Wed Jul 29 14:19:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 14:19:53 +0200 (CEST) Subject: [pypy-svn] r66662 - in pypy/branch/pyjitpl5/pypy/lib: . app_test Message-ID: <20090729121953.6EF33169E8D@codespeak.net> Author: arigo Date: Wed Jul 29 14:19:52 2009 New Revision: 66662 Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: Test and fix (correlated to CPython's dbm by changing the import at the start of the test). Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Wed Jul 29 14:19:52 2009 @@ -1,3 +1,4 @@ +import py from pypy.lib import dbm from pypy.tool.udir import udir @@ -7,3 +8,10 @@ x = d.get("42") assert x is None d.close() + +def test_set_nonstring(): + path = str(udir.join('test_dbm_extra.test_set_nonstring')) + d = dbm.open(path, 'c') + py.test.raises(TypeError, "d[123] = 'xyz'") + py.test.raises(TypeError, "d['xyz'] = 123") + d.close() Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Wed Jul 29 14:19:52 2009 @@ -57,11 +57,13 @@ raise KeyError return value - def _set(self, key, value): + def __setitem__(self, key, value): if not self._aobj: raise error('DBM object has already been closed') if not isinstance(key, str): raise TypeError("dbm mappings have string indices only") + if not isinstance(value, str): + raise TypeError("dbm mappings have string values only") dat = datum() dat.dptr = c_char_p(key) dat.dsize = c_int(len(key)) @@ -115,11 +117,6 @@ return True return False - def __setitem__(self, key, value): - if not isinstance(key, str) and isinstance(value, str): - raise error("dbm mappings have string indices only") - self._set(key, value) - def __delitem__(self, key): if not isinstance(key, str): raise error("dbm mappings have string indices only") From arigo at codespeak.net Wed Jul 29 14:40:55 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 14:40:55 +0200 (CEST) Subject: [pypy-svn] r66663 - in pypy/branch/pyjitpl5/pypy/lib: . app_test Message-ID: <20090729124055.475EB16800D@codespeak.net> Author: arigo Date: Wed Jul 29 14:40:54 2009 New Revision: 66663 Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: Fix fix fix in dbm.py, with tests, mostly related to detecting exceptional cases. Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Wed Jul 29 14:40:54 2009 @@ -9,9 +9,24 @@ assert x is None d.close() -def test_set_nonstring(): - path = str(udir.join('test_dbm_extra.test_set_nonstring')) +def test_delitem(): + path = str(udir.join('test_dbm_extra.test_delitem')) + d = dbm.open(path, 'c') + py.test.raises(KeyError, "del d['xyz']") + +def test_nonstring(): + path = str(udir.join('test_dbm_extra.test_nonstring')) d = dbm.open(path, 'c') py.test.raises(TypeError, "d[123] = 'xyz'") py.test.raises(TypeError, "d['xyz'] = 123") + py.test.raises(TypeError, "d['xyz'] = None") + py.test.raises(TypeError, "del d[123]") + py.test.raises(TypeError, "d[123]") + py.test.raises(TypeError, "123 in d") + py.test.raises(TypeError, "d.has_key(123)") + py.test.raises(TypeError, "d.setdefault(123, 'xyz')") + py.test.raises(TypeError, "d.setdefault('xyz', 123)") + py.test.raises(TypeError, "d.get(123)") + d.setdefault('xyz') + assert dict(d) == {'xyz': ''} d.close() Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Wed Jul 29 14:40:54 2009 @@ -20,6 +20,8 @@ self._aobj = dbmobj def close(self): + if not self._aobj: + raise error('DBM object has already been closed') getattr(lib, funcs['close'])(self._aobj) self._aobj = None @@ -51,42 +53,29 @@ return len(self.keys()) def __getitem__(self, key): - assert isinstance(key, str) value = self.get(key) if value is None: - raise KeyError + raise KeyError(key) return value def __setitem__(self, key, value): if not self._aobj: raise error('DBM object has already been closed') - if not isinstance(key, str): - raise TypeError("dbm mappings have string indices only") - if not isinstance(value, str): - raise TypeError("dbm mappings have string values only") dat = datum() dat.dptr = c_char_p(key) dat.dsize = c_int(len(key)) - if value == None: - status = getattr(lib, funcs['delete'])(self._aobj, dat) - if status < 0: - getattr(lib, funcs['clearerr'])(self._aobj) - raise KeyError(key) - else: - if not isinstance(value, str): - raise TypeError("dbm mappings have string indices only") - data = datum() - data.dptr = c_char_p(value) - data.dsize = c_int(len(value)) - status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_INSERT) - if status == 1: - status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_REPLACE) + data = datum() + data.dptr = c_char_p(value) + data.dsize = c_int(len(value)) + status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_INSERT) + if status == 1: + status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_REPLACE) if getattr(lib, funcs['error'])(self._aobj): getattr(lib, funcs['clearerr'])(self._aobj) raise error("") return status - def setdefault(self, key, default=None): + def setdefault(self, key, default=''): if not self._aobj: raise error('DBM object has already been closed') dat = datum() @@ -95,16 +84,14 @@ k = getattr(lib, funcs['fetch'])(self._aobj, dat) if k.dptr: return k.dptr[:k.dsize] - if default: - data = datum() - data.dptr = c_char_p(default) - data.dsize = c_int(len(default)) - status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_INSERT) - if status < 0: - getattr(lib, funcs['clearerr'])(self._aobj) - raise error("cannot add item to database") - return default - return None + data = datum() + data.dptr = c_char_p(default) + data.dsize = c_int(len(default)) + status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_INSERT) + if status < 0: + getattr(lib, funcs['clearerr'])(self._aobj) + raise error("cannot add item to database") + return default def has_key(self, key): if not self._aobj: @@ -118,12 +105,14 @@ return False def __delitem__(self, key): - if not isinstance(key, str): - raise error("dbm mappings have string indices only") + if not self._aobj: + raise error('DBM object has already been closed') dat = datum() dat.dptr = c_char_p(key) dat.dsize = c_int(len(key)) - getattr(lib, funcs['delete'])(self._aobj, dat) + status = getattr(lib, funcs['delete'])(self._aobj, dat) + if status < 0: + raise KeyError(key) ### initialization: Berkeley DB versus normal DB @@ -160,7 +149,7 @@ _init_func('fetch', restype=datum) _init_func('store', restype=c_int) _init_func('error') -_init_func('delete') +_init_func('delete', restype=c_int) lib.DBM_INSERT = 0 lib.DBM_REPLACE = 1 From arigo at codespeak.net Wed Jul 29 15:01:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 15:01:19 +0200 (CEST) Subject: [pypy-svn] r66664 - pypy/branch/pyjitpl5/pypy/lib Message-ID: <20090729130119.654CA169EAC@codespeak.net> Author: arigo Date: Wed Jul 29 15:01:16 2009 New Revision: 66664 Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: Add 'library'. Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Wed Jul 29 15:01:16 2009 @@ -141,6 +141,8 @@ lib = CDLL("/usr/lib/libdbm.dylib") # OS X _platform = 'osx' +library = "GNU gdbm" + funcs = {} _init_func('open', [c_char_p, c_int, c_int]) _init_func('close', restype=c_void_p) From arigo at codespeak.net Wed Jul 29 15:18:37 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 15:18:37 +0200 (CEST) Subject: [pypy-svn] r66665 - pypy/branch/pyjitpl5/pypy/lib Message-ID: <20090729131837.45E6C169E1F@codespeak.net> Author: arigo Date: Wed Jul 29 15:18:36 2009 New Revision: 66665 Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: Add a __del__ method to force a call to funcs['close']. Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Wed Jul 29 15:18:36 2009 @@ -25,6 +25,10 @@ getattr(lib, funcs['close'])(self._aobj) self._aobj = None + def __del__(self): + if self._aobj: + self.close() + def keys(self): if not self._aobj: raise error('DBM object has already been closed') From david at codespeak.net Wed Jul 29 17:47:53 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 17:47:53 +0200 (CEST) Subject: [pypy-svn] r66667 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090729154753.AFE58169F28@codespeak.net> Author: david Date: Wed Jul 29 17:47:51 2009 New Revision: 66667 Modified: pypy/branch/io-lang/pypy/lang/io/number.py pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_number.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: fix failing tests Modified: pypy/branch/io-lang/pypy/lang/io/number.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/number.py (original) +++ pypy/branch/io-lang/pypy/lang/io/number.py Wed Jul 29 17:47:51 2009 @@ -61,10 +61,7 @@ @register_method('Number', '==', unwrap_spec=[float, float]) def w_number_equals(space, a, b): - w_value = space.w_false - if w_number_compare(space, a, b) == 0: - w_value = space.w_true - return w_value + return space.newbool(cmp(a, b) == 0) @register_method('Number', 'compare', unwrap_spec=[float, float]) def w_number_compare(space, a, b): Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Wed Jul 29 17:47:51 2009 @@ -93,9 +93,14 @@ for i in range(start, stop, step): w_context.slots[key] = W_Number(space, i) t = body.eval(space, w_context, w_context) - if not space.is_normal_status() and not space.is_continue_status(): - space.normal_status() - break + + if not space.is_normal_status(): + if space.is_continue_status(): + space.normal_status() + else: + space.normal_status() + break + return t @register_method('Object', 'appendProto', unwrap_spec=[object, object]) Modified: pypy/branch/io-lang/pypy/lang/io/test/test_number.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_number.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_number.py Wed Jul 29 17:47:51 2009 @@ -150,11 +150,11 @@ def test_equals(): inp = '3 == 5' res, space = interpret(inp) - assert res == space.w_false + assert res is space.w_false inp = '5 == 5' res, space = interpret(inp) - assert res == space.w_true + assert res is space.w_true def test_compare(): inp = '7 compare(7)' Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Wed Jul 29 17:47:51 2009 @@ -176,8 +176,8 @@ a """ res, space = interpret(inp) - values = [x.value for x in res.items] - assert values == [1, 2 ,3, 4, 5, 6, 7, 8, 9, 10] + values = [x.value for x in space.w_lobby.slots['a'].items] + assert values == [1, 2, 4, 5, 6, 7, 8, 9, 10] def test_object_if2(): inp = """if(false, 1, 2)""" From arigo at codespeak.net Wed Jul 29 17:49:00 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 17:49:00 +0200 (CEST) Subject: [pypy-svn] r66668 - pypy/branch/pyjitpl5/pypy/rpython/test Message-ID: <20090729154900.11F83169F2C@codespeak.net> Author: arigo Date: Wed Jul 29 17:48:59 2009 New Revision: 66668 Modified: pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py Log: Some extra (passing) tests. Modified: pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py Wed Jul 29 17:48:59 2009 @@ -8,6 +8,13 @@ def __init__(self, v): self.v = v + self.w = v+1 + +class VArray(object): + _virtualizable2_ = ['lst[*]'] + + def __init__(self, lst): + self.lst = lst class BaseTest(BaseRtypingTest): def test_generate_promote_virtualizable(self): @@ -16,8 +23,34 @@ return vinst.v _, _, graph = self.gengraph(fn, [int]) block = graph.startblock - op_getfield = block.operations[-1] op_promote = block.operations[-2] + op_getfield = block.operations[-1] + assert op_getfield.opname in ('getfield', 'oogetfield') + v_inst = op_getfield.args[0] + assert op_promote.opname == 'promote_virtualizable' + assert op_promote.args[0] is v_inst + + def test_no_promote_virtualizable_for_other_fields(self): + def fn(n): + vinst = V(n) + return vinst.w + _, _, graph = self.gengraph(fn, [int]) + block = graph.startblock + op_getfield = block.operations[-1] + op_call = block.operations[-2] + assert op_getfield.opname in ('getfield', 'oogetfield') + assert op_call.opname == 'direct_call' # to V.__init__ + + def test_generate_promote_virtualizable_array(self): + def fn(n): + vinst = VArray([n, n+1]) + return vinst.lst[1] + _, _, graph = self.gengraph(fn, [int]) + block = graph.startblock + op_promote = block.operations[-3] + op_getfield = block.operations[-2] + op_getarrayitem = block.operations[-1] + assert op_getarrayitem.opname == 'direct_call' # to ll_getitem_xxx assert op_getfield.opname in ('getfield', 'oogetfield') v_inst = op_getfield.args[0] assert op_promote.opname == 'promote_virtualizable' From david at codespeak.net Wed Jul 29 17:53:41 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 17:53:41 +0200 (CEST) Subject: [pypy-svn] r66669 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090729155341.BC253169F32@codespeak.net> Author: david Date: Wed Jul 29 17:53:38 2009 New Revision: 66669 Added: pypy/branch/io-lang/pypy/lang/io/test/test_message_parser.py Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.io pypy/branch/io-lang/pypy/lang/io/parserhack.py Log: Switched to custom parser for results produced by parserhack.io Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.io ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parserhack.io (original) +++ pypy/branch/io-lang/pypy/lang/io/parserhack.io Wed Jul 29 17:53:38 2009 @@ -2,7 +2,7 @@ nil pythonize := nil Message pythonize := method( - "W_Message(space," print + "(" print addArguments next pythonize ")" print @@ -12,9 +12,9 @@ "\"" print name asMutable escape print "\"" print - ", [" print - arguments foreach(i, argument, argument pythonize; ", " print) - "]," print + "[" print + arguments foreach(i, argument, argument pythonize; "," print) + "]" print ) in := File standardInput Modified: pypy/branch/io-lang/pypy/lang/io/parserhack.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/parserhack.py (original) +++ pypy/branch/io-lang/pypy/lang/io/parserhack.py Wed Jul 29 17:53:38 2009 @@ -6,6 +6,59 @@ from pypy.lang.io.model import W_Number, parse_literal, W_Message from pypy.lang.io.objspace import ObjSpace +class MessageParser(object): + def __init__(self, space, input,): + self.space = space + self.input = input + self.position = 0 + def parse(self): + while not self._char() == ')': + self.next() + name = self._read_name() + + arguments = self.parse_arguments() + if self._char() == '(': + nmsg = self.parse() + self.next() + else: + nmsg = None + return W_Message(self.space, name, arguments, nmsg) + def _char(self): + return self.input[self.position] + def _prev_char(self): + return self.input[self.position-1] + def _next_char(self): + return self.input[self.position+1] + def _read_name(self): + name = [] + assert self._char() == '"' + self.next() + while not self._char() == '"': + if self._char() == '\\' and self._next_char() == '"': + self.next() + + name.append(self._char()) + self.next() + self.next() + return ''.join(name) + + def next(self): + self.position += 1 + + def parse_arguments(self): + arguments = [] + assert self._char() == '[' + self.next() + while not self._char() == ']': + arguments.append(self.parse()) + self.next() + assert self._char() == ',' + self.next() + + assert self._char() == ']' + self.next() + return arguments + io_file = py.magic.autopath().dirpath().join("parserhack.io") def parse(input, space=None): @@ -13,8 +66,8 @@ child_in.write(input) child_in.close() s = child_out_err.read().strip() - print s - return eval(s) + # print s + return MessageParser(space, s).parse() def interpret(code): space = ObjSpace() @@ -27,7 +80,10 @@ code = f.read() f.close() return parse(code, space) + +def extract_name(input): + re.match(input, '\"(\\"|[^"])+\"') def load_io_files(space): files = glob.glob('io/*.io') for f in files: @@ -37,4 +93,7 @@ if __name__ == '__main__': import sys space = ObjSpace() - parse(py.path.local(sys.argv[1]).read(), space) \ No newline at end of file + # print parse(py.path.local(sys.argv[1]).read(), space) + print parse(sys.argv[1], space) + + Added: pypy/branch/io-lang/pypy/lang/io/test/test_message_parser.py ============================================================================== --- (empty file) +++ pypy/branch/io-lang/pypy/lang/io/test/test_message_parser.py Wed Jul 29 17:53:38 2009 @@ -0,0 +1,28 @@ +from pypy.lang.io.model import W_Message, W_ImmutableSequence +from pypy.lang.io.parserhack import parse, MessageParser +from pypy.lang.io.objspace import ObjSpace + + +def test_parse_simple(): + space = ObjSpace() + input = '("a"[])' + ast = MessageParser(space, input).parse() + assert ast == W_Message(space, "a", []) + +def test_parse_simple_next(): + input = '("a"[]("b"[]))' + space = ObjSpace() + ast = MessageParser(space, input).parse() + assert ast == W_Message(space, "a", [], W_Message(space, 'b', [])) + +def test_parse_args(): + input = '("a"[]("+"[("b"[]),]))' + space = ObjSpace() + ast = MessageParser(space, input).parse() + assert ast == W_Message(space, "a", [], W_Message(space, '+', [W_Message(space, 'b', [])])) + +def test_parse_quoted_strings(): + input = '("setSlot"[("\\"a\\""[]),("b"[]),])' + space = ObjSpace() + ast = MessageParser(space, input).parse() + assert ast == W_Message(space, "setSlot", [W_Message(space, '"a"', []), W_Message(space, 'b', [])]) \ No newline at end of file From arigo at codespeak.net Wed Jul 29 17:59:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 17:59:13 +0200 (CEST) Subject: [pypy-svn] r66670 - pypy/branch/pyjitpl5/pypy/rpython/test Message-ID: <20090729155913.E1B70169F2B@codespeak.net> Author: arigo Date: Wed Jul 29 17:59:12 2009 New Revision: 66670 Modified: pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py Log: test_replace_promote_virtualizable_with_call. Passes. Modified: pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py (original) +++ pypy/branch/pyjitpl5/pypy/rpython/test/test_rvirtualizable2.py Wed Jul 29 17:59:12 2009 @@ -1,6 +1,7 @@ import py from pypy.rpython.lltypesystem import lltype from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rpython.rvirtualizable2 import replace_promote_virtualizable_with_call class V(object): @@ -89,6 +90,37 @@ TYPE = self.gettype(w_inst) assert 'virtualizable2_accessor' not in TYPE._hints + def test_replace_promote_virtualizable_with_call(self): + def fn(n): + vinst = V(n) + return vinst.v + _, rtyper, graph = self.gengraph(fn, [int]) + block = graph.startblock + op_getfield = block.operations[-1] + assert op_getfield.opname in ('getfield', 'oogetfield') + v_inst_ll_type = op_getfield.args[0].concretetype + # + from pypy.annotation import model as annmodel + from pypy.rpython.annlowlevel import MixLevelHelperAnnotator + def mycall(vinst_ll): + pass + annhelper = MixLevelHelperAnnotator(rtyper) + if self.type_system == 'lltype': + s_vinst = annmodel.SomePtr(v_inst_ll_type) + else: + s_vinst = annmodel.SomeOOInstance(v_inst_ll_type) + funcptr = annhelper.delayedfunction(mycall, [s_vinst], annmodel.s_None) + annhelper.finish() + replace_promote_virtualizable_with_call([graph], v_inst_ll_type, + funcptr) + # + op_promote = block.operations[-2] + op_getfield = block.operations[-1] + assert op_getfield.opname in ('getfield', 'oogetfield') + assert op_promote.opname == 'direct_call' + assert op_promote.args[0].value == funcptr + assert op_promote.args[1] == op_getfield.args[0] + class TestLLtype(LLRtypeMixin, BaseTest): prefix = 'inst_' From arigo at codespeak.net Wed Jul 29 18:10:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 18:10:06 +0200 (CEST) Subject: [pypy-svn] r66671 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090729161006.ADC41169F2C@codespeak.net> Author: arigo Date: Wed Jul 29 18:10:05 2009 New Revision: 66671 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py Log: Write the test. Still not implemented. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_virtualizable.py Wed Jul 29 18:10:05 2009 @@ -637,10 +637,43 @@ res = self.meta_interp(f, [240], policy=StopAtXPolicy(g)) assert res == f(240) - def test_external_access_sometimes(self): + def test_external_read_sometimes(self): py.test.skip("known bug: access the frame in a residual call but" " only sometimes, so that it's not seen during tracing") + jitdriver = JitDriver(greens = [], reds = ['frame'], + virtualizables = ['frame']) + + class Frame(object): + _virtualizable2_ = ['x', 'y'] + class SomewhereElse: + pass + somewhere_else = SomewhereElse() + + def g(): + somewhere_else.counter += 1 + if somewhere_else.counter == 70: + result = somewhere_else.top_frame.y # external read + debug_print(lltype.Void, '-+-+-+-+- external read:', result) + assert result == 79 + else: + result = 1 + return result + + def f(n): + frame = Frame() + frame.x = n + frame.y = 10 + somewhere_else.counter = 0 + somewhere_else.top_frame = frame + while frame.x > 0: + jitdriver.can_enter_jit(frame=frame) + jitdriver.jit_merge_point(frame=frame) + frame.x -= g() + frame.y += 1 + return frame.x + res = self.meta_interp(f, [123], policy=StopAtXPolicy(g)) + assert res == f(123) def test_promote_index_in_virtualizable_list(self): jitdriver = JitDriver(greens = [], reds = ['frame', 'n'], From arigo at codespeak.net Wed Jul 29 18:15:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 18:15:13 +0200 (CEST) Subject: [pypy-svn] r66672 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090729161513.28ADD169F3A@codespeak.net> Author: arigo Date: Wed Jul 29 18:15:12 2009 New Revision: 66672 Removed: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode3.py pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode4.py Log: Forgot to delete these two obsolete files. From arigo at codespeak.net Wed Jul 29 18:26:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 18:26:07 +0200 (CEST) Subject: [pypy-svn] r66673 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729162607.79CE9169F2B@codespeak.net> Author: arigo Date: Wed Jul 29 18:26:06 2009 New Revision: 66673 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py Log: Add VirtualArraySpecNodes. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py Wed Jul 29 18:26:06 2009 @@ -45,6 +45,33 @@ subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) +class VirtualArraySpecNode(SpecNode): + def __init__(self, arraydescr, items): + self.arraydescr = arraydescr + self.items = items # list of subspecnodes + + def equals(self, other): + if not (isinstance(other, VirtualArraySpecNode) and + len(self.items) == len(other.items)): + return False + assert self.arraydescr == other.arraydescr + for i in range(len(self.items)): + s1 = self.items[i] + s2 = other.items[i] + if not s1.equals(s2): + return False + return True + + def extract_runtime_data(self, cpu, valuebox, resultlist): + from pypy.jit.metainterp import executor, history, resoperation + for i in range(len(self.items)): + itembox = executor.execute(cpu, resoperation.rop.GETARRAYITEM_GC, + [valuebox, history.ConstInt(i)], + self.arraydescr) + subspecnode = self.items[i] + subspecnode.extract_runtime_data(cpu, itembox, resultlist) + + def equals_specnodes(specnodes1, specnodes2): assert len(specnodes1) == len(specnodes2) for i in range(len(specnodes1)): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Jul 29 18:26:06 2009 @@ -55,6 +55,9 @@ nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') + # for test_specnode + arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) + cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2)} namespace = locals() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py Wed Jul 29 18:26:06 2009 @@ -2,6 +2,7 @@ from pypy.jit.metainterp.history import AbstractDescr, BoxPtr, ConstInt from pypy.jit.metainterp.specnode import prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode +from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.specnode import equals_specnodes from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin @@ -10,6 +11,10 @@ [(LLtypeMixin.valuedescr, prebuiltNotSpecNode), (LLtypeMixin.nextdescr, prebuiltNotSpecNode)]) +def _get_aspecnode(length=2): + return VirtualArraySpecNode(LLtypeMixin.arraydescr, + [prebuiltNotSpecNode] * length) + def test_equals_specnodes(): assert equals_specnodes([prebuiltNotSpecNode, prebuiltNotSpecNode], [prebuiltNotSpecNode, prebuiltNotSpecNode]) @@ -19,6 +24,12 @@ assert not equals_specnodes([vspecnode1], [vspecnode2]) assert not equals_specnodes([vspecnode1], [prebuiltNotSpecNode]) assert not equals_specnodes([prebuiltNotSpecNode], [vspecnode2]) + aspecnode1 = _get_aspecnode(1) + aspecnode2 = _get_aspecnode(2) + assert equals_specnodes([aspecnode2], [aspecnode2]) + assert not equals_specnodes([aspecnode1], [aspecnode2]) + assert not equals_specnodes([aspecnode1], [prebuiltNotSpecNode]) + assert not equals_specnodes([prebuiltNotSpecNode], [aspecnode2]) def test_extract_runtime_data_1(): res = [] @@ -37,3 +48,15 @@ assert len(res) == 2 assert res[0].value == structure.value assert res[1].value._obj.container._as_ptr() == structure.next + +def test_extract_runtime_data_3(): + array = lltype.malloc(lltype.GcArray(lltype.Signed), 2) + array[0] = 123 + array[1] = 456 + arraybox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, array)) + aspecnode = _get_aspecnode() + res = [] + aspecnode.extract_runtime_data(LLtypeMixin.cpu, arraybox, res) + assert len(res) == 2 + assert res[0].value == 123 + assert res[1].value == 456 From david at codespeak.net Wed Jul 29 18:26:30 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 18:26:30 +0200 (CEST) Subject: [pypy-svn] r66674 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090729162630.7116D169F2D@codespeak.net> Author: david Date: Wed Jul 29 18:26:29 2009 New Revision: 66674 Modified: pypy/branch/io-lang/pypy/lang/io/message.py pypy/branch/io-lang/pypy/lang/io/test/test_message.py Log: argsEvaluatedIn on Message Modified: pypy/branch/io-lang/pypy/lang/io/message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/message.py Wed Jul 29 18:26:29 2009 @@ -16,6 +16,13 @@ # XXX TODO clone from space return W_ImmutableSequence(space, w_receiver.name) + at register_method('Message', 'argsEvaluatedIn') +def message_argsEvaluatedIn(space, w_target, w_message, w_context): + w_in = w_message.arguments[0].eval(space, w_context, w_context) + w_arguments = [arg.eval(space, w_in, w_in) for arg in w_target.arguments] + return space.w_list.clone_and_init(space, w_arguments) + + # @register_method('Message', 'setIsActivatable', unwrap_spec=[object, bool]) # def message_setIsActivatable(space, w_target, setting): # w_target.activateable = setting Modified: pypy/branch/io-lang/pypy/lang/io/test/test_message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_message.py Wed Jul 29 18:26:29 2009 @@ -30,6 +30,35 @@ res, space = interpret(inp) assert isinstance(res, W_ImmutableSequence) assert res.value == 'B' + +def test_argsEvaluatedIn(): + inp = """ + Object do( + m := method(call message argsEvaluatedIn(call sender)) + ) + x := 99 + f := Object clone do( + x := 1 + ) + f m(x) + """ + res, space = interpret(inp) + values = [x.value for x in res.items] + assert values == [99] +def test_argsEvaluatedIn2(): + inp = """ + Object do( + m := method(call message argsEvaluatedIn(call sender)) + ) + x := 99 + f := Object clone do( + x := 1 + ) + f do(return m(x)) + """ + res, space = interpret(inp) + values = [x.value for x in res.items] + assert values == [1] # def test_setIsActivatable(): # inp = "a := block(1);a setIsActivateable(true); a" From arigo at codespeak.net Wed Jul 29 18:45:33 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 18:45:33 +0200 (CEST) Subject: [pypy-svn] r66675 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729164533.C948C169F1C@codespeak.net> Author: arigo Date: Wed Jul 29 18:45:33 2009 New Revision: 66675 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Oups, a bug in handling the escaping in SETFIELD_GC. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Wed Jul 29 18:45:33 2009 @@ -105,9 +105,10 @@ def find_nodes_SETFIELD_GC(self, op): instnode = self.getnode(op.args[0]) + fieldnode = self.getnode(op.args[1]) if instnode.escaped: + fieldnode.mark_escaped() return # nothing to be gained from tracking the field - fieldnode = self.getnode(op.args[1]) field = op.descr assert isinstance(field, AbstractValue) if instnode.curfields is None: Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Jul 29 18:45:33 2009 @@ -488,6 +488,16 @@ Virtual(node_vtable, valuedescr=Not), Virtual(node_vtable, nextdescr=Not)''') + def test_find_nodes_setfield_bug(self): + ops = """ + [p1, p2] + escape(p1) + setfield_gc(p1, p2, descr=nextdescr) + p3 = new_with_vtable(ConstClass(node_vtable)) + jump(p1, p3) + """ + self.find_nodes(ops, 'Not, Not') + # ------------------------------ # Bridge tests From david at codespeak.net Wed Jul 29 18:46:39 2009 From: david at codespeak.net (david at codespeak.net) Date: Wed, 29 Jul 2009 18:46:39 +0200 (CEST) Subject: [pypy-svn] r66676 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090729164639.E28E4169F1C@codespeak.net> Author: david Date: Wed Jul 29 18:46:39 2009 New Revision: 66676 Modified: pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/test/test_map.py Log: control flow for Map foreach Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Wed Jul 29 18:46:39 2009 @@ -142,6 +142,12 @@ w_context.slots[key_name] = item.key w_context.slots[value_name] = item.value t = w_body.eval(space, w_context, w_context) + if not space.is_normal_status(): + if space.is_continue_status(): + space.normal_status() + else: + space.normal_status() + break return t def keys(self): @@ -199,6 +205,7 @@ assert w_method is not None, 'Method "%s" not found in "%s"' % (self.name, w_receiver.__class__) w_result = w_method.apply(space, w_receiver, self, w_context) if not space.is_normal_status(): + print 'Returning non default value' return space.w_return_value if self.next: #TODO: optimize Modified: pypy/branch/io-lang/pypy/lang/io/test/test_map.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_map.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_map.py Wed Jul 29 18:46:39 2009 @@ -105,6 +105,33 @@ value = sorted([(x.items[0].value, x.items[1].value) for x in res.items]) assert value == [('1', 12345), ('2', 99), ('3', 3), ('4', 234)] +def test_map_foreach_continue(): + inp = """b := Map clone do( + atPut("1", 12345) + atPut("2", 99) + atPut("3", 3) + atPut("4", 234) + ) + c := list() + b foreach(key, value, if(value == 99, continue); c append(list(key, value))); c""" + res,space = interpret(inp) + value = sorted([(x.items[0].value, x.items[1].value) for x in res.items]) + assert value == [('1', 12345), ('3', 3), ('4', 234)] + +def test_map_foreach_break(): + inp = """b := Map clone do( + atPut("1", 12345) + atPut("2", 99) + atPut("3", 3) + atPut("4", 234) + ) + c := list() + b foreach(key, value, break(99); c append(list(key, value)));""" + res,space = interpret(inp) + assert space.w_lobby.slots['c'].items == [] + assert res.value == 99 + + def test_map_foreach_leaks(): inp = """b := Map clone do( atPut("1", 12345) @@ -167,4 +194,4 @@ inp = """Map with("a", 1, "b", 2) asObject""" res, space = interpret(inp) assert res.slots['a'].value == 1 - assert res.slots['b'].value == 2 + assert res.slots['b'].value == 2 \ No newline at end of file From arigo at codespeak.net Wed Jul 29 19:25:30 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 19:25:30 +0200 (CEST) Subject: [pypy-svn] r66677 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729172530.E7D6D169E88@codespeak.net> Author: arigo Date: Wed Jul 29 19:25:28 2009 New Revision: 66677 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Unskip the test and make it pass, with an explanation about why not doing so would actually trigger a bug. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Wed Jul 29 19:25:28 2009 @@ -46,8 +46,7 @@ box.mark_escaped() def set_unique_nodes(self): - if (self.escaped or self.knownclsbox is None - # or self.fromstart, but this is implied by 'knownclsbox==None' + if (self.escaped or self.fromstart or self.knownclsbox is None or self.unique != UNIQUE_UNKNOWN): # this node is not suitable for being a virtual, or we # encounter it more than once when doing the recursion @@ -101,7 +100,9 @@ self.nodes[op.result] = instnode def find_nodes_GUARD_CLASS(self, op): - pass # prevent default handling + instnode = self.getnode(op.args[0]) + if instnode.fromstart: # only useful in this case + instnode.knownclsbox = op.args[1] def find_nodes_SETFIELD_GC(self, op): instnode = self.getnode(op.args[0]) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Jul 29 19:25:28 2009 @@ -320,7 +320,6 @@ self.find_nodes(ops, 'Not, Not') def test_find_nodes_new_mismatch(self): - py.test.skip("gives a Virtual instead of Not -- not really wrong") ops = """ [p1] guard_class(p1, ConstClass(node_vtable)) @@ -328,6 +327,8 @@ p2 = new_with_vtable(ConstClass(node_vtable2)) jump(p2) """ + # this must give 'Not', not 'Virtual', because optimizeopt.py would + # remove the guard_class for a virtual. self.find_nodes(ops, 'Not') def test_find_nodes_new_aliasing_mismatch(self): From arigo at codespeak.net Wed Jul 29 20:00:41 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 20:00:41 +0200 (CEST) Subject: [pypy-svn] r66678 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729180041.5C3A8169F31@codespeak.net> Author: arigo Date: Wed Jul 29 20:00:39 2009 New Revision: 66678 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Start on virtual arrays. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Wed Jul 29 20:00:39 2009 @@ -1,15 +1,17 @@ from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode -from pypy.jit.metainterp.history import AbstractValue +from pypy.jit.metainterp.specnode import VirtualArraySpecNode +from pypy.jit.metainterp.history import AbstractValue, ConstInt from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs # ____________________________________________________________ UNIQUE_UNKNOWN = '\x00' -UNIQUE_YES = '\x01' -UNIQUE_NO = '\x02' +UNIQUE_NO = '\x01' +UNIQUE_INST = '\x02' +UNIQUE_ARRAY = '\x03' class InstanceNode(object): """An instance of this class is used to match the start and @@ -20,10 +22,17 @@ escaped = False # if True, then all the rest of the info is pointless unique = UNIQUE_UNKNOWN # for find_unique_nodes() - # fields used to store the shape of the potential Virtual - knownclsbox = None # set only on freshly-allocated structures + # fields used to store the shape of the potential VirtualInstance + knownclsbox = None # set only on freshly-allocated or fromstart structures origfields = None # optimization; equivalent to an empty dict curfields = None # optimization; equivalent to an empty dict + + # fields used to store the shape of the potential VirtualList + arraydescr = None # set only on freshly-allocated or fromstart arrays + #arraysize = .. # valid if and only if arraydescr is not None + origitems = None # optimization; equivalent to an empty dict + curitems = None # optimization; equivalent to an empty dict + dependencies = None def __init__(self, fromstart=False): @@ -46,22 +55,29 @@ box.mark_escaped() def set_unique_nodes(self): - if (self.escaped or self.fromstart or self.knownclsbox is None - or self.unique != UNIQUE_UNKNOWN): + if self.escaped or self.fromstart or self.unique != UNIQUE_UNKNOWN: # this node is not suitable for being a virtual, or we # encounter it more than once when doing the recursion self.unique = UNIQUE_NO - else: - self.unique = UNIQUE_YES + elif self.knownclsbox is not None: + self.unique = UNIQUE_INST if self.curfields is not None: - for subnode in self.curfields.values(): + for subnode in self.curfields.itervalues(): subnode.set_unique_nodes() + elif self.arraydescr is not None: + self.unique = UNIQUE_ARRAY + if self.curitems is not None: + for subnode in self.curitems.itervalues(): + subnode.set_unique_nodes() + else: + self.unique = UNIQUE_NO def __repr__(self): flags = '' if self.escaped: flags += 'e' if self.fromstart: flags += 's' if self.knownclsbox: flags += 'c' + if self.arraydescr: flags += str(self.arraysize) return "" % (flags,) # ____________________________________________________________ @@ -99,6 +115,15 @@ instnode.knownclsbox = op.args[0] self.nodes[op.result] = instnode + def find_nodes_NEW_ARRAY(self, op): + lengthbox = op.args[0] + if not isinstance(lengthbox, ConstInt): + return # var-sized arrays are not virtual + arraynode = InstanceNode() + arraynode.arraysize = lengthbox.value + arraynode.arraydescr = op.descr + self.nodes[op.result] = arraynode + def find_nodes_GUARD_CLASS(self, op): instnode = self.getnode(op.args[0]) if instnode.fromstart: # only useful in this case @@ -137,8 +162,47 @@ return # nothing to be gained from tracking the field self.nodes[op.result] = fieldnode - def find_nodes_GETFIELD_GC_PURE(self, op): - self.find_nodes_GETFIELD_GC(op) + find_nodes_GETFIELD_GC_PURE = find_nodes_GETFIELD_GC + + def find_nodes_SETARRAYITEM_GC(self, op): + indexbox = op.args[1] + if not isinstance(indexbox, ConstInt): + self.find_nodes_default(op) # not a Const index + return + arraynode = self.getnode(op.args[0]) + itemnode = self.getnode(op.args[2]) + if arraynode.escaped: + itemnode.mark_escaped() + return # nothing to be gained from tracking the item + if arraynode.curitems is None: + arraynode.curitems = {} + arraynode.curitems[indexbox.value] = itemnode + arraynode.add_escape_dependency(itemnode) + + def find_nodes_GETARRAYITEM_GC(self, op): + indexbox = op.args[1] + if not isinstance(indexbox, ConstInt): + self.find_nodes_default(op) # not a Const index + return + arraynode = self.getnode(op.args[0]) + if arraynode.escaped: + return # nothing to be gained from tracking the item + index = indexbox.value + if arraynode.curitems is not None and index in arraynode.curitems: + itemnode = arraynode.curitems[index] + elif arraynode.origitems is not None and index in arraynode.origitems: + itemnode = arraynode.origitems[index] + elif arraynode.fromstart: + itemnode = InstanceNode(fromstart=True) + arraynode.add_escape_dependency(itemnode) + if arraynode.origitems is None: + arraynode.origitems = {} + arraynode.origitems[index] = itemnode + else: + return # nothing to be gained from tracking the item + self.nodes[op.result] = itemnode + + find_nodes_GETARRAYITEM_GC_PURE = find_nodes_GETARRAYITEM_GC def find_nodes_JUMP(self, op): # only set up the 'unique' field of the InstanceNodes; @@ -188,11 +252,18 @@ def intersect(self, inputnode, exitnode): assert inputnode.fromstart - if exitnode.unique == UNIQUE_NO or inputnode.escaped: - # give a NotSpecNode + if inputnode.escaped: return prebuiltNotSpecNode - # - assert exitnode.unique == UNIQUE_YES + unique = exitnode.unique + if unique == UNIQUE_NO: + return prebuiltNotSpecNode + if unique == UNIQUE_INST: + return self.intersect_instance(inputnode, exitnode) + if unique == UNIQUE_ARRAY: + return self.intersect_array(inputnode, exitnode) + assert 0, "unknown value for exitnode.unique: %d" % ord(unique) + + def intersect_instance(self, inputnode, exitnode): if (inputnode.knownclsbox is not None and not inputnode.knownclsbox.equals(exitnode.knownclsbox)): # unique match, but the class is known to be a mismatch @@ -224,6 +295,23 @@ fields.append((ofs, specnode)) return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) + def intersect_array(self, inputnode, exitnode): + assert inputnode.arraydescr is None + # + items = [] + for i in range(exitnode.arraysize): + if exitnode.curitems is None: + exitsubnode = self.node_escaped + else: + exitsubnode = exitnode.curitems.get(i, self.node_escaped) + if inputnode.origitems is None: + node = self.node_fromstart + else: + node = inputnode.origitems.get(i, self.node_fromstart) + specnode = self.intersect(node, exitsubnode) + items.append(specnode) + return VirtualArraySpecNode(exitnode.arraydescr, items) + # ____________________________________________________________ # A subclass of NodeFinder for bridges only @@ -252,7 +340,7 @@ if exitnode.unique == UNIQUE_NO: return False # - assert exitnode.unique == UNIQUE_YES + assert exitnode.unique == UNIQUE_INST if not self.known_class.equals(exitnode.knownclsbox): # unique match, but the class is known to be a mismatch return False @@ -273,6 +361,12 @@ return False # some key is in d but not in self.fields return True +class __extend__(VirtualArraySpecNode): + def make_instance_node(self): + xxx + def matches_instance_node(self): + xxx + class BridgeSpecializationFinder(NodeFinder): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Jul 29 20:00:39 2009 @@ -13,6 +13,7 @@ from pypy.jit.metainterp.optimizeutil import sort_descrs from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode +from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.test.oparser import parse @@ -55,7 +56,6 @@ nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') - # for test_specnode arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), @@ -85,6 +85,8 @@ nodesize = cpu.typedescrof(NODE) nodesize2 = cpu.typedescrof(NODE2) + arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) + # force a consistent order valuedescr.sort_key() nextdescr.sort_key() @@ -114,9 +116,12 @@ fields.append((self.namespace[key], value)) fields.sort(key = lambda (x, _): x.sort_key()) return VirtualInstanceSpecNode(constclass(cls_vtable), fields) + def makeVirtualArray(arraydescr, *items): + return VirtualArraySpecNode(arraydescr, items) # context = {'Not': prebuiltNotSpecNode, - 'Virtual': makeVirtual} + 'Virtual': makeVirtual, + 'VArray': makeVirtualArray} lst = eval('[' + text + ']', self.namespace, context) return lst @@ -499,6 +504,40 @@ """ self.find_nodes(ops, 'Not, Not') + def test_find_nodes_array_virtual_1(self): + ops = """ + [i1, p2] + i2 = getarrayitem_gc(p2, 1, descr=arraydescr) + escape(i2) + p3 = new_array(3, descr=arraydescr) + setarrayitem_gc(p3, 1, i1, descr=arraydescr) + jump(i1, p3) + """ + self.find_nodes(ops, 'Not, VArray(arraydescr, Not, Not, Not)') + + def test_find_nodes_array_virtual_2(self): + ops = """ + [i1, p2] + i2 = arraylen_gc(p2, descr=arraydescr) + escape(i2) + p3 = new_array(3, descr=arraydescr) + setarrayitem_gc(p3, 1, i1, descr=arraydescr) + jump(i1, p3) + """ + self.find_nodes(ops, 'Not, VArray(arraydescr, Not, Not, Not)') + + def test_find_nodes_array_nonvirtual_1(self): + ops = """ + [i1, p2] + i2 = getarrayitem_gc(p2, i1, descr=arraydescr) + escape(i2) + p3 = new_array(4, descr=arraydescr) + setarrayitem_gc(p3, i1, i2, descr=arraydescr) + jump(i1, p3) + """ + # Does not work because of the variable index, 'i1'. + self.find_nodes(ops, 'Not, Not') + # ------------------------------ # Bridge tests From arigo at codespeak.net Wed Jul 29 20:11:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 20:11:27 +0200 (CEST) Subject: [pypy-svn] r66679 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729181127.22E7C169F36@codespeak.net> Author: arigo Date: Wed Jul 29 20:11:26 2009 New Revision: 66679 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Implement and test matches_instance_node() on VirtualArraySpecNode. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Wed Jul 29 20:11:26 2009 @@ -363,10 +363,29 @@ class __extend__(VirtualArraySpecNode): def make_instance_node(self): - xxx - def matches_instance_node(self): - xxx - + raise AssertionError, "not implemented (but not used actually)" + def matches_instance_node(self, exitnode): + if exitnode.unique == UNIQUE_NO: + return False + # + assert exitnode.unique == UNIQUE_ARRAY + assert self.arraydescr == exitnode.arraydescr + if len(self.items) != exitnode.arraysize: + # the size is known to be a mismatch + return False + # + d = exitnode.curitems + for i in range(exitnode.arraysize): + try: + if d is None: + raise KeyError + itemnode = d[i] + except KeyError: + itemnode = NodeFinder.node_escaped + subspecnode = self.items[i] + if not subspecnode.matches_instance_node(itemnode): + return False + return True class BridgeSpecializationFinder(NodeFinder): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Wed Jul 29 20:11:26 2009 @@ -701,6 +701,28 @@ """ self.find_bridge(ops, 'Not', 'Not') + def test_bridge_array_virtual_1(self): + ops = """ + [i1] + p1 = new_array(3, descr=arraydescr) + setarrayitem_gc(p1, 0, i1, descr=arraydescr) + jump(p1) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Not', 'VArray(arraydescr, Not, Not, Not)') + + def test_bridge_array_virtual_2(self): + ops = """ + [i1] + p1 = new_array(3, descr=arraydescr) + setarrayitem_gc(p1, 0, i1, descr=arraydescr) + escape(p1) + jump(p1) + """ + self.find_bridge(ops, 'Not', 'Not') + self.find_bridge(ops, 'Not', 'VArray(arraydescr, Not, Not, Not)', + mismatch=True) + class TestLLtype(BaseTestOptimizeFindNode, LLtypeMixin): pass From arigo at codespeak.net Wed Jul 29 21:44:27 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 21:44:27 +0200 (CEST) Subject: [pypy-svn] r66680 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729194427.283C1169F1F@codespeak.net> Author: arigo Date: Wed Jul 29 21:44:26 2009 New Revision: 66680 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py Log: Support virtual arrays in resume.py. Slight improvement of the other tests as well. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py Wed Jul 29 21:44:26 2009 @@ -1,5 +1,5 @@ import sys -from pypy.jit.metainterp.history import Box, Const +from pypy.jit.metainterp.history import Box, Const, ConstInt from pypy.jit.metainterp.resoperation import rop # Logic to encode the chain of frames and the state of the boxes at a @@ -70,9 +70,16 @@ self.liveboxes[box] = self._getconstindex(const) def make_virtual(self, virtualbox, known_class, fielddescrs, fieldboxes): + vinfo = VirtualInfo(known_class, fielddescrs) + self._make_virtual(virtualbox, vinfo, fieldboxes) + + def make_varray(self, virtualbox, arraydescr, itemboxes): + vinfo = VArrayInfo(arraydescr, len(itemboxes)) + self._make_virtual(virtualbox, vinfo, itemboxes) + + def _make_virtual(self, virtualbox, vinfo, fieldboxes): assert self.liveboxes[virtualbox] == 0 self.liveboxes[virtualbox] = len(self.virtuals) | VIRTUAL_FLAG - vinfo = VirtualInfo(known_class, fielddescrs) self.virtuals.append(vinfo) self.vfieldboxes.append(fieldboxes) self._register_boxes(fieldboxes) @@ -118,7 +125,14 @@ return result -class VirtualInfo(object): +class AbstractVirtualInfo(object): + def allocate(self, metainterp): + raise NotImplementedError + def setfields(self, metainterp, box, fn_decode_box): + raise NotImplementedError + + +class VirtualInfo(AbstractVirtualInfo): def __init__(self, known_class, fielddescrs): self.known_class = known_class self.fielddescrs = fielddescrs @@ -135,6 +149,24 @@ [box, fieldbox], descr=self.fielddescrs[i]) +class VArrayInfo(AbstractVirtualInfo): + def __init__(self, arraydescr, length): + self.arraydescr = arraydescr + self.length = length + #self.fieldnums = ... + + def allocate(self, metainterp): + return metainterp.execute_and_record(rop.NEW_ARRAY, + [ConstInt(self.length)], + descr=self.arraydescr) + + def setfields(self, metainterp, box, fn_decode_box): + for i in range(self.length): + itembox = fn_decode_box(self.fieldnums[i]) + metainterp.execute_and_record(rop.SETARRAYITEM_GC, + [box, ConstInt(i), itembox], + descr=self.arraydescr) + class ResumeDataReader(object): i_frame_infos = 0 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py Wed Jul 29 21:44:26 2009 @@ -59,8 +59,13 @@ class MyMetaInterp: def __init__(self, cpu): self.cpu = cpu + self.trace = [] def execute_and_record(self, opnum, argboxes, descr=None): - return executor.execute(self.cpu, opnum, argboxes, descr) + resbox = executor.execute(self.cpu, opnum, argboxes, descr) + self.trace.append((opnum, + [box.value for box in argboxes], + resbox and resbox.value)) + return resbox demo55 = lltype.malloc(LLtypeMixin.NODE) demo55o = lltype.cast_opaque_ptr(llmemory.GCREF, demo55) @@ -78,14 +83,15 @@ assert liveboxes == [b1s, b2s, b3s] # b1t, b2t, b3t = [BoxInt(11), BoxPtr(demo55o), BoxInt(33)] - reader = ResumeDataReader(storage, [b1t, b2t, b3t], - MyMetaInterp(LLtypeMixin.cpu)) + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [b1t, b2t, b3t], metainterp) lst = reader.consume_boxes() assert lst == [b1t, ConstInt(1), b1t, b2t] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() assert lst == [b1t, b2t, b3t] + assert metainterp.trace == [] def test_virtual_adder_make_virtual(): @@ -124,15 +130,29 @@ b5s] # b1t, b3t, b5t = [BoxInt(11), BoxInt(33), BoxPtr(demo55o)] - reader = ResumeDataReader(storage, [b1t, b3t, b5t], - MyMetaInterp(LLtypeMixin.cpu)) + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [b1t, b3t, b5t], metainterp) lst = reader.consume_boxes() b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] + b4tx = b2t.value._obj.container._as_ptr().next + b4tx = lltype.cast_opaque_ptr(llmemory.GCREF, b4tx) + assert metainterp.trace == [ + (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr], b2t.value), + (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr2], b4tx), + (rop.SETFIELD_GC, [b2t.value, b4tx], None), + (rop.SETFIELD_GC, [b2t.value, c1s.value], None), + (rop.SETFIELD_GC, [b4tx, b2t.value], None), + (rop.SETFIELD_GC, [b4tx, b3t.value], None), + (rop.SETFIELD_GC, [b4tx, b5t.value], None), + ] + del metainterp.trace[:] lst = reader.consume_boxes() assert lst == [ConstInt(2), ConstInt(3)] + assert metainterp.trace == [] lst = reader.consume_boxes() assert lst == [b1t, b2t, b3t] + assert metainterp.trace == [] # ptr = b2t.value._obj.container._as_ptr() assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE) @@ -165,8 +185,8 @@ assert liveboxes == [b2s, b3s] # b2t, b3t = [BoxPtr(demo55o), BoxInt(33)] - reader = ResumeDataReader(storage, [b2t, b3t], - MyMetaInterp(LLtypeMixin.cpu)) + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [b2t, b3t], metainterp) lst = reader.consume_boxes() c1t = ConstInt(111) assert lst == [c1t, ConstInt(1), c1t, b2t] @@ -174,3 +194,52 @@ assert lst == [ConstInt(2), ConstInt(3)] lst = reader.consume_boxes() assert lst == [c1t, b2t, b3t] + assert metainterp.trace == [] + + +def test_virtual_adder_make_varray(): + storage = make_demo_storage() + b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxInt(4)] + c1s = ConstInt(111) + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b1s) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + modifier.make_varray(b2s, + LLtypeMixin.arraydescr, + [b4s, c1s]) # new fields + assert not modifier.is_virtual(b1s) + assert modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + assert not modifier.is_virtual(b4s) + # done + liveboxes = modifier.finish() + assert liveboxes == [b1s, + #b2s -- virtual + b3s, + b4s] + # + b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxInt(44)] + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [b1t, b3t, b4t], metainterp) + lst = reader.consume_boxes() + b2t = lst[-1] + assert lst == [b1t, ConstInt(1), b1t, b2t] + assert metainterp.trace == [ + (rop.NEW_ARRAY, [2], b2t.value), + (rop.SETARRAYITEM_GC, [b2t.value, 0, 44], None), + (rop.SETARRAYITEM_GC, [b2t.value, 1, 111], None), + ] + del metainterp.trace[:] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + assert metainterp.trace == [] + lst = reader.consume_boxes() + assert lst == [b1t, b2t, b3t] + assert metainterp.trace == [] + # + ptr = b2t.value._obj.container._as_ptr() + assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed)) + assert len(ptr) == 2 + assert ptr[0] == 44 + assert ptr[1] == 111 From arigo at codespeak.net Wed Jul 29 22:07:31 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 22:07:31 +0200 (CEST) Subject: [pypy-svn] r66681 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729200731.AB1AB169F50@codespeak.net> Author: arigo Date: Wed Jul 29 22:07:30 2009 New Revision: 66681 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Array support in optimizeopt.py (first part). Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Wed Jul 29 22:07:30 2009 @@ -29,11 +29,12 @@ LEVEL_UNKNOWN = 0 LEVEL_NONNULL = 1 -LEVEL_KNOWNCLASS = 2 +LEVEL_KNOWNCLASS = 2 # might also mean KNOWNARRAYDESCR, for arrays LEVEL_CONSTANT = 3 class InstanceValue(object): + _attrs_ = ('box', 'level') level = LEVEL_UNKNOWN def __init__(self, box): @@ -102,21 +103,31 @@ CVAL_NULLOBJ = ConstantValue(ConstObj(ConstObj.value)) -class VirtualValue(InstanceValue): +class AbstractVirtualValue(InstanceValue): + _attrs_ = ('optimizer', 'keybox', 'source_op') box = None level = LEVEL_KNOWNCLASS - def __init__(self, optimizer, known_class, keybox, source_op=None): + def __init__(self, optimizer, keybox, source_op=None): self.optimizer = optimizer - self.known_class = known_class self.keybox = keybox # only used as a key in dictionaries - self.source_op = source_op # the NEW_WITH_VTABLE operation building it - self._fields = av_newdict2() + self.source_op = source_op # the NEW_WITH_VTABLE/NEW_ARRAY operation + # that builds this box + + def get_key_box(self): + if self.box is None: + return self.keybox + return self.box + - def getfield(self, ofs): - return self._fields[ofs] +class VirtualValue(AbstractVirtualValue): - def getfield_default(self, ofs, default): + def __init__(self, optimizer, known_class, keybox, source_op=None): + AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) + self.known_class = known_class + self._fields = av_newdict2() + + def getfield(self, ofs, default): return self._fields.get(ofs, default) def setfield(self, ofs, fieldvalue): @@ -149,12 +160,70 @@ [self.known_class], self.optimizer.new_ptr_box()) - def get_key_box(self): + def get_args_for_fail(self, modifier): + if self.box is None and not modifier.is_virtual(self.keybox): + lst = self._fields.keys() + sort_descrs(lst) + fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] + modifier.make_virtual(self.keybox, self.known_class, + lst, fieldboxes) + for ofs in lst: + fieldvalue = self._fields[ofs] + fieldvalue.get_args_for_fail(modifier) + + +class VArrayValue(AbstractVirtualValue): + + def __init__(self, optimizer, arraydescr, size, keybox, source_op=None): + AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) + self.arraydescr = arraydescr + self._items = [None] * size + + def getlength(self): + return len(self._items) + + def getitem(self, index, default): + res = self._items[index] + if res is None: + res = default + return res + + def setitem(self, index, itemvalue): + assert isinstance(itemvalue, InstanceValue) + self._items[index] = itemvalue + + def force_box(self): if self.box is None: - return self.keybox + assert self.source_op is not None # otherwise, we are trying + # to force a VArray from a specnode computed by optimizefindnode. + newoperations = self.optimizer.newoperations + newoperations.append(self.source_op) + self.box = box = self.source_op.result + for index in range(len(self._items)): + subvalue = self._items[index] + if subvalue is not None: + subbox = subvalue.force_box() + op = ResOperation(rop.SETARRAYITEM_GC, + [box, ConstInt(index), subbox], None, + descr=self.arraydescr) + newoperations.append(op) return self.box + def prepare_force_box(self): + # This logic is not included in force_box() for safety reasons. + # It should only be used from teardown_virtual_node(); if we + # call force_box() from somewhere else and we get source_op=None, + # it is really a bug. + XXX + if self.box is None and self.source_op is None: + # rare case (shown by test_p123_simple) to force a Virtual + # from a specnode computed by optimizefindnode. + self.source_op = ResOperation(rop.NEW_WITH_VTABLE, + [self.known_class], + self.optimizer.new_ptr_box()) + def get_args_for_fail(self, modifier): + XXX if self.box is None and not modifier.is_virtual(self.keybox): lst = self._fields.keys() sort_descrs(lst) @@ -183,7 +252,7 @@ def teardown_virtual_node(self, optimizer, value, newexitargs): assert value.is_virtual() for ofs, subspecnode in self.fields: - subvalue = value.getfield_default(ofs, optimizer.new_const(ofs)) + subvalue = value.getfield(ofs, optimizer.new_const(ofs)) subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) @@ -224,6 +293,11 @@ self.make_equal_to(box, vvalue) return vvalue + def make_varray(self, arraydescr, size, box, source_op=None): + vvalue = VArrayValue(self, arraydescr, size, box, source_op) + self.make_equal_to(box, vvalue) + return vvalue + def new_ptr_box(self): if not self.cpu.is_oo: return BoxPtr() @@ -425,8 +499,9 @@ def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.args[0]) if value.is_virtual(): - # optimizefindnode should ensure that we don't get a KeyError - fieldvalue = value.getfield(op.descr) + # optimizefindnode should ensure that fieldvalue is found + fieldvalue = value.getfield(op.descr, None) + assert fieldvalue is not None self.make_equal_to(op.result, fieldvalue) else: value.make_nonnull() @@ -447,6 +522,47 @@ def optimize_NEW_WITH_VTABLE(self, op): self.make_virtual(op.args[0], op.result, op) + def optimize_NEW_ARRAY(self, op): + sizebox = op.args[0] + if self.is_constant(sizebox): + self.make_varray(op.descr, sizebox.getint(), op.result, op) + else: + self.optimize_default(op) + + def optimize_ARRAYLEN_GC(self, op): + value = self.getvalue(op.args[0]) + if value.is_virtual(): + assert op.result.getint() == value.getlength() + self.make_constant(op.result) + else: + value.make_nonnull() + self.optimize_default(op) + + def optimize_GETARRAYITEM_GC(self, op): + value = self.getvalue(op.args[0]) + indexbox = op.args[1] + if value.is_virtual() and self.is_constant(indexbox): + # optimizefindnode should ensure that itemvalue is found + itemvalue = value.getitem(indexbox.getint(), None) + assert itemvalue is not None + self.make_equal_to(op.result, itemvalue) + else: + value.make_nonnull() + self.optimize_default(op) + + # note: the following line does not mean that the two operations are + # completely equivalent, because GETARRAYITEM_GC_PURE is_always_pure(). + optimize_GETARRAYITEM_GC_PURE = optimize_GETARRAYITEM_GC + + def optimize_SETARRAYITEM_GC(self, op): + value = self.getvalue(op.args[0]) + indexbox = op.args[1] + if value.is_virtual() and self.is_constant(indexbox): + value.setitem(indexbox.getint(), self.getvalue(op.args[2])) + else: + value.make_nonnull() + self.optimize_default(op) + optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 29 22:07:30 2009 @@ -700,6 +700,24 @@ self.optimize_loop(ops, 'Not', expected, i1=5, boxkinds={'myptr': self.nodebox.value}) + def test_varray_1(self): + ops = """ + [i1] + p1 = new_array(3, descr=arraydescr) + i3 = arraylen_gc(p1, descr=arraydescr) + guard_value(i3, 3) + fail() + setarrayitem_gc(p1, 1, i1, descr=arraydescr) + setarrayitem_gc(p1, 0, 25, descr=arraydescr) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + jump(i2) + """ + expected = """ + [i1] + jump(i1) + """ + self.optimize_loop(ops, 'Not', expected, i3=3) + # ---------- def make_fail_descr(self): From arigo at codespeak.net Wed Jul 29 22:32:19 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 22:32:19 +0200 (CEST) Subject: [pypy-svn] r66682 - in pypy/branch/pyjitpl5/pypy/jit/backend: llgraph llvm minimal test x86 Message-ID: <20090729203219.3C248169F34@codespeak.net> Author: arigo Date: Wed Jul 29 22:32:18 2009 New Revision: 66682 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/llvm/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Log: New detail in the backends: arraydescr.is_array_of_pointers(). Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Wed Jul 29 22:32:18 2009 @@ -46,6 +46,9 @@ def is_pointer_field(self): return self.typeinfo == 'p' + def is_array_of_pointers(self): + return self.typeinfo == 'p' + def equals(self, other): if not isinstance(other, Descr): return False @@ -673,6 +676,11 @@ self.setarrayitem = setarrayitem self.getarraylength = getarraylength self.instanceof = instanceof + self._is_array_of_pointers = (history.getkind(TYPE) == 'obj') + + def is_array_of_pointers(self): + # for arrays, TYPE is the type of the array item. + return self._is_array_of_pointers def __repr__(self): return '' % self.TYPE._short_name() Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llvm/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llvm/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llvm/runner.py Wed Jul 29 22:32:18 2009 @@ -706,6 +706,8 @@ self.itemsize_index = itemsize_index # index in cpu.types_by_index self.ty_array_ptr = lltype.nullptr(llvm_rffi.LLVMTypeRef.TO) # ^^^ set by setup_once() + def is_array_of_pointers(self): + return self.itemsize_index == LLVMCPU.SIZE_GCPTR class CallDescr(AbstractDescr): ty_function_ptr = lltype.nullptr(llvm_rffi.LLVMTypeRef.TO) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/minimal/runner.py Wed Jul 29 22:32:18 2009 @@ -415,10 +415,14 @@ x = %(input)s p[index] = x """ % dict).compile() in dict2 - return ArrayDescr(dict2['new'], - dict2['length'], - dict2['getarrayitem'], - dict2['setarrayitem']) + if getkind(ARRAY.OF) == 'ptr': + Class = PtrArrayDescr + else: + Class = NonPtrArrayDescr + return Class(dict2['new'], + dict2['length'], + dict2['getarrayitem'], + dict2['setarrayitem']) # ---------- @@ -607,10 +611,14 @@ x = %(input)s a.ll_setitem_fast(index, x) """ % dict).compile() in dict2 - return ArrayDescr(dict2['new'], - dict2['length'], - dict2['getarrayitem'], - dict2['setarrayitem']) + if getkind(ARRAY.ITEM) == 'obj': + Class = PtrArrayDescr + else: + Class = NonPtrArrayDescr + return Class(dict2['new'], + dict2['length'], + dict2['getarrayitem'], + dict2['setarrayitem']) @cached_method('_methdescrcache') @@ -683,6 +691,14 @@ self.getarrayitem = getarrayitem self.setarrayitem = setarrayitem +class PtrArrayDescr(ArrayDescr): + def is_array_of_pointers(self): + return True + +class NonPtrArrayDescr(ArrayDescr): + def is_array_of_pointers(self): + return False + class CallDescr(AbstractDescr): call = None errbox = None Modified: pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/test/runner_test.py Wed Jul 29 22:32:18 2009 @@ -387,6 +387,7 @@ def test_array_basic(self): a_box, A = self.alloc_array_of(lltype.Signed, 342) arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], 'int', descr=arraydescr) assert r.value == 342 @@ -400,6 +401,7 @@ # a_box, A = self.alloc_array_of(lltype.Char, 11) arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], 'int', descr=arraydescr) assert r.value == 11 @@ -422,6 +424,7 @@ A = lltype.Ptr(A) b_box, B = self.alloc_array_of(A, 3) arraydescr = self.cpu.arraydescrof(B) + assert arraydescr.is_array_of_pointers() r = self.execute_operation(rop.ARRAYLEN_GC, [b_box], 'int', descr=arraydescr) assert r.value == 3 @@ -436,6 +439,7 @@ # Unsigned should work the same as Signed a_box, A = self.alloc_array_of(lltype.Unsigned, 342) arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], 'int', descr=arraydescr) assert r.value == 342 @@ -450,6 +454,7 @@ # Bool should work the same as Char a_box, A = self.alloc_array_of(lltype.Bool, 311) arraydescr = self.cpu.arraydescrof(A) + assert not arraydescr.is_array_of_pointers() r = self.execute_operation(rop.ARRAYLEN_GC, [a_box], 'int', descr=arraydescr) assert r.value == 311 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/x86/runner.py Wed Jul 29 22:32:18 2009 @@ -42,6 +42,9 @@ def is_pointer_field(self): return self.flag2 # for fielddescrs + def is_array_of_pointers(self): + return self.flag2 # for arraydescrs + def equals(self, other): if not isinstance(other, ConstDescr3): return False From arigo at codespeak.net Wed Jul 29 22:32:54 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 22:32:54 +0200 (CEST) Subject: [pypy-svn] r66683 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729203254.59FB9169F4C@codespeak.net> Author: arigo Date: Wed Jul 29 22:32:53 2009 New Revision: 66683 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Optimize array operations. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Wed Jul 29 22:32:53 2009 @@ -3,6 +3,7 @@ from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode +from pypy.jit.metainterp.specnode import VirtualArraySpecNode from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs from pypy.jit.metainterp import resume, compile from pypy.rlib.objectmodel import we_are_translated @@ -210,17 +211,13 @@ return self.box def prepare_force_box(self): - # This logic is not included in force_box() for safety reasons. - # It should only be used from teardown_virtual_node(); if we - # call force_box() from somewhere else and we get source_op=None, - # it is really a bug. - XXX if self.box is None and self.source_op is None: - # rare case (shown by test_p123_simple) to force a Virtual + # rare case (shown by test_p123_simple) to force a VirtualArray # from a specnode computed by optimizefindnode. - self.source_op = ResOperation(rop.NEW_WITH_VTABLE, - [self.known_class], - self.optimizer.new_ptr_box()) + self.source_op = ResOperation(rop.NEW_ARRAY, + [ConstInt(self.getlength())], + self.optimizer.new_ptr_box(), + descr=self.arraydescr) def get_args_for_fail(self, modifier): XXX @@ -255,6 +252,22 @@ subvalue = value.getfield(ofs, optimizer.new_const(ofs)) subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) +class __extend__(VirtualArraySpecNode): + def setup_virtual_node(self, optimizer, box, newinputargs): + vvalue = optimizer.make_varray(self.arraydescr, len(self.items), box) + for index in range(len(self.items)): + subbox = optimizer.new_box_item(self.arraydescr) + subspecnode = self.items[index] + subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) + vvalue.setitem(index, optimizer.getvalue(subbox)) + def teardown_virtual_node(self, optimizer, value, newexitargs): + assert value.is_virtual() + const = optimizer.new_const_item(self.arraydescr) + for index in range(len(self.items)): + subvalue = value.getitem(index, const) + subspecnode = self.items[index] + subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) + class Optimizer(object): @@ -319,6 +332,21 @@ else: return CVAL_ZERO + def new_box_item(self, arraydescr): + if arraydescr.is_array_of_pointers(): + return self.new_ptr_box() + else: + return BoxInt() + + def new_const_item(self, arraydescr): + if arraydescr.is_array_of_pointers(): + if not self.cpu.is_oo: + return CVAL_NULLPTR + else: + return CVAL_NULLOBJ + else: + return CVAL_ZERO + # ---------- def setup_virtuals(self): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 29 22:32:53 2009 @@ -718,6 +718,50 @@ """ self.optimize_loop(ops, 'Not', expected, i3=3) + def test_varray_2(self): + ops = """ + [i0, p1] + i1 = getarrayitem_gc(p1, 0, descr=arraydescr) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + i3 = int_sub(i1, i2) + guard_value(i3, 15) + fail() + p2 = new_array(2, descr=arraydescr) + setarrayitem_gc(p2, 1, i0, descr=arraydescr) + setarrayitem_gc(p2, 0, 20, descr=arraydescr) + jump(i0, p2) + """ + expected = """ + [i0, i1, i2] + i3 = int_sub(i1, i2) + guard_value(i3, 15) + fail() + jump(i0, 20, i0) + """ + self.optimize_loop(ops, 'Not, VArray(arraydescr, Not, Not)', expected, + i3=15) + + def test_p123_array(self): + ops = """ + [i1, p2, p3] + i3 = getarrayitem_gc(p3, 0, descr=arraydescr) + escape(i3) + p1 = new_array(1, descr=arraydescr) + setarrayitem_gc(p1, 0, i1, descr=arraydescr) + jump(i1, p1, p2) + """ + expected = """ + [i1, i2, p3] + i3 = getarrayitem_gc(p3, 0, descr=arraydescr) + escape(i3) + p2b = new_array(1, descr=arraydescr) + setarrayitem_gc(p2b, 0, i2, descr=arraydescr) + jump(i1, i1, p2b) + """ + # We cannot track virtuals that survive for more than two iterations. + self.optimize_loop(ops, 'Not, VArray(arraydescr, Not), Not', + expected) + # ---------- def make_fail_descr(self): From arigo at codespeak.net Wed Jul 29 22:56:32 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 22:56:32 +0200 (CEST) Subject: [pypy-svn] r66684 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090729205632.DE8AE169F20@codespeak.net> Author: arigo Date: Wed Jul 29 22:56:31 2009 New Revision: 66684 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Finish array optimization support. Write a few tests; a few more would be nice too. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Wed Jul 29 22:56:31 2009 @@ -163,6 +163,8 @@ def get_args_for_fail(self, modifier): if self.box is None and not modifier.is_virtual(self.keybox): + # modifier.is_virtual() checks for recursion: it is False unless + # we have already seen the very same keybox lst = self._fields.keys() sort_descrs(lst) fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] @@ -220,16 +222,19 @@ descr=self.arraydescr) def get_args_for_fail(self, modifier): - XXX if self.box is None and not modifier.is_virtual(self.keybox): - lst = self._fields.keys() - sort_descrs(lst) - fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - modifier.make_virtual(self.keybox, self.known_class, - lst, fieldboxes) - for ofs in lst: - fieldvalue = self._fields[ofs] - fieldvalue.get_args_for_fail(modifier) + # modifier.is_virtual() checks for recursion: it is False unless + # we have already seen the very same keybox + itemboxes = [] + const = self.optimizer.new_const_item(self.arraydescr) + for itemvalue in self._items: + if itemvalue is None: + itemvalue = const + itemboxes.append(itemvalue.get_key_box()) + modifier.make_varray(self.keybox, self.arraydescr, itemboxes) + for itemvalue in self._items: + if itemvalue is not None: + itemvalue.get_args_for_fail(modifier) class __extend__(SpecNode): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Wed Jul 29 22:56:31 2009 @@ -780,6 +780,9 @@ self.fdescr = fdescr self.namespace['fdescr'] = fdescr + def teardown_method(self, meth): + self.namespace.pop('fdescr', None) + def _verify_fail_args(self, boxes, oparse, text): import re r = re.compile(r"\bwhere\s+(\w+)\s+is a\s+(\w+)") @@ -789,9 +792,13 @@ virtuals = {} for match, end in zip(parts, ends[1:]): pvar = match.group(1) - cls_vtable = self.namespace[match.group(2)] fieldstext = text[match.end():end] - virtuals[pvar] = (cls_vtable, None, fieldstext) + if match.group(2) != 'list': + tag = ('virtual', self.namespace[match.group(2)]) + else: + arrayname, fieldstext = fieldstext.split(':', 1) + tag = ('varray', self.namespace[arrayname.strip()]) + virtuals[pvar] = (tag, None, fieldstext) # def _variables_equal(box, varname, strict): if varname not in virtuals: @@ -800,16 +807,22 @@ else: assert box.value == oparse.getvar(varname).value else: - cls_vtable, resolved, fieldstext = virtuals[varname] - if not self.cpu.is_oo: - assert box.getptr(rclass.OBJECTPTR).typeptr == cls_vtable + tag, resolved, fieldstext = virtuals[varname] + if tag[0] == 'virtual': + if not self.cpu.is_oo: + assert box.getptr(rclass.OBJECTPTR).typeptr == tag[1] + else: + root = box.getobj() + root = ootype.cast_from_object(ootype.ROOT, root) + assert ootype.classof(root) == tag[1] + elif tag[0] == 'varray': + pass # xxx check arraydescr else: - root = ootype.cast_from_object(ootype.ROOT, box.getobj()) - assert ootype.classof(root) == cls_vtable + assert 0 if resolved is not None: assert resolved.value == box.value else: - virtuals[varname] = cls_vtable, box, fieldstext + virtuals[varname] = tag, box, fieldstext # basetext = text[:ends[0]] varnames = [s.strip() for s in basetext.split(',')] @@ -819,19 +832,30 @@ # for match in parts: pvar = match.group(1) - cls_vtable, resolved, fieldstext = virtuals[pvar] + tag, resolved, fieldstext = virtuals[pvar] assert resolved is not None + index = 0 for fieldtext in fieldstext.split(','): fieldtext = fieldtext.strip() if not fieldtext: continue - fieldname, fieldvalue = fieldtext.split('=') - fielddescr = self.namespace[fieldname.strip()] - fieldbox = executor.execute(self.cpu, - rop.GETFIELD_GC, - [resolved], - descr=fielddescr) + if tag[0] == 'virtual': + fieldname, fieldvalue = fieldtext.split('=') + fielddescr = self.namespace[fieldname.strip()] + fieldbox = executor.execute(self.cpu, + rop.GETFIELD_GC, + [resolved], + descr=fielddescr) + elif tag[0] == 'varray': + fieldvalue = fieldtext + fieldbox = executor.execute(self.cpu, + rop.GETARRAYITEM_GC, + [resolved, ConstInt(index)], + descr=tag[1]) + else: + assert 0 _variables_equal(fieldbox, fieldvalue.strip(), strict=False) + index += 1 def check_expanded_fail_descr(self, expectedtext): fdescr = self.fdescr @@ -989,6 +1013,29 @@ where p0 is a node_vtable, valuedescr=i1b ''') + def test_expand_fail_varray(self): + self.make_fail_descr() + ops = """ + [i1] + p1 = new_array(3, descr=arraydescr) + setarrayitem_gc(p1, 1, i1, descr=arraydescr) + setarrayitem_gc(p1, 0, 25, descr=arraydescr) + guard_true(i1) + fail(p1, descr=fdescr) + i2 = getarrayitem_gc(p1, 1, descr=arraydescr) + jump(i2) + """ + expected = """ + [i1] + guard_true(i1) + fail(i1, descr=fdescr) + jump(1) + """ + self.optimize_loop(ops, 'Not', expected, i1=1) + self.check_expanded_fail_descr('''p1 + where p1 is a list arraydescr: 25, i1 + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Wed Jul 29 23:01:50 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 23:01:50 +0200 (CEST) Subject: [pypy-svn] r66685 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090729210150.D1B3B168457@codespeak.net> Author: arigo Date: Wed Jul 29 23:01:50 2009 New Revision: 66685 Removed: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_list_optimize.py Log: Remove dead file. From arigo at codespeak.net Wed Jul 29 23:07:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 29 Jul 2009 23:07:15 +0200 (CEST) Subject: [pypy-svn] r66686 - pypy/branch/pyjitpl5/pypy/jit/backend/llgraph Message-ID: <20090729210715.F22D0169F28@codespeak.net> Author: arigo Date: Wed Jul 29 23:07:15 2009 New Revision: 66686 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Log: RPythonification. (Not sure how it could have worked before, except as a complicated consequence of how Descr is passed around...) Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Wed Jul 29 23:07:15 2009 @@ -331,6 +331,7 @@ self.memo_cast)) def do_new(self, args, size): + assert isinstance(size, Descr) return history.BoxPtr(llimpl.do_new(size.ofs)) def do_new_with_vtable(self, args, descr=None): @@ -343,6 +344,7 @@ return history.BoxPtr(result) def do_new_array(self, args, size): + assert isinstance(size, Descr) count = args[0].getint() return history.BoxPtr(llimpl.do_new_array(size.ofs, count)) From antocuni at codespeak.net Thu Jul 30 11:41:12 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 11:41:12 +0200 (CEST) Subject: [pypy-svn] r66687 - pypy/branch/pyjitpl5/pypy/translator/cli Message-ID: <20090730094112.05294169F24@codespeak.net> Author: antocuni Date: Thu Jul 30 11:41:10 2009 New Revision: 66687 Modified: pypy/branch/pyjitpl5/pypy/translator/cli/constant.py pypy/branch/pyjitpl5/pypy/translator/cli/gencli.py Log: kill kill kill! :-) So far there were three different ways to handle pbcs in the cli backend, but two were just old experiments which turned to be slow, untested and maybe half-broken. Kill them and keep only the old "StaticFieldConstGenerator" Modified: pypy/branch/pyjitpl5/pypy/translator/cli/constant.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/constant.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/constant.py Thu Jul 30 11:41:10 2009 @@ -49,7 +49,7 @@ # Different generators implementing different techniques for loading # constants (Static fields, singleton fields, etc) -class CLIBaseConstGenerator(BaseConstantGenerator): +class CLIConstantGenerator(BaseConstantGenerator): """ Base of all CLI constant generators. It implements the oosupport constant generator in terms of the CLI interface. @@ -66,7 +66,19 @@ return gen def _end_gen_constants(self, gen, numsteps): - assert gen.ilasm is self.ilasm + + self.ilasm.begin_function('.cctor', [], 'void', False, 'static', + 'specialname', 'rtspecialname', 'default') + self.ilasm.stderr('CONST: initialization starts', DEBUG_CONST_INIT) + for i in range(numsteps): + self.ilasm.stderr('CONST: step %d of %d' % (i, numsteps), + DEBUG_CONST_INIT) + step_name = 'step%d' % i + self.ilasm.call('void %s::%s()' % (CONST_CLASS, step_name)) + self.ilasm.stderr('CONST: initialization completed', DEBUG_CONST_INIT) + self.ilasm.ret() + self.ilasm.end_function() + self.end_class() def begin_class(self): @@ -96,11 +108,6 @@ else: return BaseConstantGenerator._create_complex_const(self, value) -class FieldConstGenerator(CLIBaseConstGenerator): - pass - -class StaticFieldConstGenerator(FieldConstGenerator): - # _________________________________________________________________ # OOSupport interface @@ -127,144 +134,8 @@ gen.ilasm.ret() gen.ilasm.end_function() - def _end_gen_constants(self, gen, numsteps): - self.ilasm.begin_function('.cctor', [], 'void', False, 'static', - 'specialname', 'rtspecialname', 'default') - self.ilasm.stderr('CONST: initialization starts', DEBUG_CONST_INIT) - for i in range(numsteps): - self.ilasm.stderr('CONST: step %d of %d' % (i, numsteps), - DEBUG_CONST_INIT) - step_name = 'step%d' % i - self.ilasm.call('void %s::%s()' % (CONST_CLASS, step_name)) - self.ilasm.stderr('CONST: initialization completed', DEBUG_CONST_INIT) - self.ilasm.ret() - self.ilasm.end_function() - super(StaticFieldConstGenerator, self)._end_gen_constants( - gen, numsteps) - -class InstanceFieldConstGenerator(FieldConstGenerator): - - # _________________________________________________________________ - # OOSupport interface - - def push_constant(self, gen, const): - # load the singleton instance - gen.ilasm.opcode('ldsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) - gen.ilasm.opcode('ldfld %s %s::%s' % (const.get_type(), CONST_CLASS, const.name)) - - def _push_constant_during_init(self, gen, const): - # during initialization, we load the 'this' pointer from our - # argument rather than the singleton argument - gen.ilasm.opcode('ldarg.0') - gen.ilasm.opcode('ldfld %s %s::%s' % (const.get_type(), CONST_CLASS, const.name)) - - def _pre_store_constant(self, gen, const): - gen.ilasm.opcode('ldarg.0') - - def _store_constant(self, gen, const): - gen.ilasm.set_field((const.get_type(), CONST_CLASS, const.name)) - - # _________________________________________________________________ - # CLI interface - - def _declare_const(self, gen, all_constants): - gen.ilasm.field(const.name, const.get_type(), static=False) - - def _declare_step(self, gen, stepnum): - gen.ilasm.begin_function('step%d' % stepnum, [], 'void', False) - - def _close_step(self, gen, stepnum): - gen.ilasm.ret() - gen.ilasm.end_function() - - def _end_gen_constants(self, gen, numsteps): - - ilasm = gen.ilasm - - ilasm.begin_function('.ctor', [], 'void', False, 'specialname', 'rtspecialname', 'instance') - ilasm.opcode('ldarg.0') - ilasm.call('instance void object::.ctor()') - - ilasm.opcode('ldarg.0') - ilasm.opcode('stsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) - - for i in range(numsteps): - step_name = 'step%d' % i - ilasm.opcode('ldarg.0') - ilasm.call('instance void %s::%s()' % (CONST_CLASS, step_name)) - ilasm.ret() - ilasm.end_function() - - # declare&init the Singleton containing the constants - ilasm.field('Singleton', 'class %s' % CONST_CLASS, static=True) - ilasm.begin_function('.cctor', [], 'void', False, 'static', 'specialname', 'rtspecialname', 'default') - if SERIALIZE: - self._serialize_ctor() - else: - self._plain_ctor() - ilasm.end_function() - - super(StaticFieldConstGenerator, self)._end_gen_constants(gen, numsteps) - - def _plain_ctor(self): - self.ilasm.new('instance void class %s::.ctor()' % CONST_CLASS) - self.ilasm.pop() - self.ilasm.ret() - - def _serialize_ctor(self): - self.ilasm.opcode('ldstr "constants.dat"') - self.ilasm.call('object [pypylib]pypy.runtime.Utils::Deserialize(string)') - self.ilasm.opcode('dup') - self.ilasm.opcode('brfalse initialize') - self.ilasm.stderr('Constants deserialized successfully') - self.ilasm.opcode('stsfld class %s %s::Singleton' % (CONST_CLASS, CONST_CLASS)) - self.ilasm.ret() - self.ilasm.label('initialize') - self.ilasm.pop() - self.ilasm.stderr('Cannot deserialize constants... initialize them!') - self.ilasm.new('instance void class %s::.ctor()' % CONST_CLASS) - self.ilasm.opcode('ldstr "constants.dat"') - self.ilasm.call('void [pypylib]pypy.runtime.Utils::Serialize(object, string)') - self.ilasm.ret() - -class LazyConstGenerator(StaticFieldConstGenerator): - def push_constant(self, ilasm, const): - getter_name = '%s::%s' % (CONST_CLASS, 'get_%s' % const.name) - ilasm.call('%s %s()' % (const.get_type(), getter_name)) - - def _create_pointers(self, gen, all_constants): - # overload to do nothing since we handle everything in lazy fashion - pass - - def _initialize_data(self, gen, all_constants): - # overload to do nothing since we handle everything in lazy fashion - pass - - def _declare_const(self, gen, const): - # Declare the field - super(LazyConstGenerator, self)._declare_const(gen, const) - - # Create the method for accessing the field - getter_name = 'get_%s' % const.name - type_ = const.get_type() - self.ilasm.begin_function(getter_name, [], type_, False, 'static') - self.ilasm.load_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) - # if it's already initialized, just return it - self.ilasm.opcode('dup') - self.ilasm.opcode('brfalse', 'initialize') - self.ilasm.opcode('ret') - # else, initialize! - self.ilasm.label('initialize') - self.ilasm.opcode('pop') # discard the null value we know is on the stack - const.instantiate(ilasm) - self.ilasm.opcode('dup') # two dups because const.init pops the value at the end - self.ilasm.opcode('dup') - self.ilasm.store_static_constant(type_, CONST_NAMESPACE, CONST_CLASS, const.name) - const.init(ilasm) - self.ilasm.opcode('ret') - self.ilasm.end_function() # ______________________________________________________________________ # Mixins Modified: pypy/branch/pyjitpl5/pypy/translator/cli/gencli.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/translator/cli/gencli.py (original) +++ pypy/branch/pyjitpl5/pypy/translator/cli/gencli.py Thu Jul 30 11:41:10 2009 @@ -26,7 +26,7 @@ Database = LowLevelDatabase log = log - ConstantGenerator = constant.StaticFieldConstGenerator + ConstantGenerator = constant.CLIConstantGenerator InstanceConst = constant.CLIInstanceConst RecordConst = constant.CLIRecordConst ClassConst = constant.CLIClassConst From arigo at codespeak.net Thu Jul 30 11:49:38 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 11:49:38 +0200 (CEST) Subject: [pypy-svn] r66688 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730094938.3B63A169F4F@codespeak.net> Author: arigo Date: Thu Jul 30 11:49:37 2009 New Revision: 66688 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Test and fix. Remove _NOSIDEEFFECT_PTR because it didn't mean much. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Thu Jul 30 11:49:37 2009 @@ -105,10 +105,21 @@ self.find_nodes_default(op) def find_nodes_default(self, op): - if not op.has_no_side_effect_ptr(): - # default case: mark the arguments as escaping - for box in op.args: - self.getnode(box).mark_escaped() + # default case: mark the arguments as escaping + for box in op.args: + self.getnode(box).mark_escaped() + + def find_nodes_no_escape(self, op): + pass # for operations that don't escape their arguments + + find_nodes_OONONNULL = find_nodes_no_escape + find_nodes_OOISNULL = find_nodes_no_escape + find_nodes_OOIS = find_nodes_no_escape + find_nodes_OOISNOT = find_nodes_no_escape + find_nodes_ARRAYLEN_GC = find_nodes_no_escape + # XXX: OOIDENTITYHASH + # XXX: INSTANCEOF + # XXX: SUBCLASSOF def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/resoperation.py Thu Jul 30 11:49:37 2009 @@ -79,10 +79,6 @@ def has_no_side_effect(self): return rop._NOSIDEEFFECT_FIRST <= self.opnum <= rop._NOSIDEEFFECT_LAST - def has_no_side_effect_ptr(self): - return (rop._NOSIDEEFFECT_PTR_FIRST <= self.opnum <= - rop._NOSIDEEFFECT_PTR_LAST) - def can_raise(self): return rop._CANRAISE_FIRST <= self.opnum <= rop._CANRAISE_LAST @@ -156,7 +152,6 @@ # SAME_AS = 64 # gets a Const, turns it into a Box # - _NOSIDEEFFECT_PTR_FIRST = 70 # -- start of no_side_effect_ptr operations -- OONONNULL = 70 OOISNULL = 71 OOIS = 72 @@ -184,7 +179,6 @@ NEW = 123 NEW_WITH_VTABLE = 124 NEW_ARRAY = 125 - _NOSIDEEFFECT_PTR_LAST = 129 # -- end of no_side_effect_ptr operations -- _NOSIDEEFFECT_LAST = 129 # ----- end of no_side_effect operations ----- SETARRAYITEM_GC = 133 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 11:49:37 2009 @@ -538,6 +538,18 @@ # Does not work because of the variable index, 'i1'. self.find_nodes(ops, 'Not, Not') + def test_find_nodes_array_forced_1(self): + ops = """ + [p1, i1] + p2 = new_array(1, descr=arraydescr) + setarrayitem_gc(p2, 0, p1, descr=arraydescr) + p3 = getarrayitem_gc(p2, i1, descr=arraydescr) + p4 = new_with_vtable(ConstClass(node_vtable)) + jump(p4, i1) + """ + # escapes because getarrayitem_gc uses a non-constant index + self.find_nodes(ops, 'Not, Not') + # ------------------------------ # Bridge tests From antocuni at codespeak.net Thu Jul 30 12:16:27 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 12:16:27 +0200 (CEST) Subject: [pypy-svn] r66689 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090730101627.C9735169F6C@codespeak.net> Author: antocuni Date: Thu Jul 30 12:16:26 2009 New Revision: 66689 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: a failing test Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 12:16:26 2009 @@ -740,4 +740,14 @@ pass class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): - pass + + def test_find_nodes_instanceof(self): + py.test.skip('in-progress') + ops = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable)) + i1 = instanceof(ConstClass(node_vtable), p0) + jump(i1) + """ + boxes, getnode = self.find_nodes(ops, 'Not') + assert not getnode(boxes.p0).escaped From arigo at codespeak.net Thu Jul 30 12:23:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 12:23:13 +0200 (CEST) Subject: [pypy-svn] r66690 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730102313.C3431169F8F@codespeak.net> Author: arigo Date: Thu Jul 30 12:23:13 2009 New Revision: 66690 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py Log: Attacking now 'rop.NEW'. First step: specnode.py. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/specnode.py Thu Jul 30 12:23:13 2009 @@ -19,15 +19,12 @@ prebuiltNotSpecNode = NotSpecNode() -class VirtualInstanceSpecNode(SpecNode): - def __init__(self, known_class, fields): - self.known_class = known_class +class AbstractVirtualStructSpecNode(SpecNode): + def __init__(self, fields): self.fields = fields # list: [(fieldofs, subspecnode)] - def equals(self, other): - if not (isinstance(other, VirtualInstanceSpecNode) and - self.known_class.equals(other.known_class) and - len(self.fields) == len(other.fields)): + def equal_fields(self, other): + if len(self.fields) != len(other.fields): return False for i in range(len(self.fields)): o1, s1 = self.fields[i] @@ -45,6 +42,18 @@ subspecnode.extract_runtime_data(cpu, fieldbox, resultlist) +class VirtualInstanceSpecNode(AbstractVirtualStructSpecNode): + def __init__(self, known_class, fields): + AbstractVirtualStructSpecNode.__init__(self, fields) + self.known_class = known_class + + def equals(self, other): + if not (isinstance(other, VirtualInstanceSpecNode) and + self.known_class.equals(other.known_class)): + return False + return self.equal_fields(other) + + class VirtualArraySpecNode(SpecNode): def __init__(self, arraydescr, items): self.arraydescr = arraydescr @@ -72,6 +81,18 @@ subspecnode.extract_runtime_data(cpu, itembox, resultlist) +class VirtualStructSpecNode(AbstractVirtualStructSpecNode): + def __init__(self, typedescr, fields): + AbstractVirtualStructSpecNode.__init__(self, fields) + self.typedescr = typedescr + + def equals(self, other): + if not isinstance(other, VirtualStructSpecNode): + return False + assert self.typedescr == other.typedescr + return self.equal_fields(other) + + def equals_specnodes(specnodes1, specnodes2): assert len(specnodes1) == len(specnodes2) for i in range(len(specnodes1)): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 12:23:13 2009 @@ -58,6 +58,12 @@ arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) + # a GcStruct not inheriting from OBJECT + S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('b', lltype.Ptr(NODE))) + ssize = cpu.sizeof(S) + adescr = cpu.fielddescrof(S, 'a') + bdescr = cpu.fielddescrof(S, 'b') + cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2)} namespace = locals() @@ -87,9 +93,17 @@ arraydescr = cpu.arraydescrof(ootype.Array(ootype.Signed)) + # a plain Record + S = ootype.Record({'a': ootype.Signed, 'b': NODE}) + ssize = cpu.typedescrof(S) + adescr = cpu.fielddescrof(S, 'a') + bdescr = cpu.fielddescrof(S, 'b') + # force a consistent order valuedescr.sort_key() nextdescr.sort_key() + adescr.sort_key() + bdescr.sort_key() cpu.class_sizes = {node_vtable_adr: cpu.typedescrof(NODE), node_vtable_adr2: cpu.typedescrof(NODE2)} Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_specnode.py Thu Jul 30 12:23:13 2009 @@ -3,6 +3,7 @@ from pypy.jit.metainterp.specnode import prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode +from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.specnode import equals_specnodes from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin @@ -15,6 +16,11 @@ return VirtualArraySpecNode(LLtypeMixin.arraydescr, [prebuiltNotSpecNode] * length) +def _get_sspecnode(): + return VirtualStructSpecNode(LLtypeMixin.ssize, + [(LLtypeMixin.adescr, prebuiltNotSpecNode), + (LLtypeMixin.bdescr, prebuiltNotSpecNode)]) + def test_equals_specnodes(): assert equals_specnodes([prebuiltNotSpecNode, prebuiltNotSpecNode], [prebuiltNotSpecNode, prebuiltNotSpecNode]) @@ -30,6 +36,10 @@ assert not equals_specnodes([aspecnode1], [aspecnode2]) assert not equals_specnodes([aspecnode1], [prebuiltNotSpecNode]) assert not equals_specnodes([prebuiltNotSpecNode], [aspecnode2]) + sspecnode1 = _get_sspecnode() + assert equals_specnodes([sspecnode1], [sspecnode1]) + assert not equals_specnodes([sspecnode1], [prebuiltNotSpecNode]) + assert not equals_specnodes([prebuiltNotSpecNode], [sspecnode1]) def test_extract_runtime_data_1(): res = [] @@ -60,3 +70,16 @@ assert len(res) == 2 assert res[0].value == 123 assert res[1].value == 456 + +def test_extract_runtime_data_4(): + struct = lltype.malloc(LLtypeMixin.S) + struct.a = 123 + struct.b = lltype.malloc(LLtypeMixin.NODE) + structbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, struct)) + sspecnode = _get_sspecnode() + res = [] + sspecnode.extract_runtime_data(LLtypeMixin.cpu, structbox, res) + assert len(res) == 2 + assert res[0].value == 123 + assert (lltype.cast_opaque_ptr(lltype.Ptr(LLtypeMixin.NODE), res[1].value) + == struct.b) From antocuni at codespeak.net Thu Jul 30 12:26:55 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 12:26:55 +0200 (CEST) Subject: [pypy-svn] r66691 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730102655.5D62B169F93@codespeak.net> Author: antocuni Date: Thu Jul 30 12:26:54 2009 New Revision: 66691 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: test and fixes for find_nodes of instanceof, subclassof, ooidentityhash Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Thu Jul 30 12:26:54 2009 @@ -117,9 +117,9 @@ find_nodes_OOIS = find_nodes_no_escape find_nodes_OOISNOT = find_nodes_no_escape find_nodes_ARRAYLEN_GC = find_nodes_no_escape - # XXX: OOIDENTITYHASH - # XXX: INSTANCEOF - # XXX: SUBCLASSOF + find_nodes_INSTANCEOF = find_nodes_no_escape + find_nodes_OOIDENTITYHASH = find_nodes_no_escape + find_nodes_SUBCLASSOF = find_nodes_no_escape def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 12:26:54 2009 @@ -755,13 +755,22 @@ class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): - def test_find_nodes_instanceof(self): - py.test.skip('in-progress') + def test_find_nodes_oo_non_escape(self): ops = """ [i0] p0 = new_with_vtable(ConstClass(node_vtable)) i1 = instanceof(ConstClass(node_vtable), p0) + i2 = ooidentityhash(p0) jump(i1) """ boxes, getnode = self.find_nodes(ops, 'Not') assert not getnode(boxes.p0).escaped + + def test_find_nodes_subclassof(self): + ops = """ + [p0] + i1 = subclassof(p0, p0) + jump(p0) + """ + boxes, getnode = self.find_nodes(ops, 'Not') + assert not getnode(boxes.p0).escaped From antocuni at codespeak.net Thu Jul 30 12:37:04 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 12:37:04 +0200 (CEST) Subject: [pypy-svn] r66692 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730103704.142B3498439@codespeak.net> Author: antocuni Date: Thu Jul 30 12:37:03 2009 New Revision: 66692 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: after discussion with armin, we decided that we want subclassof and ooidentityhash to escape Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Thu Jul 30 12:37:03 2009 @@ -118,8 +118,6 @@ find_nodes_OOISNOT = find_nodes_no_escape find_nodes_ARRAYLEN_GC = find_nodes_no_escape find_nodes_INSTANCEOF = find_nodes_no_escape - find_nodes_OOIDENTITYHASH = find_nodes_no_escape - find_nodes_SUBCLASSOF = find_nodes_no_escape def find_nodes_NEW_WITH_VTABLE(self, op): instnode = InstanceNode() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 12:37:03 2009 @@ -755,22 +755,12 @@ class TestOOtype(BaseTestOptimizeFindNode, OOtypeMixin): - def test_find_nodes_oo_non_escape(self): + def test_find_nodes_instanceof(self): ops = """ [i0] p0 = new_with_vtable(ConstClass(node_vtable)) i1 = instanceof(ConstClass(node_vtable), p0) - i2 = ooidentityhash(p0) jump(i1) """ boxes, getnode = self.find_nodes(ops, 'Not') assert not getnode(boxes.p0).escaped - - def test_find_nodes_subclassof(self): - ops = """ - [p0] - i1 = subclassof(p0, p0) - jump(p0) - """ - boxes, getnode = self.find_nodes(ops, 'Not') - assert not getnode(boxes.p0).escaped From arigo at codespeak.net Thu Jul 30 13:15:51 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 13:15:51 +0200 (CEST) Subject: [pypy-svn] r66693 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730111551.7F416169F70@codespeak.net> Author: arigo Date: Thu Jul 30 13:15:49 2009 New Revision: 66693 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py Log: Support for rop.NEW in resume.py. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/resume.py Thu Jul 30 13:15:49 2009 @@ -73,6 +73,10 @@ vinfo = VirtualInfo(known_class, fielddescrs) self._make_virtual(virtualbox, vinfo, fieldboxes) + def make_vstruct(self, virtualbox, typedescr, fielddescrs, fieldboxes): + vinfo = VStructInfo(typedescr, fielddescrs) + self._make_virtual(virtualbox, vinfo, fieldboxes) + def make_varray(self, virtualbox, arraydescr, itemboxes): vinfo = VArrayInfo(arraydescr, len(itemboxes)) self._make_virtual(virtualbox, vinfo, itemboxes) @@ -132,16 +136,11 @@ raise NotImplementedError -class VirtualInfo(AbstractVirtualInfo): - def __init__(self, known_class, fielddescrs): - self.known_class = known_class +class AbstractVirtualStructInfo(AbstractVirtualInfo): + def __init__(self, fielddescrs): self.fielddescrs = fielddescrs #self.fieldnums = ... - def allocate(self, metainterp): - return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, - [self.known_class]) - def setfields(self, metainterp, box, fn_decode_box): for i in range(len(self.fielddescrs)): fieldbox = fn_decode_box(self.fieldnums[i]) @@ -149,6 +148,24 @@ [box, fieldbox], descr=self.fielddescrs[i]) +class VirtualInfo(AbstractVirtualStructInfo): + def __init__(self, known_class, fielddescrs): + AbstractVirtualStructInfo.__init__(self, fielddescrs) + self.known_class = known_class + + def allocate(self, metainterp): + return metainterp.execute_and_record(rop.NEW_WITH_VTABLE, + [self.known_class]) + +class VStructInfo(AbstractVirtualStructInfo): + def __init__(self, typedescr, fielddescrs): + AbstractVirtualStructInfo.__init__(self, fielddescrs) + self.typedescr = typedescr + + def allocate(self, metainterp): + return metainterp.execute_and_record(rop.NEW, [], + descr=self.typedescr) + class VArrayInfo(AbstractVirtualInfo): def __init__(self, arraydescr, length): self.arraydescr = arraydescr Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_resume.py Thu Jul 30 13:15:49 2009 @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.jit.metainterp.resume import * from pypy.jit.metainterp.history import BoxInt, BoxPtr, ConstInt, ConstAddr +from pypy.jit.metainterp.history import ConstPtr from pypy.jit.metainterp.test.test_optimizefindnode import LLtypeMixin from pypy.jit.metainterp import executor @@ -64,7 +65,8 @@ resbox = executor.execute(self.cpu, opnum, argboxes, descr) self.trace.append((opnum, [box.value for box in argboxes], - resbox and resbox.value)) + resbox and resbox.value, + descr)) return resbox demo55 = lltype.malloc(LLtypeMixin.NODE) @@ -138,13 +140,13 @@ b4tx = b2t.value._obj.container._as_ptr().next b4tx = lltype.cast_opaque_ptr(llmemory.GCREF, b4tx) assert metainterp.trace == [ - (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr], b2t.value), - (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr2], b4tx), - (rop.SETFIELD_GC, [b2t.value, b4tx], None), - (rop.SETFIELD_GC, [b2t.value, c1s.value], None), - (rop.SETFIELD_GC, [b4tx, b2t.value], None), - (rop.SETFIELD_GC, [b4tx, b3t.value], None), - (rop.SETFIELD_GC, [b4tx, b5t.value], None), + (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr], b2t.value, None), + (rop.NEW_WITH_VTABLE, [LLtypeMixin.node_vtable_adr2], b4tx, None), + (rop.SETFIELD_GC, [b2t.value, b4tx], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b2t.value, c1s.value],None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4tx, b2t.value], None, LLtypeMixin.nextdescr), + (rop.SETFIELD_GC, [b4tx, b3t.value], None, LLtypeMixin.valuedescr), + (rop.SETFIELD_GC, [b4tx, b5t.value], None, LLtypeMixin.otherdescr), ] del metainterp.trace[:] lst = reader.consume_boxes() @@ -226,9 +228,9 @@ b2t = lst[-1] assert lst == [b1t, ConstInt(1), b1t, b2t] assert metainterp.trace == [ - (rop.NEW_ARRAY, [2], b2t.value), - (rop.SETARRAYITEM_GC, [b2t.value, 0, 44], None), - (rop.SETARRAYITEM_GC, [b2t.value, 1, 111], None), + (rop.NEW_ARRAY, [2], b2t.value, LLtypeMixin.arraydescr), + (rop.SETARRAYITEM_GC, [b2t.value,0,44], None, LLtypeMixin.arraydescr), + (rop.SETARRAYITEM_GC, [b2t.value,1,111], None, LLtypeMixin.arraydescr), ] del metainterp.trace[:] lst = reader.consume_boxes() @@ -243,3 +245,52 @@ assert len(ptr) == 2 assert ptr[0] == 44 assert ptr[1] == 111 + + +def test_virtual_adder_make_vstruct(): + storage = make_demo_storage() + b1s, b2s, b3s, b4s = [BoxInt(1), BoxPtr(), BoxInt(3), BoxPtr()] + c1s = ConstInt(111) + modifier = ResumeDataVirtualAdder(storage, [b1s, b2s, b3s]) + assert not modifier.is_virtual(b1s) + assert not modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + modifier.make_vstruct(b2s, + LLtypeMixin.ssize, + [LLtypeMixin.adescr, LLtypeMixin.bdescr], + [c1s, b4s]) # new fields + assert not modifier.is_virtual(b1s) + assert modifier.is_virtual(b2s) + assert not modifier.is_virtual(b3s) + assert not modifier.is_virtual(b4s) + # done + liveboxes = modifier.finish() + assert liveboxes == [b1s, + #b2s -- virtual + b3s, + b4s] + # + NULL = ConstPtr.value + b1t, b3t, b4t = [BoxInt(11), BoxInt(33), BoxPtr()] + metainterp = MyMetaInterp(LLtypeMixin.cpu) + reader = ResumeDataReader(storage, [b1t, b3t, b4t], metainterp) + lst = reader.consume_boxes() + b2t = lst[-1] + assert lst == [b1t, ConstInt(1), b1t, b2t] + assert metainterp.trace == [ + (rop.NEW, [], b2t.value, LLtypeMixin.ssize), + (rop.SETFIELD_GC, [b2t.value, 111], None, LLtypeMixin.adescr), + (rop.SETFIELD_GC, [b2t.value, NULL], None, LLtypeMixin.bdescr), + ] + del metainterp.trace[:] + lst = reader.consume_boxes() + assert lst == [ConstInt(2), ConstInt(3)] + assert metainterp.trace == [] + lst = reader.consume_boxes() + assert lst == [b1t, b2t, b3t] + assert metainterp.trace == [] + # + ptr = b2t.value._obj.container._as_ptr() + assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S) + assert ptr.a == 111 + assert ptr.b == lltype.nullptr(LLtypeMixin.NODE) From arigo at codespeak.net Thu Jul 30 13:33:13 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 13:33:13 +0200 (CEST) Subject: [pypy-svn] r66694 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730113313.4C949169F8E@codespeak.net> Author: arigo Date: Thu Jul 30 13:33:12 2009 New Revision: 66694 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: Support rop.NEW in optimizefindnode. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizefindnode.py Thu Jul 30 13:33:12 2009 @@ -2,6 +2,7 @@ from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode +from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.history import AbstractValue, ConstInt from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.optimizeutil import av_newdict, _findall, sort_descrs @@ -12,6 +13,7 @@ UNIQUE_NO = '\x01' UNIQUE_INST = '\x02' UNIQUE_ARRAY = '\x03' +UNIQUE_STRUCT = '\x04' class InstanceNode(object): """An instance of this class is used to match the start and @@ -33,6 +35,11 @@ origitems = None # optimization; equivalent to an empty dict curitems = None # optimization; equivalent to an empty dict + # fields used to store the sahpe of the potential VirtualStruct + structdescr = None # set only on freshly-allocated or fromstart structs + #origfields = .. # same as above + #curfields = .. # same as above + dependencies = None def __init__(self, fromstart=False): @@ -69,6 +76,11 @@ if self.curitems is not None: for subnode in self.curitems.itervalues(): subnode.set_unique_nodes() + elif self.structdescr is not None: + self.unique = UNIQUE_STRUCT + if self.curfields is not None: + for subnode in self.curfields.itervalues(): + subnode.set_unique_nodes() else: self.unique = UNIQUE_NO @@ -78,6 +90,7 @@ if self.fromstart: flags += 's' if self.knownclsbox: flags += 'c' if self.arraydescr: flags += str(self.arraysize) + if self.structdescr: flags += 'S' return "" % (flags,) # ____________________________________________________________ @@ -124,6 +137,11 @@ instnode.knownclsbox = op.args[0] self.nodes[op.result] = instnode + def find_nodes_NEW(self, op): + instnode = InstanceNode() + instnode.structdescr = op.descr + self.nodes[op.result] = instnode + def find_nodes_NEW_ARRAY(self, op): lengthbox = op.args[0] if not isinstance(lengthbox, ConstInt): @@ -270,31 +288,27 @@ return self.intersect_instance(inputnode, exitnode) if unique == UNIQUE_ARRAY: return self.intersect_array(inputnode, exitnode) + if unique == UNIQUE_STRUCT: + return self.intersect_struct(inputnode, exitnode) assert 0, "unknown value for exitnode.unique: %d" % ord(unique) - def intersect_instance(self, inputnode, exitnode): - if (inputnode.knownclsbox is not None and - not inputnode.knownclsbox.equals(exitnode.knownclsbox)): - # unique match, but the class is known to be a mismatch - return prebuiltNotSpecNode - # + def compute_common_fields(self, orig, d): fields = [] - d = exitnode.curfields - if inputnode.origfields is not None: + if orig is not None: if d is not None: d = d.copy() else: d = av_newdict() - for ofs in inputnode.origfields: + for ofs in orig: d.setdefault(ofs, self.node_escaped) if d is not None: lst = d.keys() sort_descrs(lst) for ofs in lst: try: - if inputnode.origfields is None: + if orig is None: raise KeyError - node = inputnode.origfields[ofs] + node = orig[ofs] except KeyError: # field stored at exit, but not read at input. Must # still be allocated, otherwise it will be incorrectly @@ -302,6 +316,16 @@ node = self.node_fromstart specnode = self.intersect(node, d[ofs]) fields.append((ofs, specnode)) + return fields + + def intersect_instance(self, inputnode, exitnode): + if (inputnode.knownclsbox is not None and + not inputnode.knownclsbox.equals(exitnode.knownclsbox)): + # unique match, but the class is known to be a mismatch + return prebuiltNotSpecNode + # + fields = self.compute_common_fields(inputnode.origfields, + exitnode.curfields) return VirtualInstanceSpecNode(exitnode.knownclsbox, fields) def intersect_array(self, inputnode, exitnode): @@ -321,6 +345,13 @@ items.append(specnode) return VirtualArraySpecNode(exitnode.arraydescr, items) + def intersect_struct(self, inputnode, exitnode): + assert inputnode.structdescr is None + # + fields = self.compute_common_fields(inputnode.origfields, + exitnode.curfields) + return VirtualStructSpecNode(exitnode.structdescr, fields) + # ____________________________________________________________ # A subclass of NodeFinder for bridges only @@ -354,21 +385,23 @@ # unique match, but the class is known to be a mismatch return False # - d = exitnode.curfields - seen = 0 - for ofs, subspecnode in self.fields: - try: - if d is None: - raise KeyError - instnode = d[ofs] - seen += 1 - except KeyError: - instnode = NodeFinder.node_escaped - if not subspecnode.matches_instance_node(instnode): - return False - if d is not None and len(d) > seen: - return False # some key is in d but not in self.fields - return True + return matches_fields(self.fields, exitnode.curfields) + +def matches_fields(fields, d): + seen = 0 + for ofs, subspecnode in fields: + try: + if d is None: + raise KeyError + instnode = d[ofs] + seen += 1 + except KeyError: + instnode = NodeFinder.node_escaped + if not subspecnode.matches_instance_node(instnode): + return False + if d is not None and len(d) > seen: + return False # some key is in d but not in fields + return True class __extend__(VirtualArraySpecNode): def make_instance_node(self): @@ -396,6 +429,19 @@ return False return True +class __extend__(VirtualStructSpecNode): + def make_instance_node(self): + raise AssertionError, "not implemented (but not used actually)" + def matches_instance_node(self, exitnode): + if exitnode.unique == UNIQUE_NO: + return False + # + assert exitnode.unique == UNIQUE_STRUCT + assert self.typedescr == exitnode.structdescr + # + return matches_fields(self.fields, exitnode.curfields) + + class BridgeSpecializationFinder(NodeFinder): def find_nodes_bridge(self, bridge, specnodes=None): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 13:33:12 2009 @@ -14,6 +14,7 @@ from pypy.jit.metainterp.specnode import NotSpecNode, prebuiltNotSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode +from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.test.oparser import parse @@ -124,18 +125,25 @@ self.cpu) else: return ConstObj(ootype.cast_to_object(cls_vtable)) - def makeVirtual(cls_vtable, **kwds_fields): + def parsefields(kwds_fields): fields = [] for key, value in kwds_fields.items(): fields.append((self.namespace[key], value)) fields.sort(key = lambda (x, _): x.sort_key()) + return fields + def makeVirtual(cls_vtable, **kwds_fields): + fields = parsefields(kwds_fields) return VirtualInstanceSpecNode(constclass(cls_vtable), fields) def makeVirtualArray(arraydescr, *items): return VirtualArraySpecNode(arraydescr, items) + def makeVirtualStruct(typedescr, **kwds_fields): + fields = parsefields(kwds_fields) + return VirtualStructSpecNode(typedescr, fields) # context = {'Not': prebuiltNotSpecNode, 'Virtual': makeVirtual, - 'VArray': makeVirtualArray} + 'VArray': makeVirtualArray, + 'VStruct': makeVirtualStruct} lst = eval('[' + text + ']', self.namespace, context) return lst @@ -564,6 +572,28 @@ # escapes because getarrayitem_gc uses a non-constant index self.find_nodes(ops, 'Not, Not') + def test_find_nodes_struct_virtual_1(self): + ops = """ + [i1, p2] + i2 = getfield_gc(p2, descr=adescr) + escape(i2) + p3 = new(descr=ssize) + setfield_gc(p3, i1, descr=adescr) + jump(i1, p3) + """ + self.find_nodes(ops, 'Not, VStruct(ssize, adescr=Not)') + + def test_find_nodes_struct_nonvirtual_1(self): + ops = """ + [i1, p2] + i2 = getfield_gc(p2, descr=adescr) + escape(p2) + p3 = new(descr=ssize) + setfield_gc(p3, i1, descr=adescr) + jump(i1, p3) + """ + self.find_nodes(ops, 'Not, Not') + # ------------------------------ # Bridge tests From arigo at codespeak.net Thu Jul 30 14:08:57 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 14:08:57 +0200 (CEST) Subject: [pypy-svn] r66695 - pypy/branch/pyjitpl5/pypy/jit/backend/llgraph Message-ID: <20090730120857.A356E169F77@codespeak.net> Author: arigo Date: Thu Jul 30 14:08:55 2009 New Revision: 66695 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Log: Implement do_new for ootype backends. Not directly tested for now (there are some skipped start-of-tests in backend/test/runner_test). Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/runner.py Thu Jul 30 14:08:55 2009 @@ -489,6 +489,11 @@ assert len(args) == 1 return typedescr.create_array(args[0]) + def do_new(self, args, typedescr): + assert isinstance(typedescr, TypeDescr) + assert len(args) == 0 + return typedescr.create() + def do_runtimenew(self, args, descr): "NOT_RPYTHON" classbox = args[0] From arigo at codespeak.net Thu Jul 30 14:10:45 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 14:10:45 +0200 (CEST) Subject: [pypy-svn] r66696 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730121045.DA290169F77@codespeak.net> Author: arigo Date: Thu Jul 30 14:10:44 2009 New Revision: 66696 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Implement rop.NEW in optimizeopt. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Thu Jul 30 14:10:44 2009 @@ -2,8 +2,10 @@ from pypy.jit.metainterp.history import Const, ConstInt, ConstPtr, ConstObj from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.metainterp.specnode import SpecNode +from pypy.jit.metainterp.specnode import AbstractVirtualStructSpecNode from pypy.jit.metainterp.specnode import VirtualInstanceSpecNode from pypy.jit.metainterp.specnode import VirtualArraySpecNode +from pypy.jit.metainterp.specnode import VirtualStructSpecNode from pypy.jit.metainterp.optimizeutil import av_newdict2, _findall, sort_descrs from pypy.jit.metainterp import resume, compile from pypy.rlib.objectmodel import we_are_translated @@ -121,11 +123,10 @@ return self.box -class VirtualValue(AbstractVirtualValue): +class AbstractVirtualStructValue(AbstractVirtualValue): - def __init__(self, optimizer, known_class, keybox, source_op=None): + def __init__(self, optimizer, keybox, source_op=None): AbstractVirtualValue.__init__(self, optimizer, keybox, source_op) - self.known_class = known_class self._fields = av_newdict2() def getfield(self, ofs, default): @@ -149,6 +150,28 @@ newoperations.append(op) return self.box + def get_args_for_fail(self, modifier): + if self.box is None and not modifier.is_virtual(self.keybox): + # modifier.is_virtual() checks for recursion: it is False unless + # we have already seen the very same keybox + lst = self._fields.keys() + sort_descrs(lst) + fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] + self._make_virtual(modifier, lst, fieldboxes) + for ofs in lst: + fieldvalue = self._fields[ofs] + fieldvalue.get_args_for_fail(modifier) + + def _make_virtual(self, modifier, fielddescrs, fieldboxes): + raise NotImplementedError + + +class VirtualValue(AbstractVirtualStructValue): + + def __init__(self, optimizer, known_class, keybox, source_op=None): + AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op) + self.known_class = known_class + def prepare_force_box(self): # This logic is not included in force_box() for safety reasons. # It should only be used from teardown_virtual_node(); if we @@ -161,18 +184,28 @@ [self.known_class], self.optimizer.new_ptr_box()) - def get_args_for_fail(self, modifier): - if self.box is None and not modifier.is_virtual(self.keybox): - # modifier.is_virtual() checks for recursion: it is False unless - # we have already seen the very same keybox - lst = self._fields.keys() - sort_descrs(lst) - fieldboxes = [self._fields[ofs].get_key_box() for ofs in lst] - modifier.make_virtual(self.keybox, self.known_class, - lst, fieldboxes) - for ofs in lst: - fieldvalue = self._fields[ofs] - fieldvalue.get_args_for_fail(modifier) + def _make_virtual(self, modifier, fielddescrs, fieldboxes): + modifier.make_virtual(self.keybox, self.known_class, + fielddescrs, fieldboxes) + + +class VStructValue(AbstractVirtualStructValue): + + def __init__(self, optimizer, structdescr, keybox, source_op=None): + AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op) + self.structdescr = structdescr + + def prepare_force_box(self): + if self.box is None and self.source_op is None: + # rare case (shown by test_p123_vstruct) to force a Virtual + # from a specnode computed by optimizefindnode. + self.source_op = ResOperation(rop.NEW, [], + self.optimizer.new_ptr_box(), + descr=self.structdescr) + + def _make_virtual(self, modifier, fielddescrs, fieldboxes): + modifier.make_vstruct(self.keybox, self.structdescr, + fielddescrs, fieldboxes) class VArrayValue(AbstractVirtualValue): @@ -214,7 +247,7 @@ def prepare_force_box(self): if self.box is None and self.source_op is None: - # rare case (shown by test_p123_simple) to force a VirtualArray + # rare case (shown by test_p123_varray) to force a VirtualArray # from a specnode computed by optimizefindnode. self.source_op = ResOperation(rop.NEW_ARRAY, [ConstInt(self.getlength())], @@ -244,19 +277,29 @@ value.prepare_force_box() newexitargs.append(value.force_box()) -class __extend__(VirtualInstanceSpecNode): +class __extend__(AbstractVirtualStructSpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): - vvalue = optimizer.make_virtual(self.known_class, box) + vvalue = self._setup_virtual_node_1(optimizer, box) for ofs, subspecnode in self.fields: subbox = optimizer.new_box(ofs) subspecnode.setup_virtual_node(optimizer, subbox, newinputargs) vvalue.setfield(ofs, optimizer.getvalue(subbox)) + def _setup_virtual_node_1(self, optimizer, box): + raise NotImplementedError def teardown_virtual_node(self, optimizer, value, newexitargs): assert value.is_virtual() for ofs, subspecnode in self.fields: subvalue = value.getfield(ofs, optimizer.new_const(ofs)) subspecnode.teardown_virtual_node(optimizer, subvalue, newexitargs) +class __extend__(VirtualInstanceSpecNode): + def _setup_virtual_node_1(self, optimizer, box): + return optimizer.make_virtual(self.known_class, box) + +class __extend__(VirtualStructSpecNode): + def _setup_virtual_node_1(self, optimizer, box): + return optimizer.make_vstruct(self.typedescr, box) + class __extend__(VirtualArraySpecNode): def setup_virtual_node(self, optimizer, box, newinputargs): vvalue = optimizer.make_varray(self.arraydescr, len(self.items), box) @@ -316,6 +359,11 @@ self.make_equal_to(box, vvalue) return vvalue + def make_vstruct(self, structdescr, box, source_op=None): + vvalue = VStructValue(self, structdescr, box, source_op) + self.make_equal_to(box, vvalue) + return vvalue + def new_ptr_box(self): if not self.cpu.is_oo: return BoxPtr() @@ -555,6 +603,9 @@ def optimize_NEW_WITH_VTABLE(self, op): self.make_virtual(op.args[0], op.result, op) + def optimize_NEW(self, op): + self.make_vstruct(op.descr, op.result, op) + def optimize_NEW_ARRAY(self, op): sizebox = op.args[0] if self.is_constant(sizebox): Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 14:10:44 2009 @@ -762,6 +762,43 @@ self.optimize_loop(ops, 'Not, VArray(arraydescr, Not), Not', expected) + def test_vstruct_1(self): + ops = """ + [i1, p2] + i2 = getfield_gc(p2, descr=adescr) + escape(i2) + p3 = new(descr=ssize) + setfield_gc(p3, i1, descr=adescr) + jump(i1, p3) + """ + expected = """ + [i1, i2] + escape(i2) + jump(i1, i1) + """ + self.optimize_loop(ops, 'Not, VStruct(ssize, adescr=Not)', expected) + + def test_p123_vstruct(self): + ops = """ + [i1, p2, p3] + i3 = getfield_gc(p3, descr=adescr) + escape(i3) + p1 = new(descr=ssize) + setfield_gc(p1, i1, descr=adescr) + jump(i1, p1, p2) + """ + expected = """ + [i1, i2, p3] + i3 = getfield_gc(p3, descr=adescr) + escape(i3) + p2b = new(descr=ssize) + setfield_gc(p2b, i2, descr=adescr) + jump(i1, i1, p2b) + """ + # We cannot track virtuals that survive for more than two iterations. + self.optimize_loop(ops, 'Not, VStruct(ssize, adescr=Not), Not', + expected) + # ---------- def make_fail_descr(self): @@ -793,11 +830,17 @@ for match, end in zip(parts, ends[1:]): pvar = match.group(1) fieldstext = text[match.end():end] - if match.group(2) != 'list': - tag = ('virtual', self.namespace[match.group(2)]) - else: + if match.group(2) == 'list': arrayname, fieldstext = fieldstext.split(':', 1) tag = ('varray', self.namespace[arrayname.strip()]) + elif match.group(2) == 'struct': + if ',' in fieldstext: + structname, fieldstext = fieldstext.split(',', 1) + else: + structname, fieldstext = fieldstext, '' + tag = ('vstruct', self.namespace[structname.strip()]) + else: + tag = ('virtual', self.namespace[match.group(2)]) virtuals[pvar] = (tag, None, fieldstext) # def _variables_equal(box, varname, strict): @@ -817,6 +860,8 @@ assert ootype.classof(root) == tag[1] elif tag[0] == 'varray': pass # xxx check arraydescr + elif tag[0] == 'vstruct': + pass # xxx check typedescr else: assert 0 if resolved is not None: @@ -839,7 +884,7 @@ fieldtext = fieldtext.strip() if not fieldtext: continue - if tag[0] == 'virtual': + if tag[0] in ('virtual', 'vstruct'): fieldname, fieldvalue = fieldtext.split('=') fielddescr = self.namespace[fieldname.strip()] fieldbox = executor.execute(self.cpu, @@ -1036,6 +1081,30 @@ where p1 is a list arraydescr: 25, i1 ''') + def test_expand_fail_vstruct(self): + self.make_fail_descr() + ops = """ + [i1, p1] + p2 = new(descr=ssize) + setfield_gc(p2, i1, descr=adescr) + setfield_gc(p2, p1, descr=bdescr) + guard_true(i1) + fail(p2, descr=fdescr) + i3 = getfield_gc(p2, descr=adescr) + p3 = getfield_gc(p2, descr=bdescr) + jump(i3, p3) + """ + expected = """ + [i1, p1] + guard_true(i1) + fail(i1, p1, descr=fdescr) + jump(1, p1) + """ + self.optimize_loop(ops, 'Not, Not', expected, i1=1) + self.check_expanded_fail_descr('''p2 + where p2 is a struct ssize, adescr=i1, bdescr=p1 + ''') + class TestLLtype(BaseTestOptimizeOpt, LLtypeMixin): pass From arigo at codespeak.net Thu Jul 30 14:33:46 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 14:33:46 +0200 (CEST) Subject: [pypy-svn] r66697 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090730123346.568F8169E99@codespeak.net> Author: arigo Date: Thu Jul 30 14:33:45 2009 New Revision: 66697 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: A more complex test about mixing instances, arrays and structs in a fail() node with some sharing. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 14:33:45 2009 @@ -64,6 +64,7 @@ ssize = cpu.sizeof(S) adescr = cpu.fielddescrof(S, 'a') bdescr = cpu.fielddescrof(S, 'b') + arraydescr2 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(S))) cpu.class_sizes = {cpu.cast_adr_to_int(node_vtable_adr): cpu.sizeof(NODE), cpu.cast_adr_to_int(node_vtable_adr2): cpu.sizeof(NODE2)} @@ -99,6 +100,7 @@ ssize = cpu.typedescrof(S) adescr = cpu.fielddescrof(S, 'a') bdescr = cpu.fielddescrof(S, 'b') + arraydescr2 = cpu.arraydescrof(ootype.Array(S)) # force a consistent order valuedescr.sort_key() Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 14:33:45 2009 @@ -830,10 +830,10 @@ for match, end in zip(parts, ends[1:]): pvar = match.group(1) fieldstext = text[match.end():end] - if match.group(2) == 'list': + if match.group(2) == 'varray': arrayname, fieldstext = fieldstext.split(':', 1) tag = ('varray', self.namespace[arrayname.strip()]) - elif match.group(2) == 'struct': + elif match.group(2) == 'vstruct': if ',' in fieldstext: structname, fieldstext = fieldstext.split(',', 1) else: @@ -1078,7 +1078,7 @@ """ self.optimize_loop(ops, 'Not', expected, i1=1) self.check_expanded_fail_descr('''p1 - where p1 is a list arraydescr: 25, i1 + where p1 is a varray arraydescr: 25, i1 ''') def test_expand_fail_vstruct(self): @@ -1102,7 +1102,50 @@ """ self.optimize_loop(ops, 'Not, Not', expected, i1=1) self.check_expanded_fail_descr('''p2 - where p2 is a struct ssize, adescr=i1, bdescr=p1 + where p2 is a vstruct ssize, adescr=i1, bdescr=p1 + ''') + + def test_expand_fail_v_all_1(self): + self.make_fail_descr() + ops = """ + [i1, p1a, i2] + p6s = getarrayitem_gc(p1a, 0, descr=arraydescr2) + p7v = getfield_gc(p6s, descr=bdescr) + p5s = new(descr=ssize) + setfield_gc(p5s, i2, descr=adescr) + setfield_gc(p5s, p7v, descr=bdescr) + setarrayitem_gc(p1a, 1, p5s, descr=arraydescr2) + guard_true(i1) + fail(p1a, descr=fdescr) + p2s = new(descr=ssize) + p3v = new_with_vtable(ConstClass(node_vtable)) + p4a = new_array(2, descr=arraydescr2) + setfield_gc(p2s, i1, descr=adescr) + setfield_gc(p2s, p3v, descr=bdescr) + setfield_gc(p3v, i2, descr=valuedescr) + setarrayitem_gc(p4a, 0, p2s, descr=arraydescr2) + jump(i1, p4a, i2) + """ + expected = """ + [i1, ia, iv, pnull, i2] + guard_true(i1) + fail(ia, iv, i2, descr=fdescr) + jump(1, 1, i2, NULL, i2) + """ + self.optimize_loop(ops, ''' + Not, + VArray(arraydescr2, + VStruct(ssize, + adescr=Not, + bdescr=Virtual(node_vtable, + valuedescr=Not)), + Not), + Not''', expected, i1=1) + self.check_expanded_fail_descr('''p1a + where p1a is a varray arraydescr2: p6s, p5s + where p6s is a vstruct ssize, adescr=ia, bdescr=p7v + where p5s is a vstruct ssize, adescr=i2, bdescr=p7v + where p7v is a node_vtable, valuedescr=iv ''') From antocuni at codespeak.net Thu Jul 30 15:12:36 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 15:12:36 +0200 (CEST) Subject: [pypy-svn] r66698 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730131236.D5278169F79@codespeak.net> Author: antocuni Date: Thu Jul 30 15:12:35 2009 New Revision: 66698 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: implement instanceof in optimizeopt Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Thu Jul 30 15:12:35 2009 @@ -647,6 +647,12 @@ value.make_nonnull() self.optimize_default(op) + def optimize_INSTANCEOF(self, op): + value = self.getvalue(op.args[0]) + if value.has_constant_class(): + self.make_constant(op.result) + return + self.emit_operation(op) optimize_ops = _findall(Optimizer, 'optimize_') Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 15:12:35 2009 @@ -1153,4 +1153,17 @@ pass class TestOOtype(BaseTestOptimizeOpt, OOtypeMixin): - pass + + def test_instanceof(self): + ops = """ + [i0] + p0 = new_with_vtable(ConstClass(node_vtable)) + i1 = instanceof(p0, descr=nodesize) + jump(i1) + """ + expected = """ + [i0] + jump(1) + """ + self.optimize_loop(ops, 'Not', expected, i1=1) + From antocuni at codespeak.net Thu Jul 30 15:27:13 2009 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 30 Jul 2009 15:27:13 +0200 (CEST) Subject: [pypy-svn] r66699 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090730132713.64C03169F8D@codespeak.net> Author: antocuni Date: Thu Jul 30 15:27:12 2009 New Revision: 66699 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Log: use the proper signature of instanceof Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 15:27:12 2009 @@ -791,7 +791,7 @@ ops = """ [i0] p0 = new_with_vtable(ConstClass(node_vtable)) - i1 = instanceof(ConstClass(node_vtable), p0) + i1 = instanceof(p0, descr=nodesize) jump(i1) """ boxes, getnode = self.find_nodes(ops, 'Not') From arigo at codespeak.net Thu Jul 30 16:28:35 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 16:28:35 +0200 (CEST) Subject: [pypy-svn] r66700 - pypy/trunk/pypy/module/sys Message-ID: <20090730142835.D00023180F4@codespeak.net> Author: arigo Date: Thu Jul 30 16:28:33 2009 New Revision: 66700 Modified: pypy/trunk/pypy/module/sys/version.py Log: Protect these '$'-expanded strings in triple-quotes, in case svn expands them to something containing a single or double quote. For example in the Herbrew locale a date can contain a single-quote. Modified: pypy/trunk/pypy/module/sys/version.py ============================================================================== --- pypy/trunk/pypy/module/sys/version.py (original) +++ pypy/trunk/pypy/module/sys/version.py Thu Jul 30 16:28:33 2009 @@ -11,9 +11,9 @@ # the last item is replaced by the svn revision ^^^ TRIM_URL_UP_TO = 'svn/pypy/' -SVN_URL = "$HeadURL$"[10:-28] +SVN_URL = """$HeadURL$"""[10:-28] -REV = "$LastChangedRevision$"[22:-2] +REV = """$LastChangedRevision$"""[22:-2] import pypy From david at codespeak.net Thu Jul 30 16:57:19 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 30 Jul 2009 16:57:19 +0200 (CEST) Subject: [pypy-svn] r66701 - pypy/branch/io-lang/pypy/lang/io Message-ID: <20090730145719.8742D169F6E@codespeak.net> Author: david Date: Thu Jul 30 16:57:16 2009 New Revision: 66701 Modified: pypy/branch/io-lang/pypy/lang/io/message.py pypy/branch/io-lang/pypy/lang/io/model.py pypy/branch/io-lang/pypy/lang/io/objspace.py Log: add ImmutableSequence to objspace Modified: pypy/branch/io-lang/pypy/lang/io/message.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/message.py (original) +++ pypy/branch/io-lang/pypy/lang/io/message.py Thu Jul 30 16:57:16 2009 @@ -13,8 +13,7 @@ @register_method('Message', 'name') def message_name(space, w_receiver, w_message, w_context): - # XXX TODO clone from space - return W_ImmutableSequence(space, w_receiver.name) + return space.w_immutable_sequence.clone_and_init(w_receiver.name) @register_method('Message', 'argsEvaluatedIn') def message_argsEvaluatedIn(space, w_target, w_message, w_context): Modified: pypy/branch/io-lang/pypy/lang/io/model.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/model.py (original) +++ pypy/branch/io-lang/pypy/lang/io/model.py Thu Jul 30 16:57:16 2009 @@ -160,12 +160,20 @@ class W_ImmutableSequence(W_Object): - def __init__(self, space, string): + def __init__(self, space, string, protos=[]): + W_Object.__init__(self, space, protos) self.value = string def hash(self): return hash(self.value) + + def clone(self): + return W_ImmutableSequence(self.space, self.value, [self]) + def clone_and_init(self, value): + ims = self.clone() + ims.value = value + return ims class W_CFunction(W_Object): def __init__(self, space, function): self.function = function @@ -281,4 +289,4 @@ except ValueError: pass if literal.startswith('"') and literal.endswith('"'): - return W_ImmutableSequence(space, literal[1:-1]) \ No newline at end of file + return space.w_immutable_sequence.clone_and_init(literal[1:-1]) \ No newline at end of file Modified: pypy/branch/io-lang/pypy/lang/io/objspace.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/objspace.py (original) +++ pypy/branch/io-lang/pypy/lang/io/objspace.py Thu Jul 30 16:57:16 2009 @@ -34,6 +34,10 @@ self.w_continue = W_Object(self, [self.w_object]) self.w_return = W_Object(self, [self.w_object]) self.w_eol = W_Object(self, [self.w_object]) + + # XXX TODO: change this when Sequence is implemented + self.w_sequence = W_Object(self, [self.w_object]) + self.w_immutable_sequence = W_ImmutableSequence(self, '', [self.w_sequence]) # default stop state self.stop_status = self.w_normal self.w_return_value = self.w_nil @@ -89,6 +93,7 @@ self.w_core.protos.append(self.w_object) self.w_core.slots['Locals'] = self.w_locals self.w_core.slots['Block'] = self.w_block + self.w_core.slots['Coroutine'] = self.w_coroutine self.w_core.slots['Object'] = self.w_object self.w_core.slots['true'] = self.w_true self.w_core.slots['false'] = self.w_false @@ -96,8 +101,10 @@ self.w_core.slots['List'] = self.w_list self.w_core.slots['Call'] = self.w_call self.w_core.slots['Map'] = self.w_map + self.w_core.slots['Message'] = self.w_message self.w_core.slots['Number'] = self.w_number - self.w_core.slots['Coroutine'] = self.w_coroutine + self.w_core.slots['Sequence'] = self.w_sequence + self.w_core.slots['ImmutableSequence'] = self.w_immutable_sequence def init_w_number(self): self.w_number = instantiate(W_Number) From david at codespeak.net Thu Jul 30 16:57:59 2009 From: david at codespeak.net (david at codespeak.net) Date: Thu, 30 Jul 2009 16:57:59 +0200 (CEST) Subject: [pypy-svn] r66702 - in pypy/branch/io-lang/pypy/lang/io: . test Message-ID: <20090730145759.BBC89169F71@codespeak.net> Author: david Date: Thu Jul 30 16:57:59 2009 New Revision: 66702 Modified: pypy/branch/io-lang/pypy/lang/io/object.py pypy/branch/io-lang/pypy/lang/io/test/test_object.py Log: setSlotWithType method on Object Modified: pypy/branch/io-lang/pypy/lang/io/object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/object.py Thu Jul 30 16:57:59 2009 @@ -6,6 +6,12 @@ w_target.slots[name] = w_value return w_value + at register_method('Object', 'setSlotWithType', unwrap_spec=[object, str, object]) +def w_object_set_slot_with_type(space, w_target, name, w_value): + w_object_set_slot(space, w_target, name, w_value) + w_object_set_slot(space, w_value, "type", space.w_immutable_sequence.clone_and_init(name)) + return w_value + @register_method('Object', 'getSlot', unwrap_spec=[object, str]) def w_object_get_slot(space, w_target, name): try: Modified: pypy/branch/io-lang/pypy/lang/io/test/test_object.py ============================================================================== --- pypy/branch/io-lang/pypy/lang/io/test/test_object.py (original) +++ pypy/branch/io-lang/pypy/lang/io/test/test_object.py Thu Jul 30 16:57:59 2009 @@ -1,5 +1,5 @@ from pypy.lang.io.parserhack import parse, interpret -from pypy.lang.io.model import W_Object, W_Message, W_Number +from pypy.lang.io.model import W_Object, W_Message, W_Number, W_ImmutableSequence import py.test @@ -212,4 +212,13 @@ inp = 'stopStatus(return 42)' res, space = interpret(inp) - assert res is space.w_return \ No newline at end of file + assert res is space.w_return + +def test_set_slot_with_type(): + inp = """a := Object clone + setSlotWithType("foo", a) + """ + res, space = interpret(inp) + assert isinstance(res.slots['type'], W_ImmutableSequence) + assert res.slots['type'].value == 'foo' + assert space.w_lobby.slots['foo'] == res \ No newline at end of file From arigo at codespeak.net Thu Jul 30 17:10:06 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 17:10:06 +0200 (CEST) Subject: [pypy-svn] r66703 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730151006.E435D169F9B@codespeak.net> Author: arigo Date: Thu Jul 30 17:10:06 2009 New Revision: 66703 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Test and fix. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Thu Jul 30 17:10:06 2009 @@ -609,7 +609,11 @@ def optimize_NEW_ARRAY(self, op): sizebox = op.args[0] if self.is_constant(sizebox): - self.make_varray(op.descr, sizebox.getint(), op.result, op) + size = sizebox.getint() + if not isinstance(sizebox, ConstInt): + op = ResOperation(rop.NEW_ARRAY, [ConstInt(size)], op.result, + descr=op.descr) + self.make_varray(op.descr, size, op.result, op) else: self.optimize_default(op) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 17:10:06 2009 @@ -762,6 +762,24 @@ self.optimize_loop(ops, 'Not, VArray(arraydescr, Not), Not', expected) + def test_varray_forced_1(self): + ops = """ + [] + p2 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2, 3, descr=valuedescr) + i1 = getfield_gc(p2, descr=valuedescr) # i1 = const 3 + p1 = new_array(i1, descr=arraydescr) + escape(p1) + jump() + """ + expected = """ + [] + p1 = new_array(3, descr=arraydescr) + escape(p1) + jump() + """ + self.optimize_loop(ops, '', expected, i1=3) + def test_vstruct_1(self): ops = """ [i1, p2] From arigo at codespeak.net Thu Jul 30 20:00:25 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 20:00:25 +0200 (CEST) Subject: [pypy-svn] r66704 - in pypy/branch/pyjitpl5/pypy/jit/metainterp: . test Message-ID: <20090730180025.C0DC2169FC3@codespeak.net> Author: arigo Date: Thu Jul 30 20:00:23 2009 New Revision: 66704 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Test and fix, tentatively -- is it really doing the right thing with respect to sharing? Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/optimizeopt.py Thu Jul 30 20:00:23 2009 @@ -48,9 +48,6 @@ def force_box(self): return self.box - def prepare_force_box(self): - pass - def get_key_box(self): return self.box @@ -138,8 +135,8 @@ def force_box(self): if self.box is None: - assert self.source_op is not None # otherwise, we are trying - # to force a Virtual from a specnode computed by optimizefindnode. + if self.source_op is None: + self.prepare_force_box() newoperations = self.optimizer.newoperations newoperations.append(self.source_op) self.box = box = self.source_op.result @@ -150,6 +147,9 @@ newoperations.append(op) return self.box + def prepare_force_box(self): + raise NotImplementedError + def get_args_for_fail(self, modifier): if self.box is None and not modifier.is_virtual(self.keybox): # modifier.is_virtual() checks for recursion: it is False unless @@ -173,16 +173,16 @@ self.known_class = known_class def prepare_force_box(self): - # This logic is not included in force_box() for safety reasons. - # It should only be used from teardown_virtual_node(); if we - # call force_box() from somewhere else and we get source_op=None, - # it is really a bug. - if self.box is None and self.source_op is None: - # rare case (shown by test_p123_simple) to force a Virtual - # from a specnode computed by optimizefindnode. - self.source_op = ResOperation(rop.NEW_WITH_VTABLE, - [self.known_class], - self.optimizer.new_ptr_box()) + # rare case (shown by test_p123_simple) to force a Virtual + # from a specnode computed by optimizefindnode. + assert self.optimizer.reached_the_end + # The previous check is done for safety reasons: + # this function should only be used from teardown_virtual_node(); + # if we call force_box() from somewhere else and we get + # source_op=None, it is really a bug. + self.source_op = ResOperation(rop.NEW_WITH_VTABLE, + [self.known_class], + self.optimizer.new_ptr_box()) def _make_virtual(self, modifier, fielddescrs, fieldboxes): modifier.make_virtual(self.keybox, self.known_class, @@ -196,12 +196,12 @@ self.structdescr = structdescr def prepare_force_box(self): - if self.box is None and self.source_op is None: - # rare case (shown by test_p123_vstruct) to force a Virtual - # from a specnode computed by optimizefindnode. - self.source_op = ResOperation(rop.NEW, [], - self.optimizer.new_ptr_box(), - descr=self.structdescr) + # rare case (shown by test_p123_vstruct) to force a Virtual + # from a specnode computed by optimizefindnode. + assert self.optimizer.reached_the_end + self.source_op = ResOperation(rop.NEW, [], + self.optimizer.new_ptr_box(), + descr=self.structdescr) def _make_virtual(self, modifier, fielddescrs, fieldboxes): modifier.make_vstruct(self.keybox, self.structdescr, @@ -230,8 +230,8 @@ def force_box(self): if self.box is None: - assert self.source_op is not None # otherwise, we are trying - # to force a VArray from a specnode computed by optimizefindnode. + if self.source_op is None: + self.prepare_force_box() newoperations = self.optimizer.newoperations newoperations.append(self.source_op) self.box = box = self.source_op.result @@ -246,13 +246,13 @@ return self.box def prepare_force_box(self): - if self.box is None and self.source_op is None: - # rare case (shown by test_p123_varray) to force a VirtualArray - # from a specnode computed by optimizefindnode. - self.source_op = ResOperation(rop.NEW_ARRAY, - [ConstInt(self.getlength())], - self.optimizer.new_ptr_box(), - descr=self.arraydescr) + # rare case (shown by test_p123_varray) to force a VirtualArray + # from a specnode computed by optimizefindnode. + assert self.optimizer.reached_the_end + self.source_op = ResOperation(rop.NEW_ARRAY, + [ConstInt(self.getlength())], + self.optimizer.new_ptr_box(), + descr=self.arraydescr) def get_args_for_fail(self, modifier): if self.box is None and not modifier.is_virtual(self.keybox): @@ -274,7 +274,6 @@ def setup_virtual_node(self, optimizer, box, newinputargs): newinputargs.append(box) def teardown_virtual_node(self, optimizer, value, newexitargs): - value.prepare_force_box() newexitargs.append(value.force_box()) class __extend__(AbstractVirtualStructSpecNode): @@ -323,6 +322,7 @@ self.cpu = cpu self.loop = loop self.values = {} + self.reached_the_end = False def getvalue(self, box): try: @@ -489,6 +489,7 @@ self.emit_operation(op) def optimize_JUMP(self, op): + self.reached_the_end = True orgop = self.loop.operations[-1] exitargs = [] specnodes = orgop.jump_target.specnodes Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizefindnode.py Thu Jul 30 20:00:23 2009 @@ -90,6 +90,7 @@ nodebox2 = BoxObj(ootype.cast_to_object(node)) valuedescr = cpu.fielddescrof(NODE, 'value') nextdescr = cpu.fielddescrof(NODE, 'next') + otherdescr = cpu.fielddescrof(NODE2, 'other') nodesize = cpu.typedescrof(NODE) nodesize2 = cpu.typedescrof(NODE2) Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 20:00:23 2009 @@ -358,6 +358,39 @@ 'Not, Virtual(node_vtable, valuedescr=Not), Not', expected) + def test_p123_nested(self): + ops = """ + [i1, p2, p3] + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + p1sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p1, i1, descr=valuedescr) + setfield_gc(p1, p1sub, descr=nextdescr) + setfield_gc(p1sub, i1, descr=valuedescr) + jump(i1, p1, p2) + """ + expected = """ + [i1, i2, i2sub, p3] + i3 = getfield_gc(p3, descr=valuedescr) + escape(i3) + p2b = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p2b, i2, descr=valuedescr) + p2sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2sub, i2sub, descr=valuedescr) + setfield_gc(p2b, p2sub, descr=nextdescr) + jump(i1, i1, i1, p2b) + """ + # The same as test_p123_simple, but with a virtual containing another + # virtual. + self.optimize_loop(ops, '''Not, + Virtual(node_vtable, + valuedescr=Not, + nextdescr=Virtual(node_vtable2, + valuedescr=Not)), + Not''', + expected) + # ---------- def test_fold_guard_no_exception(self): From arigo at codespeak.net Thu Jul 30 20:20:09 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 20:20:09 +0200 (CEST) Subject: [pypy-svn] r66705 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090730182009.E9C71169FE2@codespeak.net> Author: arigo Date: Thu Jul 30 20:20:09 2009 New Revision: 66705 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Another passing test. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 20:20:09 2009 @@ -391,6 +391,33 @@ Not''', expected) + def test_p123_anti_nested(self): + ops = """ + [i1, p2, p3] + p3sub = getfield_gc(p3, descr=nextdescr) + i3 = getfield_gc(p3sub, descr=valuedescr) + escape(i3) + p1 = new_with_vtable(ConstClass(node_vtable)) + p2sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2sub, i1, descr=valuedescr) + setfield_gc(p2, p2sub, descr=nextdescr) + jump(i1, p1, p2) + """ + expected = """ + [i1, p3] + p3sub = getfield_gc(p3, descr=nextdescr) + i3 = getfield_gc(p3sub, descr=valuedescr) + escape(i3) + p2b = new_with_vtable(ConstClass(node_vtable)) + p2sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2sub, i1, descr=valuedescr) + setfield_gc(p2b, p2sub, descr=nextdescr) + jump(i1, p2b) + """ + # The same as test_p123_simple, but in the end the "old" p2 contains + # a "young" virtual p2sub. Make sure it is all forced. + self.optimize_loop(ops, 'Not, Virtual(node_vtable), Not', expected) + # ---------- def test_fold_guard_no_exception(self): From arigo at codespeak.net Thu Jul 30 20:34:07 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 20:34:07 +0200 (CEST) Subject: [pypy-svn] r66706 - pypy/branch/pyjitpl5/pypy/jit/metainterp/test Message-ID: <20090730183407.85C08169FA8@codespeak.net> Author: arigo Date: Thu Jul 30 20:34:06 2009 New Revision: 66706 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Log: Make most tests in test_optimizeopt also actually test optimizefindnode for free. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/test/test_optimizeopt.py Thu Jul 30 20:34:06 2009 @@ -6,6 +6,7 @@ from pypy.jit.metainterp.test.test_optimizefindnode import (LLtypeMixin, OOtypeMixin, BaseTest) +from pypy.jit.metainterp.optimizefindnode import PerfectSpecializationFinder from pypy.jit.metainterp.optimizeopt import optimize_loop_1 from pypy.jit.metainterp.history import AbstractDescr, ConstInt from pypy.jit.metainterp import resume, executor, compile @@ -96,10 +97,22 @@ expected.operations, remap) - def optimize_loop(self, ops, spectext, optops, boxkinds=None, **values): + def optimize_loop(self, ops, spectext, optops, + boxkinds=None, checkspecnodes=True, **values): loop = self.parse(ops, boxkinds=boxkinds) loop.setvalues(**values) - loop.specnodes = self.unpack_specnodes(spectext) + # + if checkspecnodes: + # verify that 'spectext' is indeed what optimizefindnode would + # compute for this loop + perfect_specialization_finder = PerfectSpecializationFinder() + perfect_specialization_finder.find_nodes_loop(loop) + self.check_specnodes(loop.specnodes, spectext) + else: + # for cases where we want to see how optimizeopt behaves with + # combinations different from the one computed by optimizefindnode + loop.specnodes = self.unpack_specnodes(spectext) + # assert loop.operations[-1].opnum == rop.JUMP loop.operations[-1].jump_target = loop # @@ -463,7 +476,7 @@ jump(i, i1) """ self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', - expected) + expected, checkspecnodes=False) def test_virtual_2(self): ops = """ @@ -531,7 +544,7 @@ self.optimize_loop(ops, '''Virtual(node_vtable), Virtual(node_vtable), Not''', - expected, + expected, checkspecnodes=False, i1=1, i2=0, i3=1, i4=0, i5=1, i6=0, i7=1, i8=0, i9=1, i10=0, i11=1, i12=0) # From arigo at codespeak.net Thu Jul 30 21:51:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 21:51:15 +0200 (CEST) Subject: [pypy-svn] r66708 - pypy/trunk/pypy/config Message-ID: <20090730195115.A8C03169FDB@codespeak.net> Author: arigo Date: Thu Jul 30 21:51:13 2009 New Revision: 66708 Modified: pypy/trunk/pypy/config/translationoption.py Log: Disable the --jit option on trunk, where there is no jit directory for now. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Thu Jul 30 21:51:13 2009 @@ -98,10 +98,10 @@ default=True), # JIT generation - BoolOption("jit", "generate a JIT", - default=False, cmdline="--jit", - requires=[("translation.gc", "boehm"), - ("translation.list_comprehension_operations", True)]), + #BoolOption("jit", "generate a JIT", + # default=False, cmdline="--jit", + # requires=[("translation.gc", "boehm"), + # ("translation.list_comprehension_operations", True)]), # misc BoolOption("verbose", "Print extra information", default=False), From arigo at codespeak.net Thu Jul 30 21:56:22 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 21:56:22 +0200 (CEST) Subject: [pypy-svn] r66709 - pypy/trunk/pypy/config Message-ID: <20090730195622.41844169FEB@codespeak.net> Author: arigo Date: Thu Jul 30 21:56:19 2009 New Revision: 66709 Modified: pypy/trunk/pypy/config/translationoption.py Log: Maybe better: --jit is still recognized, but using it raises an exception pointing to branch/pyjitpl5. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Thu Jul 30 21:56:19 2009 @@ -98,10 +98,12 @@ default=True), # JIT generation - #BoolOption("jit", "generate a JIT", - # default=False, cmdline="--jit", - # requires=[("translation.gc", "boehm"), - # ("translation.list_comprehension_operations", True)]), + BoolOption("jit", "generate a JIT", + default=False, cmdline="--jit", + validator=lambda config: + py.test.fail("check out branch/pyjitpl5 instead of trunk!"), + requires=[("translation.gc", "boehm"), + ("translation.list_comprehension_operations", True)]), # misc BoolOption("verbose", "Print extra information", default=False), From arigo at codespeak.net Thu Jul 30 22:01:20 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 30 Jul 2009 22:01:20 +0200 (CEST) Subject: [pypy-svn] r66710 - pypy/trunk Message-ID: <20090730200120.34B3B169FF2@codespeak.net> Author: arigo Date: Thu Jul 30 22:01:19 2009 New Revision: 66710 Modified: pypy/trunk/LICENSE Log: Small updates. Modified: pypy/trunk/LICENSE ============================================================================== --- pypy/trunk/LICENSE (original) +++ pypy/trunk/LICENSE Thu Jul 30 22:01:19 2009 @@ -116,12 +116,12 @@ Change Maker, Sweden -License for 'lib-python/2.4.1' and 'lib-python/2.4.1-modified' +License for 'lib-python/2.5.2' and 'lib-python/2.5.2-modified' ============================================================== Except when otherwise stated (look for LICENSE files or copyright/license information at the beginning of each file) the files -in the 'lib-python/2.4.1' and 'lib-python/2.4.1-modified' directories +in the 'lib-python/2.5.2' and 'lib-python/2.5.2-modified' directories are all copyrighted by the Python Software Foundation and licensed under the Python Software License of which you can find a copy here: http://www.python.org/doc/Copyright.html @@ -134,7 +134,7 @@ http://www.gnu.org/licenses/lgpl.html License for 'pypy/translator/jvm/src/jasmin.jar' -============================================= +================================================ The file 'pypy/translator/jvm/src/jasmin.jar' is copyright (c) 1996-2004 Jon Meyer and distributed with permission. The use of Jasmin by PyPy does not imply From arigo at codespeak.net Fri Jul 31 10:48:53 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 10:48:53 +0200 (CEST) Subject: [pypy-svn] r66711 - pypy/trunk/pypy/config Message-ID: <20090731084853.0789C16C17D@codespeak.net> Author: arigo Date: Fri Jul 31 10:48:50 2009 New Revision: 66711 Modified: pypy/trunk/pypy/config/translationoption.py Log: Rewrite --jit option acceptance to be based on the presence of the 'pypy.jit' package. Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Fri Jul 31 10:48:50 2009 @@ -17,6 +17,10 @@ 'distutils', ] +def check_have_jit(config): + import pypy.jit # check out branch/pyjitpl5 instead of trunk! + + translation_optiondescription = OptionDescription( "translation", "Translation Options", [ BoolOption("stackless", "enable stackless features during compilation", @@ -100,8 +104,7 @@ # JIT generation BoolOption("jit", "generate a JIT", default=False, cmdline="--jit", - validator=lambda config: - py.test.fail("check out branch/pyjitpl5 instead of trunk!"), + validator=check_have_jit, requires=[("translation.gc", "boehm"), ("translation.list_comprehension_operations", True)]), From arigo at codespeak.net Fri Jul 31 11:19:43 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 11:19:43 +0200 (CEST) Subject: [pypy-svn] r66712 - pypy/branch/pyjitpl5/pypy/lib/app_test Message-ID: <20090731091943.5B18C169F67@codespeak.net> Author: arigo Date: Fri Jul 31 11:19:42 2009 New Revision: 66712 Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Log: Don't use the default empty string in setdefault(). In some implementations of dbm in C, you cannot set keys to empty values: if you do, the key appears to be set but getting the corresponding value gives a KeyError (on CPython too!). Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Fri Jul 31 11:19:42 2009 @@ -27,6 +27,7 @@ py.test.raises(TypeError, "d.setdefault(123, 'xyz')") py.test.raises(TypeError, "d.setdefault('xyz', 123)") py.test.raises(TypeError, "d.get(123)") - d.setdefault('xyz') - assert dict(d) == {'xyz': ''} + assert dict(d) == {} + d.setdefault('xyz', '123') + assert dict(d) == {'xyz': '123'} d.close() From arigo at codespeak.net Fri Jul 31 11:30:15 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 11:30:15 +0200 (CEST) Subject: [pypy-svn] r66713 - in pypy/branch/pyjitpl5/pypy/lib: . app_test Message-ID: <20090731093015.0C082169F9D@codespeak.net> Author: arigo Date: Fri Jul 31 11:30:15 2009 New Revision: 66713 Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py pypy/branch/pyjitpl5/pypy/lib/dbm.py Log: No point in trying first with DBM_INSERT and then using DBM_REPLACE. Modified: pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/app_test/test_dbm_extra.py Fri Jul 31 11:30:15 2009 @@ -31,3 +31,12 @@ d.setdefault('xyz', '123') assert dict(d) == {'xyz': '123'} d.close() + +def test_multiple_sets(): + path = str(udir.join('test_dbm_extra.test_multiple_sets')) + d = dbm.open(path, 'c') + d['xyz'] = '12' + d['xyz'] = '3' + d['xyz'] = '546' + assert dict(d) == {'xyz': '546'} + assert d['xyz'] == '546' Modified: pypy/branch/pyjitpl5/pypy/lib/dbm.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/lib/dbm.py (original) +++ pypy/branch/pyjitpl5/pypy/lib/dbm.py Fri Jul 31 11:30:15 2009 @@ -71,9 +71,7 @@ data = datum() data.dptr = c_char_p(value) data.dsize = c_int(len(value)) - status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_INSERT) - if status == 1: - status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_REPLACE) + status = getattr(lib, funcs['store'])(self._aobj, dat, data, lib.DBM_REPLACE) if getattr(lib, funcs['error'])(self._aobj): getattr(lib, funcs['clearerr'])(self._aobj) raise error("") From arigo at codespeak.net Fri Jul 31 12:30:34 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 12:30:34 +0200 (CEST) Subject: [pypy-svn] r66714 - pypy/branch/pyjitpl5/pypy/jit/metainterp Message-ID: <20090731103034.5798E169FAA@codespeak.net> Author: arigo Date: Fri Jul 31 12:30:33 2009 New Revision: 66714 Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/heaptracker.py Log: Kill old code. Modified: pypy/branch/pyjitpl5/pypy/jit/metainterp/heaptracker.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/metainterp/heaptracker.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/metainterp/heaptracker.py Fri Jul 31 12:30:33 2009 @@ -1,58 +1,4 @@ -import re -from pypy.rpython.lltypesystem import lltype, llmemory, lloperation -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.lltypesystem import rclass -from pypy.tool.pairtype import pair, pairtype -from pypy.rlib.objectmodel import we_are_translated - -def fixupobj(EXPECTED_TYPE, x): - if isinstance(EXPECTED_TYPE, lltype.Ptr): - if lltype.typeOf(x) == llmemory.GCREF: - if x is None: - return lltype.nullptr(EXPECTED_TYPE.TO) - x = lltype.cast_opaque_ptr(EXPECTED_TYPE, x) - else: - x = x._as_ptr() - return lltype.cast_pointer(EXPECTED_TYPE, x) - #elif isinstance(EXPECTED_TYPE, ootype.Instance): - # x = realobj(x) - # return ootype._view(EXPECTED_TYPE, x) - elif EXPECTED_TYPE is lltype.Void: - return x - else: - return lltype.cast_primitive(EXPECTED_TYPE, x) - -def cast_vable(p): - T = lltype.Ptr(lltype.typeOf(p._obj._normalizedcontainer())) - p = lltype.cast_opaque_ptr(T, p) - STRUCT = cast_vable_type(T.TO) - return lltype.cast_pointer(lltype.Ptr(STRUCT), p) - -def cast_vable_type(STRUCT_OR_INST): - if isinstance(STRUCT_OR_INST, ootype.Instance): - return cast_vable_type_instance(STRUCT_OR_INST) - else: - return cast_vable_type_struct(STRUCT_OR_INST) - -def cast_vable_type_struct(STRUCT): - assert STRUCT._hints.get('virtualizable2'), \ - "not a virtualizable2: %r" % (STRUCT,) - while True: - _, PARENT = STRUCT._first_struct() - if PARENT is None or not PARENT._hints.get('virtualizable2'): - break - STRUCT = PARENT - return STRUCT - -def cast_vable_type_instance(INSTANCE): - assert INSTANCE._hints.get('virtualizable2'), \ - "not a virtualizable2: %r" % (INSTANCE,) - while True: - PARENT = INSTANCE._superclass - if PARENT is None or not PARENT._hints.get('virtualizable2'): - break - INSTANCE = PARENT - return INSTANCE +from pypy.rpython.lltypesystem import lltype, rclass def get_vtable_for_gcstruct(cpu, GCSTRUCT): From arigo at codespeak.net Fri Jul 31 12:33:29 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 12:33:29 +0200 (CEST) Subject: [pypy-svn] r66715 - pypy/branch/pyjitpl5/pypy/jit/backend/llgraph Message-ID: <20090731103329.E24A4169FAA@codespeak.net> Author: arigo Date: Fri Jul 31 12:33:29 2009 New Revision: 66715 Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Log: Kill some imports. Modified: pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/pyjitpl5/pypy/jit/backend/llgraph/llimpl.py Fri Jul 31 12:33:29 2009 @@ -10,18 +10,16 @@ from pypy.jit.metainterp.history import (ConstInt, ConstPtr, ConstAddr, ConstObj, BoxInt, BoxPtr, BoxObj) from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr -from pypy.rpython.lltypesystem import lloperation from pypy.rpython.ootypesystem import ootype from pypy.rpython.module.support import LLSupport, OOSupport from pypy.rpython.llinterp import LLException from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.jit.metainterp import heaptracker, resoperation, executor +from pypy.jit.metainterp import resoperation, executor from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.llgraph import symbolic from pypy.rlib.objectmodel import ComputedIntSymbolic -from pypy.rlib.rarithmetic import r_uint, intmask import py from pypy.tool.ansi_print import ansi_log From arigo at codespeak.net Fri Jul 31 15:10:12 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 15:10:12 +0200 (CEST) Subject: [pypy-svn] r66716 - pypy/trunk/pypy/doc/jit Message-ID: <20090731131012.B5476169FA5@codespeak.net> Author: arigo Date: Fri Jul 31 15:10:11 2009 New Revision: 66716 Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt Log: List two more blog entries. Modified: pypy/trunk/pypy/doc/jit/pyjitpl5.txt ============================================================================== --- pypy/trunk/pypy/doc/jit/pyjitpl5.txt (original) +++ pypy/trunk/pypy/doc/jit/pyjitpl5.txt Fri Jul 31 15:10:11 2009 @@ -19,3 +19,5 @@ * http://morepypy.blogspot.com/2009/04/roadmap-for-jit.html * http://morepypy.blogspot.com/2009/04/4-weeks-of-gdb.html * http://morepypy.blogspot.com/2009/05/icooolps-submissions.html +* http://morepypy.blogspot.com/2009/06/news-from-jit-front.html +* http://morepypy.blogspot.com/2009/06/jit-progress.html From cfbolz at codespeak.net Fri Jul 31 16:30:34 2009 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 31 Jul 2009 16:30:34 +0200 (CEST) Subject: [pypy-svn] r66717 - pypy/branch/shrink-multidict Message-ID: <20090731143034.EB424169F7A@codespeak.net> Author: cfbolz Date: Fri Jul 31 16:30:34 2009 New Revision: 66717 Added: pypy/branch/shrink-multidict/ - copied from r66716, pypy/trunk/ Log: A branch to try out a slightly crazy idea about how to make every (multi-)dict smaller (by saving a level of indirection in the common case). From arigo at codespeak.net Fri Jul 31 17:12:23 2009 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 31 Jul 2009 17:12:23 +0200 (CEST) Subject: [pypy-svn] r66718 - pypy/branch/pyjitpl5/pypy/module/pypyjit Message-ID: <20090731151223.2FFFC169E19@codespeak.net> Author: arigo Date: Fri Jul 31 17:12:21 2009 New Revision: 66718 Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/interp_jit.py Log: This small change is enough to add in every loop a check for the global variable set by various actions like the signal handler and the GC. This means that with smallish overhead, it should now handle Ctrl-C and app-level __del__()s correctly. Modified: pypy/branch/pyjitpl5/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/pyjitpl5/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/pyjitpl5/pypy/module/pypyjit/interp_jit.py Fri Jul 31 17:12:21 2009 @@ -6,7 +6,7 @@ import sys from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.jit import JitDriver, hint +from pypy.rlib.jit import JitDriver, hint, we_are_jitted import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, Arguments @@ -73,7 +73,11 @@ except ExitFrame: return self.popvalue() - def JUMP_ABSOLUTE(f, jumpto, next_instr, ec=None): + def JUMP_ABSOLUTE(f, jumpto, _, ec=None): + if we_are_jitted(): + f.last_instr = intmask(jumpto) + ec.bytecode_trace(f) + jumpto = r_uint(f.last_instr) pypyjitdriver.can_enter_jit(frame=f, ec=ec, next_instr=jumpto, pycode=f.getcode()) return jumpto